public commit

.dockerignore

@@ -0,0 +1,16 @@
+# files here will not be transferred to Docker daemon and therefore also not copied
+
+# ignore dot files (includes .git etc)
+.*
+# **/.git/
+# .tox/
+
+# ignore change to Dockerfile while developing, todo2
+Dockerfile
+
+runs/
+# external/
+
+experimentation/
+
+# usecases/

.github/workflows/build_doc.yml

@@ -0,0 +1,42 @@
+name: apidocs
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Setup python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install requirements for documentation generation
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install docutils pydoctor
+  
+      - name: Generate API documentation with pydoctor
+        run: |
+  
+          # Run pydoctor build
+          pydoctor \
+              --project-name="${{ github.repository }}" \
+              --project-url=https://github.com/$GITHUB_REPOSITORY \
+              --html-viewsource-base=https://github.com/$GITHUB_REPOSITORY/tree/$GITHUB_SHA \
+              --project-base-dir="$(pwd)" \
+              "${{ github.workspace }}/src/simreaduntil"
+
+      - name: Push API documentation to Github Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./apidocs
+          commit_message: "Generate API documentation"

.github/workflows/build_docker.yml

@@ -0,0 +1,138 @@
+# modified from https://docs.docker.com/build/ci/github-actions/push-multi-registries/
+
+name: build_docker
+
+on:
+  workflow_dispatch:
+  push:
+    # todo2: disable again
+    branches:
+      - "main"
+
+    tags:
+      - 'v*' # push of any tag
+#   workflow_run:
+#     workflows: ["run_tox"]
+#     branches: [main]
+#     types: 
+#     - completed
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  TEST_TAG: ${{ github.repository }}_test
+  BUILD_TARGET: dev # prod
+
+concurrency:
+  group: docker-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker:
+    permissions: 
+      packages: write
+      contents: read
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT }} # the special secrets.GITHUB_TOKEN only allows access to the current repository, so we added a personal access token with scope "repo" as a secret to this repo
+          submodules: true
+      -
+        # qemu required if building for multiple platforms with Docker (other than the platform the docker build is executed on)
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+    #   -
+    #     name: Login to Docker Hub
+    #     uses: docker/login-action@v2
+    #     with:
+    #       username: ${{ secrets.DOCKERHUB_USERNAME }}
+    #       password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          # ${{ github.repository_owner }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - 
+        name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - 
+        name: Build and export to Docker (using cache)
+        uses: docker/build-push-action@v4
+        timeout-minutes: 20
+        with:
+          context: .
+          target: ${{ env.BUILD_TARGET }}
+          load: true
+          tags: ${{ env.TEST_TAG }}
+          # todo2: disable cache eventually
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - 
+        name: Test docker image in python3.10
+        uses: addnab/docker-run-action@v3
+        with:
+          image: ${{ env.TEST_TAG }}
+          shell: /bin/bash
+          run: |
+            set -ex
+            source ~/ont_project_all/ont_project_venv/bin/activate || true
+            export PATH=~/ont_project_all/tools/bin:$PATH && which minimap2
+            cd /root/ont_project_all/ont_project
+            sed -i'.backup' -e "s@{root:uri}@file://$(pwd)@g" -e "s@[email protected]+dockerbuild$(date +%YY%mM%dD%HH%MM%SS)@" pyproject.toml
+            pip install tox
+            tox run -e py310
+      -
+        name: Build and push
+        uses: docker/build-push-action@v4
+        id: Push
+        with:
+          context: .
+          platforms: linux/amd64 #,linux/arm64
+          target: ${{ env.BUILD_TARGET }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}_${{ env.BUILD_TARGET }}
+          labels: ${{ steps.meta.outputs.labels }}
+      - 
+        name: Run full usecase (in Docker container)
+        uses: addnab/docker-run-action@v3
+        with:
+          image: ${{ env.TEST_TAG }}
+          shell: /bin/bash
+          run: |
+            set -ex
+            cd ~/ont_project_all/ont_project
+            curl https://public.bmi.inf.ethz.ch/user/mmordig/ont_project/prepackaged/usecase_data_docker.tar.gz -O
+            tar -xvzf usecase_data_docker.tar.gz
+            
+            pwd
+            ls -lR .
+            
+            cd runs/enrich_usecase
+            source ~/ont_project_all/ont_project_venv/bin/activate || true
+            export PATH=~/ont_project_all/tools/bin:$PATH && which minimap2
+            mkdir chr202122_run && cd chr202122_run
+            
+            ln -s ../data .
+            ln -s ../configs/chr202122_run/sampler_per_window configs
+            python ~/ont_project_all/ont_project/usecases/enrich_usecase.py
+
+            tar -cvzf figures.tar.gz readfish_run/figures
+      - 
+        name: Archive figures
+        uses: actions/upload-artifact@v3
+        with:
+          name: usecase-figures
+          path: figures.tar.gz

.github/workflows/publish_pypi.unyml

@@ -0,0 +1,71 @@
+# workflow currently disabled (extension .unyml) because ReadFish dependency not available on pypi, so pypi does not accept this repo
+
+name: publish_pypi
+on:
+  workflow_dispatch:
+  push:
+    # branches:
+    #   - main
+    tags:
+        - 'v*' # push of any tag
+
+# following https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+# to do it locally with hatch, do (see https://hatch.pypa.io/latest/publish/):
+# pip install hatch
+# hatch build
+# export HATCH_INDEX_AUTH=<some_token>
+# # or -r main, but does not seem to work
+# hatch publish -u '__token__' -r test
+
+# pypi refuses packages with a version derived from a normal commit (only x.x.x format), so we create a tag to have a good version name with
+# git commit, then git tag v0.0.1 && git push origin v0.0.1
+# function tag_and_push() {
+#   git tag "$1"
+#   git push origin "$1"
+# }
+
+# for trusted publishing, see https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/, set set id_token: write, and
+# need to add repository in pypi settings: https://test.pypi.org/manage/account/publishing/
+# alternatively, provide a password under the name 'TEST_PYPI_API_TOKEN' in the repository secrets under repository settings
+
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      # displayed on the github main page
+      name: testpypi
+      url: https://pypi.org/p/simreaduntil # todo
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing -> no extra pypi token required, github token enough
+      contents: read
+    steps:
+    # build package
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: >-
+        python3 -m
+        build
+        --sdist
+        --wheel
+        --outdir dist/
+
+    - name: "check tagged"
+      run: |
+        # can only upload tagged commits, see above
+        [[ "${{ github.ref_type }}" == 'tag' ]] || exit 1
+
+    - name: Publish package distributions to PyPI (commit needs to be tagged)
+    # NOTE: currently broken because pypi does not allow git repos as dependencies ("Invalid value for requires_dist. Error: Can't have direct dependency:", i.e. read-until and ont_readfish)
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        repository-url: https://test.pypi.org/legacy/

.github/workflows/run_tox.yml

@@ -0,0 +1,69 @@
+name: run_tox
+on:
+  workflow_dispatch:
+  push:
+    paths-ignore:
+    - README.md
+    - .gitignore
+  pull_request:
+  # schedule:
+  #   - cron: "0 8 * * *"
+
+concurrency:
+  group: runtests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: test with ${{ matrix.py }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        py:
+          # - "3.12"
+          # - "3.11"
+          - "3.10"
+          # - "3.9"
+          - "3.8"
+          # - "3.7" # end-of-life reached
+        os:
+          - ubuntu-latest
+          # - macos-latest
+          # - windows-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.PAT }} # the special secrets.GITHUB_TOKEN only allows access to the current repository, so we added a personal access token with scope "repo" as a secret to this repo
+          submodules: true
+      - name: Setup python for test ${{ matrix.py }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.py }}
+      - name: Install minimap2
+        run: curl -L https://github.com/lh3/minimap2/releases/download/v2.26/minimap2-2.26_x64-linux.tar.bz2 | tar -jxvf -
+      - name: Fix pyproject for tox
+        run: |
+          sed -i'.backup' "s@{root:uri}@file://$(pwd)@g" pyproject.toml
+
+          # does not work
+          # replace {root:uri} by "file:." in pyproject.toml since tox does not understand it
+          # sed -i 's/{root:uri}/file:./g' pyproject.toml
+          # remove line that contains {root:uri}
+          # sed -i '/{root:uri}/d' pyproject.toml
+      - name: Install tox
+        run: python -m pip install tox-gh>=1.2
+      - name: Setup test suite
+        run: tox -vv --notest
+      - name: Run test suite
+        run: PATH="$(pwd)/minimap2-2.26_x64-linux:$PATH" tox --skip-pkg-install
+        # run: PATH="$(pwd)/minimap2-2.26_x64-linux:$PATH" tox --skip-pkg-install -- tests/usecase_helpers/test_run_simulator_with_readfish.py
+      - name: archive if failure
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.os }}-${{ matrix.py }}-${{ github.sha }}
+          retention-days: 1
+          path: /tmp/pytest-of-runner/
+          # path: /tmp/pytest-of-runner/pytest-0/test_simulator_with_readfish0/data/run_dir/simulator_run/reads