From 3e9b8d8a76d235d9a8997b3bbfc5cbc382f5375c Mon Sep 17 00:00:00 2001 From: Chris Tomkins-Tinch Date: Tue, 12 Mar 2024 11:03:59 -0400 Subject: [PATCH] bump python version 3.8 -> 3.10 (#98) * python bumped to 3.12 for Docker image, docs build, and GitHub Actions env * change lz4 pin to allow newer versions * let's try not restricting pytest to <=8 * try more flexible pinning for a few more dependencies * verbose mamba install * remove defaults conda channel remove defaults conda channel as it is now noted to be incompatible with conda-forge; see: https://mamba.readthedocs.io/en/latest/user_guide/troubleshooting.html#mixing-the-defaults-and-conda-forge-channels * set CONDA_ENVS_PATH env var (dir containing conda envs subdirs) https://conda.io/projects/conda/en/latest/user-guide/configuration/settings.html#envs-dirs-specify-environment-directories * set number of threads for conda to fetch repodata set number of threads for conda to fetch repodata , via repodata_threads; see: https://conda.io/projects/conda/en/latest/user-guide/configuration/settings.html#configuring-number-of-threads * bump pigz * print dependency trees for packages specified (individually) * set python to 3.10 set python to 3.10, as this is the most recent version in the bioconda build matrix; see: https://github.com/bioconda/bioconda-utils/blob/master/bioconda_utils/bioconda_utils-conda_build_config.yaml#L40 * add $CONDA_CHANNEL_STRING to mamba repodata depends call * unpin lz4-c * avoid syntax warning using "is" with a string literal * add DEBUG variable-based toggle of verbose output during conda install script * quote python version * actions/setup-python@v4 -> v5 * more flexible pin of matplotlib for building docs * viral-baseimage to 0.2.2 --- .github/workflows/build.yml | 4 ++-- .readthedocs.yml | 2 +- Dockerfile | 5 +++-- docker/install-conda-dependencies.sh | 17 +++++++++++++---- docs/requirements.txt | 2 +- requirements-conda-tests.txt | 2 +- requirements-conda.txt | 14 +++++++------- util/file.py | 2 +- 8 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08e20f5c..8853d6f0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,7 +139,7 @@ jobs: needs: build_docker runs-on: ubuntu-20.04 env: - GITHUB_ACTIONS_PYTHON_VERSION: 3.8 + GITHUB_ACTIONS_PYTHON_VERSION: "3.10" PYTEST_ADDOPTS: "-rsxX -n 2 --durations=25 --fixture-durations=10 --junit-xml=pytest.xml --cov-report= --cov broad_utils --cov illumina --cov read_utils --cov reports --cov tools --cov util --cov file_utils" steps: - name: checkout repository @@ -172,7 +172,7 @@ jobs: echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV - name: install python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "${{ env.GITHUB_ACTIONS_PYTHON_VERSION }}" - name: Set up Docker Buildx diff --git a/.readthedocs.yml b/.readthedocs.yml index be45417d..626666cb 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,6 +19,6 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: 3.8 + version: 3.10 install: - requirements: docs/requirements.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2be02810..6c45dfa5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ ENV \ INSTALL_PATH="/opt/viral-ngs" \ VIRAL_NGS_PATH="/opt/viral-ngs/source" \ MINICONDA_PATH="/opt/miniconda" \ - CONDA_DEFAULT_ENV=viral-ngs-env + CONDA_DEFAULT_ENV=viral-ngs-env \ + CONDA_ENVS_PATH="$MINICONDA_PATH/envs" ENV \ PATH="$VIRAL_NGS_PATH:$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV/bin:$MINICONDA_PATH/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ CONDA_PREFIX=$MINICONDA_PATH/envs/$CONDA_DEFAULT_ENV \ @@ -27,7 +28,7 @@ ENV \ # Set it up so that this slow & heavy build layer is cached # unless the requirements* files or the install scripts actually change WORKDIR $INSTALL_PATH -RUN conda create -n $CONDA_DEFAULT_ENV python=3.7 +RUN conda create -n $CONDA_DEFAULT_ENV python=3.10 RUN echo "source activate $CONDA_DEFAULT_ENV" > ~/.bashrc RUN hash -r COPY docker $VIRAL_NGS_PATH/docker/ diff --git a/docker/install-conda-dependencies.sh b/docker/install-conda-dependencies.sh index 55d714d1..aad29b28 100755 --- a/docker/install-conda-dependencies.sh +++ b/docker/install-conda-dependencies.sh @@ -9,6 +9,7 @@ set -e -o pipefail +#DEBUG=1 # set DEBUG=1 for more verbose output CONDA_INSTALL_TIMEOUT="90m" echo "PATH: ${PATH}" @@ -17,7 +18,10 @@ echo "CONDA_PREFIX: ${CONDA_PREFIX}" echo "VIRAL_NGS_PATH: ${VIRAL_NGS_PATH}" echo "MINICONDA_PATH: ${MINICONDA_PATH}" echo "CONDA_DEFAULT_ENV: ${CONDA_DEFAULT_ENV}" -CONDA_CHANNEL_STRING="--override-channels -c broad-viral -c conda-forge -c bioconda -c defaults" +CONDA_CHANNEL_STRING="--override-channels -c broad-viral -c conda-forge -c bioconda" + +# ToDo: if confirmed working, move to conda config section of viral-baseimage +conda config --set repodata_threads $(nproc) # solving the dependency graph for a conda environment can take a while. # so long, in fact, that the conda process can run for >10 minutes without @@ -43,7 +47,7 @@ function start_keepalive { >&2 echo "Running..." # Start a process that runs as a keep-alive - # to avoid travis quitting if there is no output + # to avoid having the CI running quit if there is no output (while true; do sleep 120 >&2 echo "Still running..." @@ -70,13 +74,18 @@ sync REQUIREMENTS="" for condafile in $*; do REQUIREMENTS="$REQUIREMENTS --file $condafile" + + # print dependency tree for all packages in file + [[ $DEBUG = 1 ]] && grep -vE '^#' "${condafile}" | xargs -I {} mamba repoquery depends $CONDA_CHANNEL_STRING --quiet --pretty --recursive --tree "{}"; done # run conda install with keepalive subshell process running in background # to keep travis build going. Enforce a hard timeout via timeout GNU coreutil start_keepalive -#timeout $CONDA_INSTALL_TIMEOUT conda install -y -q $CONDA_CHANNEL_STRING -p "${CONDA_PREFIX}" $REQUIREMENTS -mamba install -y -q $CONDA_CHANNEL_STRING -p "${CONDA_PREFIX}" $REQUIREMENTS +if [[ $DEBUG = 1 ]]; then + MAMBA_DEBUG_LEVEL="-vvv" +fi +mamba install -y $MAMBA_DEBUG_LEVEL -q $CONDA_CHANNEL_STRING -p "${CONDA_PREFIX}" $REQUIREMENTS stop_keepalive # clean up diff --git a/docs/requirements.txt b/docs/requirements.txt index 0f98095a..20bf937d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,7 +2,7 @@ jinja2==3.1.2 # https://github.com/readthedocs/readthedocs.org/issues/9037#issue Sphinx==5.3.0 #override sphinx pinning done by RTD: https://docs.readthedocs.io/en/stable/build-default-versions.html#external-dependencies sphinx-argparse sphinx-rtd-theme==1.1.1 -matplotlib==2.2.4 +matplotlib>=2.2.4 PyYAML==6.0 mock==5.0.1 recommonmark diff --git a/requirements-conda-tests.txt b/requirements-conda-tests.txt index 31d73bf6..4f567798 100644 --- a/requirements-conda-tests.txt +++ b/requirements-conda-tests.txt @@ -1,7 +1,7 @@ coveralls>=1.3.0 lxml>=4.3.3 mock>=2.0.0 -pytest>=7.4.3,<=8 +pytest>=7.4.3 pytest-cov>=2.6.1 pytest-mock>=1.10.0 pytest-xdist>=1.26.1 diff --git a/requirements-conda.txt b/requirements-conda.txt index 3fc5d711..8a5ad5c3 100644 --- a/requirements-conda.txt +++ b/requirements-conda.txt @@ -1,31 +1,31 @@ bbmap>=38.56 bcftools>=1.10 bedtools>=2.29.2 -bwa=0.7.17 +bwa>=0.7.17 csvkit>=1.0.4 cd-hit=4.6.8 cd-hit-auxtools=4.6.8 fastqc>=0.11.7 gatk=3.8 jq>=1.6 -lbzip2=2.5 -lz4-c=1.8.3 +lbzip2>=2.5 +lz4-c>=1.8.3 minimap2>=2.17 mvicuna=1.0 novoalign=3.09.04 parallel>=20190922 picard=2.25.6 -pigz=2.4 -prinseq=0.20.4 +pigz>=2.4 +prinseq>=0.20.4 samtools>=1.16.1 -trimmomatic=0.38 +trimmomatic>=0.38 unzip>=6.0 zstd>=1.3.8 # Python packages below arrow>=0.12.1 biopython>=1.72 firecloud>=0.16.35 -lz4=2.2.1 +lz4>=2.2.1 jinja2>=2.11.3 matplotlib>=2.2.4 pysam>=0.20.0 diff --git a/util/file.py b/util/file.py index 98402c6b..0af1f32e 100644 --- a/util/file.py +++ b/util/file.py @@ -208,7 +208,7 @@ def extract_tarball(tarfile, out_dir=None, threads=None, compression='auto', pip else: util.file.mkdir_p(out_dir) assert compression in ('gz', 'bz2', 'lz4', 'zip', 'zst', 'none', 'auto') - if compression is 'auto': + if compression == 'auto': assert tarfile != '-' or pipe_hint, "cannot autodetect on stdin input unless pipe_hint provided" # auto-detect compression type based on file name if tarfile=='-':