From ffa1a0ff2f351fa7f9bbf8a44d1ff7e250d5cd17 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 13:42:25 -0600 Subject: [PATCH 01/20] draft main test workflow --- .github/workflows/main.yaml | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/main.yaml diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..7504e61 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,42 @@ +name: Tests + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + +permissions: + contents: read + pull-requests: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python setup.py install + + - name: Test with pytest + run: | + python -m pytest tests/ \ No newline at end of file From ec4f3d6bfac0dc858b51068499dd8142b95ad144 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 13:50:23 -0600 Subject: [PATCH 02/20] add pypi publish draft --- .github/workflows/python-publish.yaml | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/python-publish.yaml diff --git a/.github/workflows/python-publish.yaml b/.github/workflows/python-publish.yaml new file mode 100644 index 0000000..fdd6571 --- /dev/null +++ b/.github/workflows/python-publish.yaml @@ -0,0 +1,40 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI }} + verbose: true \ No newline at end of file From 1fa5fe59797229a0e8be418a2bc0ebbb627e197c Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 13:52:12 -0600 Subject: [PATCH 03/20] remove old travis files --- .bumpversion.cfg | 8 -------- .travis.yml | 37 ------------------------------------- 2 files changed, 45 deletions(-) delete mode 100644 .bumpversion.cfg delete mode 100644 .travis.yml diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 126fe0b..0000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[bumpversion] -current_version = 0.4.0 -tag = True -tag_name = {} -commit = True - -[bumpversion:file:setup.py] - diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 431b005..0000000 --- a/.travis.yml +++ /dev/null @@ -1,37 +0,0 @@ -language: python -python: -- '3.5' -- '3.6' -cache: pip -before_install: -- wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O - miniconda.sh -- export MINICONDA_PREFIX="$HOME/miniconda" -- bash miniconda.sh -b -p $MINICONDA_PREFIX -- export PATH="$MINICONDA_PREFIX/bin:$PATH" -- conda config --set always_yes yes -- conda update -q conda -- conda config --add channels bioconda -- conda config --add channels conda-forge -- conda info -a -install: -- conda create -n test-env pytest pytest-cov coveralls networkx>=2 fastspar>=0.0.7 pandas numpy scipy biom-format pandas - scikit-bio statsmodels h5py pytest-runner tqdm matplotlib seaborn armadillo=8 -- source activate test-env -- conda list -- pip install --no-cache-dir --editable . -script: -- export MPLBACKEND=Agg -- py.test --cov SCNIC --cov-report term-missing -after_success: -- coveralls -deploy: - provider: pypi - skip_existing: true - user: michael.shaffer - on: - tags: true - distributions: sdist bdist_wheel - repo: shafferm/SCNIC - password: - secure: QEK8nsJtOJo2lvpVSxCX54gscFks5OCHGARbrY1KGGihvTMAyOr9wTqcnKnz6B2Zdvrz2XMKss4FaBlfo9izKCkCtALz/6QrlpgL2H2twYwDtUSVJ8rDw5j0k2JOiX6qyuklMmGTkbTXH3lEl5SNDmUtU1DUA1YUJfr/urC7BMmR6k8QZdb6/mPWEIOUG2bzcZsSxtk3Cj5o30b3OFR4Fmpt5K0L/SxYa0VBSzKfjhS/BId7X2fQtdFtVAAfqpAdE3NSGcMPgRNT3JAv4Nqei1ePzObrsEn7diOtTPusVJlDt/DEE7oTsJUi5ZedpeHCeahBLeDjaLS8unkORlU9QAjHVbN4ibMGonDaivgwDE1xHn4pRV3QVLup3Qsru/U6C+KInT3QjrEeSgLsz+HL4SmZN0wx+x6ZnkjxufzEJKxq5p/z1Ai/fci+OiVUOFdXYIqxhlsaeAPI5ky2W74zFvVC89w2bpVvFR+wqhX6z0HfeDfK6qp8hYWOmWoogyBSBqxDo9mkejgnBRKQLgtteT9kHiU5CnXlOnvvBOqCbUfQXQ2R7gKU8lTliWV9vExD+Ni6mVpWdTrfk5bcqicdMC9mdSDmdpb+p4YxHsrOHcLlYgn1ElT8Ok/vqmINP6vyD+lsGWAwxJVRokFC902rhPCzp0Fr4rIzXKl6hGQCVOE= From 7c5be234695a4e7bdb93cc0d7ffa77ba39f32645 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:07:35 -0600 Subject: [PATCH 04/20] udpate installations --- .github/workflows/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7504e61..69a2763 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,6 +35,7 @@ jobs: python -m pip install --upgrade pip pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + conda install fastspar>=0.0.7 armadillo=8 python setup.py install - name: Test with pytest From 614e204f1205579e8cc3cd5d6d7c938e71b4efe4 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:07:51 -0600 Subject: [PATCH 05/20] draft requirements for CI --- requirements.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e59562a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +numpy +networkx>=2 +biom-format +pandas>=1 +scikit-bio +statsmodels +h5py +# fastspar>=0.0.7 +# armadillo==8 +scipy<=1.10.1,>=1.8.0 \ No newline at end of file From b6df2349bd4a663c1ea7b6b4c894ab4c033154d8 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:11:16 -0600 Subject: [PATCH 06/20] remove scipy from pip installs --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 69a2763..3b024b3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,7 +35,7 @@ jobs: python -m pip install --upgrade pip pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - conda install fastspar>=0.0.7 armadillo=8 + conda install fastspar>=0.0.7 armadillo=8 scipy<=1.10.1,>=1.8.0 python setup.py install - name: Test with pytest From 6ebf577c6149836b8f26d55758f4210ab78c29fa Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:11:22 -0600 Subject: [PATCH 07/20] remove scipy from pip installs --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e59562a..39fad7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,3 @@ statsmodels h5py # fastspar>=0.0.7 # armadillo==8 -scipy<=1.10.1,>=1.8.0 \ No newline at end of file From 96e772987ab35fc1e8ecdcf06b68626f3d3c69eb Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:34:44 -0600 Subject: [PATCH 08/20] try mamba install of needed deps --- .github/workflows/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3b024b3..bf906ad 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v3 @@ -35,7 +35,7 @@ jobs: python -m pip install --upgrade pip pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - conda install fastspar>=0.0.7 armadillo=8 scipy<=1.10.1,>=1.8.0 + mamba install "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" python setup.py install - name: Test with pytest From 21827bcc9bff2c6d55dacaa54c06f8b29aa081de Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:36:45 -0600 Subject: [PATCH 09/20] switch to conda install of needed deps --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bf906ad..9d71b61 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,7 +35,7 @@ jobs: python -m pip install --upgrade pip pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - mamba install "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" + conda install "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" python setup.py install - name: Test with pytest From d0f9f0e42e22cce3dc12cfd31e524a690bb092b7 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:43:45 -0600 Subject: [PATCH 10/20] specify channels for conda --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9d71b61..bc11140 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,7 +35,7 @@ jobs: python -m pip install --upgrade pip pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - conda install "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" + conda install -c bioconda -c conda-forge "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" python setup.py install - name: Test with pytest From 5fc419ee98a2286a4be5c340a6a74cc827e96746 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:49:53 -0600 Subject: [PATCH 11/20] reorder the conda installs andcreate a conda environment --- .github/workflows/main.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bc11140..3d1c684 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -32,10 +32,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pytest + sudo apt-get update + sudo apt-get install -y conda + conda create -n test-env python=${{ matrix.python-version }} -y + conda activate test-env + conda install -c bioconda -c conda-forge pip pytest fastspar>=0.0.7 armadillo=8 "scipy<=1.10.1,>=1.8.0" numpy networkx biom-format pandas scikit-bio statsmodels h5py if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - conda install -c bioconda -c conda-forge "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" python setup.py install - name: Test with pytest From b2ede75b6bc180074bb5d21021ca42e5e5c43a0d Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 14:52:16 -0600 Subject: [PATCH 12/20] strip the environment --- .github/workflows/main.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3d1c684..dae3323 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -32,11 +32,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt-get update - sudo apt-get install -y conda - conda create -n test-env python=${{ matrix.python-version }} -y - conda activate test-env - conda install -c bioconda -c conda-forge pip pytest fastspar>=0.0.7 armadillo=8 "scipy<=1.10.1,>=1.8.0" numpy networkx biom-format pandas scikit-bio statsmodels h5py + python -m pip install --upgrade pip + conda install -c bioconda -c conda-forge "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" + pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi python setup.py install From 117db9aa5dd87007ac0143506a27a58822b49954 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:17:09 -0600 Subject: [PATCH 13/20] setup miniconda --- .github/workflows/main.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dae3323..68f29ba 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -23,17 +23,27 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10"] - + # Set it to default use bash in interactive mode (required for miniconda setup) + defaults: + run: + shell: bash -el {0} steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: create environment with mamba + uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "*" + python-version: ${{ matrix.python-version }} + channels: conda-forge,bioconda,defaults + auto-activate-base: false + environment-file: environment.yaml - name: Install dependencies run: | python -m pip install --upgrade pip - conda install -c bioconda -c conda-forge "fastspar>=0.0.7" armadillo=8 "scipy<=1.10.1,>=1.8.0" pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi python setup.py install From 5328b458601c0866a72cd5e48b93acaab7c0b581 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:18:09 -0600 Subject: [PATCH 14/20] alter environment yaml path --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 68f29ba..b990efb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -40,7 +40,7 @@ jobs: python-version: ${{ matrix.python-version }} channels: conda-forge,bioconda,defaults auto-activate-base: false - environment-file: environment.yaml + environment-file: ../environment.yaml - name: Install dependencies run: | python -m pip install --upgrade pip From e9f84403682d2ea2529999239e10821392795b8a Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:19:53 -0600 Subject: [PATCH 15/20] yaml to yml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b990efb..087d845 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -40,7 +40,7 @@ jobs: python-version: ${{ matrix.python-version }} channels: conda-forge,bioconda,defaults auto-activate-base: false - environment-file: ../environment.yaml + environment-file: ../environment.yml - name: Install dependencies run: | python -m pip install --upgrade pip From c03d6c682d8509082a3154ace24f2d7df483ab5d Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:21:59 -0600 Subject: [PATCH 16/20] try new environment path --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 087d845..268e712 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -40,7 +40,7 @@ jobs: python-version: ${{ matrix.python-version }} channels: conda-forge,bioconda,defaults auto-activate-base: false - environment-file: ../environment.yml + environment-file: environment.yml - name: Install dependencies run: | python -m pip install --upgrade pip From 1536b5ae3e1737a3db6ad0a0ea482063e44f7394 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:26:51 -0600 Subject: [PATCH 17/20] tqdm dependency --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 39fad7e..db2e13e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,4 @@ pandas>=1 scikit-bio statsmodels h5py -# fastspar>=0.0.7 -# armadillo==8 +tqdm From 40fcf972883e7788e7ef2550a694e9f869bd32f5 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:32:36 -0600 Subject: [PATCH 18/20] more dependencies --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 8dc518c..ecebb8a 100644 --- a/environment.yml +++ b/environment.yml @@ -14,6 +14,8 @@ dependencies: - python=3 - fastspar>=0.0.7 - armadillo=8 +- seaborn +- tqdm - pip: - scipy<=1.10.1,>=1.8.0 - SCNIC \ No newline at end of file From d809027c270cf032873871a938910e954d306267 Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:49:43 -0600 Subject: [PATCH 19/20] update between tester with min_r and max_p --- tests/test_between_correls.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_between_correls.py b/tests/test_between_correls.py index af04bd1..9650e9d 100644 --- a/tests/test_between_correls.py +++ b/tests/test_between_correls.py @@ -2,6 +2,7 @@ from SCNIC.general import simulate_correls from SCNIC.between_correls import between_correls from biom.util import biom_open +import pytest def test_between_correls(tmpdir): @@ -13,7 +14,14 @@ def test_between_correls(tmpdir): with biom_open(str(loc.join("table2.biom")), 'w') as f: table2.to_hdf5(f, 'madebyme') os.chdir(str(loc)) - between_correls('table1.biom', 'table2.biom', 'out_dir', correl_method='pearson', max_p=.1) - files = os.listdir(str(loc)+'/out_dir') + + # P value based module making is not yet implemented, check this + with pytest.raises(Exception) as exc_info: + between_correls('table1.biom', 'table2.biom', 'out_dir', correl_method='pearson', max_p=.1) + assert exc_info.value.args[0] == "SCNIC does not currently support module making based on p-values." + + # Run with min r threshold and check that files exist + between_correls('table1.biom', 'table2.biom', 'out_dir_2', correl_method='pearson', min_r=.2) + files = os.listdir(str(loc)+'/out_dir_2') assert "correls.txt" in files assert "crossnet.gml" in files From 22780aad0beb1ceca9e0675a28a01d1a5f4af39c Mon Sep 17 00:00:00 2001 From: John Sterrett Date: Thu, 5 Sep 2024 16:54:46 -0600 Subject: [PATCH 20/20] set to list to keep up with bumped version of dependency in indexing --- SCNIC/annotate_correls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCNIC/annotate_correls.py b/SCNIC/annotate_correls.py index 31deffc..861234b 100644 --- a/SCNIC/annotate_correls.py +++ b/SCNIC/annotate_correls.py @@ -154,7 +154,7 @@ def do_annotate_correls(correls_loc, tre_loc, genome_loc, module_loc, output_loc correls_tip_tips = tre.tip_tip_distances(set([otu for otu_pair in correls.index for otu in otu_pair])) print("read tree") genome_frame = pd.read_csv(genome_loc, sep='\t', index_col=0) - genome_table = genome_frame_to_table(genome_frame, set([otu for otu_pair in correls.index for otu in otu_pair])) + genome_table = genome_frame_to_table(genome_frame, list(set([otu for otu_pair in correls.index for otu in otu_pair]))) print("read table") if modules_to_keep_loc is not None: modules_to_keep = get_modules_to_keep(modules_to_keep_loc)