From 20aaf6820bb77bb63a392fdae39b6f9bad4f24a9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 15:47:40 -0400 Subject: [PATCH 01/37] resolve conflicts --- .github/workflows/tests.yml | 90 ++++++++----------------------------- 1 file changed, 19 insertions(+), 71 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 799c0101e..0d7599a41 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,12 +14,12 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check" - name: Install Python 3 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -27,7 +27,7 @@ jobs: pip install flake8 - name: flake8 run: | - flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" + flake8 test: needs: lint runs-on: ubuntu-latest @@ -37,9 +37,9 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -55,88 +55,36 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./cov.xml verbose: true - update_docs: - needs: test - runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/stable') + test-distros: + needs: lint + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo, alpine] steps: - - uses: actions/checkout@v3 - with: - token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.x" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | pip install poetry poetry install - - name: Generate docs - run: | - poetry run bbot/scripts/docs.py - - name: Commit docs - uses: EndBug/add-and-commit@v9 - continue-on-error: true - with: - add: '["*.md", "docs/data/chord_graph/*.json"]' - author_name: "BBOT Docs Autopublish" - author_email: info@blacklanternsecurity.com - message: "Refresh module docs" - publish_docs: - needs: test - runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev') - steps: - - uses: actions/checkout@v3 - with: - token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - uses: actions/cache@v3 - with: - key: mkdocs-material-${{ env.cache_id }} - path: .cache - restore-keys: | - mkdocs-material- - - name: Install dependencies - run: | - pip install poetry - poetry install --only=docs - - name: Configure Git - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git fetch origin gh-pages:refs/remotes/origin/gh-pages - if git show-ref --verify --quiet refs/heads/gh-pages; then - git branch -f gh-pages origin/gh-pages - else - git branch --track gh-pages origin/gh-pages - fi - - name: Generate docs (stable branch) - if: github.ref == 'refs/heads/stable' - run: | - poetry run mike deploy Stable - - name: Generate docs (dev branch) - if: github.ref == 'refs/heads/dev' - run: | - poetry run mike deploy Dev - - name: Publish docs + - name: Run tests run: | - git switch gh-pages - git push + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . publish_code: needs: test runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable') continue-on-error: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -185,7 +133,7 @@ jobs: # runs-on: ubuntu-latest # if: github.event_name == 'push' && github.ref == 'refs/heads/stable' # steps: - # - uses: actions/checkout@v3 + # - uses: actions/checkout@v4 # with: # ref: ${{ github.head_ref }} # fetch-depth: 0 # Fetch all history for all tags and branches From f55c00441ac972dfb98acef07852c4f97cf18f3d Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 15:48:06 -0400 Subject: [PATCH 02/37] resolve conflicts --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index bbe8b7581..a6f057338 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ select = F,E722 ignore = F403,F405,F541 per-file-ignores = - */__init__.py:F401,F403 \ No newline at end of file + */__init__.py:F401,F403 From 6be3f2395432508273eae260edee11b4a3fcc122 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 18 Jun 2024 12:58:16 -0400 Subject: [PATCH 03/37] simplified run_tests.sh --- bbot/test/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/run_tests.sh b/bbot/test/run_tests.sh index 3490f5b65..39458dbf9 100755 --- a/bbot/test/run_tests.sh +++ b/bbot/test/run_tests.sh @@ -10,7 +10,7 @@ echo echo "[+] Linting with flake8" echo "=======================" -flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" "$bbot_dir" || exit 1 +flake8 "$bbot_dir" || exit 1 echo if [ "${1}x" != "x" ] ; then From c08648e0d6ca255a409f0b4f517fae987fb531cf Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 15:48:39 -0400 Subject: [PATCH 04/37] resolve conflicts --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d7599a41..bf65e13c8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,7 +48,7 @@ jobs: poetry install - name: Run tests run: | - poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . - name: Upload Code Coverage uses: codecov/codecov-action@v3 with: @@ -66,7 +66,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.x" - name: Install dependencies run: | pip install poetry From c2e69e8fbbf2a00c2cda5be20a209d22b65ccca6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 16:43:15 -0400 Subject: [PATCH 05/37] runs-on --> container: image --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf65e13c8..80a6df8eb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,7 +57,9 @@ jobs: verbose: true test-distros: needs: lint - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest + container: + image: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo, alpine] From 6875793bc646f56ead7388542a54ca928e7cac72 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 18 Oct 2024 17:34:35 -0400 Subject: [PATCH 06/37] fix python setup --- .github/workflows/tests.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 80a6df8eb..7f00ed835 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,17 +65,27 @@ jobs: os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo, alpine] steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" + - name: Install Python + run: | + if [ -f /etc/debian_version ]; then + sudo apt-get update + sudo apt-get install -y python3 python3-pip + elif [ -f /etc/arch-release ]; then + sudo pacman -Syu --noconfirm python python-pip + elif [ -f /etc/fedora-release ]; then + sudo dnf install -y python3 python3-pip + elif [ -f /etc/gentoo-release ]; then + sudo emerge dev-lang/python + elif [ -f /etc/alpine-release ]; then + sudo apk add --no-cache python3 py3-pip + fi - name: Install dependencies run: | pip install poetry poetry install - name: Run tests run: | - poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . publish_code: needs: test runs-on: ubuntu-latest From 047b9ffd1609373812704878e15e909e8d82075e Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 09:22:12 -0400 Subject: [PATCH 07/37] no sudo --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f00ed835..3d334eaba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,16 +68,16 @@ jobs: - name: Install Python run: | if [ -f /etc/debian_version ]; then - sudo apt-get update - sudo apt-get install -y python3 python3-pip + apt-get update + apt-get install -y python3 python3-pip elif [ -f /etc/arch-release ]; then - sudo pacman -Syu --noconfirm python python-pip + pacman -Syu --noconfirm python python-pip elif [ -f /etc/fedora-release ]; then - sudo dnf install -y python3 python3-pip + dnf install -y python3 python3-pip elif [ -f /etc/gentoo-release ]; then - sudo emerge dev-lang/python + emerge dev-lang/python elif [ -f /etc/alpine-release ]; then - sudo apk add --no-cache python3 py3-pip + apk add --no-cache python3 py3-pip fi - name: Install dependencies run: | From 3c6728c290f593aa57460f61420138ed349c244a Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 10:02:48 -0400 Subject: [PATCH 08/37] python things --- .github/workflows/tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3d334eaba..147fdba5f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,8 +61,9 @@ jobs: container: image: ${{ matrix.os }} strategy: + fail-fast: false matrix: - os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo, alpine] + os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo/python, python:3.10-alpine] steps: - uses: actions/checkout@v4 - name: Install Python @@ -74,10 +75,6 @@ jobs: pacman -Syu --noconfirm python python-pip elif [ -f /etc/fedora-release ]; then dnf install -y python3 python3-pip - elif [ -f /etc/gentoo-release ]; then - emerge dev-lang/python - elif [ -f /etc/alpine-release ]; then - apk add --no-cache python3 py3-pip fi - name: Install dependencies run: | From 0572ce6afb5a59ed8b1d1d15938215db3e96b5b3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 10:10:15 -0400 Subject: [PATCH 09/37] quotes --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 147fdba5f..709e11902 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo/python, python:3.10-alpine] + os: ["ubuntu-latest", "debian", "archlinux", "fedora-latest", "gentoo/python", "python:3.10-alpine"] steps: - uses: actions/checkout@v4 - name: Install Python From 5a6e68eea9368f3975b8c2055febcf656bba7124 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 10:29:53 -0400 Subject: [PATCH 10/37] testing stuff --- .github/workflows/tests.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 709e11902..83067d9ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,22 +63,25 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "debian", "archlinux", "fedora-latest", "gentoo/python", "python:3.10-alpine"] + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine"] steps: - uses: actions/checkout@v4 - - name: Install Python + - name: Install Python and Poetry run: | if [ -f /etc/debian_version ]; then apt-get update - apt-get install -y python3 python3-pip + apt-get install -y python3 python3-pip python3-poetry elif [ -f /etc/arch-release ]; then - pacman -Syu --noconfirm python python-pip + pacman -Syu --noconfirm python python-pip poetry elif [ -f /etc/fedora-release ]; then - dnf install -y python3 python3-pip + dnf install -y python3 python3-pip poetry + elif [ -f /etc/gentoo-release ]; then + emerge --update --newuse dev-python/poetry + elif [ -f /etc/alpine-release ]; then + apk add --no-cache python3 py3-pip py3-poetry fi - name: Install dependencies run: | - pip install poetry poetry install - name: Run tests run: | From ffe8ae1cc149d129e4c4989280e0330bd202bd65 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 10:39:48 -0400 Subject: [PATCH 11/37] pipx --- .github/workflows/tests.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 83067d9ca..65fb21e82 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -70,16 +70,18 @@ jobs: run: | if [ -f /etc/debian_version ]; then apt-get update - apt-get install -y python3 python3-pip python3-poetry + apt-get install -y python3 python3-pip pipx elif [ -f /etc/arch-release ]; then - pacman -Syu --noconfirm python python-pip poetry + pacman -Syu --noconfirm python python-pip pipx elif [ -f /etc/fedora-release ]; then - dnf install -y python3 python3-pip poetry + dnf install -y python3 python3-pip pipx elif [ -f /etc/gentoo-release ]; then - emerge --update --newuse dev-python/poetry + emerge --update --newuse dev-lang/python dev-python/pipx elif [ -f /etc/alpine-release ]; then - apk add --no-cache python3 py3-pip py3-poetry + apk add --no-cache python3 py3-pip py3-pipx fi + python3 -m pipx ensurepath + pipx install poetry - name: Install dependencies run: | poetry install From 46f65a4c9694b68e061295b490dcdc0f7c48654f Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 10:59:22 -0400 Subject: [PATCH 12/37] onwards --- .github/workflows/tests.yml | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65fb21e82..80f94851a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,18 +68,40 @@ jobs: - uses: actions/checkout@v4 - name: Install Python and Poetry run: | - if [ -f /etc/debian_version ]; then - apt-get update - apt-get install -y python3 python3-pip pipx + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "20.04" ]; then + # Add deadsnakes PPA for Python 3.12 + apt-get update + apt-get install -y software-properties-common + add-apt-repository ppa:deadsnakes/ppa + apt-get update + apt-get install -y python3.12 python3.12-venv python3.12-distutils python3-pip + # Update alternatives to use python3.12 + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 + update-alternatives --set python3 /usr/bin/python3.12 + python3 -m pip install --upgrade pip + python3 -m pip install pipx + else + apt-get update + apt-get install -y python3 python3-pip pipx + fi elif [ -f /etc/arch-release ]; then pacman -Syu --noconfirm python python-pip pipx elif [ -f /etc/fedora-release ]; then dnf install -y python3 python3-pip pipx elif [ -f /etc/gentoo-release ]; then + # Initialize and sync Gentoo repository + emerge-webrsync + emerge --sync emerge --update --newuse dev-lang/python dev-python/pipx elif [ -f /etc/alpine-release ]; then - apk add --no-cache python3 py3-pip py3-pipx + apk add --no-cache python3 py3-pip + python3 -m pip install --upgrade pip + python3 -m pip install pipx fi + # Add $HOME/.local/bin to PATH + export PATH="$HOME/.local/bin:$PATH" python3 -m pipx ensurepath pipx install poetry - name: Install dependencies From 50969ebb1b1f91710b5e22ec5c34fd84d58ef6f6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 11:04:33 -0400 Subject: [PATCH 13/37] distro stuff --- .github/workflows/distro_tests.yml | 67 ++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 55 ------------------------ 2 files changed, 67 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/distro_tests.yml diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml new file mode 100644 index 000000000..fd0c88efb --- /dev/null +++ b/.github/workflows/distro_tests.yml @@ -0,0 +1,67 @@ +name: Linux Distro Tests +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test-distros: + needs: lint + runs-on: ubuntu-latest + container: + image: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine"] + steps: + - uses: actions/checkout@v4 + - name: Install Python and Poetry + run: | + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "20.04" ]; then + # Add deadsnakes PPA for Python 3.12 + apt-get update + apt-get install -y software-properties-common + add-apt-repository ppa:deadsnakes/ppa + apt-get update + apt-get install -y python3.12 python3.12-venv python3.12-distutils python3-pip + # Update alternatives to use python3.12 + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 + update-alternatives --set python3 /usr/bin/python3.12 + python3 -m pip install --upgrade pip + python3 -m pip install pipx + elif [ "$ID" = "alpine" ]; then + apk add --no-cache python3 py3-pip + python3 -m pip install --upgrade pip + python3 -m pip install pipx + elif [ "$ID" = "arch" ]; then + pacman -Syu --noconfirm python python-pip pipx + elif [ "$ID" = "fedora" ]; then + dnf install -y python3 python3-pip pipx + elif [ "$ID" = "gentoo" ]; then + # Initialize and sync Gentoo repository + emerge-webrsync + emerge --sync + emerge --update --newuse dev-lang/python dev-python/pipx + else + echo "Unsupported Linux distribution" + exit 1 + fi + else + echo "Operating system not recognized" + exit 1 + fi + # Add $HOME/.local/bin to PATH + export PATH="$HOME/.local/bin:$PATH" + python3 -m pipx ensurepath + pipx install poetry + - name: Install dependencies + run: | + poetry install + - name: Run tests + run: | + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 80f94851a..3a9aecd52 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -55,61 +55,6 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./cov.xml verbose: true - test-distros: - needs: lint - runs-on: ubuntu-latest - container: - image: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine"] - steps: - - uses: actions/checkout@v4 - - name: Install Python and Poetry - run: | - if [ -f /etc/os-release ]; then - . /etc/os-release - if [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "20.04" ]; then - # Add deadsnakes PPA for Python 3.12 - apt-get update - apt-get install -y software-properties-common - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.12 python3.12-venv python3.12-distutils python3-pip - # Update alternatives to use python3.12 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 - update-alternatives --set python3 /usr/bin/python3.12 - python3 -m pip install --upgrade pip - python3 -m pip install pipx - else - apt-get update - apt-get install -y python3 python3-pip pipx - fi - elif [ -f /etc/arch-release ]; then - pacman -Syu --noconfirm python python-pip pipx - elif [ -f /etc/fedora-release ]; then - dnf install -y python3 python3-pip pipx - elif [ -f /etc/gentoo-release ]; then - # Initialize and sync Gentoo repository - emerge-webrsync - emerge --sync - emerge --update --newuse dev-lang/python dev-python/pipx - elif [ -f /etc/alpine-release ]; then - apk add --no-cache python3 py3-pip - python3 -m pip install --upgrade pip - python3 -m pip install pipx - fi - # Add $HOME/.local/bin to PATH - export PATH="$HOME/.local/bin:$PATH" - python3 -m pipx ensurepath - pipx install poetry - - name: Install dependencies - run: | - poetry install - - name: Run tests - run: | - poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . publish_code: needs: test runs-on: ubuntu-latest From b30cbaff378974f54b7e424d90dcabd87fd8773f Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 11:09:07 -0400 Subject: [PATCH 14/37] more --- .github/workflows/distro_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index fd0c88efb..2906a4771 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -8,7 +8,6 @@ concurrency: jobs: test-distros: - needs: lint runs-on: ubuntu-latest container: image: ${{ matrix.os }} From 262d5d7807ecae725ff7a0b6a88b86faddb64e30 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 11:11:37 -0400 Subject: [PATCH 15/37] just ubuntu things --- .github/workflows/distro_tests.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 2906a4771..e221a7d35 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -21,16 +21,21 @@ jobs: run: | if [ -f /etc/os-release ]; then . /etc/os-release - if [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "20.04" ]; then - # Add deadsnakes PPA for Python 3.12 - apt-get update - apt-get install -y software-properties-common - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.12 python3.12-venv python3.12-distutils python3-pip - # Update alternatives to use python3.12 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 - update-alternatives --set python3 /usr/bin/python3.12 + if [ "$ID" = "ubuntu" ]; then + if [ "$VERSION_ID" = "20.04" ]; then + # Add deadsnakes PPA for Python 3.12 + apt-get update + apt-get install -y software-properties-common + add-apt-repository ppa:deadsnakes/ppa + apt-get update + apt-get install -y python3.12 python3.12-venv python3.12-distutils python3-pip + # Update alternatives to use python3.12 + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 + update-alternatives --set python3 /usr/bin/python3.12 + else + apt-get update + apt-get install -y python3 python3-pip + fi python3 -m pip install --upgrade pip python3 -m pip install pipx elif [ "$ID" = "alpine" ]; then From cc296b7b8f012bd8c2fb27af42a6b9410c0eb602 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 11:46:15 -0400 Subject: [PATCH 16/37] tests --- .github/workflows/distro_tests.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index e221a7d35..8dc284c23 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -34,18 +34,23 @@ jobs: update-alternatives --set python3 /usr/bin/python3.12 else apt-get update - apt-get install -y python3 python3-pip + apt-get install -y python3 python3-pip python3-venv fi python3 -m pip install --upgrade pip python3 -m pip install pipx + elif [ "$ID" = "debian" ]; then + apt-get update + apt-get install -y python3 python3-pip python3-venv + python3 -m pip install --upgrade pip + python3 -m pip install pipx elif [ "$ID" = "alpine" ]; then apk add --no-cache python3 py3-pip python3 -m pip install --upgrade pip python3 -m pip install pipx elif [ "$ID" = "arch" ]; then - pacman -Syu --noconfirm python python-pip pipx + pacman -Syu --noconfirm python python-pip python-pipx elif [ "$ID" = "fedora" ]; then - dnf install -y python3 python3-pip pipx + dnf install -y python3 python3-pip pipx poetry elif [ "$ID" = "gentoo" ]; then # Initialize and sync Gentoo repository emerge-webrsync @@ -59,13 +64,13 @@ jobs: echo "Operating system not recognized" exit 1 fi - # Add $HOME/.local/bin to PATH - export PATH="$HOME/.local/bin:$PATH" python3 -m pipx ensurepath pipx install poetry - name: Install dependencies run: | + export PATH="$HOME/.local/bin:$PATH" poetry install - name: Run tests run: | + export PATH="$HOME/.local/bin:$PATH" poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . From 97ce06b29c3780c6301645b65a59baf67a3133e6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 12:04:02 -0400 Subject: [PATCH 17/37] iterating --- .github/workflows/distro_tests.yml | 32 +++++++++++------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 8dc284c23..37b26ef0b 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -21,30 +21,22 @@ jobs: run: | if [ -f /etc/os-release ]; then . /etc/os-release - if [ "$ID" = "ubuntu" ]; then - if [ "$VERSION_ID" = "20.04" ]; then - # Add deadsnakes PPA for Python 3.12 - apt-get update - apt-get install -y software-properties-common - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.12 python3.12-venv python3.12-distutils python3-pip - # Update alternatives to use python3.12 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 - update-alternatives --set python3 /usr/bin/python3.12 - else - apt-get update - apt-get install -y python3 python3-pip python3-venv - fi - python3 -m pip install --upgrade pip - python3 -m pip install pipx - elif [ "$ID" = "debian" ]; then + if [ "$ID" = "ubuntu" || "$ID" = "debian" ]; then + # Add deadsnakes PPA for Python 3.12 + apt-get update + apt-get install -y software-properties-common + add-apt-repository ppa:deadsnakes/ppa apt-get update - apt-get install -y python3 python3-pip python3-venv + apt-get install -y python3.12 python3.12-venv python3.12-distutils + # Install pip for Python 3.12 + curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 + # Update alternatives to use python3.12 + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 + update-alternatives --set python3 /usr/bin/python3.12 python3 -m pip install --upgrade pip python3 -m pip install pipx elif [ "$ID" = "alpine" ]; then - apk add --no-cache python3 py3-pip + apk add --no-cache bash python3 py3-pip python3 -m pip install --upgrade pip python3 -m pip install pipx elif [ "$ID" = "arch" ]; then From 3036629933ed1d812bf08b00d6f10c8860d94046 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 12:05:22 -0400 Subject: [PATCH 18/37] bash oopsie --- .github/workflows/distro_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 37b26ef0b..e504b8586 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -21,7 +21,7 @@ jobs: run: | if [ -f /etc/os-release ]; then . /etc/os-release - if [ "$ID" = "ubuntu" || "$ID" = "debian" ]; then + if [ "$ID" = "ubuntu" -o "$ID" = "debian" ]; then # Add deadsnakes PPA for Python 3.12 apt-get update apt-get install -y software-properties-common From 687d247f6220e219c451299dcac2760a33c3309f Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 12:22:14 -0400 Subject: [PATCH 19/37] distrosgit diff --- .github/workflows/distro_tests.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index e504b8586..9c2c744c6 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -14,17 +14,17 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine"] + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/parrot", "archlinux/archlinux"] steps: - uses: actions/checkout@v4 - name: Install Python and Poetry run: | if [ -f /etc/os-release ]; then . /etc/os-release - if [ "$ID" = "ubuntu" -o "$ID" = "debian" ]; then + if [ "$ID" = "ubuntu" ]; then # Add deadsnakes PPA for Python 3.12 apt-get update - apt-get install -y software-properties-common + apt-get install -y software-properties-common curl add-apt-repository ppa:deadsnakes/ppa apt-get update apt-get install -y python3.12 python3.12-venv python3.12-distutils @@ -36,10 +36,10 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install pipx elif [ "$ID" = "alpine" ]; then - apk add --no-cache bash python3 py3-pip + apk add --no-cache bash python3 py3-pip gcc musl-dev libffi-dev python3 -m pip install --upgrade pip python3 -m pip install pipx - elif [ "$ID" = "arch" ]; then + elif [ "$ID" = "arch" -o "$ID" = "archlinux" ]; then pacman -Syu --noconfirm python python-pip python-pipx elif [ "$ID" = "fedora" ]; then dnf install -y python3 python3-pip pipx poetry @@ -47,7 +47,10 @@ jobs: # Initialize and sync Gentoo repository emerge-webrsync emerge --sync - emerge --update --newuse dev-lang/python dev-python/pipx + emerge --update --newuse dev-lang/python dev-python/pipx poetry + elif [ "$ID" = "kali" -o "$ID" = "parrot" -o "$ID" = "debian" ]; then + apt-get update + apt-get install -y python3 python3-pip pipx else echo "Unsupported Linux distribution" exit 1 From c763c1eea262ee177c66ae56b4a774ace1eb1af5 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 12:24:43 -0400 Subject: [PATCH 20/37] alpine --- .github/workflows/distro_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 9c2c744c6..bbe32406f 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/parrot", "archlinux/archlinux"] + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security", "archlinux/archlinux"] steps: - uses: actions/checkout@v4 - name: Install Python and Poetry @@ -36,7 +36,7 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install pipx elif [ "$ID" = "alpine" ]; then - apk add --no-cache bash python3 py3-pip gcc musl-dev libffi-dev + apk add --no-cache bash python3 py3-pip gcc g++ musl-dev libffi-dev python3 -m pip install --upgrade pip python3 -m pip install pipx elif [ "$ID" = "arch" -o "$ID" = "archlinux" ]; then From 16f204c19f24d53674abfa6066614481caefedbf Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 12:26:03 -0400 Subject: [PATCH 21/37] ubuntu things --- .github/workflows/distro_tests.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index bbe32406f..a6bf7dad1 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -22,12 +22,17 @@ jobs: if [ -f /etc/os-release ]; then . /etc/os-release if [ "$ID" = "ubuntu" ]; then - # Add deadsnakes PPA for Python 3.12 apt-get update apt-get install -y software-properties-common curl - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.12 python3.12-venv python3.12-distutils + if [ "$VERSION_ID" = "24.04" ]; then + # Install Python 3.12 and its components directly from Ubuntu repositories + apt-get install -y python3.12 python3.12-venv python3.12-distutils + else + # Add deadsnakes PPA for Python 3.12 + add-apt-repository ppa:deadsnakes/ppa + apt-get update + apt-get install -y python3.12 python3.12-venv python3.12-distutils + fi # Install pip for Python 3.12 curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 # Update alternatives to use python3.12 From 10627df2d6525a801d4e27fd3295c0d92f5fa607 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 12:41:22 -0400 Subject: [PATCH 22/37] more ubuntu things --- .github/workflows/distro_tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index a6bf7dad1..61bd352d0 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -24,9 +24,14 @@ jobs: if [ "$ID" = "ubuntu" ]; then apt-get update apt-get install -y software-properties-common curl + # Install necessary Perl modules for debconf + apt-get install -y libterm-readline-gnu-perl libterm-readline-perl-perl if [ "$VERSION_ID" = "24.04" ]; then # Install Python 3.12 and its components directly from Ubuntu repositories - apt-get install -y python3.12 python3.12-venv python3.12-distutils + apt-get install -y python3.12 python3.12-venv + # Handle missing python3.12-distutils + curl -O https://bootstrap.pypa.io/pip/3.12/get-pip.py + python3.12 get-pip.py else # Add deadsnakes PPA for Python 3.12 add-apt-repository ppa:deadsnakes/ppa From f3b90e5396f94195677a1711ffc5727a076849b8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 14:36:58 -0400 Subject: [PATCH 23/37] better debian --- .github/workflows/distro_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 61bd352d0..2dbae5a5d 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -14,13 +14,14 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security", "archlinux/archlinux"] + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"] steps: - uses: actions/checkout@v4 - name: Install Python and Poetry run: | if [ -f /etc/os-release ]; then . /etc/os-release + export DEBIAN_FRONTEND=noninteractive # Set debconf to noninteractive mode if [ "$ID" = "ubuntu" ]; then apt-get update apt-get install -y software-properties-common curl From 0a6e78bf73371634f29ef8e85f20b1d0b65ee7a2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 14:40:04 -0400 Subject: [PATCH 24/37] ubuntu why? --- .github/workflows/distro_tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 2dbae5a5d..8690a0fe1 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -30,9 +30,6 @@ jobs: if [ "$VERSION_ID" = "24.04" ]; then # Install Python 3.12 and its components directly from Ubuntu repositories apt-get install -y python3.12 python3.12-venv - # Handle missing python3.12-distutils - curl -O https://bootstrap.pypa.io/pip/3.12/get-pip.py - python3.12 get-pip.py else # Add deadsnakes PPA for Python 3.12 add-apt-repository ppa:deadsnakes/ppa From 838c7811a7e077fa834cccb2813ca1c36a8f60e3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 14:43:48 -0400 Subject: [PATCH 25/37] ubuntu why? --- .github/workflows/distro_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 8690a0fe1..3c8117ffe 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -30,6 +30,7 @@ jobs: if [ "$VERSION_ID" = "24.04" ]; then # Install Python 3.12 and its components directly from Ubuntu repositories apt-get install -y python3.12 python3.12-venv + else # Add deadsnakes PPA for Python 3.12 add-apt-repository ppa:deadsnakes/ppa @@ -42,7 +43,7 @@ jobs: update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 update-alternatives --set python3 /usr/bin/python3.12 python3 -m pip install --upgrade pip - python3 -m pip install pipx + python3 -m pip install pipx --break-system-packages elif [ "$ID" = "alpine" ]; then apk add --no-cache bash python3 py3-pip gcc g++ musl-dev libffi-dev python3 -m pip install --upgrade pip From 782a68d78cbbbfd2a996e8ec4b00fd868091d3a4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 14:55:51 -0400 Subject: [PATCH 26/37] ubuntu why? --- .github/workflows/distro_tests.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 3c8117ffe..90302cc75 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -28,21 +28,15 @@ jobs: # Install necessary Perl modules for debconf apt-get install -y libterm-readline-gnu-perl libterm-readline-perl-perl if [ "$VERSION_ID" = "24.04" ]; then - # Install Python 3.12 and its components directly from Ubuntu repositories - apt-get install -y python3.12 python3.12-venv - + apt-get install -y python3.11 python3.11-venv else - # Add deadsnakes PPA for Python 3.12 add-apt-repository ppa:deadsnakes/ppa apt-get update - apt-get install -y python3.12 python3.12-venv python3.12-distutils + apt-get install -y python3.11 python3.11-venv python3.11-distutils fi - # Install pip for Python 3.12 - curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 - # Update alternatives to use python3.12 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 - update-alternatives --set python3 /usr/bin/python3.12 - python3 -m pip install --upgrade pip + curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 + update-alternatives --set python3 /usr/bin/python3.11 python3 -m pip install pipx --break-system-packages elif [ "$ID" = "alpine" ]; then apk add --no-cache bash python3 py3-pip gcc g++ musl-dev libffi-dev From 8f3ba6108fb6d912f2aa3ddb32b2c7fe38934f8e Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 15:09:38 -0400 Subject: [PATCH 27/37] ubuntu why? --- .github/workflows/distro_tests.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 90302cc75..916e60007 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -27,13 +27,9 @@ jobs: apt-get install -y software-properties-common curl # Install necessary Perl modules for debconf apt-get install -y libterm-readline-gnu-perl libterm-readline-perl-perl - if [ "$VERSION_ID" = "24.04" ]; then - apt-get install -y python3.11 python3.11-venv - else - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.11 python3.11-venv python3.11-distutils - fi + add-apt-repository ppa:deadsnakes/ppa + apt-get update + apt-get install -y python3.11 python3.11-venv python3.11-distutils curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 update-alternatives --set python3 /usr/bin/python3.11 From cdfbe7e77d2ef139a07d1aa999f5312c282efcad Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 16:29:08 -0400 Subject: [PATCH 28/37] skip dastardly --- .github/workflows/distro_tests.yml | 1 + bbot/test/test_step_2/module_tests/base.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 916e60007..879b1ea04 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -67,4 +67,5 @@ jobs: - name: Run tests run: | export PATH="$HOME/.local/bin:$PATH" + export BBOT_DISTRO_TESTS=true poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index f5d4255f6..bb63b57e5 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -89,6 +89,10 @@ def module(self): async def module_test( self, httpx_mock, bbot_httpserver, bbot_httpserver_ssl, monkeypatch, request, caplog, capsys ): + # Skip dastardly test if we're in the distro tests (because dastardly uses docker) + if os.getenv("BBOT_DISTRO_TESTS") and self.name == "dastardly": + pytest.skip("Skipping module_test for dastardly module due to BBOT_DISTRO_TESTS environment variable") + self.log.info(f"Starting {self.name} module test") module_test = self.ModuleTest( self, httpx_mock, bbot_httpserver, bbot_httpserver_ssl, monkeypatch, request, caplog, capsys From ce31264ff53d8c05e50cfbfa89187f8a3e0a130e Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 16:46:27 -0400 Subject: [PATCH 29/37] pyenv --- .github/workflows/distro_tests.yml | 54 ++++++++---------------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 879b1ea04..5e9e5441b 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -21,51 +21,25 @@ jobs: run: | if [ -f /etc/os-release ]; then . /etc/os-release - export DEBIAN_FRONTEND=noninteractive # Set debconf to noninteractive mode - if [ "$ID" = "ubuntu" ]; then - apt-get update - apt-get install -y software-properties-common curl - # Install necessary Perl modules for debconf - apt-get install -y libterm-readline-gnu-perl libterm-readline-perl-perl - add-apt-repository ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.11 python3.11-venv python3.11-distutils - curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 - update-alternatives --set python3 /usr/bin/python3.11 - python3 -m pip install pipx --break-system-packages - elif [ "$ID" = "alpine" ]; then - apk add --no-cache bash python3 py3-pip gcc g++ musl-dev libffi-dev - python3 -m pip install --upgrade pip - python3 -m pip install pipx - elif [ "$ID" = "arch" -o "$ID" = "archlinux" ]; then - pacman -Syu --noconfirm python python-pip python-pipx - elif [ "$ID" = "fedora" ]; then - dnf install -y python3 python3-pip pipx poetry - elif [ "$ID" = "gentoo" ]; then - # Initialize and sync Gentoo repository - emerge-webrsync - emerge --sync - emerge --update --newuse dev-lang/python dev-python/pipx poetry - elif [ "$ID" = "kali" -o "$ID" = "parrot" -o "$ID" = "debian" ]; then - apt-get update - apt-get install -y python3 python3-pip pipx - else - echo "Unsupported Linux distribution" - exit 1 + if [ "$ID" = "alpine" ]; then + apk add --no-cache bash gcc g++ musl-dev libffi-dev fi - else - echo "Operating system not recognized" - exit 1 fi - python3 -m pipx ensurepath + + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init --path)" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + pyenv install 3.11 + pyenv global 3.11 + python3.11 -m pip install --user spipx + python3.11 -m pipx ensurepath pipx install poetry - - name: Install dependencies - run: | - export PATH="$HOME/.local/bin:$PATH" - poetry install - name: Run tests run: | export PATH="$HOME/.local/bin:$PATH" export BBOT_DISTRO_TESTS=true + poetry env use python3.11 + poetry install poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . From 1eedaf57108aa1aedc7bee1778769569045a3d8e Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 16:47:15 -0400 Subject: [PATCH 30/37] pyenv --- .github/workflows/distro_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 5e9e5441b..f8a20e6b4 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -39,6 +39,7 @@ jobs: - name: Run tests run: | export PATH="$HOME/.local/bin:$PATH" + export PATH="$HOME/.pyenv/bin:$PATH" export BBOT_DISTRO_TESTS=true poetry env use python3.11 poetry install From bb451fd5bd42da2367c2212d55c167d939f9f813 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 16:49:52 -0400 Subject: [PATCH 31/37] curl --- .github/workflows/distro_tests.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index f8a20e6b4..e18e71031 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -21,8 +21,17 @@ jobs: run: | if [ -f /etc/os-release ]; then . /etc/os-release - if [ "$ID" = "alpine" ]; then + if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kalilinux" ] || [ "$ID" = "parrotsec" ]; then + apt-get update + apt-get -y install curl + elif [ "$ID" = "alpine" ]; then apk add --no-cache bash gcc g++ musl-dev libffi-dev + elif [ "$ID" = "archlinux" ]; then + pacman -Syu --noconfirm curl + elif [ "$ID" = "fedora" ]; then + dnf install -y curl + elif [ "$ID" = "gentoo" ]; then + emerge --update --newuse curl fi fi From 60a418e60c8f51934d7114dbceacb8f69b0237a6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 16:51:42 -0400 Subject: [PATCH 32/37] git --- .github/workflows/distro_tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index e18e71031..87ae591b0 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -22,16 +22,17 @@ jobs: if [ -f /etc/os-release ]; then . /etc/os-release if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kalilinux" ] || [ "$ID" = "parrotsec" ]; then + export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get -y install curl + apt-get -y install curl git elif [ "$ID" = "alpine" ]; then - apk add --no-cache bash gcc g++ musl-dev libffi-dev + apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git elif [ "$ID" = "archlinux" ]; then - pacman -Syu --noconfirm curl + pacman -Syu --noconfirm curl git elif [ "$ID" = "fedora" ]; then - dnf install -y curl + dnf install -y curl git elif [ "$ID" = "gentoo" ]; then - emerge --update --newuse curl + emerge --update --newuse curl git fi fi From 32abe3b4bba627e884fb805243eac1ab15df41e4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 16:55:05 -0400 Subject: [PATCH 33/37] bash --- .github/workflows/distro_tests.yml | 31 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 87ae591b0..7abf5aae7 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -24,28 +24,31 @@ jobs: if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kalilinux" ] || [ "$ID" = "parrotsec" ]; then export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get -y install curl git + apt-get -y install curl git bash elif [ "$ID" = "alpine" ]; then apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git elif [ "$ID" = "archlinux" ]; then - pacman -Syu --noconfirm curl git + pacman -Syu --noconfirm curl git bash elif [ "$ID" = "fedora" ]; then - dnf install -y curl git + dnf install -y curl git bash elif [ "$ID" = "gentoo" ]; then - emerge --update --newuse curl git + emerge --update --newuse curl git bash fi fi - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init --path)" - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - pyenv install 3.11 - pyenv global 3.11 - python3.11 -m pip install --user spipx - python3.11 -m pipx ensurepath - pipx install poetry + # Re-run the script with bash + exec bash -c " + curl https://pyenv.run | bash + export PATH=\"$HOME/.pyenv/bin:\$PATH\" + eval \"\$(pyenv init --path)\" + eval \"\$(pyenv init -)\" + eval \"\$(pyenv virtualenv-init -)\" + pyenv install 3.11 + pyenv global 3.11 + python3.11 -m pip install --user spipx + python3.11 -m pipx ensurepath + pipx install poetry + " - name: Run tests run: | export PATH="$HOME/.local/bin:$PATH" From b9d40bf6d6b7555c5429ce4db3d329a57f21c614 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 16:57:34 -0400 Subject: [PATCH 34/37] dev --- .github/workflows/distro_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 7abf5aae7..b602012e6 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -24,13 +24,13 @@ jobs: if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kalilinux" ] || [ "$ID" = "parrotsec" ]; then export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get -y install curl git bash + apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev elif [ "$ID" = "alpine" ]; then apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git elif [ "$ID" = "archlinux" ]; then - pacman -Syu --noconfirm curl git bash + pacman -Syu --noconfirm curl git bash base-devel elif [ "$ID" = "fedora" ]; then - dnf install -y curl git bash + dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel elif [ "$ID" = "gentoo" ]; then emerge --update --newuse curl git bash fi @@ -45,7 +45,7 @@ jobs: eval \"\$(pyenv virtualenv-init -)\" pyenv install 3.11 pyenv global 3.11 - python3.11 -m pip install --user spipx + python3.11 -m pip install --user pipx python3.11 -m pipx ensurepath pipx install poetry " From 51fd792285a6746ac0744ab7915a955ae13f4d65 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 17:03:00 -0400 Subject: [PATCH 35/37] path --- .github/workflows/distro_tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index b602012e6..62b3fc088 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -26,12 +26,13 @@ jobs: apt-get update apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev elif [ "$ID" = "alpine" ]; then - apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git + apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev elif [ "$ID" = "archlinux" ]; then pacman -Syu --noconfirm curl git bash base-devel elif [ "$ID" = "fedora" ]; then - dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel + dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel elif [ "$ID" = "gentoo" ]; then + emerge-webrsync emerge --update --newuse curl git bash fi fi @@ -40,6 +41,7 @@ jobs: exec bash -c " curl https://pyenv.run | bash export PATH=\"$HOME/.pyenv/bin:\$PATH\" + export PATH=\"$HOME/.local/bin:\$PATH\" eval \"\$(pyenv init --path)\" eval \"\$(pyenv init -)\" eval \"\$(pyenv virtualenv-init -)\" From f788dad306b9c5873ef57a2c0ad3a7d56d13f095 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 17:09:05 -0400 Subject: [PATCH 36/37] pyenv things --- .github/workflows/distro_tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 62b3fc088..b29752ed0 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -21,19 +21,19 @@ jobs: run: | if [ -f /etc/os-release ]; then . /etc/os-release - if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kalilinux" ] || [ "$ID" = "parrotsec" ]; then + if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then export DEBIAN_FRONTEND=noninteractive apt-get update apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev elif [ "$ID" = "alpine" ]; then apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev - elif [ "$ID" = "archlinux" ]; then + elif [ "$ID" = "arch" ]; then pacman -Syu --noconfirm curl git bash base-devel elif [ "$ID" = "fedora" ]; then dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel elif [ "$ID" = "gentoo" ]; then emerge-webrsync - emerge --update --newuse curl git bash + emerge --update --newuse dev-vcs/git curl bash fi fi @@ -47,6 +47,7 @@ jobs: eval \"\$(pyenv virtualenv-init -)\" pyenv install 3.11 pyenv global 3.11 + pyenv rehash python3.11 -m pip install --user pipx python3.11 -m pipx ensurepath pipx install poetry @@ -55,6 +56,7 @@ jobs: run: | export PATH="$HOME/.local/bin:$PATH" export PATH="$HOME/.pyenv/bin:$PATH" + export PATH="$HOME/.pyenv/shims:$PATH" export BBOT_DISTRO_TESTS=true poetry env use python3.11 poetry install From 73b00f452cb164e640d2405c74c22b511298d2cb Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 21 Oct 2024 17:18:05 -0400 Subject: [PATCH 37/37] alpine --- .github/workflows/distro_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index b29752ed0..e9eac869a 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -26,7 +26,7 @@ jobs: apt-get update apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev elif [ "$ID" = "alpine" ]; then - apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev + apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev elif [ "$ID" = "arch" ]; then pacman -Syu --noconfirm curl git bash base-devel elif [ "$ID" = "fedora" ]; then