Dev -> Stable 2.2.0

.flake8

@@ -2,4 +2,4 @@
 select = F,E722
 ignore = F403,F405,F541
 per-file-ignores =
-    */__init__.py:F401,F403
+    */__init__.py:F401,F403

.github/workflows/distro_tests.yml

@@ -0,0 +1,63 @@
+name: Linux Distro Tests
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-distros:
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Python and Poetry
+        run: |
+          if [ -f /etc/os-release ]; then
+            . /etc/os-release
+            if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update
+              apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
+            elif [ "$ID" = "alpine" ]; then
+              apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev
+            elif [ "$ID" = "arch" ]; then
+              pacman -Syu --noconfirm curl git bash base-devel
+            elif [ "$ID" = "fedora" ]; then
+              dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel
+            elif [ "$ID" = "gentoo" ]; then
+              emerge-webrsync
+              emerge --update --newuse dev-vcs/git curl bash
+            fi
+          fi
+
+          # Re-run the script with bash
+          exec bash -c "
+            curl https://pyenv.run | bash
+            export PATH=\"$HOME/.pyenv/bin:\$PATH\"
+            export PATH=\"$HOME/.local/bin:\$PATH\"
+            eval \"\$(pyenv init --path)\"
+            eval \"\$(pyenv init -)\"
+            eval \"\$(pyenv virtualenv-init -)\"
+            pyenv install 3.11
+            pyenv global 3.11
+            pyenv rehash
+            python3.11 -m pip install --user pipx
+            python3.11 -m pipx ensurepath
+            pipx install poetry
+          "
+      - name: Run tests
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          export PATH="$HOME/.pyenv/bin:$PATH"
+          export PATH="$HOME/.pyenv/shims:$PATH"
+          export BBOT_DISTRO_TESTS=true
+          poetry env use python3.11
+          poetry install
+          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG .

.github/workflows/docs_updater.yml

@@ -3,6 +3,7 @@ name: Daily Docs Update
 on:
   schedule:
     - cron: '0 0 * * *'  # Runs daily at midnight UTC
+  workflow_dispatch:  # Allows manual triggering
 
 jobs:
   update_docs:
@@ -23,18 +24,22 @@ jobs:
       - name: Generate docs
         run: |
           poetry run bbot/scripts/docs.py
-      - name: Commit changes
-        uses: EndBug/add-and-commit@v9
-        with:
-          add: '["*.md", "docs/data/chord_graph/*.json"]'
-          author_name: "BBOT Docs Autopublish"
-          author_email: [email protected]
-          message: "Refresh module docs"
-      - name: Create Pull Request
+      - name: Commit and Push Changes
+        run: |
+          git config user.name "BBOT Docs Autopublish"
+          git config user.email "[email protected]"
+          git checkout -b update-docs
+          git add "*.md" "docs/data/chord_graph/*.json"
+          git commit -m "Refresh module docs"
+          git push -u origin update-docs --force
+        env:
+          GITHUB_TOKEN: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
+      - name: Create or Update Pull Request
         uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           branch: update-docs
           base: dev
           title: "Daily Docs Update"
           body: "This is an automated pull request to update the documentation."
+          update-existing: true

.github/workflows/tests.yml

@@ -14,20 +14,20 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           options: "--check"
       - name: Install Python 3
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
         run: |
           pip install flake8
       - name: flake8
         run: |
-          flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403"
+          flake8
   test:
     needs: lint
     runs-on: ubuntu-latest
@@ -37,9 +37,9 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -48,95 +48,24 @@ jobs:
           poetry install
       - name: Run tests
         run: |
-          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./cov.xml
           verbose: true
-  update_docs:
-    needs: test
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && (github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/stable')
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install
-      - name: Generate docs
-        run: |
-          poetry run bbot/scripts/docs.py
-      - name: Commit docs
-        uses: EndBug/add-and-commit@v9
-        continue-on-error: true
-        with:
-          add: '["*.md", "docs/data/chord_graph/*.json"]'
-          author_name: "BBOT Docs Autopublish"
-          author_email: [email protected]
-          message: "Refresh module docs"
-  publish_docs:
-    needs: test
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev')
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
-      - uses: actions/cache@v3
-        with:
-          key: mkdocs-material-${{ env.cache_id }}
-          path: .cache
-          restore-keys: |
-            mkdocs-material-
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install --only=docs
-      - name: Configure Git
-        run: |
-          git config user.name github-actions
-          git config user.email [email protected]
-          git fetch origin gh-pages:refs/remotes/origin/gh-pages
-          if git show-ref --verify --quiet refs/heads/gh-pages; then
-            git branch -f gh-pages origin/gh-pages
-          else
-            git branch --track gh-pages origin/gh-pages
-          fi
-      - name: Generate docs (stable branch)
-        if: github.ref == 'refs/heads/stable'
-        run: |
-          poetry run mike deploy Stable
-      - name: Generate docs (dev branch)
-        if: github.ref == 'refs/heads/dev'
-        run: |
-          poetry run mike deploy Dev
-      - name: Publish docs
-        run: |
-          git switch gh-pages
-          git push
   publish_code:
     needs: test
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     continue-on-error: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -185,7 +114,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
   #   steps:
-  #     - uses: actions/checkout@v3
+  #     - uses: actions/checkout@v4
   #       with:
   #         ref: ${{ github.head_ref }}
   #         fetch-depth: 0 # Fetch all history for all tags and branches

bbot/modules/bufferoverrun.py

@@ -0,0 +1,48 @@
+from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
+
+
+class BufferOverrun(subdomain_enum_apikey):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive", "safe"]
+    meta = {
+        "description": "Query BufferOverrun's TLS API for subdomains",
+        "created_date": "2024-10-23",
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
+    options = {"api_key": "", "commercial": False}
+    options_desc = {"api_key": "BufferOverrun API key", "commercial": "Use commercial API"}
+
+    base_url = "https://tls.bufferover.run/dns"
+    commercial_base_url = "https://bufferover-run-tls.p.rapidapi.com/ipv4/dns"
+
+    async def setup(self):
+        self.commercial = self.config.get("commercial", False)
+        return await super().setup()
+
+    def prepare_api_request(self, url, kwargs):
+        if self.commercial:
+            kwargs["headers"]["x-rapidapi-host"] = "bufferover-run-tls.p.rapidapi.com"
+            kwargs["headers"]["x-rapidapi-key"] = self.api_key
+        else:
+            kwargs["headers"]["x-api-key"] = self.api_key
+        return url, kwargs
+
+    async def request_url(self, query):
+        url = f"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}"
+        return await self.api_request(url)
+
+    def parse_results(self, r, query):
+        j = r.json()
+        subdomains_set = set()
+        if isinstance(j, dict):
+            results = j.get("Results", [])
+            for result in results:
+                parts = result.split(",")
+                if len(parts) > 4:
+                    subdomain = parts[4].strip()
+                    if subdomain and subdomain.endswith(f".{query}"):
+                        subdomains_set.add(subdomain)
+            for subdomain in subdomains_set:
+                yield subdomain

bbot/modules/deadly/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.3.4",
+        "version": "3.3.5",
         "tags": "",
         "templates": "",
         "severity": "",

bbot/test/run_tests.sh

@@ -10,7 +10,7 @@ echo
 
 echo "[+] Linting with flake8"
 echo "======================="
-flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" "$bbot_dir" || exit 1
+flake8 "$bbot_dir" || exit 1
 echo
 
 if [ "${1}x" != "x" ] ; then

bbot/test/test_step_2/module_tests/base.py

@@ -89,6 +89,10 @@ def module(self):
     async def module_test(
         self, httpx_mock, bbot_httpserver, bbot_httpserver_ssl, monkeypatch, request, caplog, capsys
     ):
+        # Skip dastardly test if we're in the distro tests (because dastardly uses docker)
+        if os.getenv("BBOT_DISTRO_TESTS") and self.name == "dastardly":
+            pytest.skip("Skipping module_test for dastardly module due to BBOT_DISTRO_TESTS environment variable")
+
         self.log.info(f"Starting {self.name} module test")
         module_test = self.ModuleTest(
             self, httpx_mock, bbot_httpserver, bbot_httpserver_ssl, monkeypatch, request, caplog, capsys

bbot/test/test_step_2/module_tests/test_module_bufferoverrun.py

@@ -0,0 +1,35 @@
+from .base import ModuleTestBase
+
+
+class TestBufferOverrun(ModuleTestBase):
+    config_overrides = {"modules": {"bufferoverrun": {"api_key": "asdf", "commercial": False}}}
+
+    async def setup_before_prep(self, module_test):
+        # Mock response for non-commercial API
+        module_test.httpx_mock.add_response(
+            url="https://tls.bufferover.run/dns?q=.blacklanternsecurity.com",
+            match_headers={"x-api-key": "asdf"},
+            json={"Results": ["1.2.3.4,example.com,*,*,sub.blacklanternsecurity.com"]},
+        )
+
+    def check(self, module_test, events):
+        assert any(e.data == "sub.blacklanternsecurity.com" for e in events), "Failed to detect subdomain for free API"
+
+
+class TestBufferOverrunCommercial(ModuleTestBase):
+    modules_overrides = ["bufferoverrun"]
+    module_name = "bufferoverrun"
+    config_overrides = {"modules": {"bufferoverrun": {"api_key": "asdf", "commercial": True}}}
+
+    async def setup_before_prep(self, module_test):
+        # Mock response for commercial API
+        module_test.httpx_mock.add_response(
+            url="https://bufferover-run-tls.p.rapidapi.com/ipv4/dns?q=.blacklanternsecurity.com",
+            match_headers={"x-rapidapi-host": "bufferover-run-tls.p.rapidapi.com", "x-rapidapi-key": "asdf"},
+            json={"Results": ["5.6.7.8,blacklanternsecurity.com,*,*,sub.blacklanternsecurity.com"]},
+        )
+
+    def check(self, module_test, events):
+        assert any(
+            e.data == "sub.blacklanternsecurity.com" for e in events
+        ), "Failed to detect subdomain for commercial API"