Merge branch 'master' into dependabot/pip/xmltodict-0.14.2

.devcontainer/Dockerfile

@@ -1,6 +1,6 @@
 # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3/.devcontainer/base.Dockerfile
 
-# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3-bullseye, 3.10-bullseye, 3-buster, 3.10-buster, etc.
 ARG VARIANT="3.10-bullseye"
 FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
 

.devcontainer/devcontainer.json

@@ -6,7 +6,7 @@
 		"dockerfile": "Dockerfile",
 		"context": "..",
 		"args": { 
-			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, etc.
 			// Append -bullseye or -buster to pin to an OS version.
 			// Use -bullseye variants on local on arm64/Apple Silicon.
 			"VARIANT": "3.10",

.github/workflows/build.yml

@@ -21,26 +21,25 @@ jobs:
       # set to false for debugging
       fail-fast: true
       matrix:
-        # using Python 3.8 to support running across multiple operating systems including Windows 7
         include:
           - os: ubuntu-20.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
-            python_version: 3.8
+            python_version: '3.10'
           - os: ubuntu-20.04
             artifact_name: capa
             asset_name: linux-py312
-            python_version: 3.12
+            python_version: '3.12'
           - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
-            python_version: 3.8
+            python_version: '3.10'
           - os: macos-12
             # use older macOS for assumed better portability
             artifact_name: capa
             asset_name: macos
-            python_version: 3.8
+            python_version: '3.10'
     steps:
       - name: Checkout capa
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -107,7 +106,7 @@ jobs:
     # upload zipped binaries to Release page
     if: github.event_name == 'release'
     name: zip and upload ${{ matrix.asset_name }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [build]
     strategy:
       matrix:

.github/workflows/changelog.yml

@@ -14,7 +14,7 @@ jobs:
   check_changelog:
     # no need to check for dependency updates via dependabot
     if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       NO_CHANGELOG: '[x] No CHANGELOG update needed'
     steps:

.github/workflows/publish.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
-          python-version: '3.8'
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/tag.yml

@@ -9,7 +9,7 @@ permissions: read-all
 jobs:
   tag:
     name: Tag capa rules
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout capa-rules
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

.github/workflows/tests.yml

@@ -26,7 +26,7 @@ env:
 
 jobs:
   changelog_format:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - name: Checkout capa
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -37,15 +37,15 @@ jobs:
         if [ $number != 1 ]; then exit 1; fi
 
   code_style:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - name: Checkout capa
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     # use latest available python to take advantage of best performance
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - name: Install dependencies
       run: |
         pip install -r requirements.txt
@@ -64,16 +64,16 @@ jobs:
       run: pre-commit run deptry --hook-stage manual
 
   rule_linter:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         submodules: recursive
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - name: Install capa
       run: |
         pip install -r requirements.txt
@@ -90,15 +90,15 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-2019, macos-12]
         # across all operating systems
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.10", "3.11"]
         include:
           # on Ubuntu run these as well
           - os: ubuntu-20.04
-            python-version: "3.8"
+            python-version: "3.10"
           - os: ubuntu-20.04
-            python-version: "3.9"
+            python-version: "3.11"
           - os: ubuntu-20.04
-            python-version: "3.10"
+            python-version: "3.12"
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -131,7 +131,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.11"]
+        python-version: ["3.10", "3.11"]
     steps:
     - name: Checkout capa with submodules
       # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
@@ -173,7 +173,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.10", "3.11"]
         java-version: ["17"]
         ghidra-version: ["11.0.1"]
         public-version: ["PUBLIC_20240130"] # for ghidra releases

.github/workflows/web-release.yml

@@ -0,0 +1,79 @@
+name: create web release
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version number for the release (x.x.x)'
+        required: true
+        type: string
+
+jobs:
+  run-tests:
+    uses: ./.github/workflows/web-tests.yml
+
+  build-and-release:
+    needs: run-tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set release name
+      run: echo "RELEASE_NAME=capa-explorer-web-v${{ github.event.inputs.version }}-${GITHUB_SHA::7}" >> $GITHUB_ENV
+
+    - name: Check if release already exists
+      run: |
+        if ls web/explorer/releases/capa-explorer-web-v${{ github.event.inputs.version }}-* 1> /dev/null 2>&1; then
+          echo "::error:: A release with version ${{ github.event.inputs.version }} already exists"
+          exit 1
+        fi
+
+    - name: Set up Node.js
+      uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: 'web/explorer/package-lock.json'
+
+    - name: Install dependencies
+      run: npm ci
+      working-directory: web/explorer
+
+    - name: Build offline bundle
+      run: npm run build:bundle
+      working-directory: web/explorer
+
+    - name: Compress bundle
+      run: zip -r ${{ env.RELEASE_NAME }}.zip capa-explorer-web
+      working-directory: web/explorer
+
+    - name: Create releases directory
+      run: mkdir -vp web/explorer/releases
+
+    - name: Move release to releases folder
+      run: mv web/explorer/${{ env.RELEASE_NAME }}.zip web/explorer/releases
+
+    - name: Compute release SHA256 hash
+      run: |
+        echo "RELEASE_SHA256=$(sha256sum web/explorer/releases/${{ env.RELEASE_NAME }}.zip | awk '{print $1}')" >> $GITHUB_ENV
+
+    - name: Update CHANGELOG.md
+      run: |
+        echo "## ${{ env.RELEASE_NAME }}" >> web/explorer/releases/CHANGELOG.md
+        echo "- Release Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> web/explorer/releases/CHANGELOG.md
+        echo "- SHA256: ${{ env.RELEASE_SHA256 }}" >> web/explorer/releases/CHANGELOG.md
+        echo "" >> web/explorer/releases/CHANGELOG.md
+        cat web/explorer/releases/CHANGELOG.md
+
+    - name: Remove older releases
+      # keep only the latest 3 releases
+      run: ls -t capa-explorer-web-v*.zip | tail -n +4 | xargs -r rm --
+      working-directory: web/explorer/releases
+
+    - name: Commit and push release
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "Capa Bot"
+        git add -f web/explorer/releases/${{ env.RELEASE_NAME }}.zip web/explorer/releases/CHANGELOG.md
+        git add -u web/explorer/releases/
+        git commit -m ":robot: explorer web: add release ${{ env.RELEASE_NAME }}"
+        git push

.github/workflows/web-tests.yml

@@ -1,10 +1,11 @@
-name: Capa Explorer Web tests 
+name: capa Explorer Web tests 
 
 on:
   pull_request:
     branches: [ master ]
     paths:
       - 'web/explorer/**'
+  workflow_call:  # this allows the workflow to be called by other workflows
 
 jobs:
   test:
@@ -23,20 +24,20 @@ jobs:
       with:
         node-version: 20
         cache: 'npm'
-        cache-dependency-path: './web/explorer/package-lock.json'
+        cache-dependency-path: 'web/explorer/package-lock.json'
 
     - name: Install dependencies
       run: npm ci
-      working-directory: ./web/explorer
+      working-directory: web/explorer
 
     - name: Lint
       run: npm run lint
-      working-directory: ./web/explorer
+      working-directory: web/explorer
 
     - name: Format
       run: npm run format:check
-      working-directory: ./web/explorer
+      working-directory: web/explorer
 
     - name: Run unit tests
       run: npm run test
-      working-directory: ./web/explorer
+      working-directory: web/explorer

CHANGELOG.md

@@ -6,6 +6,8 @@
 
 ### Breaking Changes
 
+- remove support for Python 3.8 and use Python 3.10 as minimum now #1966 @mr-tz
+
 ### New Rules (0)
 
 -
@@ -183,6 +185,7 @@ Special thanks to our repeat and new contributors:
 - CI: update Binary Ninja version to 4.1 and use Python 3.9 to test it #2211 @xusheng6
 - CI: update tests.yml workflow to exclude web and documentation files #2263 @s-ff
 - CI: update build.yml workflow to exclude web and documentation files #2270 @s-ff
+- CI: add web releases workflow #2455 @s-ff
 
 ### Raw diffs
 

capa/capabilities/common.py

@@ -9,7 +9,7 @@
 import logging
 import itertools
 import collections
-from typing import Any, Tuple
+from typing import Any
 
 from capa.rules import Scope, RuleSet
 from capa.engine import FeatureSet, MatchResults
@@ -64,7 +64,7 @@ def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalon
 
 def find_capabilities(
     ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs
-) -> Tuple[MatchResults, Any]:
+) -> tuple[MatchResults, Any]:
     from capa.capabilities.static import find_static_capabilities
     from capa.capabilities.dynamic import find_dynamic_capabilities
 

capa/capabilities/dynamic.py

@@ -9,7 +9,7 @@
 import logging
 import itertools
 import collections
-from typing import Any, List, Tuple
+from typing import Any
 
 import capa.perf
 import capa.features.freeze as frz
@@ -24,7 +24,7 @@
 
 def find_call_capabilities(
     ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
-) -> Tuple[FeatureSet, MatchResults]:
+) -> tuple[FeatureSet, MatchResults]:
     """
     find matches for the given rules for the given call.
 
@@ -51,7 +51,7 @@ def find_call_capabilities(
 
 def find_thread_capabilities(
     ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle
-) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+) -> tuple[FeatureSet, MatchResults, MatchResults]:
     """
     find matches for the given rules within the given thread.
 
@@ -89,7 +89,7 @@ def find_thread_capabilities(
 
 def find_process_capabilities(
     ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle
-) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
+) -> tuple[MatchResults, MatchResults, MatchResults, int]:
     """
     find matches for the given rules within the given process.
 
@@ -127,15 +127,15 @@ def find_process_capabilities(
 
 def find_dynamic_capabilities(
     ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress=None
-) -> Tuple[MatchResults, Any]:
+) -> tuple[MatchResults, Any]:
     all_process_matches: MatchResults = collections.defaultdict(list)
     all_thread_matches: MatchResults = collections.defaultdict(list)
     all_call_matches: MatchResults = collections.defaultdict(list)
 
     feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=())
 
     assert isinstance(extractor, DynamicFeatureExtractor)
-    processes: List[ProcessHandle] = list(extractor.get_processes())
+    processes: list[ProcessHandle] = list(extractor.get_processes())
     n_processes: int = len(processes)
 
     with capa.helpers.CapaProgressBar(