Merge pull request #1658 from mandiant/sync-1657

sync
mandiant · Jul 20, 2023 · 5ae588d · 5ae588d
2 parents ce15a2b + a2f31ab
commit 5ae588d
Show file tree

Hide file tree

Showing 140 changed files with 1,036 additions and 753 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -159,12 +159,25 @@ The process described here has several goals:
 
 Please follow these steps to have your contribution considered by the maintainers:
 
+0. Sign the [Contributor License Agreement](#contributor-license-agreement)
 1. Follow the [styleguides](#styleguides)
 2. Update the CHANGELOG and add tests and documentation. In case they are not needed, indicate it in [the PR template](pull_request_template.md).
 3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
 
 While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
 
+### Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution,
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
 ## Styleguides
 
 ### Git Commit Messages

diff --git a/.github/flake8.ini b/.github/flake8.ini
@@ -13,8 +13,16 @@ extend-ignore =
     # B010 Do not call setattr with a constant attribute value
     B010,
     # G200 Logging statement uses exception in arguments
-    G200
-
+    G200,
+    # SIM102 Use a single if-statement instead of nested if-statements
+    # doesn't provide a space for commenting or logical separation of conditions
+    SIM102,
+    # SIM114 Use logical or and a single body
+    # makes logic trees too complex
+    SIM114,
+    # SIM117 Use 'with Foo, Bar:' instead of multiple with statements
+    # makes lines too long
+    SIM117
 
 per-file-ignores =
     # T201 print found.
@@ -26,4 +34,8 @@ per-file-ignores =
     # IDA tests emit results to output window so need to print
     tests/test_ida_features.py: T201
     # utility used to find the Binary Ninja API via invoking python.exe
-    capa/features/extractors/binja/find_binja_api.py: T201
+    capa/features/extractors/binja/find_binja_api.py: T201
+
+copyright-check = True
+copyright-min-file-size = 1 
+copyright-regexp = Copyright \(C\) 2023 Mandiant, Inc. All Rights Reserved.
diff --git a/.github/ruff.toml b/.github/ruff.toml
@@ -41,25 +41,3 @@ exclude = [
     "*_pb2.py", 
     "*_pb2.pyi"
 ]
-
-[per-file-ignores]
-# until we address #1592 and move test fixtures into conftest.py
-# then we need to ignore imports done to enable pytest fixtures.
-#
-# F401: `foo` imported but unused
-# F811 Redefinition of unused `foo`
-"tests/test_main.py" = ["F401", "F811"]
-"tests/test_proto.py" = ["F401", "F811"]
-"tests/test_freeze.py" = ["F401", "F811"]
-"tests/test_function_id.py" = ["F401", "F811"]
-"tests/test_viv_features.py" = ["F401", "F811"]
-"tests/test_cape_features.py" = ["F401", "F811"]
-"tests/test_binja_features.py" = ["F401", "F811"]
-"tests/test_pefile_features.py" = ["F401", "F811"]
-"tests/test_dnfile_features.py" = ["F401", "F811"]
-"tests/test_dotnet_features.py" = ["F401", "F811"]
-"tests/test_result_document.py" = ["F401", "F811"]
-"tests/test_dotnetfile_features.py" = ["F401", "F811"]
-"tests/test_static_freeze.py" = ["F401", "F811"]
-"tests/_test_proto.py" = ["F401", "F811"]
-"tests/_test_result_document.py" = ["F401", "F811"]
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -6,6 +6,9 @@ on:
   release:
     types: [edited, published]
 
+permissions:
+  contents: write
+
 jobs:
   build:
     name: PyInstaller for ${{ matrix.os }}

diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -7,6 +7,8 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize]
 
+permissions: read-all
+
 jobs:
   check_changelog:
     # no need to check for dependency updates via dependabot

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -6,6 +6,9 @@ on:
   release:
     types: [published]
 
+permissions:
+  contents: write
+
 jobs:
   pypi-publish:
     runs-on: ubuntu-latest
@@ -29,14 +32,7 @@ jobs:
       - name: upload package artifacts
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
-          name: ${{ matrix.asset_name }}
           path: dist/*
-      - name: upload package to GH Release
-        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN}}
-          file: dist/*
-          tag: ${{ github.ref }}
       - name: publish package
         uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
         with:

diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -4,6 +4,8 @@ on:
   release:
     types: [published]
 
+permissions: read-all
+
 jobs:
   tag:
     name: Tag capa rules

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -6,6 +6,8 @@ on:
   pull_request:
     branches: [ master, "dynamic-feature-extraction" ]
 
+permissions: read-all
+
 # save workspaces to speed up testing
 env:
   CAPA_SAVE_WORKSPACE: "True"
@@ -27,10 +29,11 @@ jobs:
     steps:
     - name: Checkout capa
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-    - name: Set up Python 3.8
+    # use latest available python to take advantage of best performance
+    - name: Set up Python 3.11
       uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install dependencies
       run: pip install -e .[dev]
     - name: Lint with ruff
@@ -51,10 +54,10 @@ jobs:
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
         submodules: recursive
-    - name: Set up Python 3.8
+    - name: Set up Python 3.11
       uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install capa
       run: pip install -e .[dev]
     - name: Run rule linter

diff --git a/.gitignore b/.gitignore
@@ -124,3 +124,5 @@ Pipfile
 Pipfile.lock
 /cache/
 .github/binja/binaryninja
+.github/binja/download_headless.py
+.github/binja/BinaryNinja-headless.zip
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,29 +1,53 @@
 # Change Log
 
 ## master (unreleased)
-- extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736
 
 ### New Features
-- Utility script to detect feature overlap between new and existing CAPA rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 - Add a dynamic feature extractor for the CAPE sandbox @yelhamer [#1535](https://github.com/mandiant/capa/issues/1535)
 - Add unit tests for the new CAPE extractor #1563 @yelhamer
 - Add a CAPE file format and CAPE-based dynamic feature extraction to scripts/show-features.py #1566 @yelhamer
 - Add a new process scope for the dynamic analysis flavor #1517 @yelhamer
 - Add a new thread scope for the dynamic analysis flavor #1517 @yelhamer
 - Add support for flavor-based rule scopes @yelhamer
-- use fancy box drawing characters for default output #1586 @williballenthin
-- use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin
-- publish via PyPI trusted publishing #1491 @williballenthin
-- migrate to pyproject.toml #1301 @williballenthin
 - Add ProcessesAddress and ThreadAddress #1612 @yelhamer
 
 ### Breaking Changes
-- Update Metadata type in capa main [#1411](https://github.com/mandiant/capa/issues/1411) [@Aayush-Goel-04](https://github.com/aayush-goel-04) @manasghandat
-- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
-- Change the old FeatureExtractor class' name into StaticFeatureExtractor, and make the former an alias for both the StaticFeatureExtractor and DynamicFeatureExtractor classes @yelhamer [#1567](https://github.com/mandiant/capa/issues/1567)
+
+
+### New Rules (1)
+
+- executable/pe/export/forwarded-export [email protected]
+-
+
+### Bug Fixes
+
+### capa explorer IDA Pro plugin
+
+### Development
+
+### Raw diffs
+- [capa v6.0.0...master](https://github.com/mandiant/capa/compare/v6.0.0...master)
+- [capa-rules v6.0.0...master](https://github.com/mandiant/capa-rules/compare/v6.0.0...master)
+
+## v6.0.0
+
+capa v6.0 brings many bug fixes and quality improvements, including 64 rule updates and 26 new rules. We're now publishing to PyPI via [Trusted Publishing](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) and have migrated to using a `pyproject.toml` file. @Aayush-Goel-04 contributed a lot of new code across many files, so please welcome them to the project, along with @anders-v @crowface28 @dkelly2e @RonnieSalomonsen and @ejfocampo as first-time rule contributors!
+
+For those that use capa as a library, we've introduced some limited breaking changes that better represent data types (versus less-structured data like dictionaries and strings). With the recent deprecation, we've also dropped support for Python 3.7.
+
+### New Features
+- add script to detect feature overlap between new and existing capa rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- extract forwarded exports from PE files #1624 @williballenthin
+- extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736
 - use fancy box drawing characters for default output #1586 @williballenthin
 
-### New Rules (23)
+### Breaking Changes
+- use a class to represent Metadata (not dict) #1411 @Aayush-Goel-04 @manasghandat
+- use pathlib.Path to represent file paths #1534 @Aayush-Goel-04
+- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
+- Require a Contributor License Agreement (CLA) for PRs going forward #1642 @williballenthin
+
+### New Rules (26)
 
 - load-code/shellcode/execute-shellcode-via-windows-callback-function [email protected] [email protected]
 - nursery/execute-shellcode-via-indirect-call [email protected]
@@ -47,7 +71,9 @@
 - host-interaction/memory/create-new-application-domain-in-dotnet [email protected]
 - host-interaction/gui/switch-active-desktop [email protected]
 - host-interaction/service/query-service-configuration @mr-tz
--
+- anti-analysis/anti-av/patch-event-tracing-for-windows-function [email protected]
+- data-manipulation/encoding/xor/covertly-decode-and-write-data-to-windows-directory-using-indirect-calls [email protected]
+- linking/runtime-linking/resolve-function-by-brute-ratel-badger-hash [email protected]
 
 ### Bug Fixes
 - extractor: add a Binary Ninja test that asserts its version #1487 @xusheng6
@@ -57,22 +83,30 @@
 - symtab: fix struct.unpack() format for 64-bit ELF files @yelhamer
 - symtab: safeguard against ZeroDivisionError for files containing a symtab with a null entry size @yelhamer
 - improve ELF strtab and needed parsing @mr-tz
-- better handle exceptional cases when parsing ELF files [#1458](https://github.com/mandiant/capa/issues/1458) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
-- Improved testing coverage for Binary Ninja Backend [#1446](https://github.com/mandiant/capa/issues/1446) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
-- Add logging and print redirect to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- better handle exceptional cases when parsing ELF files #1458 @Aayush-Goel-04
+- improved testing coverage for Binary Ninja backend #1446 @Aayush-Goel-04
+- add logging and print redirect to tqdm for capa main #749 @Aayush-Goel-04
 - extractor: fix binja installation path detection does not work with Python 3.11
 - tests: refine the IDA test runner script #1513 @williballenthin
 - output: don't leave behind traces of progress bar @williballenthin
 - import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin
+- main: don't show spinner when emitting debug messages #1636 @williballenthin
+- rules: add forwarded export characteristics to rule syntax file scope #1653 @RonnieSalomonsen
 
 ### capa explorer IDA Pro plugin
 
 ### Development
 - update ATT&CK/MBC data for linting #1568 @mr-tz
+- log time taken to analyze each function #1290 @williballenthin
+- tests: make fixture available via conftest.py #1592 @williballenthin
+- publish via PyPI trusted publishing #1491 @williballenthin
+- migrate to pyproject.toml #1301 @williballenthin
+- use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin
+
 
 ### Raw diffs
-- [capa v5.1.0...master](https://github.com/mandiant/capa/compare/v5.1.0...master)
-- [capa-rules v5.1.0...master](https://github.com/mandiant/capa-rules/compare/v5.1.0...master)
+- [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0)
+- [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0)
 
 ## v5.1.0
 capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and a output color format that's easier to read.

diff --git a/LICENSE.txt b/LICENSE.txt
@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (C) 2020 Mandiant, Inc.
+   Copyright (C) 2023 Mandiant, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-811-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-824-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)

diff --git a/capa/engine.py b/capa/engine.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

diff --git a/capa/exceptions.py b/capa/exceptions.py
@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 class UnsupportedRuntimeError(RuntimeError):
     pass
 

diff --git a/capa/features/address.py b/capa/features/address.py
@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import abc
 
 

diff --git a/capa/features/basicblock.py b/capa/features/basicblock.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

diff --git a/capa/features/common.py b/capa/features/common.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

diff --git a/capa/features/extractors/base_extractor.py b/capa/features/extractors/base_extractor.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

diff --git a/capa/features/extractors/binja/basicblock.py b/capa/features/extractors/binja/basicblock.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -130,7 +130,7 @@ def is_mov_imm_to_stack(il: MediumLevelILInstruction) -> bool:
     if il.src.operation != MediumLevelILOperation.MLIL_CONST:
         return False
 
-    if not il.dest.source_type == VariableSourceType.StackVariableSourceType:
+    if il.dest.source_type != VariableSourceType.StackVariableSourceType:
         return False
 
     return True

diff --git a/capa/features/extractors/binja/extractor.py b/capa/features/extractors/binja/extractor.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -53,9 +53,7 @@ def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
             mlil_lookup[mlil_bb.source_block.start] = mlil_bb
 
         for bb in f.basic_blocks:
-            mlil_bb = None
-            if bb.start in mlil_lookup:
-                mlil_bb = mlil_lookup[bb.start]
+            mlil_bb = mlil_lookup.get(bb.start)
 
             yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
 

diff --git a/capa/features/extractors/binja/file.py b/capa/features/extractors/binja/file.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt