add dynamic analysis

.github/mypy/mypy.ini

@@ -1,5 +1,12 @@
 [mypy]
 
+# TODO(yelhamer): remove this once proto has been added
+# for the dynamic rendering
+exclude = (?x)(
+    ^capa/render/proto/__init__.py$
+    | ^tests/_test_proto.py$
+  )
+
 [mypy-halo.*]
 ignore_missing_imports = True
 

.github/workflows/tests.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, "dynamic-feature-extraction" ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, "dynamic-feature-extraction" ]
 
 permissions: read-all
 

.gitmodules

@@ -4,3 +4,4 @@
 [submodule "tests/data"]
 	path = tests/data
 	url = ../capa-testfiles.git
+	branch = dynamic-feature-extractor

CHANGELOG.md

@@ -4,11 +4,20 @@
 
 ### New Features
 - ELF: implement file import and export name extractor #1607 @Aayush-Goel-04
+- Add a dynamic feature extractor for the CAPE sandbox @yelhamer [#1535](https://github.com/mandiant/capa/issues/1535)
+- Add unit tests for the new CAPE extractor #1563 @yelhamer
+- Add a CAPE file format and CAPE-based dynamic feature extraction to scripts/show-features.py #1566 @yelhamer
+- Add a new process scope for the dynamic analysis flavor #1517 @yelhamer
+- Add a new thread scope for the dynamic analysis flavor #1517 @yelhamer
+- Add support for flavor-based rule scopes @yelhamer
+- Add ProcessesAddress and ThreadAddress #1612 @yelhamer
+- Add dynamic capability extraction @yelhamer
+- Add support for mixed-scopes rules @yelhamer
+- Add a call scope @yelhamer
 
 ### Breaking Changes
 
 ### New Rules (4)
-
 - executable/pe/export/forwarded-export [email protected]
 - host-interaction/bootloader/get-uefi-variable [email protected]
 - host-interaction/bootloader/set-uefi-variable [email protected]

capa/features/address.py

@@ -43,6 +43,102 @@ def __hash__(self):
         return int.__hash__(self)
 
 
+class ProcessAddress(Address):
+    """an address of a process in a dynamic execution trace"""
+
+    def __init__(self, pid: int, ppid: int = 0):
+        assert ppid >= 0
+        assert pid > 0
+        self.ppid = ppid
+        self.pid = pid
+
+    def __repr__(self):
+        return "process(%s%s)" % (
+            f"ppid: {self.ppid}, " if self.ppid > 0 else "",
+            f"pid: {self.pid}",
+        )
+
+    def __hash__(self):
+        return hash((self.ppid, self.pid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) == (other.ppid, other.pid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) < (other.ppid, other.pid)
+
+
+class ThreadAddress(Address):
+    """addresses a thread in a dynamic execution trace"""
+
+    def __init__(self, process: ProcessAddress, tid: int):
+        assert tid >= 0
+        self.process = process
+        self.tid = tid
+
+    def __repr__(self):
+        return f"{self.process}, thread(tid: {self.tid})"
+
+    def __hash__(self):
+        return hash((self.process, self.tid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) == (other.process, other.tid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) < (other.process, other.tid)
+
+
+class DynamicCallAddress(Address):
+    """addesses a call in a dynamic execution trace"""
+
+    def __init__(self, thread: ThreadAddress, id: int):
+        assert id >= 0
+        self.thread = thread
+        self.id = id
+
+    def __repr__(self):
+        return f"{self.thread}, call(id: {self.id})"
+
+    def __hash__(self):
+        return hash((self.thread, self.id))
+
+    def __eq__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) == (other.thread, other.id)
+
+    def __lt__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) < (other.thread, other.id)
+
+
+class DynamicReturnAddress(Address):
+    """an address from a dynamic analysis trace"""
+
+    def __init__(self, call: DynamicCallAddress, return_address: int):
+        assert return_address >= 0
+        self.call = call
+        self.return_address = return_address
+
+    def __repr__(self):
+        return f"{self.call}, dynamic-call(return-address: 0x{self.return_address:x})"
+
+    def __hash__(self):
+        return hash((self.call, self.return_address))
+
+    def __eq__(self, other):
+        assert isinstance(other, DynamicReturnAddress)
+        return (self.call, self.return_address) == (other.call, other.return_address)
+
+    def __lt__(self, other):
+        assert isinstance(other, DynamicReturnAddress)
+        return (self.call, self.return_address) < (other.call, other.return_address)
+
+
 class RelativeVirtualAddress(int, Address):
     """a memory address relative to a base address"""
 

capa/features/common.py

@@ -457,6 +457,15 @@ def evaluate(self, ctx, **kwargs):
 FORMAT_AUTO = "auto"
 FORMAT_SC32 = "sc32"
 FORMAT_SC64 = "sc64"
+FORMAT_CAPE = "cape"
+STATIC_FORMATS = (
+    FORMAT_SC32,
+    FORMAT_SC64,
+    FORMAT_PE,
+    FORMAT_ELF,
+    FORMAT_DOTNET,
+)
+DYNAMIC_FORMATS = (FORMAT_CAPE,)
 FORMAT_FREEZE = "freeze"
 FORMAT_RESULT = "result"
 FORMAT_UNKNOWN = "unknown"