Merge branch 'master' into master

CHANGELOG.md

@@ -22,6 +22,8 @@
 
 - Fix binja backend stack string detection. #1473 @xusheng6
 - linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin 
+- OS: detect Android ELF files #1705 @williballenthin
+- ELF: fix parsing of symtab #1704 @williballenthin
 
 ### capa explorer IDA Pro plugin
 - fix unhandled exception when resolving rule path #1693 @mike-hunhoff

capa/features/extractors/elf.py

@@ -13,6 +13,8 @@
 from typing import Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
 from dataclasses import dataclass
 
+import Elf  # from vivisect
+
 logger = logging.getLogger(__name__)
 
 
@@ -54,6 +56,7 @@ class OS(str, Enum):
     CLOUD = "cloud"
     SYLLABLE = "syllable"
     NACL = "nacl"
+    ANDROID = "android"
 
 
 # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658
@@ -709,17 +712,17 @@ def get_symbols(self) -> Iterator[Symbol]:
         yield from self.symbols
 
     @classmethod
-    def from_Elf(cls, ElfBinary) -> Optional["SymTab"]:
-        endian = "<" if ElfBinary.getEndian() == 0 else ">"
-        bitness = ElfBinary.bits
+    def from_viv(cls, elf: Elf.Elf) -> Optional["SymTab"]:
+        endian = "<" if elf.getEndian() == 0 else ">"
+        bitness = elf.bits
 
         SHT_SYMTAB = 0x2
-        for section in ElfBinary.sections:
-            if section.sh_info & SHT_SYMTAB:
-                strtab_section = ElfBinary.sections[section.sh_link]
-                sh_symtab = Shdr.from_viv(section, ElfBinary.readAtOffset(section.sh_offset, section.sh_size))
+        for section in elf.sections:
+            if section.sh_type == SHT_SYMTAB:
+                strtab_section = elf.sections[section.sh_link]
+                sh_symtab = Shdr.from_viv(section, elf.readAtOffset(section.sh_offset, section.sh_size))
                 sh_strtab = Shdr.from_viv(
-                    strtab_section, ElfBinary.readAtOffset(strtab_section.sh_offset, strtab_section.sh_size)
+                    strtab_section, elf.readAtOffset(strtab_section.sh_offset, strtab_section.sh_size)
                 )
 
         try:
@@ -764,6 +767,11 @@ def guess_os_from_ph_notes(elf: ELF) -> Optional[OS]:
         elif note.name == "FreeBSD":
             logger.debug("note owner: %s", "FREEBSD")
             return OS.FREEBSD
+        elif note.name == "Android":
+            logger.debug("note owner: %s", "Android")
+            # see the following for parsing the structure:
+            # https://android.googlesource.com/platform/ndk/+/master/parse_elfnote.py
+            return OS.ANDROID
         elif note.name == "GNU":
             abi_tag = note.abi_tag
             if abi_tag:
@@ -855,6 +863,8 @@ def guess_os_from_needed_dependencies(elf: ELF) -> Optional[OS]:
             return OS.HURD
         if needed.startswith("libhurduser.so"):
             return OS.HURD
+        if needed.startswith("libandroid.so"):
+            return OS.ANDROID
 
     return None
 

capa/features/extractors/viv/function.py

@@ -38,7 +38,7 @@ def extract_function_symtab_names(fh: FunctionHandle) -> Iterator[Tuple[Feature,
         # this is in order to eliminate the computational overhead of refetching symtab each time.
         if "symtab" not in fh.ctx["cache"]:
             try:
-                fh.ctx["cache"]["symtab"] = SymTab.from_Elf(fh.inner.vw.parsedbin)
+                fh.ctx["cache"]["symtab"] = SymTab.from_viv(fh.inner.vw.parsedbin)
             except Exception:
                 fh.ctx["cache"]["symtab"] = None
 

capa/features/extractors/viv/insn.py

@@ -115,7 +115,7 @@ def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterato
                 # the symbol table gets stored as a function's attribute in order to avoid running
                 # this code everytime the call is made, thus preventing the computational overhead.
                 try:
-                    fh.ctx["cache"]["symtab"] = SymTab.from_Elf(f.vw.parsedbin)
+                    fh.ctx["cache"]["symtab"] = SymTab.from_viv(f.vw.parsedbin)
                 except Exception:
                     fh.ctx["cache"]["symtab"] = None
 

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
     "Topic :: Security",
 ]
 dependencies = [
-    "tqdm==4.65.0",
+    "tqdm==4.66.1",
     "pyyaml==6.0.1",
     "tabulate==0.9.0",
     "colorama==0.4.6",
@@ -77,10 +77,10 @@ dev = [
     "flake8-simplify==0.20.0",
     "flake8-use-pathlib==0.3.0",
     "flake8-copyright==0.2.4",
-    "ruff==0.0.282",
+    "ruff==0.0.284",
     "black==23.7.0",
     "isort==5.11.4",
-    "mypy==1.4.1",
+    "mypy==1.5.0",
     "psutil==5.9.2",
     "stix2==3.0.1",
     "requests==2.31.0",

tests/fixtures.py

@@ -308,6 +308,8 @@ def get_data_path_by_name(name) -> Path:
         return CD / "data" / "2bf18d0403677378adad9001b1243211.elf_"
     elif name.startswith("ea2876"):
         return CD / "data" / "ea2876e9175410b6f6719f80ee44b9553960758c7d0f7bed73c0fe9a78d8e669.dll_"
+    elif name.startswith("1038a2"):
+        return CD / "data" / "1038a23daad86042c66bfe6c9d052d27048de9653bde5750dc0f240c792d9ac8.elf_"
     else:
         raise ValueError(f"unexpected sample fixture: {name}")
 

tests/test_os_detection.py

@@ -80,6 +80,18 @@ def test_elf_symbol_table():
         assert capa.features.extractors.elf.detect_elf_os(f) == "linux"
 
 
+def test_elf_android_notes():
+    # DEBUG:capa.features.extractors.elf:guess: osabi: None
+    # DEBUG:capa.features.extractors.elf:guess: ph notes: OS.ANDROID
+    # DEBUG:capa.features.extractors.elf:guess: sh notes: None
+    # DEBUG:capa.features.extractors.elf:guess: linker: None
+    # DEBUG:capa.features.extractors.elf:guess: ABI versions needed: None
+    # DEBUG:capa.features.extractors.elf:guess: needed dependencies: OS.ANDROID
+    path = get_data_path_by_name("1038a2")
+    with Path(path).open("rb") as f:
+        assert capa.features.extractors.elf.detect_elf_os(f) == "android"
+
+
 def test_elf_parse_capa_pyinstaller_header():
     # error after misidentified large pydata section with address 0; fixed in #1454
     # compressed ELF header of capa-v5.1.0-linux