Code fixes before new release (#99)

## Description

Several code fixes and stability improvements including:
* switched to `rye` completely
* restored CI tests
* added plenty more tests
* fixed a circular deps problem
* code cleanup (replaced `path.open().read/write` with `path.read_text/write_text/read_bytes/write_bytes`

.github/workflows/build.yml

@@ -19,17 +19,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-2019, ubuntu-22.04, macos-13]
+        os: [windows-2019, windows-2022, ubuntu-22.04, ubuntu-24.04, macos-13]
         python-version: ['3.10', '3.11', '3.12']
     name: ${{ matrix.os }} / ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     outputs:
       windows-2019-3-10: ${{ join(steps.*.outputs.windows-2019-3-10,'') }}
       windows-2019-3-11: ${{ join(steps.*.outputs.windows-2019-3-11,'') }}
       windows-2019-3-12: ${{ join(steps.*.outputs.windows-2019-3-12,'') }}
-      ubuntu-22.04-3-10: ${{ join(steps.*.outputs.ubuntu-22.04-3-10,'') }}
-      ubuntu-22.04-3-11: ${{ join(steps.*.outputs.ubuntu-22.04-3-11,'') }}
-      ubuntu-22.04-3-12: ${{ join(steps.*.outputs.ubuntu-22.04-3-12,'') }}
+      ubuntu-22-04-3-10: ${{ join(steps.*.outputs.ubuntu-22-04-3-10,'') }}
+      ubuntu-22-04-3-11: ${{ join(steps.*.outputs.ubuntu-22-04-3-11,'') }}
+      ubuntu-22-04-3-12: ${{ join(steps.*.outputs.ubuntu-22-04-3-12,'') }}
       macos-13-3-10: ${{ join(steps.*.outputs.macos-13-3-10,'') }}
       macos-13-3-11: ${{ join(steps.*.outputs.macos-13-3-11,'') }}
       macos-13-3-12: ${{ join(steps.*.outputs.macos-13-3-12,'') }}
@@ -47,62 +47,17 @@ jobs:
         version: 'latest'
 
     - name: "Install Pre-requisite (Linux)"
-      if: matrix.os == 'ubuntu-22.04'
+      if: startsWith(matrix.os, 'ubuntu')
       shell: bash
       run: |
         sudo apt update
-        sudo apt upgrade -y
-        sudo apt install -y build-essential libegl1 libgl1-mesa-glx
-
-    # - name: "Install Pre-requisite (macOS)"
-    #   if: matrix.os == 'macos-13'
-    #   run: |
-    #     env
-
-    # - name: "Install Pre-requisite (Windows)"
-    #   if: matrix.os == 'windows-2019'
-    #   shell: pwsh
-    #   run: |
-    #     env
+        sudo apt install -y build-essential libegl1
 
     - run: rye fmt
     - run: rye lint
     - run: rye test
     - run: rye build --wheel --out ./build
 
-
-    # - name: Build artifact
-    #   shell: bash
-    #   run: |
-    #     mkdir build
-    #     mkdir build/bin
-    #     python --version
-    #     python -m pip --version
-    #     python -m pip install --upgrade pip setuptools wheel
-    #     python -m pip install --user --upgrade .[all]
-
-    # - name: "Post build Cemu (Windows)"
-    #   if: matrix.os == 'windows-2019'
-    #   shell: pwsh
-    #   run: |
-    #     Copy-Item $env:APPDATA\Python\Python*\Scripts\cemu.exe build\bin\
-
-    # - name: "Post build Cemu (Linux)"
-    #   if: matrix.os == 'ubuntu-22.04'
-    #   shell: bash
-    #   run: |
-    #     cp -v ~/.local/bin/cemu build/bin/
-
-    # - name: "Post build Cemu (macOS)"
-    #   if: matrix.os == 'macos-13'
-    #   shell: bash
-    #   run: |
-    #     cp -v ~/.local/bin/cemu build/bin/ || cp -v /Users/runner/Library/Python/${{ matrix.python-version }}/bin/cemu build/bin/
-
-    # - name: "Run tests"
-    #   run: |
-    #     python -m pytest tests/
-
     - name: Publish artifact
       id: publish_artifact
       uses: actions/upload-artifact@v4
@@ -112,35 +67,39 @@ jobs:
 
     - name: Populate the successful output (Windows)
       id: output_success_windows
-      if: ${{ matrix.os == 'windows-2019' &&  success() }}
+      if: ${{ startsWith(matrix.os, 'windows') &&  success() }}
       shell: pwsh
       run: |
+        $osVersion = "${{ matrix.os }}" -replace "\.", "-"
         $pyVersion = "${{ matrix.python-version }}" -replace "\.", "-"
-        echo "${{ matrix.os }}-$pyVersion=✅ ${{ matrix.os }} ${{ matrix.python-version }}" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+        echo "${osVersion}-$pyVersion=✅ ${{ matrix.os }} ${{ matrix.python-version }}" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
 
     - name: Populate the successful output (Other)
       id: output_success_other
-      if: ${{matrix.os != 'windows-2019' && success() }}
+      if: ${{startsWith(matrix.os, 'windows') == false && success() }}
       shell: bash
       run: |
+        osVersion="$(echo -n ${{ matrix.os }} | tr . -)"
         pyVersion="$(echo -n ${{ matrix.python-version }} | tr . -)"
-        echo "${{ matrix.os }}-${pyVersion}=✅ ${{ matrix.os }} ${{ matrix.python-version }}" >> $GITHUB_OUTPUT
+        echo "${osVersion}-${pyVersion}=✅ ${{ matrix.os }} ${{ matrix.python-version }}" >> $GITHUB_OUTPUT
 
     - name: Populate the failure output (Windows)
       id: output_failure_windows
-      if: ${{matrix.os == 'windows-2019' && failure() }}
+      if: ${{startsWith(matrix.os, 'windows') && failure() }}
       shell: pwsh
       run: |
+        $osVersion = "${{ matrix.os }}" -replace "\.", "-"
         $pyVersion = "${{ matrix.python-version }}" -replace "\.", "-"
-        echo "${{ matrix.os }}-${pyVersion}=❌ ${{ matrix.os }} ${{ matrix.python-version }}" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+        echo "${osVersion}-${pyVersion}=❌ ${{ matrix.os }} ${{ matrix.python-version }}" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
 
     - name: Populate the failure output (Other)
       id: output_failure_other
-      if: ${{matrix.os != 'windows-2019' && failure() }}
+      if: ${{startsWith(matrix.os, 'windows') && failure() }}
       shell: bash
       run: |
+        osVersion="$(echo -n ${{ matrix.os }} | tr . -)"
         pyVersion="$(echo -n ${{ matrix.python-version }} | tr . -)"
-        echo "${{ matrix.os }}-$pyVersion=❌ ${{ matrix.os }} ${{ matrix.python-version }}" >> $GITHUB_OUTPUT
+        echo "${osVersion}-$pyVersion=❌ ${{ matrix.os }} ${{ matrix.python-version }}" >> $GITHUB_OUTPUT
 
   notify:
     env:
@@ -167,9 +126,9 @@ jobs:
           ${{ needs.build.outputs.windows-2019-3-10 }}
           ${{ needs.build.outputs.windows-2019-3-11 }}
           ${{ needs.build.outputs.windows-2019-3-12 }}
-          ${{ needs.build.outputs.ubuntu-22.04-3-10 }}
-          ${{ needs.build.outputs.ubuntu-22.04-3-11 }}
-          ${{ needs.build.outputs.ubuntu-22.04-3-12 }}
+          ${{ needs.build.outputs.ubuntu-22-04-3-10 }}
+          ${{ needs.build.outputs.ubuntu-22-04-3-11 }}
+          ${{ needs.build.outputs.ubuntu-22-04-3-12 }}
           ${{ needs.build.outputs.macos-13-3-10 }}
           ${{ needs.build.outputs.macos-13-3-11 }}
           ${{ needs.build.outputs.macos-13-3-12 }}

src/cemu/__main__.py

@@ -20,9 +20,7 @@ def setup_remote_debug(port: int = cemu.const.DEBUG_DEBUGPY_PORT):
 
 
 def main(argv: list[str]):
-    parser = argparse.ArgumentParser(
-                        prog=cemu.const.PROGNAME,
-                        description=cemu.const.DESCRIPTION)
+    parser = argparse.ArgumentParser(prog=cemu.const.PROGNAME, description=cemu.const.DESCRIPTION)
     parser.add_argument("filename")
     parser.add_argument("--debug", action="store_true")
     parser.add_argument("--attach", action="store_true")
@@ -46,6 +44,7 @@ def main(argv: list[str]):
 
 if __name__ == "__main__":
     import sys
+
     path = pathlib.Path(__file__).absolute().parent.parent
     sys.path.append(str(path))
     main(sys.argv)

src/cemu/arch/__init__.py

@@ -1,19 +1,24 @@
+from dataclasses import dataclass
 import enum
 import importlib
+import pathlib
 from typing import Optional, TYPE_CHECKING
 
 import capstone
 import keystone
 import unicorn
 
+import cemu.errors
 from cemu.const import SYSCALLS_PATH
-from ..ui.utils import popup, PopupType
+from cemu.log import dbg, error
+from cemu.utils import DISASSEMBLY_DEFAULT_BASE_ADDRESS
+
 
 if TYPE_CHECKING:
     import cemu.core
 
 
-class Endianness(enum.Enum):
+class Endianness(enum.IntEnum):
     LITTLE_ENDIAN = 1
     BIG_ENDIAN = 2
 
@@ -27,7 +32,7 @@ def __int__(self) -> int:
         return self.value
 
 
-class Syntax(enum.Enum):
+class Syntax(enum.IntEnum):
     INTEL = 1
     ATT = 2
 
@@ -71,26 +76,29 @@ def syscalls(self):
         if not self.__context:
             import cemu.core
 
+            assert cemu.core.context
             self.__context = cemu.core.context
             assert isinstance(self.__context, cemu.core.GlobalContext)
 
         if not self.__syscalls:
-            syscall_dir = SYSCALLS_PATH / str(self.__context.os)
+            syscall_dir = SYSCALLS_PATH / str(self.__context.os).lower()
 
             try:
                 fpath = syscall_dir / (self.syscall_filename + ".csv")
             except ValueError as e:
-                popup(str(e), PopupType.Error, "No Syscall File Error")
+                error(f"No Syscall File Error: {e}")
                 return {}
 
             self.__syscalls = {}
-            if fpath.exists():
-                with fpath.open("r") as fd:
-                    for row in fd.readlines():
-                        row = [x.strip() for x in row.strip().split(",")]
-                        syscall_number = int(row[0])
-                        syscall_name = row[1].lower()
-                        self.__syscalls[syscall_name] = self.syscall_base + syscall_number
+            if not fpath.exists():
+                raise FileNotFoundError(fpath)
+
+            with fpath.open("r") as fd:
+                for row in fd.readlines():
+                    row = [x.strip() for x in row.strip().split(",")]
+                    syscall_number = int(row[0])
+                    syscall_name = row[1].lower()
+                    self.__syscalls[syscall_name] = self.syscall_base + syscall_number
 
         return self.__syscalls
 
@@ -264,3 +272,118 @@ def is_sparc64(a: Architecture):
 
 def is_ppc(a: Architecture):
     return isinstance(a, PowerPC)
+
+
+def format_address(addr: int, arch: Optional[Architecture] = None) -> str:
+    """Format an address to string, aligned to the given architecture
+
+    Args:
+        addr (int): _description_
+        arch (Optional[Architecture], optional): _description_. Defaults to None.
+
+    Raises:
+        ValueError: _description_
+
+    Returns:
+        str: _description_
+    """
+    if arch is None:
+        import cemu.core
+
+        if not cemu.core.context:
+            ptrsize = 8
+        else:
+            ptrsize = cemu.core.context.architecture.ptrsize
+    else:
+        ptrsize = arch.ptrsize
+
+    match ptrsize:
+        case 2:
+            return f"{addr:#04x}"
+        case 4:
+            return f"{addr:#08x}"
+        case 8:
+            return f"{addr:#016x}"
+        case _:
+            raise ValueError(f"Invalid pointer size value of {ptrsize}")
+
+
+@dataclass
+class Instruction:
+    address: int
+    mnemonic: str
+    operands: str
+    bytes: bytes
+
+    @property
+    def size(self):
+        return len(self.bytes)
+
+    @property
+    def end(self) -> int:
+        return self.address + self.size
+
+    def __str__(self):
+        return f'Instruction({self.address:#x}, "{self.mnemonic} {self.operands}")'
+
+
+def disassemble(raw_data: bytes, count: int = -1, base: int = DISASSEMBLY_DEFAULT_BASE_ADDRESS) -> list[Instruction]:
+    """Disassemble the code given as raw data, with the given architecture.
+
+    Args:
+        raw_data (bytes): the raw byte code to disassemble
+        arch (Architecture): the architecture to use for disassembling
+        count (int, optional): the maximum number of instruction to disassemble. Defaults to -1.
+        base (int, optional): the disassembled code base address. Defaults to DISASSEMBLY_DEFAULT_BASE_ADDRESS
+
+    Returns:
+        str: the text representation of the disassembled code
+    """
+    assert cemu.core.context
+    arch = cemu.core.context.architecture
+    insns: list[Instruction] = []
+    for idx, ins in enumerate(arch.cs.disasm(raw_data, base)):
+        insn = Instruction(ins.address, ins.mnemonic, ins.op_str, ins.bytes)
+        insns.append(insn)
+        if idx == count:
+            break
+
+    dbg(f"{insns=}")
+    return insns
+
+
+def disassemble_file(fpath: pathlib.Path) -> list[Instruction]:
+    return disassemble(fpath.read_bytes())
+
+
+def assemble(code: str, base_address: int = DISASSEMBLY_DEFAULT_BASE_ADDRESS) -> list[Instruction]:
+    """
+    Helper function to assemble code receive in parameter `asm_code` using Keystone.
+
+    @param code : assembly code in bytes (multiple instructions must be separated by ';')
+    @param base_address : (opt) the base address to use
+
+    @return a list of Instruction
+    """
+    assert cemu.core.context
+    arch = cemu.core.context.architecture
+
+    #
+    # Compile the entire given code
+    #
+    bytecode, assembled_insn_count = arch.ks.asm(code, as_bytes=True, addr=base_address)
+    if not bytecode or assembled_insn_count == 0:
+        raise cemu.errors.AssemblyException("Not instruction compiled")
+
+    assert isinstance(bytecode, bytes)
+
+    #
+    # Decompile it and return the stuff
+    #
+    insns = disassemble(bytecode, base=base_address)
+    dbg(f"{insns=}")
+    return insns
+
+
+def assemble_file(fpath: pathlib.Path) -> list[Instruction]:
+    return assemble(fpath.read_text())