diff --git a/.github/workflows/test_lib.yml b/.github/workflows/test_lib.yml index 747194b2..2bc5a250 100644 --- a/.github/workflows/test_lib.yml +++ b/.github/workflows/test_lib.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v4 - name: Install local splat - run: python3 -m pip install . + run: python3 -m pip install .[dev] - name: Test run: splat capy diff --git a/README.md b/README.md index 219be683..66d2fbb5 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,30 @@ # splat + +[![PyPI](https://img.shields.io/pypi/v/splat64)](https://pypi.org/project/splat64/) + A binary splitting tool to assist with decompilation and modding projects Currently, only N64, PSX, and PS2 binaries are supported. Please check out the [wiki](https://github.com/ethteck/splat/wiki) for more information including [examples](https://github.com/ethteck/splat/wiki/Examples) of projects that use splat. -### Requirements -splat requires Python 3.8+. Package requirements can be installed via `pip3 install -U -r requirements.txt` +## Installing + +The recommended way to install is using from the PyPi release, via `pip`: + +```bash +python3 -m pip install -U splat64[mips] +``` + +The brackets corresponds to the optional dependencies to install while installing splat. Refer to [Optional dependencies](#optional-dependencies) to see the list of available groups. + +If you use a `requirements.txt` file in your repository, then you can add this library with the following line: + +```txt +splat64[mips]>=0.21.0,<1.0.0 +``` + +### Optional dependencies + +- `mips`: Required when using the N64, PSX or PS2 platforms. +- `dev`: Installs all the available dependencies groups and other packages for development. diff --git a/create_config.py b/create_config.py index aa751a1a..49428b9e 100755 --- a/create_config.py +++ b/create_config.py @@ -1,315 +1,9 @@ #! /usr/bin/env python3 -import argparse -import sys from pathlib import Path -from src.splat.util.gc import gcinfo -from src.splat.util.n64 import find_code_length, rominfo -from src.splat.util.psx import psxexeinfo - -parser = argparse.ArgumentParser( - description="Create a splat config from an N64 ROM, PSX executable, or a GameCube disc image." -) -parser.add_argument( - "file", help="Path to a .z64/.n64 ROM, PSX executable, or .iso/.gcm GameCube image" -) - - -def main(file_path: Path): - if not file_path.exists(): - sys.exit(f"File {file_path} does not exist ({file_path.absolute()})") - if file_path.is_dir(): - sys.exit(f"Path {file_path} is a directory ({file_path.absolute()})") - - # Check for N64 ROM - if file_path.suffix.lower() == ".n64" or file_path.suffix.lower() == ".z64": - create_n64_config(file_path) - return - - file_bytes = file_path.read_bytes() - - # Check for GC disc image - if int.from_bytes(file_bytes[0x1C:0x20], byteorder="big") == 0xC2339F3D: - create_gc_config(file_path, file_bytes) - return - - # Check for PSX executable - if file_bytes[0:8] == b"PS-X EXE": - create_psx_config(file_path, file_bytes) - return - - -def create_n64_config(rom_path: Path): - rom_bytes = rominfo.read_rom(rom_path) - - rom = rominfo.get_info(rom_path, rom_bytes) - basename = rom.name.replace(" ", "").lower() - - header = f"""\ -name: {rom.name.title()} ({rom.get_country_name()}) -sha1: {rom.sha1} -options: - basename: {basename} - target_path: {rom_path.with_suffix(".z64")} - elf_path: build/{basename}.elf - base_path: . - platform: n64 - compiler: {rom.compiler} - - # asm_path: asm - # src_path: src - # build_path: build - # create_asm_dependencies: True - - ld_script_path: {basename}.ld - ld_dependencies: True - - find_file_boundaries: True - header_encoding: {rom.header_encoding} - - o_as_suffix: True - use_legacy_include_asm: False - mips_abi_float_regs: o32 - - asm_function_macro: glabel - asm_jtbl_label_macro: jlabel - asm_data_macro: dlabel - - # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] - - symbol_addrs_path: - - symbol_addrs.txt - reloc_addrs_path: - - reloc_addrs.txt - - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto_path: undefined_syms_auto.txt - - extensions_path: tools/splat_ext - - # string_encoding: ASCII - # data_string_encoding: ASCII - rodata_string_guesser_level: 2 - data_string_guesser_level: 2 - # libultra_symbols: True - # hardware_regs: True - # gfx_ucode: # one of [f3d, f3db, f3dex, f3dexb, f3dex2] -""" - - first_section_end = find_code_length.run(rom_bytes, 0x1000, rom.entry_point) - - segments = f"""\ -segments: - - name: header - type: header - start: 0x0 - - - name: boot - type: bin - start: 0x40 - - - name: entry - type: code - start: 0x1000 - vram: 0x{rom.entry_point:X} - subsegments: - - [0x1000, hasm] - - - name: main - type: code - start: 0x{0x1000 + rom.entrypoint_info.entry_size:X} - vram: 0x{rom.entry_point + rom.entrypoint_info.entry_size:X} - follows_vram: entry -""" - - if rom.entrypoint_info.bss_size is not None: - segments += f"""\ - bss_size: 0x{rom.entrypoint_info.bss_size:X} -""" - - segments += f"""\ - subsegments: - - [0x{0x1000 + rom.entrypoint_info.entry_size:X}, asm] -""" - - if ( - rom.entrypoint_info.bss_size is not None - and rom.entrypoint_info.bss_start_address is not None - ): - bss_start = rom.entrypoint_info.bss_start_address - rom.entry_point + 0x1000 - # first_section_end points to the start of data - segments += f"""\ - - [0x{first_section_end:X}, data] - - {{ start: 0x{bss_start:X}, type: bss, vram: 0x{rom.entrypoint_info.bss_start_address:08X} }} -""" - # Point next segment to the detected end of the main one - first_section_end = bss_start - - segments += f"""\ - - - type: bin - start: 0x{first_section_end:X} - follows_vram: main - - [0x{rom.size:X}] -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - - -def create_gc_config(iso_path: Path, iso_bytes: bytes): - gc = gcinfo.get_info(iso_path, iso_bytes) - basename = gc.system_code + gc.game_code + gc.region_code + gc.publisher_code - - header = f"""\ -name: \"{gc.name.title()} ({gc.get_region_name()})\" -system_code: {gc.system_code} -game_code: {gc.game_code} -region_code: {gc.region_code} -publisher_code: {gc.publisher_code} -sha1: {gc.sha1} -options: - filesystem_path: filesystem - basename: {basename} - target_path: {iso_path.with_suffix(".iso")} - base_path: . - compiler: {gc.compiler} - platform: gc - # undefined_funcs_auto: True - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto: True - # undefined_syms_auto_path: undefined_syms_auto.txt - # symbol_addrs_path: symbol_addrs.txt - # asm_path: asm - # src_path: src - # build_path: build - # extensions_path: tools/splat_ext - # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] -""" - - segments = f"""\ -segments: - - name: filesystem - type: fst - path: filesystem/sys/fst.bin - - name: bootinfo - type: bootinfo - path: filesystem/sys/boot.bin - - name: bi2 - type: bi2 - path: filesystem/sys/bi2.bin - - name: apploader - type: apploader - path: filesystem/sys/apploader.img - - name: main - type: dol - path: filesystem/sys/main.dol -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - - -def create_psx_config(exe_path: Path, exe_bytes: bytes): - exe = psxexeinfo.PsxExe.get_info(exe_path, exe_bytes) - basename = exe_path.name.replace(" ", "").lower() - - header = f"""\ -name: {exe_path.name} -sha1: {exe.sha1} -options: - basename: {basename} - target_path: {exe_path} - base_path: . - platform: psx - compiler: GCC - - # asm_path: asm - # src_path: src - # build_path: build - # create_asm_dependencies: True - - ld_script_path: {basename}.ld - - find_file_boundaries: False - gp_value: 0x{exe.initial_gp:08X} - - o_as_suffix: True - use_legacy_include_asm: False - - asm_function_macro: glabel - asm_jtbl_label_macro: jlabel - asm_data_macro: dlabel - - section_order: [".rodata", ".text", ".data", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] - - symbol_addrs_path: - - symbol_addrs.txt - reloc_addrs_path: - - reloc_addrs.txt - - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto_path: undefined_syms_auto.txt - - extensions_path: tools/splat_ext - - subalign: 2 - - string_encoding: ASCII - data_string_encoding: ASCII - rodata_string_guesser_level: 2 - data_string_guesser_level: 2 -""" - - segments = f"""\ -segments: - - name: header - type: header - start: 0x0 - - - name: main - type: code - start: 0x800 - vram: 0x{exe.destination_vram:X} - # bss_size: Please fill out this value when you figure out the bss size - subsegments: -""" - text_offset = exe.text_offset - if text_offset != 0x800: - segments += f"""\ - - [0x800, rodata, 800] -""" - segments += f"""\ - - [0x{text_offset:X}, asm, {text_offset:X}] # estimated -""" - - if exe.data_offset != 0: - data_offset = exe.data_offset - segments += f"""\ - - [0x{data_offset:X}, data, {data_offset:X}] # estimated -""" - - segments += f"""\ - - [0x{exe.size:X}] -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - +import src.splat as splat if __name__ == "__main__": - args = parser.parse_args() - main(Path(args.file)) + args = splat.scripts.create_config.parser.parse_args() + splat.scripts.create_config.main(Path(args.file)) diff --git a/docs/Configuration.md b/docs/Configuration.md index 3eb2cf0e..3d3ef07c 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -32,12 +32,13 @@ target_path: path/to/target/binary Path to the final elf target #### Default -Path to the binary that was used as the input to `create_config.py` +Path to the binary that was used as the input to `python3 -m splat create_config` ### platform -The target platform for the binary. Options are +The target platform for the binary. Options are: + - `n64` (Nintendo 64) - `psx` (PlayStation 1) - `ps2` (PlayStation 2) diff --git a/docs/Quickstart.md b/docs/Quickstart.md index f8711446..591918fe 100644 --- a/docs/Quickstart.md +++ b/docs/Quickstart.md @@ -1,3 +1,5 @@ +# Quickstart + > **Note**: This quickstart is written with N64 ROMs in mind, and the assumption that you are using Ubuntu 20.04 either natively, via WSL2 or via Docker. For the purposes of this quickstart, we will assume that we are going to split a game called `mygame` and we have the ROM in `.z64` format named `baserom.z64`. @@ -10,59 +12,37 @@ mkdir -p ${HOME}/mygame && cd ${HOME}/mygame Copy the `baserom.z64` file into the `mygame` directory inside your home directory. -### System packages +## System packages -#### Python 3.8 +### Python 3.8 Ensure you are have **Python 3.8** or higher installed: ```sh -$ python3 --version +python3 --version Python 3.8.10 ``` If you get `bash: python3: command not found` install it with the following command: ```sh -sudo apt-get update && sudo apt-get install -y python3 python3-pip -``` - -#### Git - -Ensure you have **git**: - -```sh -$ git --version -``` - -If you get `bash: git: command not found`, install it with the following command: - -```sh -sudo apt-get update && sudo apt-get install -y git +sudo apt update && sudo apt install -y python3 python3-pip ``` -## Checkout the repository +## Install splat -We will clone **splat** into a `tools` directory to keep things organised: +We'll install `splat` using `pip` and enable its `mips` dependencies: ```sh -git clone https://github.com/ethteck/splat.git tools/splat -``` - -## Python packages - -Run the following to install the prerequisite Python packages: - -```sh -python3 -m pip install -r ./tools/splat/requirements.txt +python3 -m pip install -U splat64[mips] ``` ## Create a config file for your baserom -**splat** has a script that will generate a `yaml` file for your ROM. +`splat` has a script that will generate a `yaml` file for your ROM. ```sh -python3 tools/splat/create_config.py baserom.z64 +python3 -m splat create_config baserom.z64 ``` The `yaml` file generated will be named based upon the name of the ROM (taken from its header). The example below is for Super Mario 64: @@ -110,11 +90,12 @@ This is a bare-bones configuration and there is a lot of work required to map ou ## Run splat with your configuration ```sh -python3 tools/splat/split.py supermario64.yaml +python3 -m splat split supermario64.yaml ``` The output will look something like this: -``` + +```plain_text splat 0.7.10.1 Loading and processing symbols Starting scan @@ -139,11 +120,10 @@ Split 943 KB (11.24%) in defined segments unknown: 7 MB (88.76%) from unknown bin files ``` -Notice that **splat** has found some potential file splits (function start/end with 16 byte alignment padded with nops). +Notice that `splat` has found some potential file splits (function start/end with 16 byte alignment padded with nops). It's up to you to figure out the layout of the ROM. - ## Next Steps The reassembly of the ROM is currently out of scope of this quickstart, as is switching out the `asm` segments for `c`. diff --git a/pyproject.toml b/pyproject.toml index c0bd2395..6326bec2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,28 @@ classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", ] -dynamic = ["dependencies"] +dependencies = [ + "PyYAML", + "pylibyaml", + "tqdm", + "intervaltree", + "colorama", +] + +[project.optional-dependencies] +mips = [ + "spimdisasm>=1.18.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py + "rabbitizer>=1.8.0,<2.0.0", + "pygfxd", + "n64img>=0.1.4", + "crunch64>=0.2.0,<1.0.0", +] +dev = [ + "splat64[mips]", + "mypy", + "black", + "types-PyYAML", +] [project.urls] Repository = "https://github.com/ethteck/splat" @@ -18,12 +39,9 @@ Issues = "https://github.com/ethteck/splat/issues" Changelog = "https://github.com/ethteck/splat/blob/master/CHANGELOG.md" [build-system] -requires = ["hatchling", "hatch-requirements-txt"] +requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.metadata.hooks.requirements_txt] -files = ["requirements.txt"] - [tool.hatch.build.targets.wheel] packages = ["src/splat"]