Add dockerfile for toolchain, minor internal changes

.devcontainer/devcontainer.json

@@ -0,0 +1,45 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
+{
+	"name": "Existing Docker Compose (Extend)",
+
+	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
+	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+	"dockerComposeFile": [
+		"../ASM/docker-compose.yml"
+	],
+
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+	"service": "toolchain",
+
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
+	"workspaceFolder": "/app/OoT-Randomizer",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line if you want start specific services in your Docker Compose config.
+	// "runServices": [],
+
+	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+	// "shutdownAction": "none",
+
+	// Uncomment the next line to run commands after the container is created.
+	// "postCreateCommand": "cat /etc/os-release",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// On Linux these users will have their IDs modified to the host user's ID to attempt to prevent
+	// file permissions conflicts on bind mounts.
+	"remoteUser": "ootr",
+	"containerUser": "ootr",
+
+	"overrideCommand": true
+}

ASM/.gitignore

@@ -4,3 +4,4 @@
 /c/*.o
 armips*
 !.gitkeep
+!dmaTable.dat

ASM/Dockerfile

@@ -0,0 +1,72 @@
+FROM debian:bookworm
+
+ARG USERNAME=ootr
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+# Create the user
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME -s /bin/bash
+
+# Install build tools
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes curl gnupg ca-certificates \
+  && curl https://practicerom.com/public/packages/debian/pgp.pub | apt-key add - \
+  && echo deb http://practicerom.com/public/packages/debian staging main >/etc/apt/sources.list.d/practicerom.list \
+  && apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes \
+    build-essential \
+    nodejs \
+    npm \
+    git \
+    cmake \
+    gdb-multiarch
+
+# Install toolchain. Compile from source if not on Intel/AMD.
+RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes n64-ultra; \
+  else \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes \
+      wget \
+      tar \
+      make \
+      diffutils \
+      texinfo \
+      gcc \
+      g++ \
+      lua5.3 \
+      libjansson4 \
+      libusb-1.0-0 \
+      libgmp10 \
+      liblua5.3-dev \
+      libjansson-dev \
+      libusb-1.0-0-dev \
+      libgmp-dev \
+      libmpfr-dev \
+      zlib1g-dev; \
+  fi
+RUN if [ "$(dpkg --print-architecture)" != "amd64" ]; then \
+    mkdir -p build \
+    && cd build \
+    && git clone --recursive https://github.com/glankk/n64.git \
+    && cd n64 \
+    && ./configure --enable-vc --prefix=/opt/n64 \
+    && make install-toolchain \
+    && make && make install \
+    && make install-sys \
+    && echo 'export PATH="/opt/n64/bin:$PATH"' >> /etc/profile; \
+  fi
+
+# Build armips
+WORKDIR /build
+RUN git clone --recursive https://github.com/Kingcom/armips.git
+WORKDIR /build/armips
+RUN mkdir build
+WORKDIR /build/armips/build
+RUN cmake -DCMAKE_BUILD_TYPE=Release .. \
+  && cmake --build . \
+  && mv ./armips /usr/local/bin/ \
+  && chmod a+rx /usr/local/bin/armips
+
+# Run as non-root user
+USER $USERNAME

ASM/README.md

@@ -1,5 +1,23 @@
 Advanced modifications to the Randomzier source require a bit more software than what is needed for running it.
 
+# Global Prequisites
+- Put the ROM you want to patch at `roms/base.z64`. This needs to be an uncompressed ROM; OoTRandomizer will create it when you run it with a compressed ROM.
+
+# Option 1: Docker
+Using Docker will automatically download and set up the tools required to compile and run all code in the randomizer, on any OS. Setting up Docker also enables the use of the [Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) extension in VSCode (see `/Notes/vscmode.md` for more details).
+- Install Docker Desktop
+  - Windows: https://docs.docker.com/desktop/install/windows-install/
+  - macOS: https://docs.docker.com/desktop/install/mac-install/
+  - Linux: https://docs.docker.com/desktop/install/linux/
+    - *(Optional)* Install [Docker CE](https://docs.docker.com/engine/install/) and the [Compose plugin](https://docs.docker.com/compose/install/linux/) in place of Desktop if desired, or use podman.
+    - Do *NOT* use the Ubuntu snap package.
+- On Windows, run `build.bat`
+- On Linux or macOS, run `build.sh`
+- Alternatively, run `docker compose up` in the `ASM` folder.
+  - Linux users must supply a user ID and group ID pair for the container to run as so compiled code is not owned by root. See `build.sh` for details. This does not affect Windows or macOS.
+- All ASM and C code will be recompiled each time the build scripts are run.
+
+# Option 2: Manual setup
 ## Assembly: armips
 ### Windows Prerequisite
 - Download and install the [Visual Studio 2015-202x Visual C++ Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170#visual-studio-2015-2017-2019-and-2022) package.
@@ -10,7 +28,6 @@ Advanced modifications to the Randomzier source require a bit more software than
   - [Windows automated builds](https://buildbot.orphis.net/armips/)
   - On other platforms you'll need either `clang` or `gcc`, `cmake`, and either `ninja` or `make` installed. All of these should be available in the package repositories of every major Linux distribution and Homebrew on macOS. After, follow the [building from source instructions](https://github.com/Kingcom/armips#22-building-from-source).
 - Put the armips executable in the `tools` directory, or somewhere in your PATH.
-- Put the ROM you want to patch at `roms/base.z64`. This needs to be an uncompressed ROM; OoTRandomizer will produce one at ZOOTDEC.z64 when you run it with a compressed ROM.
 - Run `python build.py --no-compile-c`, which will:
   - create `roms/patched.z64`
   - update some `txt` files in `build/` and in `../data/generated/`. Check `git status` to see which ones have changed. Make sure you submit them all together!

ASM/build.bat

@@ -0,0 +1,2 @@
+@echo off
+docker compose up

ASM/build.sh

@@ -0,0 +1,8 @@
+#/bin/bash
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    export DOCKER_UID="$(id -u)"
+    export DOCKER_GID="$(id -g)"
+    export DOCKER_UID_GID="$(id -u):$(id -g)"
+fi
+cd "$(dirname "$0")"
+docker compose up

ASM/docker-compose.yml

@@ -0,0 +1,30 @@
+name: ootr-dev
+
+services:
+  toolchain:
+    build:
+      context: .
+      args:
+        USERNAME: ootr
+        USER_UID: ${DOCKER_UID:-1000}
+        USER_GID: ${DOCKER_GID:-1000}
+    user: ${DOCKER_UID_GID:-}
+    volumes:
+      - ..:/app/OoT-Randomizer:cached
+      - ./build:/app/OoT-Randomizer/ASM/build:consistent
+      - ./roms:/app/OoT-Randomizer/ASM/roms:consistent
+      - ../Output:/app/OoT-Randomizer/Output:consistent
+      - ../data/generated:/app/OoT-Randomizer/data/generated:consistent
+      - ../Logs:/app/OoT-Randomizer/Logs:consistent
+    network_mode: host
+    extra_hosts:
+      - host.docker.internal:host-gateway
+
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp:unconfined
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: python3 /app/OoT-Randomizer/ASM/build.py --compile-c

GUI/electron/src/preload.ts

@@ -153,12 +153,24 @@ post.on('browseForFile', function (event) {
   if (!data || typeof (data) != "object" || Object.keys(data).length != 1 || !data["fileTypes"] || typeof (data["fileTypes"]) != "object")
     return false;
 
-  return remote.dialog.showOpenDialogSync({ filters: data.fileTypes, properties: ["openFile", "treatPackageAsDirectory"]});
+  let res = remote.dialog.showOpenDialogSync({ filters: data.fileTypes, properties: ["openFile", "treatPackageAsDirectory"]});
+
+  // Canceled dialog
+  if (!res || res.length != 1 || !res[0] || res[0].length < 1) return res;
+
+  // Use relative paths whenever possible
+  return [path.relative(pythonSourcePath, res[0])];
 });
 
 
 post.on('browseForDirectory', function (event) {
-  return remote.dialog.showOpenDialogSync({ properties: ["openDirectory", "createDirectory", "treatPackageAsDirectory"] });
+  let res = remote.dialog.showOpenDialogSync({ properties: ["openDirectory", "createDirectory", "treatPackageAsDirectory"] });
+
+  // Canceled dialog
+  if (!res || res.length != 1 || !res[0] || res[0].length < 1) return res;
+
+  // Use relative paths whenever possible
+  return [path.relative(pythonSourcePath, res[0])];
 });
 
 post.on('createAndOpenPath', function (event) {

MBSDIFFPatch.py

@@ -57,7 +57,7 @@ def apply_ootr_3_web_patch(settings, rom: Rom) -> None:
 
         # Patch the base ROM.
         decompressed_patched_rom_file = output_path + "_patched.z64"
-        run_process(logger, [minibsdiff_path, "app", local_path('ZOOTDEC.z64'), decompressed_patch_file, decompressed_patched_rom_file])
+        run_process(logger, [minibsdiff_path, "app", local_path('ASM/roms/base.z64'), decompressed_patch_file, decompressed_patched_rom_file])
         os.remove(decompressed_patch_file)
 
         # Read the ROM back in and check for changes.

Main.py

@@ -241,7 +241,22 @@ def compress_rom(input_file: str, output_file: str, delete_input: bool = False)
         logger.info("OS not supported for ROM compression.")
         raise Exception("This operating system does not support ROM compression. You may only output patch files or uncompressed ROMs.")
 
-    run_process(logger, [compressor_path, input_file, output_file])
+    # Runs compressor with a working directory of ASM/roms/
+    # to make file permissions for dev containers easier to
+    # define. If the ARCHIVE.bin file created by the compressor
+    # was not generated before the container ran, it would create
+    # an ARCHIVE.bin/ folder instead. The ASM/roms/ folder already
+    # exists in the repository and is used to contain other
+    # working files for ROM generation. The compressor does not
+    # have an option to define where to store ARCHIVE.bin short
+    # of recompiling, always using the working directory.
+    #
+    # `cwd` argument for Popen behaves differently with relative
+    # paths for Windows/macOS and Linux when searching for the
+    # executable. Behavior is consistent with absolute paths.
+    # Input/output files are already defined as absolute through
+    # `Utils.default_output_path()`.
+    run_process(logger, [os.path.realpath(compressor_path), input_file, output_file], working_dir=os.path.realpath('./ASM/roms/'))
     if delete_input:
         os.remove(input_file)
 

Notes/Chest Textures/Custom Chest Textures.txt

@@ -26,7 +26,7 @@ These compress each RGB channel from 8 bits to 5 bits as well as the alpha chann
 
 RGBA5551-encoded images can be converted back to PNGs using the utility here: https://github.com/krimtonz/rgba5551topng. A compiled copy for ubuntu 20.04 is included in this folder, dependent on libpng. If on Windows, it is recommended to use WSL for this as well as the n64 toolchain.
 
-If any part of vanilla OOT textures are used for custom textures, the copyrighted portions of the image must be removed before contributing to the randomizer. This can be achieved by simply subtracting each byte of the vanilla texture from the custom texture. diff_bytes.py in this folder is provided as an example. When generating a randomizer patch file or rom, read the texture bytes from the user-provided rom and dynamically add these bytes back in to the custom texture before patching. Search for '# Apply chest texture diffs' in Patches.py for an example. Vanilla texture addresses can be found via decomp and/or the cloudmodding wiki, usually under objects. In an uncompressed rom, such as ZOOTDEC.z64 as generated by the randomizer, the wooden chest front texture starts at 0xFEC798 (length 4096 or 0x1000 bytes), and the wooden chest base texture starts at 0xFED798 (length 2048 or 0x800 bytes). To extract, just copy the desired bytes using your preferred hex editor.
+If any part of vanilla OOT textures are used for custom textures, the copyrighted portions of the image must be removed before contributing to the randomizer. This can be achieved by simply subtracting each byte of the vanilla texture from the custom texture. diff_bytes.py in this folder is provided as an example. When generating a randomizer patch file or rom, read the texture bytes from the user-provided rom and dynamically add these bytes back in to the custom texture before patching. Search for '# Apply chest texture diffs' in Patches.py for an example. Vanilla texture addresses can be found via decomp and/or the cloudmodding wiki, usually under objects. In an uncompressed rom, such as the one generated by the randomizer at ASM/roms/base.z64, the wooden chest front texture starts at 0xFEC798 (length 4096 or 0x1000 bytes), and the wooden chest base texture starts at 0xFED798 (length 2048 or 0x800 bytes). To extract, just copy the desired bytes using your preferred hex editor.
 
 OOTR custom chest textures work by modifying the wooden chest display list to allow switching textures based on chest actor type. The main logic for this is in ASM/c/chests.c as of version 6.2.19. draw_chest specifies which display list to use depending on chest type (boss key texture or wooden), then stores the corresponding texture for the chest type if it is not a boss key chest. Chest types are defined as follows in ASM/c/item_table.h and Patches.py (search for '#Fix chest animations'):
 

Notes/Chest Textures/extract_wooden_chest_textures.py

@@ -1,4 +1,4 @@
-with open('../../ZOOTDEC.z64', 'rb') as rom:
+with open('../../ASM/roms/base.z64', 'rb') as rom:
     rom.seek(0xFEC798, 0)
     front = rom.read(4096)
     base = rom.read(2048)

Notes/vscode.md

@@ -75,6 +75,20 @@ VS Code has a panel to run and debug Python unit tests. `pytest` is recommended
 
 The testing panel can be opened by clicking the flask icon in the left toolbar. To run all tests, DO NOT use the Run All Tests button. Click the Run Test button next to Unittest.py in the test tree. For more information on VS Code unit testing, see [the official VS Code Python testing page](https://code.visualstudio.com/docs/python/testing#_run-tests).
 
+## Dev Containers
+
+If the [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension is installed, it should automatically detect the configuration file in `/.devcontainer/devcontainer.json`. If you allow it to activate, it will create a Debian container with python, node, glankk's N64 toolchain, and armips for compiling and running all code for the randomizer. To manually activate it:
+
+- Bring up the command palette with F1
+- Select `Dev Container: Open Folder in Container...`
+- Select the root of the randomizer repository and wait for the container to build.
+
+Before using any of the build scripts, ensure an uncompressed v1.0 NTSC US ROM is placed at `ASM/roms/base.z64` relative to the repository's root. The randomizer will create this file for you when generating a patched ROM or wad.
+
+Debugging and building options described below will work with no additional changes while inside the container. Alternatively, any new terminal prompt in VSCode will allow you to run commands inside the container. Randomizer source code is mounted to `/app/OoT-Randomizer`.
+
+Running the GUI from the container is not currently supported. It is technically possible by either running an X server in the container or sharing the host's X server, but this approach is not cross-platform-friendly. The GUI can still be run from the host outside of VSCode. It will share code changes with the container through the container's mounted folders.
+
 ## Building the ASM/C code
 
 Instead of manually running the build script in a terminal, you can add a build task to VS Code by creating a `.vscode/tasks.json` file with the following content:
@@ -88,7 +102,8 @@ Instead of manually running the build script in a terminal, you can add a build
             "type": "shell",
             "command": "python3",
             "args": [
-                "${workspaceFolder}/ASM/build.py"
+                "${workspaceFolder}/ASM/build.py",
+                "--compile-c"
             ],
             "group": {
                 "kind": "build",

ProcessActors.py

@@ -360,7 +360,7 @@ def process_pot(actor_bytes):
         'item_id': item_dict[item_id],
     }
 
-#rom = Rom('ZOOTDEC.z64')
+#rom = Rom('ASM/roms/base.z64')
 rom = Rom('zeloot_mqdebug.z64')
 pots = get_crates(rom)
 

Rom.py

@@ -33,7 +33,7 @@ def __init__(self, file: Optional[str] = None) -> None:
         if file is None:
             return
 
-        decompressed_file: str = local_path('ZOOTDEC.z64')
+        decompressed_file: str = local_path('ASM/roms/base.z64')
 
         os.chdir(local_path())
 

Unittest.py

@@ -204,7 +204,7 @@ def test_excess_starting_items(self):
 
     def test_rom_patching(self):
         # This makes sure there are no crashes while patching.
-        if not os.path.isfile('./ZOOTDEC.z64'):
+        if not os.path.isfile('./ASM/roms/base.z64'):
             self.skipTest("Base ROM file not available.")
         filename = "plando-ammo-max-out-of-bounds"
         logic_rules_settings = ['glitchless', 'glitched', 'none']
@@ -555,7 +555,7 @@ def test_skip_zelda(self):
         self.assertIn('Hyrule Castle', woth)
 
     def test_ganondorf(self):
-        if not os.path.isfile('./ZOOTDEC.z64'):
+        if not os.path.isfile('./ASM/roms/base.z64'):
             self.skipTest("Base ROM file not available.")
         filenames = [
             "light-arrows-1",
@@ -855,9 +855,9 @@ def test_fuzzer(self):
 
 class TestTextShuffle(unittest.TestCase):
     def test_text_shuffle(self):
-        if not os.path.isfile('./ZOOTDEC.z64'):
+        if not os.path.isfile('./ASM/roms/base.z64'):
             self.skipTest("Base ROM file not available.")
-        rom = Rom("./ZOOTDEC.z64")
+        rom = Rom("./ASM/roms/base.z64")
         messages = read_messages(rom)
         shuffle_messages(messages)
         shuffle_messages(messages, False)

Utils.py

@@ -57,7 +57,11 @@ def default_output_path(path: str) -> str:
 
     if not os.path.exists(path):
         os.mkdir(path)
-    return path
+
+    # Convert output path to absolute path if necessary
+    abs_path = os.path.realpath(path)
+
+    return abs_path
 
 
 def read_logic_file(file_path: str):
@@ -199,8 +203,8 @@ def subprocess_args(include_stdout: bool = True) -> dict[str, Any]:
     return ret
 
 
-def run_process(logger: logging.Logger, args: Sequence[str], stdin: Optional[AnyStr] = None) -> None:
-    process = subprocess.Popen(args, bufsize=1, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+def run_process(logger: logging.Logger, args: Sequence[str], stdin: Optional[AnyStr] = None, working_dir: Optional[str] = None) -> None:
+    process = subprocess.Popen(args, bufsize=1, stdout=subprocess.PIPE, stdin=subprocess.PIPE, cwd=working_dir)
     filecount = None
     if stdin is not None:
         process.communicate(input=stdin)