Add a Pad segment and make LinkerEntry a bit more flexible (#319)

* Allow changing the behavior of each emited linker entry by making a subclass of LinkerEntry

* a bit more of cleanup

* Remove hardcoded `lib` stuff from `linker_entry`

* Implement pad segment

* black

* write docs for `pad`

* ld_generate_symbol_per_data_segment

* black

* version bump

CHANGELOG.md

@@ -1,5 +1,15 @@
 # splat Release Notes
 
+### 0.20.0
+
+* Add a pad segment that advances the linker script instead of dumping a binary / generating an assembly file.
+* Move the logic of writing the entry to the linker script from `LinkerWriter` to `LinkerEntry`
+  * This allows to have custom behavior for an entry without needing to hardcode extra checks on `LinkerWriter`.
+  * Extension segments can make a subclass of `LinkerEntry` and override its methods to have custom linker script behavior.
+* New yaml option: `ld_generate_symbol_per_data_segment`
+  * If enabled, the generated linker script will have a linker symbol for each data file.
+  * Defaults to `True`.
+
 ### 0.19.7
 
 * Ensure the directory exists when extracting a palette segment.

docs/Configuration.md

@@ -442,6 +442,11 @@ Setting this to `True` will make the `*_END` linker symbol of every section to b
 
 Defaults to `True`.
 
+### ld_generate_symbol_per_data_segment
+
+If enabled, the generated linker script will have a linker symbol for each data file.
+
+Defaults to `True`.
 
 ## C file options
 

docs/Segments.md

@@ -257,6 +257,20 @@ These segments will parse the image data and dump out a `png` file.
   flip_y: no
 ```
 
+## `pad`
+
+`pad` is a segment that represents a rom region that's filled with zeroes and decomping it doesn't have much value.
+
+This segment does not generate an assembly (`.s`) or binary (`.bin`) file, it simply increments the position of the linker script, avoding to build zero-filled files.
+
+While this kind of segment can be represented by other segment types ([`asm`](#asm), [`data`](#data), etc), it is better practice to use this segment instead to better reflect the contents of the file.
+
+**Example:**
+
+```yaml
+- [0x00B250, pad, nops_00B250]
+```
+
 ## General segment options
 
 All splat's segments can be passed extra options for finer configuration. Note that those extra options require to rewrite the entry using the dictionary yaml notation instead of the list one.

segtypes/common/lib.py

@@ -1,13 +1,34 @@
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List
 
 from util import log, options
 
-from segtypes.linker_entry import LinkerEntry
-from segtypes.n64.segment import N64Segment
+from segtypes.linker_entry import LinkerEntry, LinkerWriter
+from segtypes.common.segment import CommonSegment
 
+from segtypes.segment import Segment
 
-class CommonSegLib(N64Segment):
+
+class LinkerEntryLib(LinkerEntry):
+    def __init__(
+        self,
+        segment: Segment,
+        src_paths: List[Path],
+        object_path: Path,
+        section_order: str,
+        section_link: str,
+        noload: bool,
+    ):
+        super().__init__(
+            segment, src_paths, object_path, section_order, section_link, noload
+        )
+        self.object_path = object_path
+
+    def emit_entry(self, linker_writer: LinkerWriter):
+        self.emit_path(linker_writer)
+
+
+class CommonSegLib(CommonSegment):
     def __init__(
         self,
         rom_start: Optional[int],
@@ -45,13 +66,13 @@ def __init__(
     def get_linker_section(self) -> str:
         return self.section
 
-    def get_linker_entries(self):
+    def get_linker_entries(self) -> List[LinkerEntry]:
         path = options.opts.lib_path / self.name
 
         object_path = Path(f"{path}.a:{self.object}.o")
 
         return [
-            LinkerEntry(
+            LinkerEntryLib(
                 self,
                 [path],
                 object_path,

segtypes/common/pad.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+from typing import List
+
+from segtypes.common.segment import CommonSegment
+from segtypes.linker_entry import LinkerEntry, LinkerWriter
+from segtypes.segment import Segment
+
+
+class LinkerEntryPad(LinkerEntry):
+    def __init__(
+        self,
+        segment: Segment,
+    ):
+        super().__init__(segment, [], Path(), "pad", "pad", False)
+        self.object_path = None
+
+    def emit_entry(self, linker_writer: LinkerWriter):
+        linker_writer._writeln(f". += 0x{self.segment.size:X};")
+
+
+class CommonSegPad(CommonSegment):
+    def get_linker_entries(self) -> List[LinkerEntry]:
+        return [LinkerEntryPad(self)]

segtypes/linker_entry.py

@@ -2,7 +2,7 @@
 import re
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, OrderedDict, Set, Tuple, Union
+from typing import Dict, List, OrderedDict, Set, Tuple, Union, Optional
 
 from util import options
 
@@ -114,6 +114,9 @@ def get_segment_vram_end_symbol_name(segment: Segment) -> str:
     return get_segment_vram_end(segment.get_cname())
 
 
+regex_data_segment_normalizer = re.compile(r"[^0-9a-zA-Z_]")
+
+
 class LinkerEntry:
     def __init__(
         self,
@@ -130,12 +133,7 @@ def __init__(
         self.section_link = section_link
         self.noload = noload
         self.bss_contains_common = segment.bss_contains_common
-        if self.section_link == "linker" or self.section_link == "linker_offset":
-            self.object_path = None
-        elif self.segment.type == "lib":
-            self.object_path = object_path
-        else:
-            self.object_path = path_to_object_path(object_path)
+        self.object_path: Optional[Path] = path_to_object_path(object_path)
 
     @property
     def section_order_type(self) -> str:
@@ -151,6 +149,38 @@ def section_link_type(self) -> str:
         else:
             return self.section_link
 
+    def emit_symbol_for_data(self, linker_writer: "LinkerWriter"):
+        if not options.opts.ld_generate_symbol_per_data_segment:
+            return
+
+        if self.object_path and self.section_link_type == ".data":
+            path_cname = regex_data_segment_normalizer.sub(
+                "_",
+                str(self.segment.dir / self.segment.name)
+                + ".".join(self.object_path.suffixes[:-1]),
+            )
+            linker_writer._write_symbol(path_cname, ".")
+
+    def emit_path(self, linker_writer: "LinkerWriter"):
+        assert (
+            self.object_path is not None
+        ), f"{self.segment.name}, {self.segment.rom_start}"
+
+        if self.noload and self.bss_contains_common:
+            linker_writer._write_object_path_section(
+                self.object_path, ".bss COMMON .scommon"
+            )
+        else:
+            wildcard = "*" if options.opts.ld_wildcard_sections else ""
+
+            linker_writer._write_object_path_section(
+                self.object_path, f"{self.section_link}{wildcard}"
+            )
+
+    def emit_entry(self, linker_writer: "LinkerWriter"):
+        self.emit_symbol_for_data(linker_writer)
+        self.emit_path(linker_writer)
+
 
 class LinkerWriter:
     def __init__(self, is_partial: bool = False):
@@ -282,7 +312,7 @@ def add_legacy(self, segment: Segment, entries: List[LinkerEntry]):
                 self._begin_section(seg_name, entry.section_order_type)
                 started_sections[entry.section_order_type] = True
 
-            self._write_linker_entry(entry)
+            entry.emit_entry(self)
 
             if entry in last_seen_sections:
                 self._end_section(seg_name, entry.section_order_type, segment)
@@ -300,7 +330,7 @@ def add_legacy(self, segment: Segment, entries: List[LinkerEntry]):
                     self._begin_section(seg_name, entry.section_order_type)
                     started_sections[entry.section_order_type] = True
 
-                self._write_linker_entry(entry)
+                entry.emit_entry(self)
 
                 if entry in last_seen_sections:
                     self._end_section(seg_name, entry.section_order_type, segment)
@@ -341,7 +371,7 @@ def add_referenced_partial_segment(
                     segment, [], segments_path / f"{seg_name}.o", l, l, noload=False
                 )
                 self.dependencies_entries.append(entry)
-                self._write_linker_entry(entry)
+                entry.emit_entry(self)
             is_first = False
 
         if any(e.noload for e in entries):
@@ -369,7 +399,7 @@ def add_referenced_partial_segment(
             )
             entry.bss_contains_common = bss_contains_common
             self.dependencies_entries.append(entry)
-            self._write_linker_entry(entry)
+            entry.emit_entry(self)
 
         self._end_segment(segment, all_bss=not any_load)
 
@@ -405,7 +435,7 @@ def add_partial_segment(self, segment: Segment):
             self._begin_section(seg_name, section_name)
 
             for entry in entries:
-                self._write_linker_entry(entry)
+                entry.emit_entry(self)
 
             self._end_section(seg_name, section_name, segment)
 
@@ -496,6 +526,9 @@ def _write_symbol(self, symbol: str, value: Union[str, int]):
 
         self.header_symbols.add(symbol)
 
+    def _write_object_path_section(self, object_path: Path, section: str):
+        self._writeln(f"{object_path}({section});")
+
     def _begin_segment(
         self, segment: Segment, seg_name: str, noload: bool, is_first: bool
     ):
@@ -594,32 +627,6 @@ def _end_section(self, seg_name: str, cur_section: str, segment: Segment) -> Non
             f"ABSOLUTE({section_end} - {section_start})",
         )
 
-    def _write_linker_entry(self, entry: LinkerEntry):
-        if entry.section_link_type == "linker_offset":
-            self._write_symbol(f"{entry.segment.get_cname()}_OFFSET", ".")
-            return
-
-        # TODO: option to turn this off?
-        if (
-            entry.object_path
-            and entry.section_link_type == ".data"
-            and entry.segment.type != "lib"
-        ):
-            path_cname = re.sub(
-                r"[^0-9a-zA-Z_]",
-                "_",
-                str(entry.segment.dir / entry.segment.name)
-                + ".".join(entry.object_path.suffixes[:-1]),
-            )
-            self._write_symbol(path_cname, ".")
-
-        if entry.noload and entry.bss_contains_common:
-            self._writeln(f"{entry.object_path}(.bss COMMON .scommon);")
-        else:
-            wildcard = "*" if options.opts.ld_wildcard_sections else ""
-
-            self._writeln(f"{entry.object_path}({entry.section_link}{wildcard});")
-
     def _write_segment_sections(
         self,
         segment: Segment,
@@ -643,5 +650,5 @@ def _write_segment_sections(
 
             self._begin_section(seg_name, section_name)
             for entry in entries:
-                self._write_linker_entry(entry)
+                entry.emit_entry(self)
             self._end_section(seg_name, section_name, segment)

segtypes/n64/linker_offset.py

@@ -1,14 +1,23 @@
 from pathlib import Path
+from typing import List
 
 from segtypes.n64.segment import N64Segment
+from segtypes.linker_entry import LinkerEntry, LinkerWriter
+from segtypes.segment import Segment
+
+
+class LinkerEntryOffset(LinkerEntry):
+    def __init__(
+        self,
+        segment: Segment,
+    ):
+        super().__init__(segment, [], Path(), "linker_offset", "linker_offset", False)
+        self.object_path = None
+
+    def emit_entry(self, linker_writer: LinkerWriter):
+        linker_writer._write_symbol(f"{self.segment.get_cname()}_OFFSET", ".")
 
 
 class N64SegLinker_offset(N64Segment):
-    def get_linker_entries(self):
-        from segtypes.linker_entry import LinkerEntry
-
-        return [
-            LinkerEntry(
-                self, [], Path(self.name), "linker_offset", "linker_offset", False
-            )
-        ]
+    def get_linker_entries(self) -> List[LinkerEntry]:
+        return [LinkerEntryOffset(self)]

split.py

@@ -24,7 +24,7 @@
 from segtypes.segment import Segment
 from util import log, options, palettes, symbols, relocs
 
-VERSION = "0.19.7"
+VERSION = "0.20.0"
 
 parser = argparse.ArgumentParser(
     description="Split a rom given a rom, a config, and output directory"

util/options.py

@@ -135,6 +135,8 @@ class SplatOpts:
     ld_align_segment_vram_end: bool
     # Allows to toggle aligning the `*_END` linker symbol for each section of each section.
     ld_align_section_vram_end: bool
+    # If enabled, the generated linker script will have a linker symbol for each data file
+    ld_generate_symbol_per_data_segment: bool
 
     ################################################################################
     # C file options
@@ -448,6 +450,9 @@ def parse_endianness() -> Literal["big", "little"]:
         ld_bss_is_noload=p.parse_opt("ld_bss_is_noload", bool, True),
         ld_align_segment_vram_end=p.parse_opt("ld_align_segment_vram_end", bool, True),
         ld_align_section_vram_end=p.parse_opt("ld_align_section_vram_end", bool, True),
+        ld_generate_symbol_per_data_segment=p.parse_opt(
+            "ld_generate_symbol_per_data_segment", bool, True
+        ),
         create_c_files=p.parse_opt("create_c_files", bool, True),
         auto_decompile_empty_functions=p.parse_opt(
             "auto_decompile_empty_functions", bool, True