Refactor PCH + script dependencies and use a 2-stage pipeline

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all release debug check rom target format clean distclean setup_release setup_debug configure
+.PHONY: all release debug check rom data target format clean distclean setup_release setup_debug configure
 
 MESON ?= meson
 NINJA ?= ninja
@@ -61,22 +61,31 @@ release: setup_release rom
 
 .NOTPARALLEL: debug
 debug: setup_debug rom
-	$(MESON) compile -C $(BUILD) debug.nef overlay.map
+	$(NINJA) -C $(BUILD) debug.nef overlay.map
 
 check: rom
 	$(MESON) test -C $(BUILD)
 
-rom: $(BUILD)/build.ninja
-	$(MESON) compile -C $(BUILD) pokeplatinum.us.nds
+.NOTPARALLEL: rom
+# We use a two-stage pipeline to absolve what appears to be a bug in Ninja
+# where generated headers that are listed as order-only dependencies are not
+# respected by the depfiles generated by compiling source code. To that end, we
+# generate data-targets first (archives and generated headers), then proceed
+# with compiling the ROM code.
+rom: $(BUILD)/build.ninja data
+	$(NINJA) -C $(BUILD) pokeplatinum.us.nds
+
+data: $(BUILD)/build.ninja
+	$(NINJA) -C $(BUILD) data
 
 target: $(BUILD)/build.ninja
-	$(MESON) compile -C $(BUILD) $(MESON_TARGET)
+	$(NINJA) -C $(BUILD) $(MESON_TARGET)
 
 format: $(BUILD)/build.ninja
 	$(NINJA) -C $(BUILD) clang-format
 
 clean: $(BUILD)/build.ninja
-	$(MESON) compile -C $(BUILD) --clean
+	$(NINJA) -C $(BUILD) --clean
 
 update: $(BUILD)/build.ninja
 	$(MESON) subprojects update

include/pch/global_pch.h

@@ -7,8 +7,6 @@
 #include <stddef.h>
 #include <string.h>
 
-#include "constants/pokemon.h"
-
 #include "global/assert.h"
 #include "global/config.h" // MUST COME BEFORE ANY OTHER GAMEFREAK HEADERS
 #include "global/pm_version.h"

include/pokemon.h

@@ -3,6 +3,7 @@
 
 #include <nitro/rtc.h>
 
+#include "constants/pokemon.h"
 #include "constants/sound.h"
 
 #include "struct_decls/cell_actor_data.h"

meson.build

@@ -146,6 +146,8 @@ subdir('lib')
 ############################################################
 subdir('res')
 
+# Phony-like target to build all generated data files
+alias_target('data', nitrofs_files, gen_species_headers)
 
 ############################################################
 ###                     ARM9 BINARY                      ###

res/battle/scripts/meson.build

@@ -17,7 +17,7 @@ waza_seq_private_dir = relative_build_dir / waza_seq_target_name + '.p'
 
 be_seq_narc = custom_target(be_seq_target_name,
     output: be_seq_target_name,
-    input: s_to_bin_gen.process(
+    input: script_bin_gen.process(
         effect_script_files,
         extra_args: ['--out-dir', be_seq_private_dir]
     ),
@@ -33,7 +33,7 @@ sub_seq_narc = custom_target(sub_seq_narc_name,
         sub_seq_narc_name,
         sub_seq_naix_name,
     ],
-    input: s_to_bin_gen.process(
+    input: script_bin_gen.process(
         subscript_files,
         extra_args: ['--out-dir', sub_seq_private_dir]
     ),
@@ -48,7 +48,7 @@ sub_seq_narc = custom_target(sub_seq_narc_name,
 
 waza_seq_narc = custom_target(waza_seq_target_name,
     output: waza_seq_target_name,
-    input: s_to_bin_gen.process(
+    input: script_bin_gen.process(
         move_script_files,
         extra_args: ['--out-dir', waza_seq_private_dir]
     ),
@@ -60,7 +60,7 @@ waza_seq_narc = custom_target(waza_seq_target_name,
 )
 
 nitrofs_files += be_seq_narc
-nitrofs_files += sub_seq_narc[0]
+nitrofs_files += sub_seq_narc
 nitrofs_files += waza_seq_narc
 
 naix_headers += sub_seq_narc[1]

res/field/events/meson.build

@@ -558,5 +558,5 @@ events_narc = custom_target('zone_event.narc',
     ]
 )
 
-nitrofs_files += events_narc[0]
+nitrofs_files += events_narc
 naix_headers += events_narc[1]

res/field/scripts/meson.build

@@ -1131,30 +1131,14 @@ scr_seq_files = files(
     'scripts_route_230.s',
 )
 
-# Field scripts declare a direct dependency on `trdata.naix` for use in the
-# `StartTrainerBattle` command. Ordinarily, we would try to declare this as an
-# order-only dependency just to ensure that it is built before this generator
-# can be run, then let the individual script files include it as needed, similar
-# to message banks. But, there is only one `trdata.naix` file, and script files
-# are fast to build anyways, so this is the preferred solution.
-field_script_gen = generator(make_script_bin_sh,
-    arguments: make_script_bin_args,
-    output: '@BASENAME@',
-    depfile: '@[email protected]',
-    depends: [
-        message_banks_index, # TODO: Can this be an order-only dependency...?
-        trdata_naix,
-    ],
-)
-
 scr_seq_narc_order = files('scripts.order')
 
 scr_seq_narc = custom_target('scr_seq.narc',
     output: [
         'scr_seq.narc',
         'scr_seq.naix',
     ],
-    input: field_script_gen.process(
+    input: script_bin_gen.process(
         scr_seq_files,
         extra_args: ['--depfile', '--out-dir', scr_seq_private_dir]
     ),
@@ -1167,5 +1151,5 @@ scr_seq_narc = custom_target('scr_seq.narc',
     ],
 )
 
-nitrofs_files += scr_seq_narc[0]
+nitrofs_files += scr_seq_narc
 naix_headers += scr_seq_narc[1]

res/fonts/meson.build

@@ -109,5 +109,5 @@ pl_font_narc = custom_target('pl_font.narc',
     ],
 )
 
-nitrofs_files += pl_font_narc[0]
+nitrofs_files += pl_font_narc
 naix_headers += pl_font_narc[1]

res/graphics/options_menu/meson.build

@@ -28,5 +28,5 @@ config_gra_narc = custom_target('config_gra.narc',
     ]
 )
 
-nitrofs_files += config_gra_narc[0]
+nitrofs_files += config_gra_narc
 naix_headers += config_gra_narc[1]

res/graphics/pokemon_summary_screen/meson.build

@@ -235,5 +235,5 @@ pokemon_summary_screen_narc = custom_target('pl_pst_gra.narc',
     ]
 )
 
-nitrofs_files += pokemon_summary_screen_narc[0]
+nitrofs_files += pokemon_summary_screen_narc
 naix_headers += pokemon_summary_screen_narc[1]

res/graphics/signposts/meson.build

@@ -88,5 +88,5 @@ field_board_narc = custom_target('field_board.narc',
     ]
 )
 
-nitrofs_files += field_board_narc[0]
+nitrofs_files += field_board_narc
 naix_headers += field_board_narc[1]

res/graphics/windows/meson.build

@@ -86,5 +86,5 @@ pl_winframe_narc = custom_target('pl_winframe.narc',
     ]
 )
 
-nitrofs_files += pl_winframe_narc[0]
+nitrofs_files += pl_winframe_narc
 naix_headers += pl_winframe_narc[1]

res/meson.build

@@ -20,28 +20,33 @@ copy_gen = generator(find_program('cp'),
     output: '@PLAINNAME@'
 )
 
-make_script_bin_args = [
-    '-i', relative_source_root / 'include',
-    '-i', relative_source_root / 'asm',
-    '-i', '.' / 'res' / 'text',
-    '-i', '.' / 'res',
-    '-i', '.',
-    '--assembler', arm_none_eabi_gcc_exe.full_path(),
-    '--objcopy', arm_none_eabi_objcopy_exe.full_path(),
-    '@EXTRA_ARGS@',
-    '@INPUT@',
-]
-
-make_script_bin_deps = [
-    message_banks_index, # for GMM headers
-    asm_consts_generators, # for ASM headers
-    c_consts_generators, # for C headers
-]
-
-s_to_bin_gen = generator(make_script_bin_sh,
-    arguments: make_script_bin_args,
-    depends: make_script_bin_deps,
-    output: '@BASENAME@'
+# NOTE: The members of the `depends` clause below will always be modified by the
+# postconf script to be order-only dependencies. This means that this generator
+# will only *wait* to run until after these files have been generated, and it
+# *breaks* the dependency-chain if any of these files are edited. However, because
+# this generator produces a depfile, the build back-end will still see the correct
+# granular headers on which each input source file depends.
+script_bin_gen = generator(make_script_bin_sh,
+    arguments: [
+        '-i', relative_source_root / 'include',
+        '-i', relative_source_root / 'asm',
+        '-i', '.' / 'res' / 'text',
+        '-i', '.' / 'res',
+        '-i', '.',
+        '--depfile',
+        '--assembler', arm_none_eabi_gcc_exe.full_path(),
+        '--objcopy', arm_none_eabi_objcopy_exe.full_path(),
+        '@EXTRA_ARGS@',
+        '@INPUT@',
+    ],
+    depends: [
+        message_banks_index,
+        asm_consts_generators,
+        c_consts_generators,
+        trdata_naix,
+    ],
+    output: '@BASENAME@',
+    depfile: '@[email protected]',
 )
 
 ncgr_gen = generator(nitrogfx_exe,

res/pokemon/meson.build

@@ -204,12 +204,6 @@ datagen_species_out = custom_target('datagen_species_out',
         species_data_files,
     ],
 )
-pl_personal_narc = datagen_species_out[0]
-evo_narc = datagen_species_out[1]
-wotbl_narc = datagen_species_out[2]
-ppark_narc = datagen_species_out[3]
-height_narc = datagen_species_out[4]
-pl_poke_data_narc = datagen_species_out[5]
 tutorable_moves_h = datagen_species_out[6]
 species_learnsets_by_tutor_h = datagen_species_out[7]
 
@@ -346,9 +340,7 @@ pokedex_data_giratina_altered_narc = custom_target('zukan_data_gira.narc',
 species_header_target = meson.current_build_dir()
 
 gen_species_headers = custom_target('gen_species_headers',
-    output: [
-        'footprint_data.h',
-    ],
+    output: 'footprint_data.h',
     input: species_data_files,
     env: json2bin_env,
     depends: [ py_consts_generators ],
@@ -359,16 +351,11 @@ gen_species_headers = custom_target('gen_species_headers',
     ]
 )
 
-nitrofs_files += pl_personal_narc
-nitrofs_files += evo_narc
-nitrofs_files += wotbl_narc
-nitrofs_files += ppark_narc
-nitrofs_files += height_narc
+nitrofs_files += datagen_species_out
 
 nitrofs_files += pl_poke_icon_narc
 nitrofs_files += pl_pokegra_narc
 nitrofs_files += pl_otherpoke_narc
-nitrofs_files += pl_poke_data_narc
 nitrofs_files += pl_pokezukan_narc
 nitrofs_files += shinzukan_narc
 nitrofs_files += pl_growtbl_narc

res/text/meson.build

@@ -783,5 +783,5 @@ message_banks = custom_target('pl_msg.narc',
 )
 message_banks_index = message_banks[1]
 
-nitrofs_files += message_banks[0]
+nitrofs_files += message_banks
 naix_headers += message_banks_index

res/trainers/meson.build

@@ -943,7 +943,7 @@ trainer_party_gen = generator(
 
 trainers_order = files('trainers.order')
 
-trdata_target = custom_target('trdata.narc',
+trdata_narc = custom_target('trdata.narc',
     output: [
         'trdata.narc',
         'trdata.naix',
@@ -959,8 +959,7 @@ trdata_target = custom_target('trdata.narc',
         '@PRIVATE_DIR@',
     ],
 )
-trdata_narc = trdata_target[0]
-trdata_naix = trdata_target[1]
+trdata_naix = trdata_narc[1]
 
 # Do not generate an NAIX for `trpoke.narc`; it would go wholly unused
 trpoke_narc = custom_target('trpoke.narc',

tools/postconf/postconf.py

@@ -22,18 +22,41 @@ def is_wsl_accessing_windows() -> bool:
     return ("microsoft" in platform.uname()[2].lower() and os.path.realpath(os.path.abspath(__file__)).startswith("/mnt/"))
 
 
+def bytecode_scripts_order_only_deps(fileString: str) -> str:
+    '''Express bytecode-script dependencies on generated headers as order-only'''
+    return re.sub(
+        r"build ([\w\/\.]+): (\w+) ([\w\/\.]+) \| ([\w\/\.]+\/make_script_bin\.sh) ([\w\s\/\.]+)",
+        r"build \1: \2 \3 | \4 || \5",
+        fileString
+    )
+
+def pch_order_only_deps(fileString: str) -> str:
+    '''
+    Express dependencies attached to the PCH as order-only.
+    This is to address a Meson bug where the PCH is flagged as having implicit
+    dependencies on generated sources and headers.
+    '''
+    return re.sub(
+        r"build main.nef.p/global_pch.h.mch: c_PCH ../include/pch/global_pch.h \| ([\w\s\/\.]+)",
+        r"build main.nef.p/global_pch.h.mch: c_PCH ../include/pch/global_pch.h || \1",
+        fileString
+    )
+
+
 def main():
     BUILD_NINJA         = f'{BUILD_DIRECTORY}/build.ninja'
     COMPILE_COMMANDS    = f'{BUILD_DIRECTORY}/compile_commands.json'
 
     with open(BUILD_NINJA, 'r') as build_ninja_in, open(COMPILE_COMMANDS, 'r') as compile_commands_in:
         build_ninja_string = build_ninja_in.read()
         compile_commands_string = compile_commands_in.read()
-    
+
     # build.ninja edits
     build_ninja_string = backslash_to_forward_slash(build_ninja_string)
     build_ninja_string = fix_static_libs(build_ninja_string)
     build_ninja_string = nasm_to_asm(build_ninja_string)
+    build_ninja_string = bytecode_scripts_order_only_deps(build_ninja_string)
+    build_ninja_string = pch_order_only_deps(build_ninja_string)
 
     # compile_commands.json edits
     compile_commands_string = backslash_to_forward_slash(compile_commands_string)