From 03103c18d4bcf79efb919401b52ed7f59e1463f1 Mon Sep 17 00:00:00 2001 From: Rachel Date: Sat, 25 Jan 2025 21:20:05 -0800 Subject: [PATCH] Refactor PCH + script dependencies and use a 2-stage pipeline --- Makefile | 21 +++++--- include/pch/global_pch.h | 2 - include/pokemon.h | 1 + meson.build | 2 + res/battle/scripts/meson.build | 8 +-- res/field/events/meson.build | 2 +- res/field/scripts/meson.build | 20 +------- res/fonts/meson.build | 2 +- res/graphics/options_menu/meson.build | 2 +- .../pokemon_summary_screen/meson.build | 2 +- res/graphics/signposts/meson.build | 2 +- res/graphics/windows/meson.build | 2 +- res/meson.build | 49 ++++++++++--------- res/pokemon/meson.build | 17 +------ res/text/meson.build | 2 +- res/trainers/meson.build | 5 +- tools/postconf/postconf.py | 25 +++++++++- 17 files changed, 86 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index 21161a2b75..23fc916d2d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all release debug check rom target format clean distclean setup_release setup_debug configure +.PHONY: all release debug check rom data target format clean distclean setup_release setup_debug configure MESON ?= meson NINJA ?= ninja @@ -61,22 +61,31 @@ release: setup_release rom .NOTPARALLEL: debug debug: setup_debug rom - $(MESON) compile -C $(BUILD) debug.nef overlay.map + $(NINJA) -C $(BUILD) debug.nef overlay.map check: rom $(MESON) test -C $(BUILD) -rom: $(BUILD)/build.ninja - $(MESON) compile -C $(BUILD) pokeplatinum.us.nds +.NOTPARALLEL: rom +# We use a two-stage pipeline to absolve what appears to be a bug in Ninja +# where generated headers that are listed as order-only dependencies are not +# respected by the depfiles generated by compiling source code. To that end, we +# generate data-targets first (archives and generated headers), then proceed +# with compiling the ROM code. +rom: $(BUILD)/build.ninja data + $(NINJA) -C $(BUILD) pokeplatinum.us.nds + +data: $(BUILD)/build.ninja + $(NINJA) -C $(BUILD) data target: $(BUILD)/build.ninja - $(MESON) compile -C $(BUILD) $(MESON_TARGET) + $(NINJA) -C $(BUILD) $(MESON_TARGET) format: $(BUILD)/build.ninja $(NINJA) -C $(BUILD) clang-format clean: $(BUILD)/build.ninja - $(MESON) compile -C $(BUILD) --clean + $(NINJA) -C $(BUILD) --clean update: $(BUILD)/build.ninja $(MESON) subprojects update diff --git a/include/pch/global_pch.h b/include/pch/global_pch.h index 8218d34b0d..bc7b7578d6 100644 --- a/include/pch/global_pch.h +++ b/include/pch/global_pch.h @@ -7,8 +7,6 @@ #include #include -#include "constants/pokemon.h" - #include "global/assert.h" #include "global/config.h" // MUST COME BEFORE ANY OTHER GAMEFREAK HEADERS #include "global/pm_version.h" diff --git a/include/pokemon.h b/include/pokemon.h index 145c9970b7..31986377f5 100644 --- a/include/pokemon.h +++ b/include/pokemon.h @@ -3,6 +3,7 @@ #include +#include "constants/pokemon.h" #include "constants/sound.h" #include "struct_decls/cell_actor_data.h" diff --git a/meson.build b/meson.build index 407a53a1a2..228a796e18 100644 --- a/meson.build +++ b/meson.build @@ -146,6 +146,8 @@ subdir('lib') ############################################################ subdir('res') +# Phony-like target to build all generated data files +alias_target('data', nitrofs_files, gen_species_headers) ############################################################ ### ARM9 BINARY ### diff --git a/res/battle/scripts/meson.build b/res/battle/scripts/meson.build index 9e6d9ae984..54e2a10862 100644 --- a/res/battle/scripts/meson.build +++ b/res/battle/scripts/meson.build @@ -17,7 +17,7 @@ waza_seq_private_dir = relative_build_dir / waza_seq_target_name + '.p' be_seq_narc = custom_target(be_seq_target_name, output: be_seq_target_name, - input: s_to_bin_gen.process( + input: script_bin_gen.process( effect_script_files, extra_args: ['--out-dir', be_seq_private_dir] ), @@ -33,7 +33,7 @@ sub_seq_narc = custom_target(sub_seq_narc_name, sub_seq_narc_name, sub_seq_naix_name, ], - input: s_to_bin_gen.process( + input: script_bin_gen.process( subscript_files, extra_args: ['--out-dir', sub_seq_private_dir] ), @@ -48,7 +48,7 @@ sub_seq_narc = custom_target(sub_seq_narc_name, waza_seq_narc = custom_target(waza_seq_target_name, output: waza_seq_target_name, - input: s_to_bin_gen.process( + input: script_bin_gen.process( move_script_files, extra_args: ['--out-dir', waza_seq_private_dir] ), @@ -60,7 +60,7 @@ waza_seq_narc = custom_target(waza_seq_target_name, ) nitrofs_files += be_seq_narc -nitrofs_files += sub_seq_narc[0] +nitrofs_files += sub_seq_narc nitrofs_files += waza_seq_narc naix_headers += sub_seq_narc[1] diff --git a/res/field/events/meson.build b/res/field/events/meson.build index 8ae1b69a37..069fbe607c 100644 --- a/res/field/events/meson.build +++ b/res/field/events/meson.build @@ -558,5 +558,5 @@ events_narc = custom_target('zone_event.narc', ] ) -nitrofs_files += events_narc[0] +nitrofs_files += events_narc naix_headers += events_narc[1] diff --git a/res/field/scripts/meson.build b/res/field/scripts/meson.build index 92804f23a5..1c96adfcf1 100644 --- a/res/field/scripts/meson.build +++ b/res/field/scripts/meson.build @@ -1131,22 +1131,6 @@ scr_seq_files = files( 'scripts_route_230.s', ) -# Field scripts declare a direct dependency on `trdata.naix` for use in the -# `StartTrainerBattle` command. Ordinarily, we would try to declare this as an -# order-only dependency just to ensure that it is built before this generator -# can be run, then let the individual script files include it as needed, similar -# to message banks. But, there is only one `trdata.naix` file, and script files -# are fast to build anyways, so this is the preferred solution. -field_script_gen = generator(make_script_bin_sh, - arguments: make_script_bin_args, - output: '@BASENAME@', - depfile: '@BASENAME@.d', - depends: [ - message_banks_index, # TODO: Can this be an order-only dependency...? - trdata_naix, - ], -) - scr_seq_narc_order = files('scripts.order') scr_seq_narc = custom_target('scr_seq.narc', @@ -1154,7 +1138,7 @@ scr_seq_narc = custom_target('scr_seq.narc', 'scr_seq.narc', 'scr_seq.naix', ], - input: field_script_gen.process( + input: script_bin_gen.process( scr_seq_files, extra_args: ['--depfile', '--out-dir', scr_seq_private_dir] ), @@ -1167,5 +1151,5 @@ scr_seq_narc = custom_target('scr_seq.narc', ], ) -nitrofs_files += scr_seq_narc[0] +nitrofs_files += scr_seq_narc naix_headers += scr_seq_narc[1] diff --git a/res/fonts/meson.build b/res/fonts/meson.build index 622b5f8424..70b2818a50 100644 --- a/res/fonts/meson.build +++ b/res/fonts/meson.build @@ -109,5 +109,5 @@ pl_font_narc = custom_target('pl_font.narc', ], ) -nitrofs_files += pl_font_narc[0] +nitrofs_files += pl_font_narc naix_headers += pl_font_narc[1] diff --git a/res/graphics/options_menu/meson.build b/res/graphics/options_menu/meson.build index b558bc7cad..6332a81e9c 100644 --- a/res/graphics/options_menu/meson.build +++ b/res/graphics/options_menu/meson.build @@ -28,5 +28,5 @@ config_gra_narc = custom_target('config_gra.narc', ] ) -nitrofs_files += config_gra_narc[0] +nitrofs_files += config_gra_narc naix_headers += config_gra_narc[1] diff --git a/res/graphics/pokemon_summary_screen/meson.build b/res/graphics/pokemon_summary_screen/meson.build index 2be4c67707..9193f23044 100644 --- a/res/graphics/pokemon_summary_screen/meson.build +++ b/res/graphics/pokemon_summary_screen/meson.build @@ -235,5 +235,5 @@ pokemon_summary_screen_narc = custom_target('pl_pst_gra.narc', ] ) -nitrofs_files += pokemon_summary_screen_narc[0] +nitrofs_files += pokemon_summary_screen_narc naix_headers += pokemon_summary_screen_narc[1] diff --git a/res/graphics/signposts/meson.build b/res/graphics/signposts/meson.build index 8fb35de0ac..ad415c347d 100644 --- a/res/graphics/signposts/meson.build +++ b/res/graphics/signposts/meson.build @@ -88,5 +88,5 @@ field_board_narc = custom_target('field_board.narc', ] ) -nitrofs_files += field_board_narc[0] +nitrofs_files += field_board_narc naix_headers += field_board_narc[1] diff --git a/res/graphics/windows/meson.build b/res/graphics/windows/meson.build index 72ae931ee4..c02881f7bf 100644 --- a/res/graphics/windows/meson.build +++ b/res/graphics/windows/meson.build @@ -86,5 +86,5 @@ pl_winframe_narc = custom_target('pl_winframe.narc', ] ) -nitrofs_files += pl_winframe_narc[0] +nitrofs_files += pl_winframe_narc naix_headers += pl_winframe_narc[1] diff --git a/res/meson.build b/res/meson.build index 391c941c12..9390e405e6 100644 --- a/res/meson.build +++ b/res/meson.build @@ -20,28 +20,33 @@ copy_gen = generator(find_program('cp'), output: '@PLAINNAME@' ) -make_script_bin_args = [ - '-i', relative_source_root / 'include', - '-i', relative_source_root / 'asm', - '-i', '.' / 'res' / 'text', - '-i', '.' / 'res', - '-i', '.', - '--assembler', arm_none_eabi_gcc_exe.full_path(), - '--objcopy', arm_none_eabi_objcopy_exe.full_path(), - '@EXTRA_ARGS@', - '@INPUT@', -] - -make_script_bin_deps = [ - message_banks_index, # for GMM headers - asm_consts_generators, # for ASM headers - c_consts_generators, # for C headers -] - -s_to_bin_gen = generator(make_script_bin_sh, - arguments: make_script_bin_args, - depends: make_script_bin_deps, - output: '@BASENAME@' +# NOTE: The members of the `depends` clause below will always be modified by the +# postconf script to be order-only dependencies. This means that this generator +# will only *wait* to run until after these files have been generated, and it +# *breaks* the dependency-chain if any of these files are edited. However, because +# this generator produces a depfile, the build back-end will still see the correct +# granular headers on which each input source file depends. +script_bin_gen = generator(make_script_bin_sh, + arguments: [ + '-i', relative_source_root / 'include', + '-i', relative_source_root / 'asm', + '-i', '.' / 'res' / 'text', + '-i', '.' / 'res', + '-i', '.', + '--depfile', + '--assembler', arm_none_eabi_gcc_exe.full_path(), + '--objcopy', arm_none_eabi_objcopy_exe.full_path(), + '@EXTRA_ARGS@', + '@INPUT@', + ], + depends: [ + message_banks_index, + asm_consts_generators, + c_consts_generators, + trdata_naix, + ], + output: '@BASENAME@', + depfile: '@BASENAME@.d', ) ncgr_gen = generator(nitrogfx_exe, diff --git a/res/pokemon/meson.build b/res/pokemon/meson.build index 07f3fc65c5..ad6d454403 100644 --- a/res/pokemon/meson.build +++ b/res/pokemon/meson.build @@ -204,12 +204,6 @@ datagen_species_out = custom_target('datagen_species_out', species_data_files, ], ) -pl_personal_narc = datagen_species_out[0] -evo_narc = datagen_species_out[1] -wotbl_narc = datagen_species_out[2] -ppark_narc = datagen_species_out[3] -height_narc = datagen_species_out[4] -pl_poke_data_narc = datagen_species_out[5] tutorable_moves_h = datagen_species_out[6] species_learnsets_by_tutor_h = datagen_species_out[7] @@ -346,9 +340,7 @@ pokedex_data_giratina_altered_narc = custom_target('zukan_data_gira.narc', species_header_target = meson.current_build_dir() gen_species_headers = custom_target('gen_species_headers', - output: [ - 'footprint_data.h', - ], + output: 'footprint_data.h', input: species_data_files, env: json2bin_env, depends: [ py_consts_generators ], @@ -359,16 +351,11 @@ gen_species_headers = custom_target('gen_species_headers', ] ) -nitrofs_files += pl_personal_narc -nitrofs_files += evo_narc -nitrofs_files += wotbl_narc -nitrofs_files += ppark_narc -nitrofs_files += height_narc +nitrofs_files += datagen_species_out nitrofs_files += pl_poke_icon_narc nitrofs_files += pl_pokegra_narc nitrofs_files += pl_otherpoke_narc -nitrofs_files += pl_poke_data_narc nitrofs_files += pl_pokezukan_narc nitrofs_files += shinzukan_narc nitrofs_files += pl_growtbl_narc diff --git a/res/text/meson.build b/res/text/meson.build index caebf7f88b..9bb4b58149 100644 --- a/res/text/meson.build +++ b/res/text/meson.build @@ -783,5 +783,5 @@ message_banks = custom_target('pl_msg.narc', ) message_banks_index = message_banks[1] -nitrofs_files += message_banks[0] +nitrofs_files += message_banks naix_headers += message_banks_index diff --git a/res/trainers/meson.build b/res/trainers/meson.build index 78156fb21e..cf06d15780 100644 --- a/res/trainers/meson.build +++ b/res/trainers/meson.build @@ -943,7 +943,7 @@ trainer_party_gen = generator( trainers_order = files('trainers.order') -trdata_target = custom_target('trdata.narc', +trdata_narc = custom_target('trdata.narc', output: [ 'trdata.narc', 'trdata.naix', @@ -959,8 +959,7 @@ trdata_target = custom_target('trdata.narc', '@PRIVATE_DIR@', ], ) -trdata_narc = trdata_target[0] -trdata_naix = trdata_target[1] +trdata_naix = trdata_narc[1] # Do not generate an NAIX for `trpoke.narc`; it would go wholly unused trpoke_narc = custom_target('trpoke.narc', diff --git a/tools/postconf/postconf.py b/tools/postconf/postconf.py index b5e9a5074e..b01739aa77 100644 --- a/tools/postconf/postconf.py +++ b/tools/postconf/postconf.py @@ -22,6 +22,27 @@ def is_wsl_accessing_windows() -> bool: return ("microsoft" in platform.uname()[2].lower() and os.path.realpath(os.path.abspath(__file__)).startswith("/mnt/")) +def bytecode_scripts_order_only_deps(fileString: str) -> str: + '''Express bytecode-script dependencies on generated headers as order-only''' + return re.sub( + r"build ([\w\/\.]+): (\w+) ([\w\/\.]+) \| ([\w\/\.]+\/make_script_bin\.sh) ([\w\s\/\.]+)", + r"build \1: \2 \3 | \4 || \5", + fileString + ) + +def pch_order_only_deps(fileString: str) -> str: + ''' + Express dependencies attached to the PCH as order-only. + This is to address a Meson bug where the PCH is flagged as having implicit + dependencies on generated sources and headers. + ''' + return re.sub( + r"build main.nef.p/global_pch.h.mch: c_PCH ../include/pch/global_pch.h \| ([\w\s\/\.]+)", + r"build main.nef.p/global_pch.h.mch: c_PCH ../include/pch/global_pch.h || \1", + fileString + ) + + def main(): BUILD_NINJA = f'{BUILD_DIRECTORY}/build.ninja' COMPILE_COMMANDS = f'{BUILD_DIRECTORY}/compile_commands.json' @@ -29,11 +50,13 @@ def main(): with open(BUILD_NINJA, 'r') as build_ninja_in, open(COMPILE_COMMANDS, 'r') as compile_commands_in: build_ninja_string = build_ninja_in.read() compile_commands_string = compile_commands_in.read() - + # build.ninja edits build_ninja_string = backslash_to_forward_slash(build_ninja_string) build_ninja_string = fix_static_libs(build_ninja_string) build_ninja_string = nasm_to_asm(build_ninja_string) + build_ninja_string = bytecode_scripts_order_only_deps(build_ninja_string) + build_ninja_string = pch_order_only_deps(build_ninja_string) # compile_commands.json edits compile_commands_string = backslash_to_forward_slash(compile_commands_string)