From 8dec97b4347fb0a6e1a930935f45327fc338cb3f Mon Sep 17 00:00:00 2001 From: Alessandro De Maria Date: Sat, 17 Aug 2024 18:41:30 +0000 Subject: [PATCH 1/2] Add mp-method arg --- kapitan/cli.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index 54a32e886..5cd5a7917 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -101,6 +101,7 @@ def trigger_compile(args): def build_parser(): parser = argparse.ArgumentParser(prog=PROJECT_NAME, description=DESCRIPTION) parser.add_argument("--version", action="version", version=VERSION) + parser.add_argument("--mp-method", action="store", default=from_dot_kapitan("global", "mp-method", "spawn"), help="set multiprocessing start method", choices=["spawn", "fork", "forkserver"]) subparser = parser.add_subparsers(help="commands", dest="subparser_name") inventory_backend_parser = argparse.ArgumentParser(add_help=False) @@ -601,16 +602,17 @@ def build_parser(): def main(): """main function for command line usage""" + + parser = build_parser() + args = parser.parse_args() + try: - multiprocessing.set_start_method("spawn") + multiprocessing.set_start_method(args.mp_method) # main() is explicitly multiple times in tests # and will raise RuntimeError except RuntimeError: pass - - parser = build_parser() - args = parser.parse_args() - + if getattr(args, "func", None) == generate_inventory and args.pattern and args.target_name == "": parser.error("--pattern requires --target_name") From 365fa9c5e9560e1063d3f8bb4c23bc7f6f19f736 Mon Sep 17 00:00:00 2001 From: Alessandro De Maria Date: Sat, 17 Aug 2024 19:57:22 +0000 Subject: [PATCH 2/2] Do not read multiple times --- kapitan/resources.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kapitan/resources.py b/kapitan/resources.py index 71696b533..3bc358f16 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -22,6 +22,7 @@ import kapitan.cached as cached from kapitan import __file__ as kapitan_install_path +from kapitan.inputs.kadet import Dict from kapitan.errors import CompileError, InventoryError, KapitanError from kapitan.inventory import Inventory, ReclassInventory, AVAILABLE_BACKENDS from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string @@ -334,6 +335,13 @@ def get_inventory(inventory_path, ignore_class_not_found: bool = False) -> Inven cached.inv = inventory_backend cached.global_inv = cached.inv.inventory + + # if we use forked processes, we need to load the inventory for kadet once + # and pass it to the children, to avoid re-reading the inventory for each child + # TODO(adenaria): Improve to only do it for kadet + if cached.args.mp_method != "spawn": + cached.inventory_global_kadet = Dict(cached.global_inv) + # migrate inventory to selected inventory backend if hasattr(cached.args, "migrate") and cached.args.migrate: inventory_backend.migrate()