From 66d9c86bb1adccf240a9d167e6d5632e3ec72b4f Mon Sep 17 00:00:00 2001 From: Douglas Jacobsen Date: Fri, 10 Jan 2025 15:27:45 -0700 Subject: [PATCH] Abstract container modifiers to use a container-base modifier This commit extracts some of the shared logic for container based modifiers into a common container-base base_modifier. This allows more explicit standardization between the container modifiers. --- .../container-base/base_modifier.py | 172 ++++++++++++++++++ .../builtin/modifiers/apptainer/modifier.py | 160 ++-------------- .../builtin/modifiers/docker/modifier.py | 142 +-------------- .../modifiers/pyxis-enroot/modifier.py | 145 ++------------- 4 files changed, 209 insertions(+), 410 deletions(-) create mode 100644 var/ramble/repos/builtin/base_modifiers/container-base/base_modifier.py diff --git a/var/ramble/repos/builtin/base_modifiers/container-base/base_modifier.py b/var/ramble/repos/builtin/base_modifiers/container-base/base_modifier.py new file mode 100644 index 000000000..78a9781d4 --- /dev/null +++ b/var/ramble/repos/builtin/base_modifiers/container-base/base_modifier.py @@ -0,0 +1,172 @@ +# Copyright 2022-2025 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os + +from ramble.modkit import * + + +class ContainerBase(BasicModifier): + """This base modifier contains many of the variable and method definitions + that containerized runtimes need to implement individual modifiers. It is + used as a layer to standardize the interface for the containerized runtimes, + to give them a consistent behavior within Ramble.""" + + name = "container-base" + + tags("container") + + maintainers("douglasjacobsen") + + mode("standard", description="Standard execution mode") + default_mode("standard") + + required_variable( + "container_uri", + description="The variable controls the URI the container is pulled from. " + "This should be of the format accepted by this container runtime.", + ) + + modifier_variable( + "container_mounts", + default="", + description="Comma delimited list of mount points for the container. Filled in by modifier", + modes=["standard"], + ) + + modifier_variable( + "container_env_vars", + default="", + description="Comma delimited list of environments to import into container. Filled in by modifier", + modes=["standard"], + ) + + modifier_variable( + "container_extract_dir", + default="{workload_input_dir}", + description="Directory where the extracted paths will be stored", + modes=["standard"], + ) + + modifier_variable( + "container_extract_paths", + default="[]", + description="List of paths to extract from the sqsh file into the {workload_input_dir}. " + + "Will have paths of {workload_input_dir}/enroot_extractions/{path_basename}", + modes=["standard"], + track_used=False, + ) + + def _build_runner( + self, runtime, check_software_env=False, app_inst=None, dry_run=False + ): + """Construct command runner for container runtime""" + + runner_name = f"{runtime}_runner" + + if ( + not hasattr(self, runner_name) + or getattr(self, runner_name) is None + ): + path = None + # If using spack, load spack environment before getting container runtime exec path + if check_software_env and app_inst.package_manager is not None: + if app_inst.package_manager.spec_prefix() == "spack": + app_inst.package_manager.runner.activate() + _, base = app_inst.package_manager.runner.get_package_path( + runtime + ) + app_inst.package_manager.runner.deactivate() + + if base and os.path.exists(base): + test_path = os.path.join(base, "bin") + if os.path.isdir(test_path): + path = test_path + + exec_runner = CommandRunner( + name=runtime, + command=runtime, + dry_run=dry_run, + path=path, + ) + + setattr(self, runner_name, exec_runner) + + register_phase( + "define_container_variables", + pipeline="setup", + run_before=["get_inputs"], + ) + + def _define_container_variables(self, workspace, app_inst=None): + """Define helper variables for working with containerized experiments + + To ensure it is defined properly, construct a comma delimited list of + environment variable names that will be added into the + container_env_vars variable. + """ + + def extract_names(itr, name_set=set()): + """Extract names of environment variables from the environment variable action sets + + Given an iterator over environment variable action sets, extract + the names of the environment variables. + + Modifies the name_set argument inplace. + """ + for action, conf in itr: + if action in ["set", "unset"]: + for name in conf: + name_set.add(name) + elif action == "prepend": + for group in conf: + for name in group["paths"]: + name_set.add(name) + elif action == "append": + for group in conf: + for name in group["vars"]: + name_set.add(name) + + # Only define variables if mode is standard + if self._usage_mode == "standard": + # Define container_env-vars + set_names = set() + + for env_var_set in app_inst._env_variable_sets: + extract_names(env_var_set.items(), set_names) + + for mod_inst in app_inst._modifier_instances: + extract_names(mod_inst.all_env_var_modifications(), set_names) + + env_var_list = ",".join(set_names) + app_inst.define_variable("container_env_vars", env_var_list) + + # Define container_mounts + input_mounts = app_inst.expander.expand_var("{container_mounts}") + + exp_mount = "{experiment_run_dir}:{experiment_run_dir}" + expanded_exp_mount = app_inst.expander.expand_var(exp_mount) + + if ( + exp_mount not in input_mounts + and expanded_exp_mount not in input_mounts + ): + add_mod = self._usage_mode not in self.variable_modifications + add_mod = add_mod or ( + self._usage_mode in self.variable_modifications + and "container_mounts" + not in self.variable_modifications[self._usage_mode] + ) + if add_mod: + self.variable_modification( + "container_mounts", + modification=exp_mount, + separator=",", + method="append", + mode=self._usage_mode, + ) diff --git a/var/ramble/repos/builtin/modifiers/apptainer/modifier.py b/var/ramble/repos/builtin/modifiers/apptainer/modifier.py index 8a312981f..592e056c0 100644 --- a/var/ramble/repos/builtin/modifiers/apptainer/modifier.py +++ b/var/ramble/repos/builtin/modifiers/apptainer/modifier.py @@ -12,11 +12,12 @@ from ramble.modkit import * from ramble.util.hashing import hash_string from spack.util.path import canonicalize_path +from ramble.base_mod.builtin.container_base import ContainerBase import llnl.util.filesystem as fs -class Apptainer(BasicModifier): +class Apptainer(ContainerBase): """Apptainer is a container platform. It allows you to create and run containers that package up pieces of software in a way that is portable and reproducible. You can build a container using Apptainer on your laptop, and @@ -27,41 +28,19 @@ class Apptainer(BasicModifier): different operating system.""" container_extension = "sif" + _runtime = "apptainer" name = "apptainer" tags("container") maintainers("douglasjacobsen") - mode("standard", description="Standard execution mode for apptainer") - default_mode("standard") - required_variable( "container_name", description="The variable controls the name of the resulting container file. " "It will be of the format {container_name}.{container_extension}.", ) - required_variable( - "container_uri", - description="The variable controls the URI the container is pulled from. " - "This should be of the format that would be input into `apptainer pull `.", - ) - - modifier_variable( - "container_mounts", - default="", - description="Comma delimited list of mount points for the container. Filled in by modifier", - modes=["standard"], - ) - - modifier_variable( - "container_env_vars", - default="", - description="Comma delimited list of environments to import into container. Filled in by modifier", - modes=["standard"], - ) - modifier_variable( "container_dir", default="{workload_input_dir}", @@ -69,13 +48,6 @@ class Apptainer(BasicModifier): modes=["standard"], ) - modifier_variable( - "container_extract_dir", - default="{workload_input_dir}", - description="Directory where the extracted paths will be stored", - modes=["standard"], - ) - modifier_variable( "container_path", default="{container_dir}/{container_name}." + container_extension, @@ -83,15 +55,6 @@ class Apptainer(BasicModifier): modes=["standard"], ) - modifier_variable( - "container_extract_paths", - default="[]", - description="List of paths to extract from the sqsh file into the {workload_input_dir}. " - + "Will have paths of {workload_input_dir}/enroot_extractions/{path_basename}", - modes=["standard"], - track_used=False, - ) - modifier_variable( "apptainer_run_args", default="--bind {container_mounts}", @@ -106,126 +69,23 @@ class Apptainer(BasicModifier): modes=["standard"], ) - def __init__(self, file_path): - super().__init__(file_path) - - self.apptainer_runner = None - - def _build_commands(self, app_inst=None, dry_run=False): - """Construct command runner for apptainer""" - - if self.apptainer_runner is None: - path = None - # If using spack, load spack environment before getting apptainer exec path - if app_inst.package_manager is not None: - if app_inst.package_manager.spec_prefix() == "spack": - app_inst.package_manager.runner.activate() - _, base = app_inst.package_manager.runner.get_package_path( - "apptainer" - ) - app_inst.package_manager.runner.deactivate() - - if base and os.path.exists(base): - test_path = os.path.join(base, "bin") - if os.path.isdir(test_path): - path = test_path - - self.apptainer_runner = CommandRunner( - name="apptainer", - command="apptainer", - dry_run=dry_run, - path=path, - ) - - register_phase( - "define_container_variables", - pipeline="setup", - run_before=["get_inputs"], - ) - - def _define_container_variables(self, workspace, app_inst=None): - """Define helper variables for working with enroot experiments - - To ensure it is defined properly, construct a comma delimited list of - environment variable names that will be added into the - container_env_vars variable. - """ - - def extract_names(itr, name_set=set()): - """Extract names of environment variables from the environment variable action sets - - Given an iterator over environment variable action sets, extract - the names of the environment variables. - - Modifies the name_set argument inplace. - """ - for action, conf in itr: - if action in ["set", "unset"]: - for name in conf: - name_set.add(name) - elif action == "prepend": - for group in conf: - for name in group["paths"]: - name_set.add(name) - elif action == "append": - for group in conf: - for name in group["vars"]: - name_set.add(name) - - # Only define variables if mode is standard - if self._usage_mode == "standard": - # Define container_env-vars - set_names = set() - - for env_var_set in app_inst._env_variable_sets: - extract_names(env_var_set.items(), set_names) - - for mod_inst in app_inst._modifier_instances: - extract_names(mod_inst.all_env_var_modifications(), set_names) - - env_var_list = ",".join(set_names) - app_inst.define_variable("container_env_vars", env_var_list) - - # Define container_mounts - input_mounts = app_inst.expander.expand_var("{container_mounts}") - - exp_mount = "{experiment_run_dir}:{experiment_run_dir}" - expanded_exp_mount = app_inst.expander.expand_var(exp_mount) - - if ( - exp_mount not in input_mounts - and expanded_exp_mount not in input_mounts - ): - add_mod = self._usage_mode not in self.variable_modifications - add_mod = add_mod or ( - self._usage_mode in self.variable_modifications - and "container_mounts" - not in self.variable_modifications[self._usage_mode] - ) - if add_mod: - self.variable_modification( - "container_mounts", - modification=exp_mount, - separator=",", - method="append", - mode=self._usage_mode, - ) - register_phase( - "pull_sif", + "pull_container", pipeline="setup", run_after=["get_inputs"], run_before=["make_experiments"], ) - def _pull_sif(self, workspace, app_inst=None): + def _pull_container(self, workspace, app_inst=None): """Import the container uri as an apptainer sif file Extract the container uri and path from the experiment, and import (using apptainer) into the target container_dir. """ - self._build_commands(app_inst, workspace.dry_run) + self._build_runner( + runtime=self._runtime, app_inst=app_inst, dry_run=workspace.dry_run + ) uri = self.expander.expand_var_name("container_uri") @@ -258,7 +118,9 @@ def artifact_inventory(self, workspace, app_inst=None): (dict): Artifact inventory for container attributes """ - self._build_commands(app_inst, workspace.dry_run) + self._build_runner( + runtime=self._runtime, app_inst=app_inst, dry_run=workspace.dry_run + ) id_regex = re.compile(r"\s*ID:\s*(?P\S+)") container_name = self.expander.expand_var_name("container_name") diff --git a/var/ramble/repos/builtin/modifiers/docker/modifier.py b/var/ramble/repos/builtin/modifiers/docker/modifier.py index 62a7bf175..66c5594d1 100644 --- a/var/ramble/repos/builtin/modifiers/docker/modifier.py +++ b/var/ramble/repos/builtin/modifiers/docker/modifier.py @@ -10,9 +10,10 @@ from ramble.modkit import * from ramble.util.hashing import hash_string +from ramble.base_mod.builtin.container_base import ContainerBase -class Docker(BasicModifier): +class Docker(ContainerBase): """Docker is a set of platform as a service (PaaS) products that use OS-level virtualization to deliver software in packages called containers. The service has both free and premium tiers. The software @@ -21,49 +22,11 @@ class Docker(BasicModifier): name = "docker" - tags("container") + _runtime = "docker" + _pull_command = "docker pull" maintainers("douglasjacobsen") - mode("standard", description="Standard execution mode for docker") - default_mode("standard") - - required_variable( - "container_uri", - description="The variable controls the URI the container is pulled from. " - "This should be of the format that would be input into `docker pull `.", - ) - - modifier_variable( - "container_mounts", - default="", - description="Comma delimited list of mount points for the container. Filled in by modifier", - modes=["standard"], - ) - - modifier_variable( - "container_env_vars", - default="", - description="Comma delimited list of environments to import into container. Filled in by modifier", - modes=["standard"], - ) - - modifier_variable( - "container_extract_dir", - default="{workload_input_dir}", - description="Directory where the extracted paths will be stored", - modes=["standard"], - ) - - modifier_variable( - "container_extract_paths", - default="[]", - description="List of paths to extract from the sqsh file into the {workload_input_dir}. " - + "Will have paths of {workload_input_dir}/enroot_extractions/{path_basename}", - modes=["standard"], - track_used=False, - ) - modifier_variable( "docker_run_args", default="-v {container_mounts}", @@ -78,95 +41,6 @@ class Docker(BasicModifier): modes=["standard"], ) - def __init__(self, file_path): - super().__init__(file_path) - - self.docker_runner = None - - def _build_commands(self, app_inst=None, dry_run=False): - """Construct command runner for docker""" - - if self.docker_runner is None: - self.docker_runner = CommandRunner( - name="docker", - command="docker", - dry_run=dry_run, - ) - - register_phase( - "define_container_variables", - pipeline="setup", - run_before=["get_inputs"], - ) - - def _define_container_variables(self, workspace, app_inst=None): - """Define helper variables for working with enroot experiments - - To ensure it is defined properly, construct a comma delimited list of - environment variable names that will be added into the - container_env_vars variable. - """ - - def extract_names(itr, name_set=set()): - """Extract names of environment variables from the environment variable action sets - - Given an iterator over environment variable action sets, extract - the names of the environment variables. - - Modifies the name_set argument inplace. - """ - for action, conf in itr: - if action in ["set", "unset"]: - for name in conf: - name_set.add(name) - elif action == "prepend": - for group in conf: - for name in group["paths"]: - name_set.add(name) - elif action == "append": - for group in conf: - for name in group["vars"]: - name_set.add(name) - - # Only define variables if mode is standard - if self._usage_mode == "standard": - # Define container_env-vars - set_names = set() - - for env_var_set in app_inst._env_variable_sets: - extract_names(env_var_set.items(), set_names) - - for mod_inst in app_inst._modifier_instances: - extract_names(mod_inst.all_env_var_modifications(), set_names) - - env_var_list = ",".join(set_names) - app_inst.define_variable("container_env_vars", env_var_list) - - # Define container_mounts - input_mounts = app_inst.expander.expand_var("{container_mounts}") - - exp_mount = "{experiment_run_dir}:{experiment_run_dir}" - expanded_exp_mount = app_inst.expander.expand_var(exp_mount) - - if ( - exp_mount not in input_mounts - and expanded_exp_mount not in input_mounts - ): - add_mod = self._usage_mode not in self.variable_modifications - add_mod = add_mod or ( - self._usage_mode in self.variable_modifications - and "container_mounts" - not in self.variable_modifications[self._usage_mode] - ) - if add_mod: - self.variable_modification( - "container_mounts", - modification=exp_mount, - separator=",", - method="append", - mode=self._usage_mode, - ) - register_phase( "pull_container", pipeline="setup", @@ -177,7 +51,9 @@ def extract_names(itr, name_set=set()): def _pull_container(self, workspace, app_inst=None): """Pull the container uri using docker""" - self._build_commands(app_inst, workspace.dry_run) + self._build_runner( + runtime=self._runtime, app_inst=app_inst, dry_run=workspace.dry_run + ) uri = self.expander.expand_var_name("container_uri") @@ -196,7 +72,9 @@ def artifact_inventory(self, workspace, app_inst=None): (dict): Artifact inventory for container attributes """ - self._build_commands(app_inst, workspace.dry_run) + self._build_runner( + runtime=self._runtime, app_inst=app_inst, dry_run=workspace.dry_run + ) id_regex = re.compile(r'Id.*sha256:(?P\S+)"') container_uri = self.expander.expand_var_name("container_uri") diff --git a/var/ramble/repos/builtin/modifiers/pyxis-enroot/modifier.py b/var/ramble/repos/builtin/modifiers/pyxis-enroot/modifier.py index 8b4fe2b08..522735195 100644 --- a/var/ramble/repos/builtin/modifiers/pyxis-enroot/modifier.py +++ b/var/ramble/repos/builtin/modifiers/pyxis-enroot/modifier.py @@ -11,11 +11,12 @@ from ramble.modkit import * from ramble.util.hashing import hash_file, hash_string from spack.util.path import canonicalize_path +from ramble.base_mod.builtin.container_base import ContainerBase import llnl.util.filesystem as fs -class PyxisEnroot(BasicModifier): +class PyxisEnroot(ContainerBase): """Modifier to aid configuring pyxis-enroot based execution environments Pyxis is a container plugin for slurm developed by NVIDIA. @@ -41,6 +42,8 @@ class PyxisEnroot(BasicModifier): - container_env_vars """ + _runtime = "enroot" + _unsquash = "unsquashfs" container_extension = "sqsh" container_hash_file_extension = "sha256" @@ -51,28 +54,15 @@ class PyxisEnroot(BasicModifier): maintainers("douglasjacobsen") - mode("standard", description="Standard execution mode for pyxis-enroot") mode( "no_provenance", description="Standard execution mode without provenance tracking", ) - default_mode("standard") - required_variable("container_name") - required_variable("container_uri") - - modifier_variable( - "container_mounts", - default="", - description="Comma delimited list of mount points for the container. Filled in by modifier", - modes=["standard", "no_provenance"], - ) - - modifier_variable( - "container_env_vars", - default="", - description="Comma delimited list of environments to import into container. Filled in by modifier", - modes=["standard", "no_provenance"], + required_variable( + "container_name", + description="The variable controls the name of the resulting container file. " + "It will be of the format {container_name}.{container_extension}.", ) modifier_variable( @@ -82,13 +72,6 @@ class PyxisEnroot(BasicModifier): modes=["standard", "no_provenance"], ) - modifier_variable( - "container_extract_dir", - default="{workload_input_dir}", - description="Directory where the extracted paths will be stored", - modes=["standard", "no_provenance"], - ) - modifier_variable( "container_path", default="{container_dir}/{container_name}." + container_extension, @@ -96,108 +79,6 @@ class PyxisEnroot(BasicModifier): modes=["standard", "no_provenance"], ) - modifier_variable( - "container_extract_paths", - default="[]", - description="List of paths to extract from the sqsh file into the {workload_input_dir}. " - + "Will have paths of {workload_input_dir}/enroot_extractions/{path_basename}", - modes=["standard", "no_provenance"], - track_used=False, - ) - - def __init__(self, file_path): - super().__init__(file_path) - - self.enroot_runner = None - self.unsquashfs_runner = None - - def _build_commands(self, dry_run=False): - """Construct command runners for enroot and unsquashfs""" - if self.enroot_runner is None: - self.enroot_runner = CommandRunner( - name="enroot", command="enroot", dry_run=dry_run - ) - - if self.unsquashfs_runner is None: - self.unsquashfs_runner = CommandRunner( - name="unsquashfs", command="unsquashfs", dry_run=dry_run - ) - - register_phase( - "define_container_variables", - pipeline="setup", - run_before=["get_inputs"], - ) - - def _define_container_variables(self, workspace, app_inst=None): - """Define helper variables for working with enroot experiments - - To ensure it is defined properly, construct a comma delimited list of - environment variable names that will be added into the - container_env_vars variable. - """ - - def extract_names(itr, name_set=set()): - """Extract names of environment variables from the environment variable action sets - - Given an iterator over environment variable action sets, extract - the names of the environment variables. - - Modifies the name_set argument inplace. - """ - for action, conf in itr: - if action in ["set", "unset"]: - for name in conf: - name_set.add(name) - elif action == "prepend": - for group in conf: - for name in group["paths"]: - name_set.add(name) - elif action == "append": - for group in conf: - for name in group["vars"]: - name_set.add(name) - - # Only define variables if mode is standard - if self._usage_mode == "standard": - # Define container_env-vars - set_names = set() - - for env_var_set in app_inst._env_variable_sets: - extract_names(env_var_set.items(), set_names) - - for mod_inst in app_inst._modifier_instances: - extract_names(mod_inst.all_env_var_modifications(), set_names) - - env_var_list = ",".join(set_names) - app_inst.define_variable("container_env_vars", env_var_list) - - # Define container_mounts - input_mounts = app_inst.expander.expand_var("{container_mounts}") - - exp_mount = "{experiment_run_dir}:{experiment_run_dir}" - expanded_exp_mount = app_inst.expander.expand_var(exp_mount) - - if ( - exp_mount not in input_mounts - and expanded_exp_mount not in input_mounts - ): - add_mod = self._usage_mode not in self.variable_modifications - add_mod = ( - add_mod - or self._usage_mode in self.variable_modifications - and "container_mounts" - not in self.variable_modifications[self._usage_mode] - ) - if add_mod: - self.variable_modification( - "container_mounts", - modification=exp_mount, - separator=",", - method="append", - mode=self._usage_mode, - ) - register_phase( "import_sqsh", pipeline="setup", @@ -212,7 +93,9 @@ def _import_sqsh(self, workspace, app_inst=None): (using enroot) into the target container_dir. """ - self._build_commands(workspace.dry_run) + self._build_runner( + runtime=self._runtime, app_inst=app_inst, dry_run=workspace.dry_run + ) uri = self.expander.expand_var_name("container_uri") @@ -242,7 +125,11 @@ def _import_sqsh(self, workspace, app_inst=None): def _extract_from_sqsh(self, workspace, app_inst=None): """Extract paths from the sqsh file into the workload inputs path""" - self._build_commands(workspace.dry_run) + self._build_runner( + runtime=self._unsquash, + app_inst=app_inst, + dry_run=workspace.dry_run, + ) extract_paths = self.expander.expand_var_name( "container_extract_paths", typed=True, merge_used_stage=False