From 5580ea270d1843c310c3aa40f5648e38e8c211ff Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Tue, 6 Aug 2024 15:40:45 -0700 Subject: [PATCH] Issue a warning when mirror with pip package manager Previous attempt at https://github.com/GoogleCloudPlatform/ramble/pull/595 was to error out, however as @douglasjacobsen pointed out, mirror may be provided outside of Ramble so it would be annoying to hard fail. Instead only issue a warning per package manager (currently only for pip.) --- .../test/end_to_end/warn_mirror_support.py | 67 +++++++++++++++++++ .../package_managers/pip/package_manager.py | 9 +++ 2 files changed, 76 insertions(+) create mode 100644 lib/ramble/ramble/test/end_to_end/warn_mirror_support.py diff --git a/lib/ramble/ramble/test/end_to_end/warn_mirror_support.py b/lib/ramble/ramble/test/end_to_end/warn_mirror_support.py new file mode 100644 index 000000000..533ff977e --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/warn_mirror_support.py @@ -0,0 +1,67 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import glob +import os + +import pytest + +import ramble.workspace +from ramble.main import RambleCommand +from ramble.test.dry_run_helpers import search_files_for_string + +pytestmark = pytest.mark.usefixtures( + "mutable_config", + "mutable_mock_workspace_path", +) + +workspace = RambleCommand("workspace") + + +def assert_text_in_mirror_logs(ws, text): + mirror_logs = glob.glob(os.path.join(ws.log_dir, "**", "*.out")) + assert search_files_for_string(mirror_logs, text) + + +def test_warn_mirror_support(tmpdir): + test_config = """ +ramble: + variants: + package_manager: pip + variables: + mpi_command: '' + batch_submit: '{execute_experiment}' + processes_per_node: 1 + n_ranks: 1 + applications: + hostname: + workloads: + local: + experiments: + test: {} + software: + packages: {} + environments: {} +""" + ws_name = "test_pip_mirror_support" + ws = ramble.workspace.create(ws_name) + ramble.workspace.activate(ws) + ws.write() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path, "w+") as f: + f.write(test_config) + + ws._re_read() + + mirror_path = os.path.join(tmpdir, ws_name) + workspace("mirror", "--dry-run", "-d", mirror_path) + assert_text_in_mirror_logs( + ws, "Warning: Mirroring software using pip is not currently supported" + ) diff --git a/var/ramble/repos/builtin/package_managers/pip/package_manager.py b/var/ramble/repos/builtin/package_managers/pip/package_manager.py index 3f71667b1..e700d6014 100644 --- a/var/ramble/repos/builtin/package_managers/pip/package_manager.py +++ b/var/ramble/repos/builtin/package_managers/pip/package_manager.py @@ -197,6 +197,15 @@ def populate_inventory( } ) + register_phase("warn_mirror_support", pipeline="mirror") + + def _warn_mirror_support(self, workspace, app_inst=None): + del workspace, app_inst # unused arguments + logger.warn( + f"Mirroring software using {self.name} is not currently supported. " + "If a software mirror is required, it needs to be set up outside of Ramble" + ) + package_name_regex = re.compile( r"\s*(?P[A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9]).*", re.IGNORECASE