From b5d6df3e0e82f06ca7c7dcde44d580d4d9103b81 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Tue, 30 Jul 2024 10:32:10 -0700 Subject: [PATCH 1/2] Clean up older (3.5 and below) python syntax under /lib/ramble/ramble Most cleanups are done automatically via: ``` find lib/ramble/ramble/ -name '*.py' -exec pyupgrade --py36-plus {} \; ``` The above `pyupgrade` call is invoked twice (as the first time it changes string interpolation to format calls, and the second time to f-strings.) Now invoking that same command should see no changes. Some manual cleanups are guided by the existing format check (mostly on unused imports.) --- lib/ramble/ramble/application.py | 12 +-- lib/ramble/ramble/caches.py | 2 +- lib/ramble/ramble/cmd/__init__.py | 9 +- lib/ramble/ramble/cmd/attributes.py | 9 +- lib/ramble/ramble/cmd/commands.py | 25 +++-- lib/ramble/ramble/cmd/common/info.py | 1 - lib/ramble/ramble/cmd/common/list.py | 21 ++--- lib/ramble/ramble/cmd/config.py | 4 +- lib/ramble/ramble/cmd/debug.py | 5 +- lib/ramble/ramble/cmd/deployment.py | 8 +- lib/ramble/ramble/cmd/edit.py | 2 +- lib/ramble/ramble/cmd/flake8.py | 22 ++--- lib/ramble/ramble/cmd/license.py | 11 +-- lib/ramble/ramble/cmd/mirror.py | 2 +- lib/ramble/ramble/cmd/on.py | 7 -- lib/ramble/ramble/cmd/python.py | 2 +- lib/ramble/ramble/cmd/repo.py | 1 - lib/ramble/ramble/cmd/style.py | 19 ++-- lib/ramble/ramble/cmd/unit_test.py | 14 ++- lib/ramble/ramble/cmd/workspace.py | 9 +- lib/ramble/ramble/config.py | 47 +++++----- lib/ramble/ramble/context.py | 2 +- lib/ramble/ramble/error.py | 3 +- lib/ramble/ramble/expander.py | 16 ++-- lib/ramble/ramble/experiment_set.py | 2 +- lib/ramble/ramble/fetch_strategy.py | 94 +++++++++---------- lib/ramble/ramble/filters.py | 2 +- lib/ramble/ramble/graphs.py | 5 +- lib/ramble/ramble/keywords.py | 2 +- lib/ramble/ramble/language/language_base.py | 10 +- lib/ramble/ramble/main.py | 34 +++---- lib/ramble/ramble/mirror.py | 18 ++-- lib/ramble/ramble/modifier.py | 11 +-- lib/ramble/ramble/package_manager.py | 9 +- lib/ramble/ramble/pipeline.py | 8 +- lib/ramble/ramble/renderer.py | 4 +- lib/ramble/ramble/repeats.py | 2 +- lib/ramble/ramble/repository.py | 64 ++++++------- lib/ramble/ramble/software_environments.py | 6 +- lib/ramble/ramble/spec.py | 14 +-- lib/ramble/ramble/stage.py | 32 +++---- lib/ramble/ramble/success_criteria.py | 7 +- lib/ramble/ramble/test/cmd/clean.py | 2 +- lib/ramble/ramble/test/cmd/config.py | 4 +- lib/ramble/ramble/test/cmd/mirror.py | 8 +- lib/ramble/ramble/test/concretize_builtin.py | 2 +- lib/ramble/ramble/test/conftest.py | 16 ++-- .../test/end_to_end/analyze_fom_output.py | 2 +- .../chained_experiment_var_inheritance.py | 4 +- .../end_to_end/config_section_env_vars.py | 2 +- .../test/end_to_end/custom_executables.py | 2 +- .../test/end_to_end/define_package_paths.py | 8 +- .../end_to_end/dryrun_chained_experiments.py | 8 +- .../end_to_end/dryrun_copies_external_env.py | 2 +- .../dryrun_series_contains_package_paths.py | 2 +- .../ramble/test/end_to_end/env_var_builtin.py | 8 +- .../test/end_to_end/expanded_fom_dry_run.py | 2 +- .../test/end_to_end/experiment_excludes.py | 6 +- .../test/end_to_end/experiment_repeats.py | 2 +- .../ramble/test/end_to_end/explicit_zips.py | 6 +- .../test/end_to_end/formatted_executables.py | 2 +- .../test/end_to_end/globbing_patterns.py | 4 +- .../test/end_to_end/gromacs_size_expansion.py | 2 +- .../test/end_to_end/merge_config_files.py | 2 +- .../test/end_to_end/package_manager_config.py | 2 +- .../package_manager_requirements.py | 2 +- .../test/end_to_end/passthrough_variables.py | 2 +- .../phase_selection_with_dependencies.py | 2 +- .../ramble/test/end_to_end/shared_context.py | 2 +- .../ramble/test/end_to_end/spack_env_cache.py | 8 +- .../test/end_to_end/test_configvar_dry_run.py | 2 +- .../ramble/test/end_to_end/wrfv4_dry_run.py | 12 +-- lib/ramble/ramble/test/expander.py | 8 +- lib/ramble/ramble/test/experiment_set.py | 2 +- lib/ramble/ramble/test/gcs_fetch.py | 2 +- lib/ramble/ramble/test/mirror_tests.py | 2 +- .../ramble/test/modifier_application.py | 4 +- .../experiment_modification.py | 2 +- .../mock_modifier_phases.py | 2 +- .../mock_modifier_spack_configs.py | 2 +- .../modifier_helpers.py | 4 +- .../modifier_prepare_analysis.py | 2 +- lib/ramble/ramble/test/spec_basic.py | 2 +- lib/ramble/ramble/test/stage.py | 6 +- lib/ramble/ramble/test/success_criteria.py | 4 +- .../success_criteria/always_print_foms.py | 2 +- .../success_criteria/repeat_success_strict.py | 6 +- .../success_fom_comparison.py | 2 +- .../success_criteria/success_fom_globbing.py | 2 +- .../success_criteria/success_functions.py | 2 +- .../success_criteria/success_modifiers.py | 4 +- .../success_variable_fom_comparison.py | 2 +- lib/ramble/ramble/test/util/editor.py | 2 +- lib/ramble/ramble/util/editor.py | 4 +- lib/ramble/ramble/util/executable.py | 2 +- lib/ramble/ramble/util/file_cache.py | 6 +- lib/ramble/ramble/util/graph.py | 2 +- .../ramble/util/imp/importlib_importer.py | 6 +- lib/ramble/ramble/util/lock.py | 14 +-- lib/ramble/ramble/util/logger.py | 2 +- lib/ramble/ramble/util/naming.py | 11 +-- lib/ramble/ramble/util/stats.py | 2 +- lib/ramble/ramble/util/web.py | 37 +++----- lib/ramble/ramble/util/yaml_generation.py | 2 +- lib/ramble/ramble/workload.py | 6 +- lib/ramble/ramble/workspace/shell.py | 10 +- lib/ramble/ramble/workspace/workspace.py | 34 +++---- 107 files changed, 412 insertions(+), 495 deletions(-) diff --git a/lib/ramble/ramble/application.py b/lib/ramble/ramble/application.py index 31af8e266..3ecdf3a66 100644 --- a/lib/ramble/ramble/application.py +++ b/lib/ramble/ramble/application.py @@ -72,7 +72,7 @@ def _get_context_display_name(context): ) -class ApplicationBase(object, metaclass=ApplicationMeta): +class ApplicationBase(metaclass=ApplicationMeta): name = None _builtin_name = "builtin::{name}" _builtin_required_key = "required" @@ -346,7 +346,7 @@ def _long_print(self): out_str.append(rucolor.nested_1(" %s:\n" % name)) for key, val in info.items(): if val: - out_str.append(" %s = %s\n" % (key, val.replace("@", "@@"))) + out_str.append(" {} = {}\n".format(key, val.replace("@", "@@"))) return out_str @@ -639,7 +639,7 @@ def create_experiment_chain(self, workspace): # Build initial stack. Uses a reversal of the current instance's # chained experiments parent_namespace = self.expander.experiment_namespace - classes_in_stack = set([self]) + classes_in_stack = {self} chain_idx = 0 chain_stack = [] for exp in reversed(self.chained_experiments): @@ -1386,7 +1386,7 @@ def populate_inventory(self, workspace, force_compute=False, require_exist=False inventory_file = os.path.join(experiment_run_dir, self._inventory_file_name) if os.path.exists(inventory_file) and not force_compute: - with open(inventory_file, "r") as f: + with open(inventory_file) as f: self.hash_inventory = spack.util.spack_json.load(f) else: @@ -1593,7 +1593,7 @@ def format_context(context_match, context_format): logger.debug(f"Skipping analysis of non-existent file: {file}") continue - with open(file, "r") as f: + with open(file) as f: for line in f.readlines(): logger.debug(f"Line: {line}") @@ -2038,7 +2038,7 @@ def read_status(self): ) if os.path.isfile(status_path): - with open(status_path, "r") as f: + with open(status_path) as f: status_data = spack.util.spack_json.load(f) self.variables[self.keywords.experiment_status] = status_data[ self.keywords.experiment_status diff --git a/lib/ramble/ramble/caches.py b/lib/ramble/ramble/caches.py index 90ed6005c..7e1cd3a43 100644 --- a/lib/ramble/ramble/caches.py +++ b/lib/ramble/ramble/caches.py @@ -56,7 +56,7 @@ def _fetch_cache(): return ramble.fetch_strategy.FsCache(path) -class MirrorCache(object): +class MirrorCache: def __init__(self, root): self.root = os.path.abspath(root) diff --git a/lib/ramble/ramble/cmd/__init__.py b/lib/ramble/ramble/cmd/__init__.py index 87f7ddeb0..1867683f3 100644 --- a/lib/ramble/ramble/cmd/__init__.py +++ b/lib/ramble/ramble/cmd/__init__.py @@ -6,7 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import os import re @@ -113,7 +112,7 @@ def get_module(cmd_name): try: # Try to import the command from the built-in directory - module_name = "%s.%s" % (__name__, pname) + module_name = f"{__name__}.{pname}" module = __import__(module_name, fromlist=[pname, SETUP_PARSER, DESCRIPTION], level=0) logger.debug(f"Imported {pname} from built-in commands") except ImportError: @@ -190,7 +189,7 @@ class PythonNameError(ramble.error.RambleError): def __init__(self, name): self.name = name - super(PythonNameError, self).__init__("{0} is not a permissible Python name.".format(name)) + super().__init__(f"{name} is not a permissible Python name.") class CommandNameError(ramble.error.RambleError): @@ -198,9 +197,7 @@ class CommandNameError(ramble.error.RambleError): def __init__(self, name): self.name = name - super(CommandNameError, self).__init__( - "{0} is not a permissible Ramble command name.".format(name) - ) + super().__init__(f"{name} is not a permissible Ramble command name.") ######################################## diff --git a/lib/ramble/ramble/cmd/attributes.py b/lib/ramble/ramble/cmd/attributes.py index 103f04e1c..56dbcdce4 100644 --- a/lib/ramble/ramble/cmd/attributes.py +++ b/lib/ramble/ramble/cmd/attributes.py @@ -6,7 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import argparse from collections import defaultdict @@ -91,7 +90,7 @@ def objects_to_attributes( if not object_names: object_names = ramble.repository.paths[object_type].all_object_names() - app_to_users = defaultdict(lambda: set()) + app_to_users = defaultdict(set) for name in object_names: cls = ramble.repository.paths[object_type].get_obj_class(name) for user in getattr(cls, attr_name): @@ -103,7 +102,7 @@ def objects_to_attributes( def attributes_to_objects( users=None, attr_name=default_attr, object_type=ramble.repository.default_type ): - user_to_apps = defaultdict(lambda: []) + user_to_apps = defaultdict(list) object_names = ramble.repository.paths[object_type].all_object_names() for name in object_names: cls = ramble.repository.paths[object_type].get_obj_class(name) @@ -163,7 +162,7 @@ def attributes(parser, args): args.object_or_attr, attr_name=attr_name, object_type=object_type ) for user, objects in sorted(attributes.items()): - color.cprint("@c{%s}: %s" % (user, ", ".join(sorted(objects)))) + color.cprint("@c{{{}}}: {}".format(user, ", ".join(sorted(objects)))) return 0 if attributes else 1 else: @@ -171,7 +170,7 @@ def attributes(parser, args): args.object_or_attr, attr_name=attr_name, object_type=object_type ) for app, attributes in sorted(objects.items()): - color.cprint("@c{%s}: %s" % (app, ", ".join(sorted(attributes)))) + color.cprint("@c{{{}}}: {}".format(app, ", ".join(sorted(attributes)))) return 0 if objects else 1 if args.by_attribute: diff --git a/lib/ramble/ramble/cmd/commands.py b/lib/ramble/ramble/cmd/commands.py index de9e74ba3..7225b7faa 100644 --- a/lib/ramble/ramble/cmd/commands.py +++ b/lib/ramble/ramble/cmd/commands.py @@ -6,7 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import argparse import copy @@ -102,15 +101,15 @@ def __init__( rst_levels=["-", "-", "^", "~", ":", "`"], ): out = sys.stdout if out is None else out - super(RambleArgparseRstWriter, self).__init__(prog, out, aliases, rst_levels) + super().__init__(prog, out, aliases, rst_levels) self.documented = documented_commands def usage(self, *args): - string = super(RambleArgparseRstWriter, self).usage(*args) + string = super().usage(*args) cmd = self.parser.prog.replace(" ", "-") if cmd in self.documented: - string += "\n:ref:`More documentation `\n".format(cmd) + string += f"\n:ref:`More documentation `\n" return string @@ -141,9 +140,9 @@ def body(self, positionals, optionals, subcommands): return """ if $list_options then - {0} + {} else - {1} + {} fi """.format( self.optionals(optionals), self.positionals(positionals) @@ -152,16 +151,16 @@ def body(self, positionals, optionals, subcommands): return """ if $list_options then - {0} + {} else - {1} + {} fi """.format( self.optionals(optionals), self.subcommands(subcommands) ) else: return """ - {0} + {} """.format( self.optionals(optionals) ) @@ -177,10 +176,10 @@ def positionals(self, positionals): return 'RAMBLE_COMREPLY=""' def optionals(self, optionals): - return 'RAMBLE_COMPREPLY="{0}"'.format(" ".join(optionals)) + return 'RAMBLE_COMPREPLY="{}"'.format(" ".join(optionals)) def subcommands(self, subcommands): - return 'RAMBLE_COMPREPLY="{0}"'.format(" ".join(subcommands)) + return 'RAMBLE_COMPREPLY="{}"'.format(" ".join(subcommands)) @formatter @@ -200,7 +199,7 @@ def rst_index(out): dmax = max(len(section_descriptions.get(s, s)) for s in sections) + 2 cmax = max(len(c) for _, c in sections.items()) + 60 - row = "%s %s\n" % ("=" * dmax, "=" * cmax) + row = "{} {}\n".format("=" * dmax, "=" * cmax) line = "%%-%ds %%s\n" % dmax out.write(row) @@ -211,7 +210,7 @@ def rst_index(out): for i, cmd in enumerate(sorted(commands)): description = description.capitalize() if i == 0 else "" - ref = ":ref:`%s `" % (cmd, cmd) + ref = f":ref:`{cmd} `" comma = "," if i != len(commands) - 1 else "" bar = "| " if i % 8 == 0 else " " out.write(line % (description, bar + ref + comma)) diff --git a/lib/ramble/ramble/cmd/common/info.py b/lib/ramble/ramble/cmd/common/info.py index fd243db6e..cf0730c34 100644 --- a/lib/ramble/ramble/cmd/common/info.py +++ b/lib/ramble/ramble/cmd/common/info.py @@ -6,7 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import llnl.util.tty.color as color diff --git a/lib/ramble/ramble/cmd/common/list.py b/lib/ramble/ramble/cmd/common/list.py index ad817ac6f..26ef11c1c 100644 --- a/lib/ramble/ramble/cmd/common/list.py +++ b/lib/ramble/ramble/cmd/common/list.py @@ -6,8 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function -from __future__ import division import argparse import fnmatch @@ -22,10 +20,7 @@ import ramble.cmd.common.arguments as arguments from ramble.util.logger import logger -if sys.version_info > (3, 1): - from html import escape # novm -else: - from cgi import escape +from html import escape # novm formatters = {} @@ -179,7 +174,7 @@ def head(n, span_id, title, anchor=None): out.write('\n' if i % 2 == 0 else '\n') for name in row: out.write("\n") - out.write('%s\n' % (name, name)) + out.write(f'{name}\n') out.write("\n") out.write("\n") out.write("\n") @@ -204,13 +199,11 @@ def head(n, span_id, title, anchor=None): out.write(f'
Ramble {obj_def["dir_name"]}:
\n') out.write('
\n") diff --git a/lib/ramble/ramble/cmd/config.py b/lib/ramble/ramble/cmd/config.py index 60506e5ef..4b32a4ffd 100644 --- a/lib/ramble/ramble/cmd/config.py +++ b/lib/ramble/ramble/cmd/config.py @@ -280,7 +280,7 @@ def config_update(args): if cannot_overwrite: msg = "Detected permission issues with the following scopes:\n\n" for scope, cfg_file in cannot_overwrite: - msg += "\t[scope={0}, cfg={1}]\n".format(scope.name, cfg_file) + msg += f"\t[scope={scope.name}, cfg={cfg_file}]\n" msg += ( "\nEither ensure that you have sufficient permissions to " "modify these files or do not include these scopes in the " @@ -304,7 +304,7 @@ def config_update(args): ) for scope in updates: cfg_file = ramble.config.config.get_config_filename(scope.name, args.section) - msg += "\t[scope={0}, file={1}]\n".format(scope.name, cfg_file) + msg += f"\t[scope={scope.name}, file={cfg_file}]\n" msg += ( "\nIf the configuration files are updated, versions of Ramble " "that are older than this version may not be able to read " diff --git a/lib/ramble/ramble/cmd/debug.py b/lib/ramble/ramble/cmd/debug.py index 534685cfb..b3983d6c8 100644 --- a/lib/ramble/ramble/cmd/debug.py +++ b/lib/ramble/ramble/cmd/debug.py @@ -6,7 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import os import platform @@ -52,9 +51,9 @@ def _debug_tarball_suffix(): commit = git("rev-parse", "--short", "HEAD", output=str).strip() if symbolic == commit: - return "nobranch.%s.%s" % (commit, suffix) + return f"nobranch.{commit}.{suffix}" else: - return "%s.%s.%s" % (symbolic, commit, suffix) + return f"{symbolic}.{commit}.{suffix}" def report(args): diff --git a/lib/ramble/ramble/cmd/deployment.py b/lib/ramble/ramble/cmd/deployment.py index 2f6b5efcd..378e212f9 100644 --- a/lib/ramble/ramble/cmd/deployment.py +++ b/lib/ramble/ramble/cmd/deployment.py @@ -7,7 +7,6 @@ # except according to those terms. import os -import sys import llnl.util.filesystem as fs @@ -26,11 +25,6 @@ import ramble.pipeline import ramble.filters -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm noqa: F401 -else: - from collections import Sequence # noqa: F401 - description = "(experimental) manage workspace deployments" section = "workspaces" @@ -130,7 +124,7 @@ def pull_file(src, dest): pull_file(remote_index_path, local_index_path) - with open(local_index_path, "r") as f: + with open(local_index_path) as f: index_data = sjson.load(f) for file in index_data[push_cls.index_namespace]: diff --git a/lib/ramble/ramble/cmd/edit.py b/lib/ramble/ramble/cmd/edit.py index cc559e5f0..a02e00cce 100644 --- a/lib/ramble/ramble/cmd/edit.py +++ b/lib/ramble/ramble/cmd/edit.py @@ -127,7 +127,7 @@ def edit(parser, args): blacklist = [".pyc", "~"] # blacklist binaries and backups files = list(filter(lambda x: all(s not in x for s in blacklist), files)) if len(files) > 1: - m = "Multiple files exist with the name {0}.".format(name) + m = f"Multiple files exist with the name {name}." m += " Please specify a suffix. Files are:\n\n" for f in files: m += " " + os.path.basename(f) + "\n" diff --git a/lib/ramble/ramble/cmd/flake8.py b/lib/ramble/ramble/cmd/flake8.py index 62d68a724..4a76db91c 100644 --- a/lib/ramble/ramble/cmd/flake8.py +++ b/lib/ramble/ramble/cmd/flake8.py @@ -6,7 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import deprecation import re @@ -91,13 +90,12 @@ def is_application(f): } # compile all regular expressions. -pattern_exemptions = dict( - ( - re.compile(file_pattern), - dict((code, [re.compile(p) for p in patterns]) for code, patterns in error_dict.items()), - ) +pattern_exemptions = { + re.compile(file_pattern): { + code: [re.compile(p) for p in patterns] for code, patterns in error_dict.items() + } for file_pattern, error_dict in pattern_exemptions.items() -) +} def changed_files(base=None, untracked=True, all_files=False): @@ -108,7 +106,7 @@ def changed_files(base=None, untracked=True, all_files=False): if base is None: base = os.environ.get("GITHUB_BASE_REF", "develop") - range = "{0}...".format(base) + range = f"{base}..." git_args = [ # Add changed files committed since branching off of develop @@ -172,9 +170,9 @@ def add_pattern_exemptions(line, codes): # append exemption to line if "# noqa: " in line: - line += ",{0}".format(exemptions) + line += f",{exemptions}" elif line: # ignore noqa on empty lines - line += " # noqa: {0}".format(exemptions) + line += f" # noqa: {exemptions}" # if THIS made the line too long, add an exemption for that if len(line) > max_line_length and orig_len <= max_line_length: @@ -315,7 +313,7 @@ def prefix_relative(path): print() print("Modified files:") for filename in file_list: - print(" {0}".format(filename.strip())) + print(f" {filename.strip()}") print("=======================================================") # run flake8 on the temporary tree, once for core, once for apps @@ -372,7 +370,7 @@ def prefix_relative(path): else: # print results relative to current working directory def cwd_relative(path): - return "{0}: [".format( + return "{}: [".format( os.path.relpath(os.path.join(ramble.paths.prefix, path.group(1)), os.getcwd()) ) diff --git a/lib/ramble/ramble/cmd/license.py b/lib/ramble/ramble/cmd/license.py index 3d53020c3..3a31200f1 100644 --- a/lib/ramble/ramble/cmd/license.py +++ b/lib/ramble/ramble/cmd/license.py @@ -5,7 +5,6 @@ # , at your # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import os import re @@ -99,7 +98,7 @@ def list_files(args): OLD_LICENSE, SPDX_MISMATCH, GENERAL_MISMATCH = range(1, 4) -class LicenseError(object): +class LicenseError: def __init__(self): self.error_counts = defaultdict(int) @@ -154,7 +153,7 @@ def _check_license(lines, path): def old_license(line, path): if re.search("This program is free software", line): - print("{0}: has old LGPL license header".format(path)) + print(f"{path}: has old LGPL license header") return OLD_LICENSE # If the SPDX identifier is present, then there is a mismatch (since it @@ -163,8 +162,8 @@ def wrong_spdx_identifier(line, path): m = re.search(r"SPDX-License-Identifier: ([^\n]*)", line) if m and m.group(1) != apache2_mit_spdx: print( - "{0}: SPDX license identifier mismatch" - "(expecting {1}, found {2})".format(path, apache2_mit_spdx, m.group(1)) + "{}: SPDX license identifier mismatch" + "(expecting {}, found {})".format(path, apache2_mit_spdx, m.group(1)) ) return SPDX_MISMATCH @@ -176,7 +175,7 @@ def wrong_spdx_identifier(line, path): if error: return error - print("{0}: the license does not match the expected format".format(path)) + print(f"{path}: the license does not match the expected format") return GENERAL_MISMATCH diff --git a/lib/ramble/ramble/cmd/mirror.py b/lib/ramble/ramble/cmd/mirror.py index 59c3147f2..ad757ce26 100644 --- a/lib/ramble/ramble/cmd/mirror.py +++ b/lib/ramble/ramble/cmd/mirror.py @@ -176,7 +176,7 @@ def mirror_list(args): def _read_specs_from_file(filename): specs = [] - with open(filename, "r") as stream: + with open(filename) as stream: for i, string in enumerate(stream): try: s = ramble.Spec(string) diff --git a/lib/ramble/ramble/cmd/on.py b/lib/ramble/ramble/cmd/on.py index a0af35fca..54c430600 100644 --- a/lib/ramble/ramble/cmd/on.py +++ b/lib/ramble/ramble/cmd/on.py @@ -6,8 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -import sys - import ramble.workspace import ramble.expander import ramble.pipeline @@ -15,11 +13,6 @@ import ramble.cmd.common.arguments as arguments -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm noqa: F401 -else: - from collections import Sequence # noqa: F401 - description = '"And now\'s the time, the time is now" (execute workspace experiments)' section = "workspaces" diff --git a/lib/ramble/ramble/cmd/python.py b/lib/ramble/ramble/cmd/python.py index b1191f217..f51035f9e 100644 --- a/lib/ramble/ramble/cmd/python.py +++ b/lib/ramble/ramble/cmd/python.py @@ -106,7 +106,7 @@ def ipython_interpreter(args): elif args.python_command: IPython.start_ipython(argv=["-c", args.python_command]) else: - header = "Ramble version %s\nPython %s, %s %s" % ( + header = "Ramble version {}\nPython {}, {} {}".format( ramble.ramble_version, platform.python_version(), platform.system(), diff --git a/lib/ramble/ramble/cmd/repo.py b/lib/ramble/ramble/cmd/repo.py index 0fc9b97c6..63f2b378c 100644 --- a/lib/ramble/ramble/cmd/repo.py +++ b/lib/ramble/ramble/cmd/repo.py @@ -6,7 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import os import sys diff --git a/lib/ramble/ramble/cmd/style.py b/lib/ramble/ramble/cmd/style.py index 8ffb78973..c5cce32d6 100644 --- a/lib/ramble/ramble/cmd/style.py +++ b/lib/ramble/ramble/cmd/style.py @@ -104,13 +104,12 @@ def is_object(f): } # compile all regular expressions. -pattern_exemptions = dict( - ( - re.compile(file_pattern), - dict((code, [re.compile(p) for p in patterns]) for code, patterns in error_dict.items()), - ) +pattern_exemptions = { + re.compile(file_pattern): { + code: [re.compile(p) for p in patterns] for code, patterns in error_dict.items() + } for file_pattern, error_dict in pattern_exemptions.items() -) +} # Tools run in the given order, with flake8 as the last check. tool_names = ["black", "flake8"] @@ -136,7 +135,7 @@ def changed_files(base=None, untracked=True, all_files=False): if base is None: base = os.environ.get("GITHUB_BASE_REF", "develop") - range = "{0}...".format(base) + range = f"{base}..." git_args = [ # Add changed files committed since branching off of develop @@ -267,7 +266,7 @@ def print_output(output, args): else: # print results relative to current working directory def cwd_relative(path): - return "{0}: [".format( + return "{}: [".format( os.path.relpath(os.path.join(ramble.paths.prefix, path.group(1)), os.getcwd()) ) @@ -300,9 +299,9 @@ def add_pattern_exemptions(line, codes): # append exemption to line if "# noqa: " in line: - line += ",{0}".format(exemptions) + line += f",{exemptions}" elif line: # ignore noqa on empty lines - line += " # noqa: {0}".format(exemptions) + line += f" # noqa: {exemptions}" # if THIS made the line too long, add an exemption for that if len(line) > max_line_length and orig_len <= max_line_length: diff --git a/lib/ramble/ramble/cmd/unit_test.py b/lib/ramble/ramble/cmd/unit_test.py index 939a0e3df..4a1106232 100644 --- a/lib/ramble/ramble/cmd/unit_test.py +++ b/lib/ramble/ramble/cmd/unit_test.py @@ -6,8 +6,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function -from __future__ import division import collections import io @@ -119,7 +117,7 @@ def do_list(args, extra_args): sys.stdout = old_output lines = output.getvalue().split("\n") - tests = collections.defaultdict(lambda: set()) + tests = collections.defaultdict(set) prefix = [] print("All lines =") @@ -141,19 +139,19 @@ def do_list(args, extra_args): if nodetype.endswith("Function"): key = tuple(prefix) tests[key].add(name) - print("added test {0}={1} from {2}".format(key, name, nodetype)) + print(f"added test {key}={name} from {nodetype}") else: prefix = prefix[:depth] prefix.append(name) - print("added prefix {0}".format(name)) + print(f"added prefix {name}") def colorize(c, prefix): if isinstance(prefix, tuple): - return "::".join(color.colorize("@%s{%s}" % (c, p)) for p in prefix if p != "()") - return color.colorize("@%s{%s}" % (c, prefix)) + return "::".join(color.colorize(f"@{c}{{{p}}}") for p in prefix if p != "()") + return color.colorize(f"@{c}{{{prefix}}}") if args.list == "list": - files = set(prefix[0] for prefix in tests) + files = {prefix[0] for prefix in tests} color_files = [colorize("B", file) for file in sorted(files)] colify(color_files) diff --git a/lib/ramble/ramble/cmd/workspace.py b/lib/ramble/ramble/cmd/workspace.py index 2264d207d..599af3f5b 100644 --- a/lib/ramble/ramble/cmd/workspace.py +++ b/lib/ramble/ramble/cmd/workspace.py @@ -34,11 +34,6 @@ import ramble.util.colors as rucolor from ramble.util.logger import logger -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm noqa: F401 -else: - from collections import Sequence # noqa: F401 - description = "manage experiment workspaces" section = "workspaces" @@ -343,12 +338,12 @@ def _workspace_create( workspace.write(inputs_dir=inputs_dir, software_dir=software_dir) if config: - with open(config, "r") as f: + with open(config) as f: workspace._read_config("workspace", f) workspace._write_config("workspace") if template_execute: - with open(template_execute, "r") as f: + with open(template_execute) as f: _, file_name = os.path.split(template_execute) template_name = os.path.splitext(file_name)[0] workspace._read_template(template_name, f.read()) diff --git a/lib/ramble/ramble/config.py b/lib/ramble/ramble/config.py index a218a7315..8cefcfb75 100644 --- a/lib/ramble/ramble/config.py +++ b/lib/ramble/ramble/config.py @@ -107,9 +107,7 @@ # Same as above, but including keys for workspaces # this allows us to unify config reading between configs and workspaces all_schemas = copy.deepcopy(section_schemas) -all_schemas.update( - dict((key, ramble.schema.workspace.schema) for key in ramble.schema.workspace.keys) -) +all_schemas.update({key: ramble.schema.workspace.schema for key in ramble.schema.workspace.keys}) #: Builtin paths to configuration files in ramble configuration_paths = ( @@ -168,7 +166,7 @@ def first_existing(dictionary, keys): raise KeyError("None of %s is in dict!" % keys) -class ConfigScope(object): +class ConfigScope: """This class represents a configuration scope. A scope is one directory containing named configuration files. @@ -208,7 +206,7 @@ def _write_section(self, section): mkdirp(self.path) with open(filename, "w") as f: syaml.dump_config(data, stream=f, default_flow_style=False) - except (yaml.YAMLError, IOError) as e: + except (yaml.YAMLError, OSError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) def clear(self): @@ -216,7 +214,7 @@ def clear(self): self.sections = syaml.syaml_dict() def __repr__(self): - return "" % (self.name, self.path) + return f"" class SingleFileScope(ConfigScope): @@ -239,7 +237,7 @@ def __init__(self, name, path, schema, yaml_path=None): config: install_tree: $ramble/opt/ramble """ - super(SingleFileScope, self).__init__(name, path) + super().__init__(name, path) self._raw_data = None self.schema = schema self.yaml_path = yaml_path or [] @@ -338,11 +336,11 @@ def _write_section(self, section): syaml.dump_config(data_to_write, stream=f, default_flow_style=False) rename(tmp, self.path) - except (yaml.YAMLError, IOError) as e: + except (yaml.YAMLError, OSError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) def __repr__(self): - return "" % (self.name, self.path) + return f"" class ImmutableConfigScope(ConfigScope): @@ -355,7 +353,7 @@ def _write_section(self, section): raise ConfigError("Cannot write to immutable scope %s" % self) def __repr__(self): - return "" % (self.name, self.path) + return f"" class InternalConfigScope(ConfigScope): @@ -367,7 +365,7 @@ class InternalConfigScope(ConfigScope): """ def __init__(self, name, data=None): - super(InternalConfigScope, self).__init__(name, None) + super().__init__(name, None) self.sections = syaml.syaml_dict() if data: @@ -433,7 +431,7 @@ def _method(self, *args, **kwargs): return _method -class Configuration(object): +class Configuration: """A full Ramble configuration, from a hierarchy of config files. This class makes it easy to add a new scope on top of an existing one. @@ -532,7 +530,7 @@ def _validate_scope(self, scope): else: raise ValueError( - "Invalid config scope: '%s'. Must be one of %s" % (scope, self.scopes.keys()) + f"Invalid config scope: '{scope}'. Must be one of {self.scopes.keys()}" ) def get_config_filename(self, scope, section): @@ -736,8 +734,7 @@ def set(self, path, value, scope=None): def __iter__(self): """Iterate over scopes in this configuration.""" - for scope in self.scopes.values(): - yield scope + yield from self.scopes.values() def print_section(self, section, blame=False): """Print a configuration to stdout.""" @@ -745,7 +742,7 @@ def print_section(self, section, blame=False): data = syaml.syaml_dict() data[section] = self.get_config(section) syaml.dump_config(data, stream=sys.stdout, default_flow_style=False, blame=blame) - except (yaml.YAMLError, IOError): + except (yaml.YAMLError, OSError): raise ConfigError("Error reading configuration: %s" % section) @@ -768,10 +765,10 @@ def override(path_or_scope, value=None): else: base_name = overrides_base_name # Ensure the new override gets a unique scope name - current_overrides = [s.name for s in config.matching_scopes(r"^{0}".format(base_name))] + current_overrides = [s.name for s in config.matching_scopes(rf"^{base_name}")] num_overrides = len(current_overrides) while True: - scope_name = "{0}{1}".format(base_name, num_overrides) + scope_name = f"{base_name}{num_overrides}" if scope_name in current_overrides: num_overrides += 1 else: @@ -1037,10 +1034,10 @@ def read_config_file(filename, schema=None): raise ConfigFileError("Config file is empty or is not a valid YAML dict: %s" % filename) except MarkedYAMLError as e: - raise ConfigFileError("Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) + raise ConfigFileError(f"Error parsing yaml{str(e.context_mark)}: {e.problem}") - except IOError as e: - raise ConfigFileError("Error reading configuration file %s: %s" % (filename, str(e))) + except OSError as e: + raise ConfigFileError(f"Error reading configuration file {filename}: {str(e)}") def _override(string): @@ -1186,14 +1183,14 @@ def they_are(t): def process_config_path(path): result = [] if path.startswith(":"): - raise syaml.rambleYAMLError("Illegal leading `:' in path `{0}'".format(path), "") + raise syaml.rambleYAMLError(f"Illegal leading `:' in path `{path}'", "") seen_override_in_path = False while path: front, sep, path = path.partition(":") if (sep and not path) or path.startswith(":"): if seen_override_in_path: raise syaml.RambleYAMLError( - "Meaningless second override" " indicator `::' in path `{0}'".format(path), "" + "Meaningless second override" " indicator `::' in path `{}'".format(path), "" ) path = path.lstrip(":") front = syaml.syaml_str(front) @@ -1309,7 +1306,7 @@ def _config_from(scopes_or_paths): # Otherwise we need to construct it path = os.path.normpath(scope_or_path) - assert os.path.isdir(path), '"{0}" must be a directory'.format(path) + assert os.path.isdir(path), f'"{path}" must be a directory' name = os.path.basename(path) scopes.append(ConfigScope(name, path)) @@ -1349,7 +1346,7 @@ def __init__(self, validation_error, data, filename=None, line=None): location = f"{filename}" if line is not None: location += f":{line}" - with open(filename, "r") as file: + with open(filename) as file: lines = file.readlines() location += f"\n{lines[mark.line]}" diff --git a/lib/ramble/ramble/context.py b/lib/ramble/ramble/context.py index ca645a40e..585d0de5c 100644 --- a/lib/ramble/ramble/context.py +++ b/lib/ramble/ramble/context.py @@ -12,7 +12,7 @@ import spack.util.spack_yaml as syaml -class Context(object): +class Context: """Class to represent a context This class contains variable definitions to store any individual context diff --git a/lib/ramble/ramble/error.py b/lib/ramble/ramble/error.py index cf7f340ad..67036e1ea 100644 --- a/lib/ramble/ramble/error.py +++ b/lib/ramble/ramble/error.py @@ -5,7 +5,6 @@ # , at your # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function import sys import inspect @@ -24,7 +23,7 @@ class RambleError(Exception): """ def __init__(self, message, long_message=None): - super(RambleError, self).__init__() + super().__init__() self.message = message self._long_message = long_message diff --git a/lib/ramble/ramble/expander.py b/lib/ramble/ramble/expander.py index 49efbd6f5..4df3567a2 100644 --- a/lib/ramble/ramble/expander.py +++ b/lib/ramble/ramble/expander.py @@ -71,7 +71,7 @@ def _or(a, b): formatter = string.Formatter() -class ExpansionDelimiter(object): +class ExpansionDelimiter: """Class representing the delimiters for ramble expansion strings""" left = "{" @@ -79,14 +79,14 @@ class ExpansionDelimiter(object): escape = "\\" -class VformatDelimiter(object): +class VformatDelimiter: """Class representing the delimiters for the string.Formatter class""" left = "{" right = "}" -class ExpansionNode(object): +class ExpansionNode: """Class representing a node in a ramble expansion graph""" def __init__(self, left_idx, right_idx): @@ -239,7 +239,7 @@ def define_value( self.value = self.value.replace("\\{", "{").replace("\\}", "}") -class ExpansionGraph(object): +class ExpansionGraph: """Class representing a graph of ExpansionNodes""" def __init__(self, in_str): @@ -301,7 +301,7 @@ def __str__(self): lines = [] lines.append(f"Processing string: {self.str}") for node in self.walk(): - lines.append((f"{node}")) + lines.append(f"{node}") return "\n".join(lines) @@ -310,7 +310,7 @@ def __missing__(self, key): return "{" + key + "}" -class Expander(object): +class Expander: """A class that will track and expand keyword arguments This class will track variables and their definitions, to allow for @@ -394,14 +394,14 @@ def application_namespace(self): @property def workload_namespace(self): if not self._workload_namespace: - self._workload_namespace = "%s.%s" % (self.application_name, self.workload_name) + self._workload_namespace = f"{self.application_name}.{self.workload_name}" return self._workload_namespace @property def experiment_namespace(self): if not self._experiment_namespace: - self._experiment_namespace = "%s.%s.%s" % ( + self._experiment_namespace = "{}.{}.{}".format( self.application_name, self.workload_name, self.experiment_name, diff --git a/lib/ramble/ramble/experiment_set.py b/lib/ramble/ramble/experiment_set.py index 97844aa02..575249366 100644 --- a/lib/ramble/ramble/experiment_set.py +++ b/lib/ramble/ramble/experiment_set.py @@ -26,7 +26,7 @@ import spack.util.naming -class ExperimentSet(object): +class ExperimentSet: """Class to represent a full set of experiments This class contains logic to take sets of variable definitions and generate diff --git a/lib/ramble/ramble/fetch_strategy.py b/lib/ramble/ramble/fetch_strategy.py index ce1aa67ee..bd0328ebd 100644 --- a/lib/ramble/ramble/fetch_strategy.py +++ b/lib/ramble/ramble/fetch_strategy.py @@ -99,7 +99,7 @@ def fetcher(cls): return cls -class FetchStrategy(object): +class FetchStrategy: """Superclass of all fetch strategies.""" #: The URL attribute must be specified either at the package class @@ -235,9 +235,7 @@ class FetchStrategyComposite(pattern.Composite): matches = FetchStrategy.matches def __init__(self): - super(FetchStrategyComposite, self).__init__( - ["fetch", "check", "expand", "reset", "archive", "cachable", "mirror_id"] - ) + super().__init__(["fetch", "check", "expand", "reset", "archive", "cachable", "mirror_id"]) def source_id(self): component_ids = tuple(i.source_id() for i in self) @@ -260,7 +258,7 @@ class URLFetchStrategy(FetchStrategy): optional_attrs = list(crypto.hashes.keys()) + ["checksum"] def __init__(self, url=None, checksum=None, **kwargs): - super(URLFetchStrategy, self).__init__(**kwargs) + super().__init__(**kwargs) # Prefer values in kwargs to the positionals. self.url = kwargs.get("url", url) @@ -319,7 +317,7 @@ def candidate_urls(self): @_needs_stage def fetch(self): if self.archive_file: - logger.debug("Already downloaded {0}".format(self.archive_file)) + logger.debug(f"Already downloaded {self.archive_file}") return url = None @@ -343,7 +341,7 @@ def fetch(self): raise FailedDownloadError(url) def _existing_url(self, url): - logger.debug("Checking existence of {0}".format(url)) + logger.debug(f"Checking existence of {url}") if ramble.config.get("config:url_fetch_method") == "curl": curl = self.curl @@ -360,7 +358,7 @@ def _existing_url(self, url): url, headers, response = ramble.util.web.read_from_url(url) except ramble.util.web.SpackWebError as werr: msg = "Urllib fetch failed to verify url\ - {0}\n with error {1}".format( + {}\n with error {}".format( url, werr ) raise FailedDownloadError(url, msg) @@ -385,7 +383,7 @@ def _fetch_urllib(self, url): save_file = None if self.stage.save_filename: save_file = self.stage.save_filename - logger.msg("Fetching {0}".format(url)) + logger.msg(f"Fetching {url}") # Check if we're about to try and open a broken simlink, and if so # remove that file to avoid a bad situation where a file "exists" but @@ -402,7 +400,7 @@ def _fetch_urllib(self, url): os.remove(self.archive_file) if save_file and os.path.exists(save_file): os.remove(save_file) - msg = "urllib failed to fetch with error {0}".format(e) + msg = f"urllib failed to fetch with error {e}" raise FailedDownloadError(url, msg) with open(save_file, "wb") as _open_file: @@ -418,7 +416,7 @@ def _fetch_curl(self, url): if self.stage.save_filename: save_file = self.stage.save_filename partial_file = self.stage.save_filename + ".part" - logger.msg("Fetching {0}".format(url)) + logger.msg(f"Fetching {url}") if partial_file: save_args = [ "-C", @@ -602,8 +600,8 @@ def check(self): checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s" % (checker.hash_name, self.archive_file), - "Expected %s but got %s" % (self.digest, checker.sum), + f"{checker.hash_name} checksum failed for {self.archive_file}", + f"Expected {self.digest} but got {checker.sum}", ) @_needs_stage @@ -628,7 +626,7 @@ def reset(self): def __repr__(self): url = self.url if self.url else "no url" - return "%s<%s>" % (self.__class__.__name__, url) + return f"{self.__class__.__name__}<{url}>" def __str__(self): if self.url: @@ -684,12 +682,12 @@ class VCSFetchStrategy(FetchStrategy): """ def __init__(self, **kwargs): - super(VCSFetchStrategy, self).__init__(**kwargs) + super().__init__(**kwargs) # Set a URL based on the type of fetch strategy. self.url = kwargs.get(self.url_attr, None) if not self.url: - raise ValueError("%s requires %s argument." % (self.__class__, self.url_attr)) + raise ValueError(f"{self.__class__} requires {self.url_attr} argument.") for attr in self.optional_attrs: setattr(self, attr, kwargs.get(attr, None)) @@ -731,7 +729,7 @@ def __str__(self): return "VCS: %s" % self.url def __repr__(self): - return "%s<%s>" % (self.__class__, self.url) + return f"{self.__class__}<{self.url}>" @fetcher @@ -756,7 +754,7 @@ def __init__(self, **kwargs): # call to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) - super(GoFetchStrategy, self).__init__(**forwarded_args) + super().__init__(**forwarded_args) self._go = None @@ -785,7 +783,7 @@ def fetch(self): self.go("get", "-v", "-d", self.url, env=env) def archive(self, destination): - super(GoFetchStrategy, self).archive(destination, exclude=".git") + super().archive(destination, exclude=".git") @_needs_stage def expand(self): @@ -843,7 +841,7 @@ def __init__(self, **kwargs): # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) - super(GitFetchStrategy, self).__init__(**forwarded_args) + super().__init__(**forwarded_args) self._git = None self.submodules = kwargs.get("submodules", False) @@ -899,13 +897,13 @@ def _repo_info(self): args = "" if self.commit: - args = " at commit {0}".format(self.commit) + args = f" at commit {self.commit}" elif self.tag: - args = " at tag {0}".format(self.tag) + args = f" at tag {self.tag}" elif self.branch: - args = " on branch {0}".format(self.branch) + args = f" on branch {self.branch}" - return "{0}{1}".format(self.url, args) + return f"{self.url}{args}" @_needs_stage def fetch(self): @@ -1034,7 +1032,7 @@ def clone(self, dest=None, commit=None, branch=None, tag=None, bare=False): git(*args) def archive(self, destination): - super(GitFetchStrategy, self).archive(destination, exclude=".git") + super().archive(destination, exclude=".git") @_needs_stage def reset(self): @@ -1055,7 +1053,7 @@ def protocol_supports_shallow_clone(self): return not (self.url.startswith("http://") or self.url.startswith("/")) def __str__(self): - return "[git] {0}".format(self._repo_info()) + return f"[git] {self._repo_info()}" @fetcher @@ -1083,7 +1081,7 @@ def __init__(self, **kwargs): # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) - super(CvsFetchStrategy, self).__init__(**forwarded_args) + super().__init__(**forwarded_args) self._cvs = None if self.branch is not None: @@ -1163,7 +1161,7 @@ def _remove_untracked_files(self): os.unlink(path) def archive(self, destination): - super(CvsFetchStrategy, self).archive(destination, exclude="CVS") + super().archive(destination, exclude="CVS") @_needs_stage def reset(self): @@ -1198,7 +1196,7 @@ def __init__(self, **kwargs): # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) - super(SvnFetchStrategy, self).__init__(**forwarded_args) + super().__init__(**forwarded_args) self._svn = None if self.revision is not None: @@ -1257,7 +1255,7 @@ def _remove_untracked_files(self): shutil.rmtree(path, ignore_errors=True) def archive(self, destination): - super(SvnFetchStrategy, self).archive(destination, exclude=".svn") + super().archive(destination, exclude=".svn") @_needs_stage def reset(self): @@ -1300,7 +1298,7 @@ def __init__(self, **kwargs): # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop("name", None) - super(HgFetchStrategy, self).__init__(**forwarded_args) + super().__init__(**forwarded_args) self._hg = None @@ -1361,7 +1359,7 @@ def fetch(self): shutil.move(repo_name, self.stage.source_path) def archive(self, destination): - super(HgFetchStrategy, self).archive(destination, exclude=".hg") + super().archive(destination, exclude=".hg") @_needs_stage def reset(self): @@ -1390,7 +1388,7 @@ class S3FetchStrategy(URLFetchStrategy): def __init__(self, *args, **kwargs): try: - super(S3FetchStrategy, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) except ValueError: if not kwargs.get("url"): raise ValueError("S3FetchStrategy requires a url for fetching.") @@ -1435,7 +1433,7 @@ class GCSFetchStrategy(URLFetchStrategy): def __init__(self, *args, **kwargs): try: - super(GCSFetchStrategy, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) except ValueError: if not kwargs.get("url"): raise ValueError("GCSFetchStrategy requires a url for fetching.") @@ -1522,10 +1520,10 @@ def check_pkg_attributes(pkg): """ # a single package cannot have URL attributes for multiple VCS fetch # strategies *unless* they are the same attribute. - conflicts = set([s.url_attr for s in all_strategies if hasattr(pkg, s.url_attr)]) + conflicts = {s.url_attr for s in all_strategies if hasattr(pkg, s.url_attr)} # URL isn't a VCS fetch method. We can use it with a VCS method. - conflicts -= set(["url"]) + conflicts -= {"url"} if len(conflicts) > 1: raise FetcherConflict( @@ -1540,10 +1538,10 @@ def _check_version_attributes(fetcher, pkg, version): This assumes that we have already determined the fetcher for the specific version using ``for_package_version()`` """ - all_optionals = set(a for s in all_strategies for a in s.optional_attrs) + all_optionals = {a for s in all_strategies for a in s.optional_attrs} args = pkg.versions[version] - extra = set(args) - set(fetcher.optional_attrs) - set([fetcher.url_attr, "no_cache"]) + extra = set(args) - set(fetcher.optional_attrs) - {fetcher.url_attr, "no_cache"} extra.intersection_update(all_optionals) if extra: @@ -1669,9 +1667,7 @@ def from_url_scheme(url, *args, **kwargs): if url_attr and url_attr == scheme: return fetcher(url, *args, **kwargs) - raise ValueError( - 'No FetchStrategy found for url with scheme: "{SCHEME}"'.format(SCHEME=parsed_url.scheme) - ) + raise ValueError(f'No FetchStrategy found for url with scheme: "{parsed_url.scheme}"') def from_list_url(pkg): @@ -1707,7 +1703,7 @@ def from_list_url(pkg): logger.msg("Could not determine url from list_url.") -class FsCache(object): +class FsCache: def __init__(self, root): self.root = os.path.abspath(root) @@ -1745,7 +1741,7 @@ class FailedDownloadError(FetchError): """Raised when a download fails.""" def __init__(self, url, msg=""): - super(FailedDownloadError, self).__init__("Failed to fetch file from URL: %s" % url, msg) + super().__init__("Failed to fetch file from URL: %s" % url, msg) self.url = url @@ -1771,11 +1767,11 @@ class InvalidArgsError(FetchError): def __init__(self, pkg=None, version=None, **args): msg = "Could not guess a fetch strategy" if pkg: - msg += " for {pkg}".format(pkg=pkg) + msg += f" for {pkg}" if version: - msg += "@{version}".format(version=version) - long_msg = "with arguments: {args}".format(args=args) - super(InvalidArgsError, self).__init__(msg, long_msg) + msg += f"@{version}" + long_msg = f"with arguments: {args}" + super().__init__(msg, long_msg) class ChecksumError(FetchError): @@ -1786,6 +1782,4 @@ class NoStageError(FetchError): """Raised when fetch operations are called before set_stage().""" def __init__(self, method): - super(NoStageError, self).__init__( - "Must call FetchStrategy.set_stage() before calling %s" % method.__name__ - ) + super().__init__("Must call FetchStrategy.set_stage() before calling %s" % method.__name__) diff --git a/lib/ramble/ramble/filters.py b/lib/ramble/ramble/filters.py index 7e63187b3..9c4a7c9cc 100644 --- a/lib/ramble/ramble/filters.py +++ b/lib/ramble/ramble/filters.py @@ -9,7 +9,7 @@ import itertools -class Filters(object): +class Filters: """Object containing filters for limiting various operations in Ramble""" def __init__( diff --git a/lib/ramble/ramble/graphs.py b/lib/ramble/ramble/graphs.py index c89aa909f..d5e9d2341 100644 --- a/lib/ramble/ramble/graphs.py +++ b/lib/ramble/ramble/graphs.py @@ -15,7 +15,7 @@ from ramble.util.logger import logger -class AttributeGraph(object): +class AttributeGraph: node_type = "object" @@ -124,8 +124,7 @@ def walk(self): ) self._prepared = True - for node in self._sorted: - yield node + yield from self._sorted def get_node(self, key): """Given a key, return the node containing this key diff --git a/lib/ramble/ramble/keywords.py b/lib/ramble/ramble/keywords.py index 2000b44df..0a2bf9e92 100644 --- a/lib/ramble/ramble/keywords.py +++ b/lib/ramble/ramble/keywords.py @@ -50,7 +50,7 @@ } -class Keywords(object): +class Keywords: """Class to represent known ramble keywords. Each keyword contains a dictionary of its attributes. Currently, these include: diff --git a/lib/ramble/ramble/language/language_base.py b/lib/ramble/ramble/language/language_base.py index 5f8d0deba..b905d7172 100644 --- a/lib/ramble/ramble/language/language_base.py +++ b/lib/ramble/ramble/language/language_base.py @@ -11,17 +11,13 @@ """ import functools -import sys import llnl.util.lang import llnl.util.tty.color import ramble.error -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm -else: - from collections import Sequence +from collections.abc import Sequence # novm __all__ = ["DirectiveMeta", "DirectiveError"] @@ -74,7 +70,7 @@ def __new__(cls, name, bases, attr_dict): ) DirectiveMeta._directives_to_be_executed = [] - return super(DirectiveMeta, cls).__new__(cls, name, bases, attr_dict) + return super().__new__(cls, name, bases, attr_dict) def __init__(cls, name, bases, attr_dict): # The instance is being initialized: if it is a package we must ensure @@ -112,7 +108,7 @@ def __init__(cls, name, bases, attr_dict): # directives by clearing out the queue they're appended to DirectiveMeta._directives_to_be_executed = [] - super(DirectiveMeta, cls).__init__(name, bases, attr_dict) + super().__init__(name, bases, attr_dict) @classmethod def directive(cls, dicts=None): diff --git a/lib/ramble/ramble/main.py b/lib/ramble/ramble/main.py index 551464c55..e6d141687 100644 --- a/lib/ramble/ramble/main.py +++ b/lib/ramble/ramble/main.py @@ -11,7 +11,6 @@ In a normal Ramble installation, this is invoked from the bin/ramble script after the system path is set up. """ -from __future__ import print_function import argparse import inspect @@ -179,7 +178,7 @@ def index_commands(): class RambleHelpFormatter(argparse.RawTextHelpFormatter): def _format_actions_usage(self, actions, groups): """Formatter with more concise usage strings.""" - usage = super(RambleHelpFormatter, self)._format_actions_usage(actions, groups) + usage = super()._format_actions_usage(actions, groups) # Eliminate any occurrence of two or more consecutive spaces usage = re.sub(r"[ ]{2,}", " ", usage) @@ -189,12 +188,12 @@ def _format_actions_usage(self, actions, groups): chars = "".join(re.findall(r"\[-(.)\]", usage)) usage = re.sub(r"\[-.\] ?", "", usage) if chars: - usage = "[-%s] %s" % (chars, usage) + usage = f"[-{chars}] {usage}" return usage.strip() def add_arguments(self, actions): actions = sorted(actions, key=operator.attrgetter("option_strings")) - super(RambleHelpFormatter, self).add_arguments(actions) + super().add_arguments(actions) class RambleArgumentParser(argparse.ArgumentParser): @@ -229,10 +228,10 @@ def add_group(group): def add_subcommand_group(title, commands): """Add informational help group for a specific subcommand set.""" - cmd_set = set(c for c in commands) + cmd_set = {c for c in commands} # make a dict of commands of interest - cmds = dict((a.dest, a) for a in self.actions if a.dest in cmd_set) + cmds = {a.dest: a for a in self.actions if a.dest in cmd_set} # add commands to a group in order, and add the group group = argparse._ArgumentGroup(self, title=title) @@ -245,9 +244,9 @@ def add_subcommand_group(title, commands): # select only the options for the particular level we're showing. show_options = options_by_level[level] if show_options != "all": - opts = dict( - (opt.option_strings[0].strip("-"), opt) for opt in self._optionals._group_actions - ) + opts = { + opt.option_strings[0].strip("-"): opt for opt in self._optionals._group_actions + } new_actions = [opts[letter] for letter in show_options] self._optionals._group_actions = new_actions @@ -314,7 +313,7 @@ def add_subparsers(self, **kwargs): if sys.version_info[:2] > (3, 6): kwargs.setdefault("required", True) - sp = super(RambleArgumentParser, self).add_subparsers(**kwargs) + sp = super().add_subparsers(**kwargs) # This monkey patching is needed for Python 3.5 and 3.6, which support # having a required subparser but don't expose the API used above if sys.version_info[:2] == (3, 5) or sys.version_info[:2] == (3, 6): @@ -367,13 +366,13 @@ def format_help(self, level="short"): return self.format_help_sections(level) else: # in subparsers, self.prog is, e.g., 'ramble list' - return super(RambleArgumentParser, self).format_help() + return super().format_help() def _check_value(self, action, value): # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: cols = llnl.util.tty.colify.colified(sorted(action.choices), indent=4, tty=True) - msg = "invalid choice: %r choose from:\n%s" % (value, cols) + msg = f"invalid choice: {value!r} choose from:\n{cols}" raise argparse.ArgumentError(action, msg) @@ -680,7 +679,7 @@ def _invoke_command(command, parser, args, unknown_args): return 0 if return_val is None else return_val -class RambleCommand(object): +class RambleCommand: """Callable object that invokes a ramble command (for testing). Example usage:: @@ -818,9 +817,9 @@ def print_setup_info(*info): def shell_set(var, value): if shell == "sh": - print("%s='%s'" % (var, value)) + print(f"{var}='{value}'") elif shell == "csh": - print("set %s = '%s'" % (var, value)) + print(f"set {var} = '{value}'") else: logger.die("shell must be sh or csh") @@ -1022,10 +1021,7 @@ def main(argv=None): raise sys.stderr.write("\n") logger.error("Keyboard interrupt.") - if sys.version_info >= (3, 5): - return signal.SIGINT.value - else: - return signal.SIGINT + return signal.SIGINT.value except SystemExit as e: if ramble.config.get("config:debug"): diff --git a/lib/ramble/ramble/mirror.py b/lib/ramble/ramble/mirror.py index b5d15369a..a3e9e9a54 100644 --- a/lib/ramble/ramble/mirror.py +++ b/lib/ramble/ramble/mirror.py @@ -52,7 +52,7 @@ def _display_mirror_entry(size, name, url, type_=None): print("%-*s%s%s" % (size + 4, name, url, type_)) -class Mirror(object): +class Mirror: """Represents a named location for storing input tarballs. Mirrors have a fetch_url that indicate where and how artifacts are fetched @@ -117,16 +117,16 @@ def __str__(self): name = ' "%s"' % name if self._push_url is None: - return "[Mirror%s (%s)]" % (name, self._fetch_url) + return f"[Mirror{name} ({self._fetch_url})]" - return "[Mirror%s (fetch: %s, push: %s)]" % (name, self._fetch_url, self._push_url) + return f"[Mirror{name} (fetch: {self._fetch_url}, push: {self._push_url})]" def __repr__(self): return "".join( ( "Mirror(", ", ".join( - "%s=%s" % (k, repr(v)) + f"{k}={repr(v)}" for k, v in ( ("fetch_url", self._fetch_url), ("push_url", self._push_url), @@ -324,7 +324,7 @@ def _determine_extension(fetcher): return ext -class MirrorReference(object): +class MirrorReference: """A ``MirrorReference`` stores the relative paths where you can store a resource in a mirror directory. @@ -466,7 +466,7 @@ def remove(name, scope): logger.msg(f"Removed mirror {name}.") -class MirrorStats(object): +class MirrorStats: def __init__(self): self.present = {} self.new = {} @@ -546,7 +546,7 @@ def push_url_from_mirror_name(mirror_name): """Given a mirror name, return the URL on which to push resources.""" mirror = ramble.mirror.MirrorCollection().lookup(mirror_name) if mirror.name == "": - raise ValueError('no mirror named "{0}"'.format(mirror_name)) + raise ValueError(f'no mirror named "{mirror_name}"') return url_util.format(mirror.push_url) @@ -554,7 +554,7 @@ def push_url_from_mirror_url(mirror_url): """Given a mirror URL, return the URL on which to push resources.""" scheme = url_util.parse(mirror_url, scheme="").scheme if scheme == "": - raise ValueError('"{0}" is not a valid URL'.format(mirror_url)) + raise ValueError(f'"{mirror_url}" is not a valid URL') mirror = ramble.mirror.MirrorCollection().lookup(mirror_url) return url_util.format(mirror.push_url) @@ -563,4 +563,4 @@ class MirrorError(ramble.error.RambleError): """Superclass of all mirror-creation related errors.""" def __init__(self, msg, long_msg=None): - super(MirrorError, self).__init__(msg, long_msg) + super().__init__(msg, long_msg) diff --git a/lib/ramble/ramble/modifier.py b/lib/ramble/ramble/modifier.py index 80135378a..5d8e8e37f 100644 --- a/lib/ramble/ramble/modifier.py +++ b/lib/ramble/ramble/modifier.py @@ -24,7 +24,7 @@ from ramble.util.logger import logger -class ModifierBase(object, metaclass=ModifierMeta): +class ModifierBase(metaclass=ModifierMeta): name = None _builtin_name = "modifier_builtin::{obj_name}::{name}" _mod_prefix_builtin = r"modifier_builtin::" @@ -294,18 +294,15 @@ def all_env_var_modifications(self): if self._usage_mode not in self.env_var_modifications: return - for action, conf in self.env_var_modifications[self._usage_mode].items(): - yield action, conf + yield from self.env_var_modifications[self._usage_mode].items() def all_package_manager_requirements(self): if self._usage_mode in self.package_manager_requirements: - for req in self.package_manager_requirements[self._usage_mode]: - yield req + yield from self.package_manager_requirements[self._usage_mode] def all_pipeline_phases(self, pipeline): if pipeline in self.phase_definitions: - for phase_name, phase_node in self.phase_definitions[pipeline].items(): - yield phase_name, phase_node + yield from self.phase_definitions[pipeline].items() def no_expand_vars(self): """Iterator over non-expandable variables in current mode diff --git a/lib/ramble/ramble/package_manager.py b/lib/ramble/ramble/package_manager.py index e33d11e9d..7a0b7cbdd 100644 --- a/lib/ramble/ramble/package_manager.py +++ b/lib/ramble/ramble/package_manager.py @@ -25,7 +25,7 @@ import spack.util.naming -class PackageManagerBase(object, metaclass=PackageManagerMeta): +class PackageManagerBase(metaclass=PackageManagerMeta): name = None _builtin_name = "package_manager_builtin::{obj_name}::{name}" _pkgman_prefix_builtin = r"package_manager_builtin::" @@ -147,7 +147,9 @@ def _long_print(self): out_str.append(rucolor.nested_1(" %s:\n" % name)) for key in self._spec_keys: if key in info and info[key]: - out_str.append(" %s = %s\n" % (key, info[key].replace("@", "@@"))) + out_str.append( + " {} = {}\n".format(key, info[key].replace("@", "@@")) + ) return out_str @@ -188,8 +190,7 @@ def all_pipeline_phases(self, pipeline): phase_note (GraphNode): Object representing a node in the phase graph """ if pipeline in self.phase_definitions: - for phase_name, phase_node in self.phase_definitions[pipeline].items(): - yield phase_name, phase_node + yield from self.phase_definitions[pipeline].items() def set_application(self, app_inst): """Add an internal reference to the application instance this package diff --git a/lib/ramble/ramble/pipeline.py b/lib/ramble/ramble/pipeline.py index bbd31f5e9..940bc5e8e 100644 --- a/lib/ramble/ramble/pipeline.py +++ b/lib/ramble/ramble/pipeline.py @@ -42,7 +42,7 @@ logger.die("Module `tqdm` is not found. Ensure requirements.txt are installed.") -class Pipeline(object): +class Pipeline: """Base Class for all pipeline objects""" name = "base" @@ -83,7 +83,7 @@ def _construct_hash(self): files_exist = os.path.exists(workspace_inventory) and os.path.exists(workspace_hash_file) if not self.force_inventory and files_exist: - with open(workspace_inventory, "r") as f: + with open(workspace_inventory) as f: self.workspace.hash_inventory = sjson.load(f) self.workspace.workspace_hash = ramble.util.hashing.hash_json( @@ -314,7 +314,7 @@ def _prepare(self): if not self.archive_prefix: self.archive_prefix = os.path.basename(self.workspace.path) - self.archive_name = "%s-archive-%s" % (self.archive_prefix, date_str) + self.archive_name = f"{self.archive_prefix}-archive-{date_str}" archive_path = os.path.join(self.workspace.archive_dir, self.archive_name) fs.mkdirp(archive_path) @@ -585,7 +585,7 @@ def _execute(self): aux_repo_conf = os.path.join(aux_software_dir, repo_conf[1]) repo_data = syaml.syaml_dict() if os.path.exists(aux_repo_conf): - with open(aux_repo_conf, "r") as f: + with open(aux_repo_conf) as f: repo_data = syaml.load_config(f.read()) else: repo_data[repo_conf[0]] = [] diff --git a/lib/ramble/ramble/renderer.py b/lib/ramble/ramble/renderer.py index 61c05f41d..3eb160aa3 100644 --- a/lib/ramble/ramble/renderer.py +++ b/lib/ramble/ramble/renderer.py @@ -17,7 +17,7 @@ from ramble.util.logger import logger -class RenderGroup(object): +class RenderGroup: _obj_types = ["experiment", "package", "environment"] _actions = ["create", "exclude"] @@ -113,7 +113,7 @@ def from_dict(self, name_template, in_dict): return extracted -class Renderer(object): +class Renderer: def render_objects(self, render_group, exclude_where=None, ignore_used=True, fatal=True): """Render objects based on the input variables and matrices diff --git a/lib/ramble/ramble/repeats.py b/lib/ramble/ramble/repeats.py index d02d9136c..d91e11d61 100644 --- a/lib/ramble/ramble/repeats.py +++ b/lib/ramble/ramble/repeats.py @@ -9,7 +9,7 @@ from ramble.util.logger import logger -class Repeats(object): +class Repeats: """Class to represent configuration of experiment repeats""" def __init__(self): diff --git a/lib/ramble/ramble/repository.py b/lib/ramble/ramble/repository.py index e7b0c8680..b65c8486b 100644 --- a/lib/ramble/ramble/repository.py +++ b/lib/ramble/ramble/repository.py @@ -26,7 +26,7 @@ try: from collections.abc import Mapping # novm except ImportError: - from collections import Mapping + from collections.abc import Mapping from enum import Enum @@ -302,7 +302,7 @@ class ObjectNamespace(types.ModuleType): """Allow lazy loading of modules.""" def __init__(self, namespace): - super(ObjectNamespace, self).__init__(namespace) + super().__init__(namespace) self.__file__ = "(ramble namespace)" self.__path__ = [] self.__name__ = namespace @@ -524,7 +524,7 @@ def write(self, stream): self.index.to_json(stream) -class RepoIndex(object): +class RepoIndex: """Container class that manages a set of Indexers for a Repo. This class is responsible for checking objects in a repository for @@ -588,7 +588,7 @@ def _build_index(self, name, indexer): """Determine which objects need an update, and update indexes.""" # Filename of the provider index cache (we assume they're all json) - cache_filename = "{0}/{1}-index.json".format(name, self.namespace) + cache_filename = f"{name}/{self.namespace}-index.json" # Compute which objects needs to be updated in the cache misc_cache = ramble.caches.misc_cache @@ -608,7 +608,7 @@ def _build_index(self, name, indexer): indexer.read(old) if old else indexer.create() for obj_name in needs_update: - namespaced_name = "%s.%s" % (self.namespace, obj_name) + namespaced_name = f"{self.namespace}.{obj_name}" indexer.update(namespaced_name) indexer.write(new) @@ -616,7 +616,7 @@ def _build_index(self, name, indexer): return indexer.index -class RepoPath(object): +class RepoPath: """A RepoPath is a list of repos that function as one. It functions exactly like a Repo, but it operates on the combined @@ -870,7 +870,7 @@ def __contains__(self, obj_name): return self.exists(obj_name) -class Repo(object): +class Repo: """Class representing a object repository in the filesystem. Each object repository must have a top-level configuration file @@ -911,7 +911,7 @@ def check(condition, msg): self.config_name = config self.config_file = config_file check(self.config_file, "No valid config file found") - check(os.path.isfile(self.config_file), "No %s found in '%s'" % (self.config_name, root)) + check(os.path.isfile(self.config_file), f"No {self.config_name} found in '{root}'") # Read configuration and validate namespace config = self._read_config() @@ -923,7 +923,7 @@ def check(condition, msg): self.namespace = config["namespace"] check( re.match(r"[a-zA-Z][a-zA-Z0-9_.]+", self.namespace), - ("Invalid namespace '%s' in repo '%s'. " % (self.namespace, self.root)) + (f"Invalid namespace '{self.namespace}' in repo '{self.root}'. ") + "Namespaces must be valid python identifiers separated by '.'", ) @@ -936,7 +936,7 @@ def check(condition, msg): self.objects_path = os.path.join(self.root, objects_dir) check( os.path.isdir(self.objects_path), - "No directory '%s' found in '%s'" % (objects_dir, root), + f"No directory '{objects_dir}' found in '{root}'", ) # Set up 'full_namespace' to include the super-namespace @@ -1052,11 +1052,11 @@ def load_module(self, fullname): elif namespace == self.full_namespace: real_name = self.real_name(module_name) if not real_name: - raise ImportError("No module %s in %s" % (module_name, self)) + raise ImportError(f"No module {module_name} in {self}") module = self._get_obj_module(real_name) else: - raise ImportError("No module %s in %s" % (fullname, self)) + raise ImportError(f"No module {fullname} in {self}") module.__loader__ = self sys.modules[fullname] = module @@ -1082,7 +1082,7 @@ def _read_config(self): return yaml_data["repo"] - except IOError: + except OSError: logger.die(f"Error reading {self.config_file} when opening {self.root}") @autospec @@ -1251,7 +1251,7 @@ def _get_obj_module(self, obj_name): logger.die(f"Cannot read '{file_path}'!") # e.g., ramble.app.builtin.mpich - fullname = "%s.%s" % (self.full_namespace, obj_name) + fullname = f"{self.full_namespace}.{obj_name}" try: module = ramble.util.imp.load_source(fullname, file_path) @@ -1260,7 +1260,7 @@ def _get_obj_module(self, obj_name): # manually construct the error message in order to give the # user the correct .py where the syntax error is # located - raise SyntaxError("invalid syntax in {0:}, line {1:}".format(file_path, e.lineno)) + raise SyntaxError(f"invalid syntax in {file_path}, line {e.lineno}") module.__object__ = self.full_namespace module.__loader__ = self @@ -1278,7 +1278,7 @@ def get_obj_class(self, obj_name): namespace, _, obj_name = obj_name.rpartition(".") if namespace and (namespace != self.namespace): raise InvalidNamespaceError( - "Invalid namespace for %s repo: %s" % (self.namespace, namespace) + f"Invalid namespace for {self.namespace} repo: {namespace}" ) class_name = nm.mod_to_class(obj_name) @@ -1292,7 +1292,7 @@ def get_obj_class(self, obj_name): return cls def __str__(self): - return "[Repo '%s' at '%s']" % (self.namespace, self.root) + return f"[Repo '{self.namespace}' at '{self.root}']" def __repr__(self): return self.__str__() @@ -1365,7 +1365,7 @@ def create_repo( if subdir is not None: config.write(f" subdirectory: '{subdir}'\n") - except (IOError, OSError) as e: + except OSError as e: # try to clean up. if existed: shutil.rmtree(config_path, ignore_errors=True) @@ -1379,7 +1379,7 @@ def create_repo( shutil.rmtree(root, ignore_errors=True) raise BadRepoError( - "Failed to create new repository in %s." % root, "Caused by %s: %s" % (type(e), e) + "Failed to create new repository in %s." % root, f"Caused by {type(e)}: {e}" ) return full_path, namespace @@ -1412,7 +1412,7 @@ class RepositoryNamespace(types.ModuleType): """Allow lazy loading of modules.""" def __init__(self, namespace): - super(RepositoryNamespace, self).__init__(namespace) + super().__init__(namespace) self.__file__ = "(repository namespace)" self.__path__ = [] self.__name__ = namespace @@ -1432,17 +1432,17 @@ def __getattr__(self, name): class _PrependFileLoader(importlib.machinery.SourceFileLoader): def __init__(self, fullname, path, prepend=None): - super(_PrependFileLoader, self).__init__(fullname, path) + super().__init__(fullname, path) self.prepend = prepend def path_stats(self, path): - stats = super(_PrependFileLoader, self).path_stats(path) + stats = super().path_stats(path) if self.prepend: stats["size"] += len(self.prepend) + 1 return stats def get_data(self, path): - data = super(_PrependFileLoader, self).get_data(path) + data = super().get_data(path) if path != self.path or self.prepend is None: return data else: @@ -1460,12 +1460,10 @@ def __init__(self, fullname, repo, object_name): self.object_name = object_name self.object_py = repo.filename_for_object_name(object_name) self.fullname = fullname - super(RepoLoader, self).__init__( - self.fullname, self.object_py, prepend=self._object_prepend - ) + super().__init__(self.fullname, self.object_py, prepend=self._object_prepend) -class RepositoryNamespaceLoader(object): +class RepositoryNamespaceLoader: def create_module(self, spec): return RepositoryNamespace(spec.name) @@ -1473,7 +1471,7 @@ def exec_module(self, module): module.__loader__ = self -class ReposFinder(object): +class ReposFinder: """MetaPathFinder class that loads a Python module corresponding to an object Return a loader based on the inspection of the current global repository list. @@ -1485,7 +1483,7 @@ def __init__(self, object_type=default_type): def find_spec(self, fullname, python_path, target=None): # "target" is not None only when calling importlib.reload() if target is not None: - raise RuntimeError('cannot reload module "{0}"'.format(fullname)) + raise RuntimeError(f'cannot reload module "{fullname}"') # Preferred API from https://peps.python.org/pep-0451/ if not fullname.startswith("ramble."): @@ -1574,7 +1572,7 @@ def __init__(self, name, repo=None, object_type="Object"): else: msg = f"Attempting to retrieve anonymous {object_type}." - super(UnknownObjectError, self).__init__(msg, long_msg) + super().__init__(msg, long_msg) self.name = name @@ -1582,17 +1580,17 @@ class UnknownNamespaceError(UnknownEntityError): """Raised when we encounter an unknown namespace""" def __init__(self, namespace): - super(UnknownNamespaceError, self).__init__("Unknown namespace: %s" % namespace) + super().__init__("Unknown namespace: %s" % namespace) class FailedConstructorError(RepoError): """Raised when an object's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb, object_type=None): - super(FailedConstructorError, self).__init__( + super().__init__( f"Class constructor failed for {object_type} '%s'." % name, "\nCaused by:\n" - + ("%s: %s\n" % (exc_type.__name__, exc_obj)) + + (f"{exc_type.__name__}: {exc_obj}\n") + "".join(traceback.format_tb(exc_tb)), ) self.name = name diff --git a/lib/ramble/ramble/software_environments.py b/lib/ramble/ramble/software_environments.py index 46e6e58a3..c54033066 100644 --- a/lib/ramble/ramble/software_environments.py +++ b/lib/ramble/ramble/software_environments.py @@ -35,7 +35,7 @@ def _is_dict_empty(rendered: defaultdict): return True -class SoftwarePackage(object): +class SoftwarePackage: """Class to represent a single software package""" def __init__( @@ -287,7 +287,7 @@ def add_rendered_package(self, new_package: object, all_packages: dict, pm_name: all_packages[pm_name][new_package.name] = new_package -class SoftwareEnvironment(object): +class SoftwareEnvironment: """Class representing a single software environment""" def __init__(self, name: str): @@ -508,7 +508,7 @@ def add_rendered_environment( template_pkg.add_rendered_package(rendered_pkg, all_packages, pm_name) -class SoftwareEnvironments(object): +class SoftwareEnvironments: """Class representing a group of software environments""" def __init__(self, workspace): diff --git a/lib/ramble/ramble/spec.py b/lib/ramble/ramble/spec.py index a29df4e46..3d1015740 100644 --- a/lib/ramble/ramble/spec.py +++ b/lib/ramble/ramble/spec.py @@ -32,7 +32,7 @@ class SpecLexer(spack.parse.Lexer): """Parses tokens that make up spack specs.""" def __init__(self): - super(SpecLexer, self).__init__( + super().__init__( [ (r"\^", lambda scanner, val: self.token(DEP, val)), (r"\@", lambda scanner, val: self.token(AT, val)), @@ -76,7 +76,7 @@ def __init__(self, initial_spec=None): superfluous Spec object in the Spec constructor. """ logger.debug(f"Starting parser with spec {initial_spec}") - super(SpecParser, self).__init__(_lexer) + super().__init__(_lexer) self.previous = None self._initial = initial_spec @@ -116,10 +116,10 @@ def check_identifier(self, id=None): if not id: id = self.token.value if "." in id: - self.last_token_error("{0}: Identifier cannot contain '.'".format(id)) + self.last_token_error(f"{id}: Identifier cannot contain '.'") -class Spec(object): +class Spec: def __init__(self, spec_like=None): """Create a new Spec. @@ -216,7 +216,7 @@ def write_attribute(spec, attribute, color): except AttributeError: parent = ".".join(parts[:idx]) m = "Attempted to format attribute %s." % attribute - m += "Spec %s has no attribute %s" % (parent, part) + m += f"Spec {parent} has no attribute {part}" raise SpecFormatStringError(m) if callable(current): @@ -273,7 +273,7 @@ def __str__(self): @property def fullname(self): return ( - ("%s.%s" % (self.namespace, self.name)) + (f"{self.namespace}.{self.name}") if self.namespace else (self.name if self.name else "") ) @@ -311,7 +311,7 @@ class SpecParseError(ramble.error.SpecError): """Wrapper for ParseError for when we're parsing specs.""" def __init__(self, parse_error): - super(SpecParseError, self).__init__(parse_error.message) + super().__init__(parse_error.message) self.string = parse_error.string diff --git a/lib/ramble/ramble/stage.py b/lib/ramble/ramble/stage.py index 621b1343a..627515f6f 100644 --- a/lib/ramble/ramble/stage.py +++ b/lib/ramble/ramble/stage.py @@ -7,8 +7,6 @@ # except according to those terms. -from __future__ import print_function - import errno import getpass import glob @@ -176,7 +174,7 @@ def _mirror_roots(): ] -class InputStage(object): +class InputStage: """Manages a stage directory for containing input files. A Stage object is a context manager that handles a directory where @@ -450,14 +448,12 @@ def fetch(self, mirror_only=False, err_msg=None): fetchers.insert(0, cache_fetcher) def generate_fetchers(): - for fetcher in fetchers: - yield fetcher + yield from fetchers # The search function may be expensive, so wait until now to # call it so the user can stop if a prior fetcher succeeded if self.search_fn and not mirror_only: dynamic_fetchers = self.search_fn() - for fetcher in dynamic_fetchers: - yield fetcher + yield from dynamic_fetchers def print_errors(errors): for msg in errors: @@ -474,11 +470,11 @@ def print_errors(errors): # Don't bother reporting when something is not cached. continue except ramble.error.RambleError as e: - errors.append("Fetching from {0} failed.".format(fetcher)) + errors.append(f"Fetching from {fetcher} failed.") logger.debug(e) continue except spack.util.web.SpackWebError as e: - errors.append("Fetching from {0} failed.".format(fetcher)) + errors.append(f"Fetching from {fetcher} failed.") logger.debug(e) continue @@ -631,16 +627,16 @@ def destroy(self): class ResourceStage(InputStage): def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): - super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) + super().__init__(url_or_fetch_strategy, **kwargs) self.root_stage = root self.resource = resource def restage(self): - super(ResourceStage, self).restage() + super().restage() self._add_to_root_stage() def expand_archive(self): - super(ResourceStage, self).expand_archive() + super().expand_archive() self._add_to_root_stage() def _add_to_root_stage(self): @@ -701,7 +697,7 @@ class StageComposite(pattern.Composite): # def __init__(self): - super(StageComposite, self).__init__( + super().__init__( [ "fetch", "create", @@ -747,7 +743,7 @@ def archive_file(self): return self[0].archive_file -class DIYStage(object): +class DIYStage: """ Simple class that allows any directory to be a ramble input stage. Consequently, it does not expect or require that the source path adhere to @@ -842,7 +838,7 @@ def get_checksums_for_versions( num_ver = len(sorted_versions) logger.msg( - "Found {0} version{1} of {2}:".format(num_ver, "" if num_ver == 1 else "s", name), + "Found {} version{} of {}:".format(num_ver, "" if num_ver == 1 else "s", name), "", *llnl.util.lang.elide_list( ["{0:{1}} {2}".format(str(v), max_len, url_dict[v]) for v in sorted_versions] @@ -887,7 +883,7 @@ def get_checksums_for_versions( ) i += 1 except FailedDownloadError: - errors.append("Failed to fetch {0}".format(url)) + errors.append(f"Failed to fetch {url}") except Exception as e: logger.msg(f"Something failed on {url}, skipping. ({e})") @@ -903,14 +899,14 @@ def get_checksums_for_versions( # Generate the version directives to put in a package.py version_lines = "\n".join( [ - " version('{0}', {1}sha256='{2}')".format(v, " " * (max_len - len(str(v))), h) + " version('{}', {}sha256='{}')".format(v, " " * (max_len - len(str(v))), h) for v, h in version_hashes ] ) num_hash = len(version_hashes) logger.debug( - "Checksummed {0} version{1} of {2}:".format(num_hash, "" if num_hash == 1 else "s", name) + "Checksummed {} version{} of {}:".format(num_hash, "" if num_hash == 1 else "s", name) ) return version_lines diff --git a/lib/ramble/ramble/success_criteria.py b/lib/ramble/ramble/success_criteria.py index 82476bb09..b52b028b5 100644 --- a/lib/ramble/ramble/success_criteria.py +++ b/lib/ramble/ramble/success_criteria.py @@ -12,7 +12,7 @@ from ramble.util.logger import logger -class ScopedCriteriaList(object): +class ScopedCriteriaList: """A scoped list of success criteria This object represents a list of success criteria. The criteria are scoped @@ -88,8 +88,7 @@ def passed(self): def all_criteria(self): for scope in self._valid_scopes: - for criteria in self.criteria[scope]: - yield criteria + yield from self.criteria[scope] def find_criteria(self, name): for scope in self._valid_scopes: @@ -99,7 +98,7 @@ def find_criteria(self, name): return None -class SuccessCriteria(object): +class SuccessCriteria: """A single success criteria object This object represents a single criteria for success for a Ramble diff --git a/lib/ramble/ramble/test/cmd/clean.py b/lib/ramble/ramble/test/cmd/clean.py index 0661f43ae..0e0458fe1 100644 --- a/lib/ramble/ramble/test/cmd/clean.py +++ b/lib/ramble/ramble/test/cmd/clean.py @@ -23,7 +23,7 @@ def mock_calls_for_clean(monkeypatch): counts = {} - class Counter(object): + class Counter: def __init__(self, name): self.name = name counts[name] = 0 diff --git a/lib/ramble/ramble/test/cmd/config.py b/lib/ramble/ramble/test/cmd/config.py index 4ace38fea..1b0a517ab 100644 --- a/lib/ramble/ramble/test/cmd/config.py +++ b/lib/ramble/ramble/test/cmd/config.py @@ -581,7 +581,7 @@ def test_config_add_invalid_file_fails(tmpdir): with open(file, "w") as f: f.write(contents) - with pytest.raises((ramble.config.ConfigFormatError)): + with pytest.raises(ramble.config.ConfigFormatError): config("add", "-f", file) @@ -828,7 +828,7 @@ def _editor(*args, **kwargs): def section_args(section_name): - class TestArgs(object): + class TestArgs: scope = None section = section_name config_command = "edit" diff --git a/lib/ramble/ramble/test/cmd/mirror.py b/lib/ramble/ramble/test/cmd/mirror.py index 598629a90..957b202c3 100644 --- a/lib/ramble/ramble/test/cmd/mirror.py +++ b/lib/ramble/ramble/test/cmd/mirror.py @@ -29,14 +29,12 @@ def tmp_scope(): """Creates a temporary configuration scope""" base_name = "internal-testing-scope" - current_overrides = set( - x.name for x in ramble.config.config.matching_scopes(r"^{0}".format(base_name)) - ) + current_overrides = {x.name for x in ramble.config.config.matching_scopes(rf"^{base_name}")} num_overrides = 0 scope_name = base_name while scope_name in current_overrides: - scope_name = "{0}{1}".format(base_name, num_overrides) + scope_name = f"{base_name}{num_overrides}" num_overrides += 1 with ramble.config.override(ramble.config.InternalConfigScope(scope_name)): @@ -124,7 +122,7 @@ def test_mirror_destroy( tmpdir, ): mirror_dir = tmpdir.join("mirror_dir") - mirror_url = "file://{0}".format(mirror_dir.strpath) + mirror_url = f"file://{mirror_dir.strpath}" mirror("add", "atest", mirror_url) fs.mkdirp(mirror_dir.strpath) diff --git a/lib/ramble/ramble/test/concretize_builtin.py b/lib/ramble/ramble/test/concretize_builtin.py index 3437f79e7..08e01ee20 100644 --- a/lib/ramble/ramble/test/concretize_builtin.py +++ b/lib/ramble/ramble/test/concretize_builtin.py @@ -91,7 +91,7 @@ def test_concretize_does_not_set_required(mutable_config, mutable_mock_workspace ws._re_read() req_test = True - with open(config_path, "r") as f: + with open(config_path) as f: for line in f.readlines(): if re.match(r"^[^#]*required", line): req_test = False diff --git a/lib/ramble/ramble/test/conftest.py b/lib/ramble/ramble/test/conftest.py index 0f77a41a4..d2fa77a34 100644 --- a/lib/ramble/ramble/test/conftest.py +++ b/lib/ramble/ramble/test/conftest.py @@ -378,7 +378,7 @@ def _store_dir_and_cache(tmpdir_factory): return store, cache -class MockLayout(object): +class MockLayout: def __init__(self, root): self.root = root @@ -401,7 +401,7 @@ def create_layout(root): yield create_layout -class MockConfig(object): +class MockConfig: def __init__(self, configuration, writer_key): self._configuration = configuration self.writer_key = writer_key @@ -413,7 +413,7 @@ def writer_configuration(self): return self.configuration()[self.writer_key] -class ConfigUpdate(object): +class ConfigUpdate: def __init__(self, root_for_conf, writer_mod, writer_key, monkeypatch): self.root_for_conf = root_for_conf self.writer_mod = writer_mod @@ -441,7 +441,7 @@ def __call__(self, filename): ########## -class MockBundle(object): +class MockBundle: has_code = False name = "mock-bundle" versions = {} @@ -528,8 +528,8 @@ def mock_archive(request, tmpdir_factory): # Archive it with tmpdir.as_cwd(): - archive_name = "{0}{1}".format(ramble.stage._input_subdir, request.param[0]) - tar("-c{0}f".format(request.param[1]), archive_name, ramble.stage._input_subdir) + archive_name = f"{ramble.stage._input_subdir}{request.param[0]}" + tar(f"-c{request.param[1]}f", archive_name, ramble.stage._input_subdir) Archive = collections.namedtuple( "Archive", ["url", "path", "archive_file", "expanded_archive_basedir"] @@ -559,7 +559,7 @@ def install_mockery_mutable_config(mutable_config, mock_applications): yield -class MockCache(object): +class MockCache: def store(self, copy_cmd, relative_dest): pass @@ -567,7 +567,7 @@ def fetcher(self, target_path, digest, **kwargs): return MockCacheFetcher() -class MockCacheFetcher(object): +class MockCacheFetcher: def fetch(self): raise FetchError("Mock cache always fails for tests") diff --git a/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py b/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py index 3fbaafafa..5b7f965d4 100644 --- a/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py +++ b/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py @@ -90,7 +90,7 @@ def test_analyze_fom_output(): workspace("analyze", "-p", global_args=["-w", workspace_name]) result_file = glob.glob(os.path.join(ws.root, "results.latest.txt"))[0] - with open(result_file, "r") as f: + with open(result_file) as f: content = f.read() assert "default (null) context figures of merit" in content assert "possible hostname = test-user.c.googlers.com" in content diff --git a/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py b/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py index 294ca09e5..4c1435820 100644 --- a/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py +++ b/lib/ramble/ramble/test/end_to_end/chained_experiment_var_inheritance.py @@ -145,7 +145,7 @@ def test_chained_experiment_variable_inheritance(mutable_config, mutable_mock_wo assert os.path.exists(script) # Check all chained experiments have the correct arguments - with open(script, "r") as f: + with open(script) as f: parent_script_data = f.read() for chain_idx in [1, 3, 5]: @@ -158,5 +158,5 @@ def test_chained_experiment_variable_inheritance(mutable_config, mutable_mock_wo assert os.path.exists(chained_script) assert chained_script in parent_script_data - with open(chained_script, "r") as f: + with open(chained_script) as f: assert "mpirun -n 20 -ppn 10" in f.read() diff --git a/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py b/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py index 64eaadce0..0e87226bf 100644 --- a/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py +++ b/lib/ramble/ramble/test/end_to_end/config_section_env_vars.py @@ -67,7 +67,7 @@ def test_config_section_env_vars(mutable_config, mutable_mock_workspace_path, mo export_regex = re.compile(r"export MY_VAR=TEST") # Assert experiment 1 has exports before commands - with open(exp1_script, "r") as f: + with open(exp1_script) as f: export_found = False for line in f.readlines(): if export_regex.search(line): diff --git a/lib/ramble/ramble/test/end_to_end/custom_executables.py b/lib/ramble/ramble/test/end_to_end/custom_executables.py index 061b93c14..08c386517 100644 --- a/lib/ramble/ramble/test/end_to_end/custom_executables.py +++ b/lib/ramble/ramble/test/end_to_end/custom_executables.py @@ -115,7 +115,7 @@ def test_custom_executables(mutable_config, mutable_mock_workspace_path, mock_ap ] # Assert command order is: lscpu -> export -> foo - with open(exp_script, "r") as f: + with open(exp_script) as f: custom_found = False cmd_found = False export_found = False diff --git a/lib/ramble/ramble/test/end_to_end/define_package_paths.py b/lib/ramble/ramble/test/end_to_end/define_package_paths.py index d049e401c..d01c22b75 100644 --- a/lib/ramble/ramble/test/end_to_end/define_package_paths.py +++ b/lib/ramble/ramble/test/end_to_end/define_package_paths.py @@ -82,7 +82,7 @@ def test_define_package_paths(): test1_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_bare.test1.out") gromacs_log_line = _spack_loc_log_line("gromacs") impi_log_line = _spack_loc_log_line("intel-oneapi-mpi@2021.11.0") - with open(test1_log, "r") as f: + with open(test1_log) as f: content = f.read() assert "Executing phase define_package_paths" in content assert content.count(gromacs_log_line) == 1 @@ -90,7 +90,7 @@ def test_define_package_paths(): # test2 should use cached paths without invoking spack. test2_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_bare.test2.out") - with open(test2_log, "r") as f: + with open(test2_log) as f: content = f.read() assert "Executing phase define_package_paths" in content assert gromacs_log_line not in content @@ -152,7 +152,7 @@ def test_package_path_variables(): test1_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_bare.test1.out") gromacs_log_line = _spack_loc_log_line("gromacs") impi_log_line = _spack_loc_log_line("intel-oneapi-mpi@2021.11.0") - with open(test1_log, "r") as f: + with open(test1_log) as f: content = f.read() assert "Executing phase define_package_paths" in content assert content.count(gromacs_log_line) == 1 @@ -160,7 +160,7 @@ def test_package_path_variables(): # test2 should not use a path, as test1 has a variable for the path defined test2_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_bare.test2.out") - with open(test2_log, "r") as f: + with open(test2_log) as f: content = f.read() assert "Executing phase define_package_paths" in content assert gromacs_log_line in content diff --git a/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py b/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py index 4536f40da..4dca0a11b 100644 --- a/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py +++ b/lib/ramble/ramble/test/end_to_end/dryrun_chained_experiments.py @@ -141,7 +141,7 @@ def test_dryrun_chained_experiments(mutable_config, mutable_mock_workspace_path) assert os.path.exists(script) # Check all chained experiments are referenced - with open(script, "r") as f: + with open(script) as f: parent_script_data = f.read() for chain_idx in [1, 3, 5]: @@ -156,7 +156,7 @@ def test_dryrun_chained_experiments(mutable_config, mutable_mock_workspace_path) # Check that experiment 1 has n_ranks = 4 instead of 2 if chain_idx == 3: - with open(chained_script, "r") as f: + with open(chained_script) as f: assert "mpirun -n 4" in f.read() expected_order = [ @@ -169,7 +169,7 @@ def test_dryrun_chained_experiments(mutable_config, mutable_mock_workspace_path) ] # Check prepend / append order is correct - with open(script, "r") as f: + with open(script) as f: for line in f.readlines(): if expected_order[0].match(line): @@ -207,7 +207,7 @@ def test_dryrun_chained_experiments(mutable_config, mutable_mock_workspace_path) names = ["results.latest.json", "results.latest.yaml"] loaders = [sjson.load, syaml.load] for name, loader in zip(names, loaders): - with open(os.path.join(ws.root, name), "r") as f: + with open(os.path.join(ws.root, name)) as f: data = loader(f) assert "experiments" in data diff --git a/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py b/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py index 31117e8f1..5ac946acc 100644 --- a/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py +++ b/lib/ramble/ramble/test/end_to_end/dryrun_copies_external_env.py @@ -82,5 +82,5 @@ def test_dryrun_copies_external_env(mutable_config, mutable_mock_workspace_path, assert os.path.exists(env_file) - with open(env_file, "r") as f: + with open(env_file) as f: assert "wrf" in f.read() diff --git a/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py b/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py index 7bf54c046..dc22cf82b 100644 --- a/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py +++ b/lib/ramble/ramble/test/end_to_end/dryrun_series_contains_package_paths.py @@ -81,5 +81,5 @@ def test_dryrun_series_contains_package_paths( ) assert os.path.exists(script) - with open(script, "r") as f: + with open(script) as f: assert r"{zlib}" not in f.read() diff --git a/lib/ramble/ramble/test/end_to_end/env_var_builtin.py b/lib/ramble/ramble/test/end_to_end/env_var_builtin.py index 81fed966d..a73235f7d 100644 --- a/lib/ramble/ramble/test/end_to_end/env_var_builtin.py +++ b/lib/ramble/ramble/test/end_to_end/env_var_builtin.py @@ -92,7 +92,7 @@ def test_env_var_builtin(mutable_config, mutable_mock_workspace_path, mock_appli cmd3_regex = re.compile("foo >>") # Assert experiment 1 has exports before commands - with open(exp1_script, "r") as f: + with open(exp1_script) as f: cmd_found = False export_found = False for line in f.readlines(): @@ -104,7 +104,7 @@ def test_env_var_builtin(mutable_config, mutable_mock_workspace_path, mock_appli assert cmd_found and export_found # Assert experiment 2 has commands before exports - with open(exp2_script, "r") as f: + with open(exp2_script) as f: cmd_found = False export_found = False for line in f.readlines(): @@ -116,7 +116,7 @@ def test_env_var_builtin(mutable_config, mutable_mock_workspace_path, mock_appli assert cmd_found and export_found # Assert experiment 3 has exports before commands - with open(exp3_script, "r") as f: + with open(exp3_script) as f: cmd_found = False export_found = False for line in f.readlines(): @@ -164,5 +164,5 @@ def test_env_var_from_app_only(mutable_config, mutable_mock_workspace_path, mock experiment_root = ws.experiment_dir exp1_dir = os.path.join(experiment_root, "interleved-env-vars", "test_wl", "simple_test") - with open(os.path.join(exp1_dir, "execute_experiment"), "r") as f: + with open(os.path.join(exp1_dir, "execute_experiment")) as f: assert "FROM_DIRECTIVE" in f.read() diff --git a/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py b/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py index 7d70618ce..21456cd2b 100644 --- a/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py +++ b/lib/ramble/ramble/test/end_to_end/expanded_fom_dry_run.py @@ -76,7 +76,7 @@ def test_expanded_foms_dry_run(mutable_config, mutable_mock_workspace_path, mock print(output) text_results_files = glob.glob(os.path.join(ws1.root, "results*.txt")) - with open(text_results_files[0], "r") as f: + with open(text_results_files[0]) as f: data = f.read() for expected in expected_expansions: assert f"test_fom {expected} = 567.8" in data diff --git a/lib/ramble/ramble/test/end_to_end/experiment_excludes.py b/lib/ramble/ramble/test/end_to_end/experiment_excludes.py index 91a3b3da1..3bfa44f1a 100644 --- a/lib/ramble/ramble/test/end_to_end/experiment_excludes.py +++ b/lib/ramble/ramble/test/end_to_end/experiment_excludes.py @@ -214,7 +214,7 @@ def test_wrfv4_exclusions(mutable_config, mutable_mock_workspace_path): assert os.path.exists(os.path.join(exp_dir, "execute_experiment")) assert os.path.exists(os.path.join(exp_dir, "full_command")) - with open(os.path.join(exp_dir, "full_command"), "r") as f: + with open(os.path.join(exp_dir, "full_command")) as f: data = f.read() # Test the required environment variables exist @@ -231,7 +231,7 @@ def test_wrfv4_exclusions(mutable_config, mutable_mock_workspace_path): # Test the run script has a reference to the experiment log file assert os.path.join(exp_dir, f"{exp}.out") in data - with open(os.path.join(exp_dir, "execute_experiment"), "r") as f: + with open(os.path.join(exp_dir, "execute_experiment")) as f: data = f.read() # Test the required environment variables exist @@ -275,7 +275,7 @@ def test_wrfv4_exclusions(mutable_config, mutable_mock_workspace_path): assert len(json_results_files) == 2 assert len(yaml_results_files) == 2 - with open(text_results_files[0], "r") as f: + with open(text_results_files[0]) as f: data = f.read() assert "Average Timestep Time = 33.3 s" in data assert "Cumulative Timestep Time = 166.5 s" in data diff --git a/lib/ramble/ramble/test/end_to_end/experiment_repeats.py b/lib/ramble/ramble/test/end_to_end/experiment_repeats.py index c68d83a85..accb31349 100644 --- a/lib/ramble/ramble/test/end_to_end/experiment_repeats.py +++ b/lib/ramble/ramble/test/end_to_end/experiment_repeats.py @@ -175,7 +175,7 @@ def test_gromacs_repeats(mutable_config, mutable_mock_workspace_path): assert len(yaml_results_files) == 2 for text_result in text_results_files: - with open(text_result, "r") as f: + with open(text_result) as f: data = f.read() assert "Core Time = 11.111 s" in data assert "Core Time = 22.222 s" in data diff --git a/lib/ramble/ramble/test/end_to_end/explicit_zips.py b/lib/ramble/ramble/test/end_to_end/explicit_zips.py index 37c0cde7b..7c98d5632 100644 --- a/lib/ramble/ramble/test/end_to_end/explicit_zips.py +++ b/lib/ramble/ramble/test/end_to_end/explicit_zips.py @@ -201,7 +201,7 @@ def test_wrfv4_explicit_zips(mutable_config, mutable_mock_workspace_path): assert os.path.exists(os.path.join(exp_dir, "execute_experiment")) assert os.path.exists(os.path.join(exp_dir, "full_command")) - with open(os.path.join(exp_dir, "full_command"), "r") as f: + with open(os.path.join(exp_dir, "full_command")) as f: data = f.read() # Test the required environment variables exist @@ -218,7 +218,7 @@ def test_wrfv4_explicit_zips(mutable_config, mutable_mock_workspace_path): # Test the run script has a reference to the experiment log file assert os.path.join(exp_dir, f"{exp}.out") in data - with open(os.path.join(exp_dir, "execute_experiment"), "r") as f: + with open(os.path.join(exp_dir, "execute_experiment")) as f: data = f.read() # Test the required environment variables exist @@ -262,7 +262,7 @@ def test_wrfv4_explicit_zips(mutable_config, mutable_mock_workspace_path): assert len(json_results_files) == 2 assert len(yaml_results_files) == 2 - with open(text_results_files[0], "r") as f: + with open(text_results_files[0]) as f: data = f.read() assert "Average Timestep Time = 33.3 s" in data assert "Cumulative Timestep Time = 166.5 s" in data diff --git a/lib/ramble/ramble/test/end_to_end/formatted_executables.py b/lib/ramble/ramble/test/end_to_end/formatted_executables.py index 22dffb82c..a700d4d9c 100644 --- a/lib/ramble/ramble/test/end_to_end/formatted_executables.py +++ b/lib/ramble/ramble/test/end_to_end/formatted_executables.py @@ -82,7 +82,7 @@ def test_formatted_executables(mutable_config, mutable_mock_workspace_path, mock exp_dir = os.path.join(experiment_root, "basic", "working_wl", "simple_test") exp_script = os.path.join(exp_dir, "execute_experiment") - with open(exp_script, "r") as f: + with open(exp_script) as f: data = f.read() assert "from_app echo" in data assert ";" + " " * 9 + "from_ws echo" in data diff --git a/lib/ramble/ramble/test/end_to_end/globbing_patterns.py b/lib/ramble/ramble/test/end_to_end/globbing_patterns.py index 9708f1a85..6263140f3 100644 --- a/lib/ramble/ramble/test/end_to_end/globbing_patterns.py +++ b/lib/ramble/ramble/test/end_to_end/globbing_patterns.py @@ -92,7 +92,7 @@ def test_globbing_patterns( glob_var_mod_regex = re.compile("var_mod_modified") glob_env_var_mod_regex = re.compile("env_var_mod=modded") - with open(exp1_script, "r") as f: + with open(exp1_script) as f: # Check for only 'test' executable command test_cmd_found = False glob_cmd_not_found = True @@ -137,7 +137,7 @@ def test_globbing_patterns( assert test_wl_var_found and glob_wl_var_found and baz_wl_var_not_found assert test_env_var_found and glob_env_var_found and baz_env_var_not_found - with open(exp2_script, "r") as f: + with open(exp2_script) as f: # Check for executables matching 'test*' glob pattern test_cmd_found = False glob_cmd_found = False diff --git a/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py b/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py index 36b8c61ff..aa4a6beb0 100644 --- a/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py +++ b/lib/ramble/ramble/test/end_to_end/gromacs_size_expansion.py @@ -76,5 +76,5 @@ def test_gromacs_size_expansion(mutable_config, mutable_mock_workspace_path): ws1.experiment_dir, "gromacs", "water_bare", "expansion_test", "execute_experiment" ) - with open(exec_script_path, "r") as f: + with open(exec_script_path) as f: assert "0000.96" in f.read() diff --git a/lib/ramble/ramble/test/end_to_end/merge_config_files.py b/lib/ramble/ramble/test/end_to_end/merge_config_files.py index 2cd1498d2..6a0a03534 100644 --- a/lib/ramble/ramble/test/end_to_end/merge_config_files.py +++ b/lib/ramble/ramble/test/end_to_end/merge_config_files.py @@ -95,7 +95,7 @@ def test_merge_config_files(mutable_config, mutable_mock_workspace_path, mock_ap ws._re_read() - with open(config_path, "r") as f: + with open(config_path) as f: data = f.read() assert "ensure_installed" in data diff --git a/lib/ramble/ramble/test/end_to_end/package_manager_config.py b/lib/ramble/ramble/test/end_to_end/package_manager_config.py index 728491969..93137bde5 100644 --- a/lib/ramble/ramble/test/end_to_end/package_manager_config.py +++ b/lib/ramble/ramble/test/end_to_end/package_manager_config.py @@ -64,7 +64,7 @@ def test_package_manager_config_zlib(mock_applications): assert os.path.isfile(spack_yaml) - with open(spack_yaml, "r") as f: + with open(spack_yaml) as f: data = f.read() assert "config:" in data assert "debug: true" in data diff --git a/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py b/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py index 1da07cf26..a7f1b781c 100644 --- a/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py +++ b/lib/ramble/ramble/test/end_to_end/package_manager_requirements.py @@ -74,7 +74,7 @@ def test_package_manager_requirements_zlib(mock_applications, mock_modifiers): assert os.path.isfile(spack_yaml) - with open(spack_yaml, "r") as f: + with open(spack_yaml) as f: data = f.read() assert "config:" in data assert "debug: true" in data diff --git a/lib/ramble/ramble/test/end_to_end/passthrough_variables.py b/lib/ramble/ramble/test/end_to_end/passthrough_variables.py index 8c2c0b200..1abf74408 100644 --- a/lib/ramble/ramble/test/end_to_end/passthrough_variables.py +++ b/lib/ramble/ramble/test/end_to_end/passthrough_variables.py @@ -69,7 +69,7 @@ def test_passthrough_variables(mutable_config, mutable_mock_workspace_path, mock undefined_regex = re.compile(r"{undefined_var}") # Assert undefined variable is found - with open(exp1_script, "r") as f: + with open(exp1_script) as f: undefined_found = False for line in f.readlines(): if undefined_regex.search(line): diff --git a/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py b/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py index 0d43fad9d..2f08ece27 100644 --- a/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py +++ b/lib/ramble/ramble/test/end_to_end/phase_selection_with_dependencies.py @@ -164,7 +164,7 @@ def test_workspace_phase_selection_with_dependencies( for file in out_files: found = [False, False, False] cur_found = 0 - with open(file, "r") as f: + with open(file) as f: for line in f.readlines(): if expected_phase_order[cur_found] in line: found[cur_found] = True diff --git a/lib/ramble/ramble/test/end_to_end/shared_context.py b/lib/ramble/ramble/test/end_to_end/shared_context.py index ea627f797..77e35fb6c 100644 --- a/lib/ramble/ramble/test/end_to_end/shared_context.py +++ b/lib/ramble/ramble/test/end_to_end/shared_context.py @@ -78,7 +78,7 @@ def test_shared_contexts( results_files = glob.glob(os.path.join(ws.root, "results.latest.txt")) - with open(results_files[0], "r") as f: + with open(results_files[0]) as f: data = f.read() assert "matched_shared_context" in data # find the merged context assert "test_fom = 123.4" in data # from the app diff --git a/lib/ramble/ramble/test/end_to_end/spack_env_cache.py b/lib/ramble/ramble/test/end_to_end/spack_env_cache.py index 3bef4c743..fd0ae7971 100644 --- a/lib/ramble/ramble/test/end_to_end/spack_env_cache.py +++ b/lib/ramble/ramble/test/end_to_end/spack_env_cache.py @@ -99,26 +99,26 @@ def test_spack_env_cache(): # First encounter of an env_name (test1 -> gromacs, test2 -> g2) requires spack usage. test1_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_bare.test1.out") - with open(test1_log, "r") as f: + with open(test1_log) as f: content = f.read() assert "spack install" in content assert "spack concretize" in content test2_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_bare.test2.out") - with open(test2_log, "r") as f: + with open(test2_log) as f: content = f.read() assert "spack install" in content assert "spack concretize" in content # Envs should already exist and can skip spack calls. test3_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_bare.test3.out") - with open(test3_log, "r") as f: + with open(test3_log) as f: content = f.read() assert "spack install" not in content assert "spack concretize" not in content test4_log = os.path.join(ws.log_dir, "setup.latest", "gromacs.water_gmx50.test4.out") - with open(test4_log, "r") as f: + with open(test4_log) as f: content = f.read() assert "spack install" not in content assert "spack concretize" not in content diff --git a/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py b/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py index 42182612f..78b3c85ce 100644 --- a/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py +++ b/lib/ramble/ramble/test/end_to_end/test_configvar_dry_run.py @@ -106,7 +106,7 @@ def test_configvar_dry_run(mutable_config, mutable_mock_workspace_path): assert os.path.isdir(exp_dir) assert os.path.exists(os.path.join(exp_dir, "execute_experiment")) - with open(os.path.join(exp_dir, "execute_experiment"), "r") as f: + with open(os.path.join(exp_dir, "execute_experiment")) as f: data = f.read() # Test the license exists assert f"export {var_name2}{i}={var_val}" in data diff --git a/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py b/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py index 07669f1ee..d5006ecd5 100644 --- a/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py +++ b/lib/ramble/ramble/test/end_to_end/wrfv4_dry_run.py @@ -190,7 +190,7 @@ def test_wrfv4_spack_dry_run(mutable_config, mutable_mock_workspace_path): assert os.path.exists(os.path.join(exp_dir, "full_command")) license_inc_path = os.path.join(ws1.root, "shared", "licenses", "wrfv4", "license.inc") - with open(os.path.join(exp_dir, "full_command"), "r") as f: + with open(os.path.join(exp_dir, "full_command")) as f: data = f.read() # Test the license exists assert f". {license_inc_path}" in data @@ -214,7 +214,7 @@ def test_wrfv4_spack_dry_run(mutable_config, mutable_mock_workspace_path): # Test the license is added to the include file assert "export WRF_LICENSE=port@server" in data - with open(os.path.join(exp_dir, "execute_experiment"), "r") as f: + with open(os.path.join(exp_dir, "execute_experiment")) as f: data = f.read() # Test the required environment variables exist @@ -268,7 +268,7 @@ def test_wrfv4_spack_dry_run(mutable_config, mutable_mock_workspace_path): assert len(yaml_results_files) == 2 for text_result in text_results_files: - with open(text_result, "r") as f: + with open(text_result) as f: data = f.read() assert "Tags =" in data assert "Average Timestep Time = 33.3 s" in data @@ -417,7 +417,7 @@ def test_wrfv4_no_pkg_man_dry_run(mutable_config, mutable_mock_workspace_path): assert os.path.exists(os.path.join(exp_dir, "full_command")) license_inc_path = os.path.join(ws1.root, "shared", "licenses", "wrfv4", "license.inc") - with open(os.path.join(exp_dir, "full_command"), "r") as f: + with open(os.path.join(exp_dir, "full_command")) as f: data = f.read() # Test the license exists assert f". {license_inc_path}" in data @@ -441,7 +441,7 @@ def test_wrfv4_no_pkg_man_dry_run(mutable_config, mutable_mock_workspace_path): # Test the license is added to the include file assert "export WRF_LICENSE=port@server" in data - with open(os.path.join(exp_dir, "execute_experiment"), "r") as f: + with open(os.path.join(exp_dir, "execute_experiment")) as f: data = f.read() # Test the required environment variables exist @@ -492,7 +492,7 @@ def test_wrfv4_no_pkg_man_dry_run(mutable_config, mutable_mock_workspace_path): assert len(yaml_results_files) == 2 for text_result in text_results_files: - with open(text_result, "r") as f: + with open(text_result) as f: data = f.read() assert "Tags =" in data assert "Average Timestep Time = 33.3 s" in data diff --git a/lib/ramble/ramble/test/expander.py b/lib/ramble/ramble/test/expander.py index f51470adb..d344f3c08 100644 --- a/lib/ramble/ramble/test/expander.py +++ b/lib/ramble/ramble/test/expander.py @@ -54,7 +54,7 @@ def exp_dict(): ("{{n_ranks}*{var{processes_per_node}}:05d}", "00012", set(), 1), ("{{n_ranks}-1}", "3", set(), 1), ("{{{n_ranks}/2}:0.0f}", "2", set(), 1), - ("{size}", "0000.96", set(["size"]), 1), + ("{size}", "0000.96", {"size"}, 1), ("CPU(s)", "CPU(s)", set(), 1), ("str(1.5)", "1.5", set(), 1), ("int(1.5)", "1", set(), 1), @@ -72,7 +72,7 @@ def exp_dict(): (r"\\{experiment_name\\}", "baz", set(), 3), ('"2.1.1" in ["2.1.1", "3.1.1", "4.2.1"]', "True", set(), 1), ('"2.1.2" in ["2.1.1", "3.1.1", "4.2.1"]', "False", set(), 1), - ("{test_mask}", "0x0", set(["test_mask"]), 1), + ("{test_mask}", "0x0", {"test_mask"}, 1), ], ) def test_expansions(input, output, no_expand_vars, passes): @@ -108,7 +108,7 @@ def test_expansions(input, output, no_expand_vars, passes): ("{{n_ranks}*{var{processes_per_node}}:05d}", "00012", set(), 1), ("{{n_ranks}-1}", 3, set(), 1), ("{{{n_ranks}/2}:0.0f}", 2, set(), 1), - ("{size}", 0.96, set(["size"]), 1), + ("{size}", 0.96, {"size"}, 1), ("CPU(s)", "CPU(s)", set(), 1), ("str(1.5)", 1.5, set(), 1), ("int(1.5)", 1, set(), 1), @@ -126,7 +126,7 @@ def test_expansions(input, output, no_expand_vars, passes): (r"\\{experiment_name\\}", "baz", set(), 3), ('"2.1.1" in ["2.1.1", "3.1.1", "4.2.1"]', True, set(), 1), ('"2.1.2" in ["2.1.1", "3.1.1", "4.2.1"]', False, set(), 1), - ("{test_mask}", 0, set(["test_mask"]), 1), + ("{test_mask}", 0, {"test_mask"}, 1), ("{var3} // {processes_per_node}", 1, set(), 1), ("-5 % 3", 1, set(), 1), ("2 and 1", 1, set(), 1), diff --git a/lib/ramble/ramble/test/experiment_set.py b/lib/ramble/ramble/test/experiment_set.py index 289439743..2d9678ed3 100644 --- a/lib/ramble/ramble/test/experiment_set.py +++ b/lib/ramble/ramble/test/experiment_set.py @@ -1151,7 +1151,7 @@ def test_modifiers_set_correctly(mutable_mock_workspace_path, mock_modifiers, ca app_inst = exp_set.experiments["basic.test_wl.test1"] assert app_inst.modifiers is not None - expected_modifier_modes = set(["app-scope", "wl-scope", "exp-scope"]) + expected_modifier_modes = {"app-scope", "wl-scope", "exp-scope"} for mod_def in app_inst.modifiers: assert mod_def["mode"] in expected_modifier_modes expected_modifier_modes.remove(mod_def["mode"]) diff --git a/lib/ramble/ramble/test/gcs_fetch.py b/lib/ramble/ramble/test/gcs_fetch.py index e982ba0c6..73642d1ee 100644 --- a/lib/ramble/ramble/test/gcs_fetch.py +++ b/lib/ramble/ramble/test/gcs_fetch.py @@ -52,7 +52,7 @@ class Archived_GCSFS(ramble.fetch_strategy.GCSFetchStrategy): def archive_file(self): return archive - url = "gcs:///{0}".format(archive) + url = f"gcs:///{archive}" fetcher = Archived_GCSFS(url=url) with ramble.stage.InputStage(fetcher, name="test", path=testpath): fetcher.fetch() diff --git a/lib/ramble/ramble/test/mirror_tests.py b/lib/ramble/ramble/test/mirror_tests.py index 0d93990c2..fc333f9aa 100644 --- a/lib/ramble/ramble/test/mirror_tests.py +++ b/lib/ramble/ramble/test/mirror_tests.py @@ -35,7 +35,7 @@ ] -class MockFetcher(object): +class MockFetcher: """Mock fetcher object which implements the necessary functionality for testing MirrorCache """ diff --git a/lib/ramble/ramble/test/modifier_application.py b/lib/ramble/ramble/test/modifier_application.py index 2dabe2f87..44b80f57d 100644 --- a/lib/ramble/ramble/test/modifier_application.py +++ b/lib/ramble/ramble/test/modifier_application.py @@ -79,13 +79,13 @@ def test_wrfv4_aps_test(mutable_config, mutable_mock_workspace_path): workspace("setup", "--dry-run", global_args=["-w", workspace_name]) software_path = os.path.join(ws1.software_dir, "wrfv4", "spack.yaml") - with open(software_path, "r") as f: + with open(software_path) as f: assert "intel-oneapi-vtune" in f.read() execute_script = os.path.join( ws1.experiment_dir, "wrfv4", "CONUS_12km", "modifier_test", "execute_experiment" ) - with open(execute_script, "r") as f: + with open(execute_script) as f: data = f.read() assert "aps -c mpi" in data assert "aps-report" in data diff --git a/lib/ramble/ramble/test/modifier_functionality/experiment_modification.py b/lib/ramble/ramble/test/modifier_functionality/experiment_modification.py index f3ac60c4a..af3d74d34 100644 --- a/lib/ramble/ramble/test/modifier_functionality/experiment_modification.py +++ b/lib/ramble/ramble/test/modifier_functionality/experiment_modification.py @@ -66,6 +66,6 @@ def test_experiment_modification( assert os.path.exists(exp_script) - with open(exp_script, "r") as f: + with open(exp_script) as f: data = f.read() assert f"mpirun -n {n_ranks}" in data diff --git a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_phases.py b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_phases.py index a689014ec..b57450522 100644 --- a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_phases.py +++ b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_phases.py @@ -63,7 +63,7 @@ def test_gromacs_dry_run_mock_mod_phase( make_experiments_regex = re.compile("Executing phase make_experiments") after_make_experiments_regex = re.compile("Executing phase after_make_experiments") - with open(out_file, "r") as f: + with open(out_file) as f: for line in f.readlines(): if first_phase_regex.search(line): assert not found_phase diff --git a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py index 295adfc4a..717f665a9 100644 --- a/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py +++ b/lib/ramble/ramble/test/modifier_functionality/mock_modifier_spack_configs.py @@ -56,7 +56,7 @@ def test_gromacs_mock_spack_config_mod( spack_yaml = os.path.join(ws1.software_dir, "gromacs", "spack.yaml") assert os.path.isfile(spack_yaml) - with open(spack_yaml, "r") as f: + with open(spack_yaml) as f: data = f.read() assert "debug: true" in data diff --git a/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py b/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py index 79f772e00..aa62ec43c 100644 --- a/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py +++ b/lib/ramble/ramble/test/modifier_functionality/modifier_helpers.py @@ -28,13 +28,13 @@ def check_software_env(base_dir, tests): spack_file = os.path.join(env_dir, "spack.yaml") assert os.path.isfile(spack_file) - with open(spack_file, "r") as f: + with open(spack_file) as f: assert test_content in f.read() def check_execute_script(script_path, tests): assert os.path.isfile(script_path) - with open(script_path, "r") as f: + with open(script_path) as f: data = f.read() for test in tests: assert test in data diff --git a/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py b/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py index 500d7a7aa..e574a766c 100644 --- a/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py +++ b/lib/ramble/ramble/test/modifier_functionality/modifier_prepare_analysis.py @@ -57,7 +57,7 @@ def test_basic_dry_run_mock_prepare_analysis_mod( expected_regex = re.compile(".*This test worked") found_str = False - with open(os.path.join(ws1.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws1.root, "results.latest.txt")) as f: for line in f.readlines(): if expected_regex.match(line): found_str = True diff --git a/lib/ramble/ramble/test/spec_basic.py b/lib/ramble/ramble/test/spec_basic.py index a98037061..e141fc9fb 100644 --- a/lib/ramble/ramble/test/spec_basic.py +++ b/lib/ramble/ramble/test/spec_basic.py @@ -10,7 +10,7 @@ from ramble.spec import Spec -class TestSpec(object): +class TestSpec: def test_spec_examples(self): app_name = "test_application" diff --git a/lib/ramble/ramble/test/stage.py b/lib/ramble/ramble/test/stage.py index ece1ac1f5..bf4cb6575 100644 --- a/lib/ramble/ramble/test/stage.py +++ b/lib/ramble/ramble/test/stage.py @@ -331,7 +331,7 @@ def fetch(self): def search_fn(): """Returns a search function that always succeeds.""" - class _Mock(object): + class _Mock: performed_search = False def __call__(self): @@ -375,7 +375,7 @@ def check_stage_dir_perms(prefix, path): assert p_status.st_mode & stat.S_IRWXU == stat.S_IRWXU -class TestStage(object): +class TestStage: stage_name = "ramble-test-stage" @@ -798,7 +798,7 @@ def test_resolve_paths(self): assert ramble.stage._resolve_paths(paths) == can_paths # resolved path with node including user does not append user - paths = [os.path.join(os.path.sep, "spack-{0}".format(user), "stage")] + paths = [os.path.join(os.path.sep, f"spack-{user}", "stage")] assert ramble.stage._resolve_paths(paths) == paths tempdir = "$tempdir" diff --git a/lib/ramble/ramble/test/success_criteria.py b/lib/ramble/ramble/test/success_criteria.py index 4f8cb697e..7f43a6edb 100644 --- a/lib/ramble/ramble/test/success_criteria.py +++ b/lib/ramble/ramble/test/success_criteria.py @@ -32,7 +32,7 @@ def test_single_criteria(tmpdir): "test", "string", r".*Success string.*", log_path ) - with open(log_path, "r") as f: + with open(log_path) as f: for line in f.readlines(): if new_criteria.passed(line): new_criteria.mark_found() @@ -58,7 +58,7 @@ def test_criteria_list(tmpdir): criteria_list.add_criteria("workspace", "test-ws", "string", r".*Into a log file.*", log_path) - with open(log_path, "r") as f: + with open(log_path) as f: for line in f.readlines(): for criteria in criteria_list.all_criteria(): if criteria.passed(line): diff --git a/lib/ramble/ramble/test/success_criteria/always_print_foms.py b/lib/ramble/ramble/test/success_criteria/always_print_foms.py index b37684cde..b757d0aa0 100644 --- a/lib/ramble/ramble/test/success_criteria/always_print_foms.py +++ b/lib/ramble/ramble/test/success_criteria/always_print_foms.py @@ -57,7 +57,7 @@ def test_always_print_foms(mutable_config, mutable_mock_workspace_path, mock_app ramble_on(global_args=["-w", workspace_name]) workspace("analyze", "--always-print-foms", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() assert "FAILED" in data assert "0.9 s" in data diff --git a/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py b/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py index f7e9249cb..01817337a 100644 --- a/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py +++ b/lib/ramble/ramble/test/success_criteria/repeat_success_strict.py @@ -60,7 +60,7 @@ def test_repeat_success_strict(mutable_config, mutable_mock_workspace_path, mock ramble_on(global_args=["-w", workspace_name]) workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() print(data) assert "FAILED" not in data @@ -75,7 +75,7 @@ def test_repeat_success_strict(mutable_config, mutable_mock_workspace_path, mock workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() print(data) assert "SUCCESS" in data @@ -91,7 +91,7 @@ def test_repeat_success_strict(mutable_config, mutable_mock_workspace_path, mock workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() print(data) assert "SUCCESS" not in data diff --git a/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py b/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py index 3c069e47c..d2752e9f9 100644 --- a/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py +++ b/lib/ramble/ramble/test/success_criteria/success_fom_comparison.py @@ -94,6 +94,6 @@ def test_success_fom_comparison( workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() assert result in data diff --git a/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py b/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py index 414c7297b..a7e3317bb 100644 --- a/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py +++ b/lib/ramble/ramble/test/success_criteria/success_fom_globbing.py @@ -95,6 +95,6 @@ def test_success_fom_globbing( workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() assert result in data diff --git a/lib/ramble/ramble/test/success_criteria/success_functions.py b/lib/ramble/ramble/test/success_criteria/success_functions.py index a67652c92..628455dac 100644 --- a/lib/ramble/ramble/test/success_criteria/success_functions.py +++ b/lib/ramble/ramble/test/success_criteria/success_functions.py @@ -57,7 +57,7 @@ def test_success_function(mutable_config, mutable_mock_workspace_path, mock_appl ramble_on(global_args=["-w", workspace_name]) workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() assert "FAILED" in data assert "0.9 s" not in data diff --git a/lib/ramble/ramble/test/success_criteria/success_modifiers.py b/lib/ramble/ramble/test/success_criteria/success_modifiers.py index 50491539e..927645146 100644 --- a/lib/ramble/ramble/test/success_criteria/success_modifiers.py +++ b/lib/ramble/ramble/test/success_criteria/success_modifiers.py @@ -76,7 +76,7 @@ def test_success_modifier( workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() assert result in data @@ -128,7 +128,7 @@ def test_success_criteria_with_multiple_experiments(mock_applications, mock_modi workspace("analyze", global_args=["-w", workspace_name]) result_file = os.path.join(ws.root, "results.latest.txt") - with open(result_file, "r") as f: + with open(result_file) as f: content = f.read() assert "FAILED" not in content assert "default (null) context figures of merit" in content diff --git a/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py b/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py index e489d8929..56e90b1f2 100644 --- a/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py +++ b/lib/ramble/ramble/test/success_criteria/success_variable_fom_comparison.py @@ -99,6 +99,6 @@ def test_success_variable_fom_comparison( workspace("analyze", global_args=["-w", workspace_name]) - with open(os.path.join(ws.root, "results.latest.txt"), "r") as f: + with open(os.path.join(ws.root, "results.latest.txt")) as f: data = f.read() assert result in data diff --git a/lib/ramble/ramble/test/util/editor.py b/lib/ramble/ramble/test/util/editor.py index 29011e38d..9ef2a2da1 100644 --- a/lib/ramble/ramble/test/util/editor.py +++ b/lib/ramble/ramble/test/util/editor.py @@ -127,7 +127,7 @@ def test_editor_both_bad(nosuch_exe, vim_exe): os.environ["VISUAL"] = nosuch_exe os.environ["EDITOR"] = nosuch_exe - os.environ["PATH"] = "%s%s%s" % (os.path.dirname(vim_exe), os.pathsep, os.environ["PATH"]) + os.environ["PATH"] = "{}{}{}".format(os.path.dirname(vim_exe), os.pathsep, os.environ["PATH"]) def assert_exec(exe, args): assert exe == vim_exe diff --git a/lib/ramble/ramble/util/editor.py b/lib/ramble/ramble/util/editor.py index 4d5ea5dc3..d87c610c0 100644 --- a/lib/ramble/ramble/util/editor.py +++ b/lib/ramble/ramble/util/editor.py @@ -93,7 +93,7 @@ def try_exec(exe, args, var=None): # Show variable we were trying to use, if it's from one if var: - exe = "$%s (%s)" % (var, exe) + exe = f"${var} ({exe})" logger.warn(f"Could not execute {exe} due to error: {e}") return False @@ -128,7 +128,7 @@ def try_env_var(var): return # Fail if nothing could be found - raise EnvironmentError( + raise OSError( "No text editor found! Please set the VISUAL and/or EDITOR " "environment variable(s) to your preferred text editor." ) diff --git a/lib/ramble/ramble/util/executable.py b/lib/ramble/ramble/util/executable.py index 75b7ef3bf..34cf36b39 100644 --- a/lib/ramble/ramble/util/executable.py +++ b/lib/ramble/ramble/util/executable.py @@ -54,7 +54,7 @@ def which(*args, **kwargs): return PrefixedExecutable(exe) if exe else None -class CommandExecutable(object): +class CommandExecutable: """CommandExecutable class This class is used to represent internal executables in Ramble. diff --git a/lib/ramble/ramble/util/file_cache.py b/lib/ramble/ramble/util/file_cache.py index 30edbe51b..a3a10625b 100644 --- a/lib/ramble/ramble/util/file_cache.py +++ b/lib/ramble/ramble/util/file_cache.py @@ -15,7 +15,7 @@ from ramble.util.lock import Lock, ReadTransaction, WriteTransaction -class FileCache(object): +class FileCache: """This class manages cached data in the filesystem. - Cache files are fetched and stored by unique keys. Keys can be relative @@ -123,13 +123,13 @@ def write_transaction(self, key): # TODO: is pretty hard to reason about in llnl.util.lock. At some # TODO: point we should just replace it with functions and simplify # TODO: the locking code. - class WriteContextManager(object): + class WriteContextManager: def __enter__(cm): # noqa cm.orig_filename = self.cache_path(key) cm.orig_file = None if os.path.exists(cm.orig_filename): - cm.orig_file = open(cm.orig_filename, "r") + cm.orig_file = open(cm.orig_filename) cm.tmp_filename = self.cache_path(key) + ".tmp" cm.tmp_file = open(cm.tmp_filename, "w") diff --git a/lib/ramble/ramble/util/graph.py b/lib/ramble/ramble/util/graph.py index a84435f73..410804ddb 100644 --- a/lib/ramble/ramble/util/graph.py +++ b/lib/ramble/ramble/util/graph.py @@ -7,7 +7,7 @@ # except according to those terms. -class GraphNode(object): +class GraphNode: """Class representing a node of a graph, where the node can have an attribute attached to it. diff --git a/lib/ramble/ramble/util/imp/importlib_importer.py b/lib/ramble/ramble/util/imp/importlib_importer.py index 3d0436a06..4592ab17e 100644 --- a/lib/ramble/ramble/util/imp/importlib_importer.py +++ b/lib/ramble/ramble/util/imp/importlib_importer.py @@ -16,17 +16,17 @@ class PrependFileLoader(SourceFileLoader): def __init__(self, full_name, path, prepend=None): - super(PrependFileLoader, self).__init__(full_name, path) + super().__init__(full_name, path) self.prepend = prepend def path_stats(self, path): - stats = super(PrependFileLoader, self).path_stats(path) + stats = super().path_stats(path) if self.prepend: stats["size"] += len(self.prepend) + 1 return stats def get_data(self, path): - data = super(PrependFileLoader, self).get_data(path) + data = super().get_data(path) if path != self.path or self.prepend is None: return data else: diff --git a/lib/ramble/ramble/util/lock.py b/lib/ramble/ramble/util/lock.py index 1f73db8c6..00bb00fb9 100644 --- a/lib/ramble/ramble/util/lock.py +++ b/lib/ramble/ramble/util/lock.py @@ -27,27 +27,27 @@ class Lock(llnl.util.lock.Lock): """ def __init__(self, *args, **kwargs): - super(Lock, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._enable = ramble.config.get("config:locks", True) def _lock(self, op, timeout=0): if self._enable: - return super(Lock, self)._lock(op, timeout) + return super()._lock(op, timeout) else: return 0, 0 def _unlock(self): """Unlock call that always succeeds.""" if self._enable: - super(Lock, self)._unlock() + super()._unlock() def _debug(self, *args): if self._enable: - super(Lock, self)._debug(*args) + super()._debug(*args) def cleanup(self, *args): if self._enable: - super(Lock, self).cleanup(*args) + super().cleanup(*args) def check_lock_safety(path): @@ -74,9 +74,9 @@ def check_lock_safety(path): writable = "world" if writable: - msg = "Refusing to disable locks: ramble is {0}-writable.".format(writable) + msg = f"Refusing to disable locks: ramble is {writable}-writable." long_msg = ( "Running a shared ramble without locks is unsafe. You must " - "restrict permissions on {0} or enable locks." + "restrict permissions on {} or enable locks." ).format(path) raise ramble.error.RambleError(msg, long_msg) diff --git a/lib/ramble/ramble/util/logger.py b/lib/ramble/ramble/util/logger.py index 45e573f38..c156a8106 100644 --- a/lib/ramble/ramble/util/logger.py +++ b/lib/ramble/ramble/util/logger.py @@ -14,7 +14,7 @@ from pathlib import Path -class Logger(object): +class Logger: """Logger class This class providers additional functionality on top of LLNL's tty utility. diff --git a/lib/ramble/ramble/util/naming.py b/lib/ramble/ramble/util/naming.py index 9d087c1f9..a8aff9abc 100644 --- a/lib/ramble/ramble/util/naming.py +++ b/lib/ramble/ramble/util/naming.py @@ -7,7 +7,6 @@ # except according to those terms. # Need this because of ramble.util.string -from __future__ import absolute_import import io import string import itertools @@ -168,7 +167,7 @@ class InvalidModuleNameError(ramble.error.RambleError): """Raised when we encounter a bad module name.""" def __init__(self, name): - super(InvalidModuleNameError, self).__init__("Invalid module name: " + name) + super().__init__("Invalid module name: " + name) self.name = name @@ -176,15 +175,13 @@ class InvalidFullyQualifiedModuleNameError(ramble.error.RambleError): """Raised when we encounter a bad full package name.""" def __init__(self, name): - super(InvalidFullyQualifiedModuleNameError, self).__init__( - "Invalid fully qualified package name: " + name - ) + super().__init__("Invalid fully qualified package name: " + name) self.name = name -class NamespaceTrie(object): +class NamespaceTrie: - class Element(object): + class Element: def __init__(self, value): self.value = value diff --git a/lib/ramble/ramble/util/stats.py b/lib/ramble/ramble/util/stats.py index 16ec276db..4386127c1 100644 --- a/lib/ramble/ramble/util/stats.py +++ b/lib/ramble/ramble/util/stats.py @@ -29,7 +29,7 @@ def max_decimal_places(list): return max -class StatsBase(object): +class StatsBase: min_count = 1 def compute(self, values): diff --git a/lib/ramble/ramble/util/web.py b/lib/ramble/ramble/util/web.py index 72adca78b..ef8faa103 100644 --- a/lib/ramble/ramble/util/web.py +++ b/lib/ramble/ramble/util/web.py @@ -6,7 +6,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function +from html.parser import HTMLParser import codecs import errno @@ -38,18 +38,12 @@ from spack.util.path import convert_to_posix_path #: User-Agent used in Request objects -SPACK_USER_AGENT = "Spackbot/{0}".format(spack.spack_version) +SPACK_USER_AGENT = f"Spackbot/{spack.spack_version}" -if sys.version_info < (3, 0): - # Python 2 had these in the HTMLParser package. - from HTMLParser import HTMLParseError, HTMLParser # novm -else: - # In Python 3, things moved to html.parser - from html.parser import HTMLParser - # Also, HTMLParseError is deprecated and never raised. - class HTMLParseError(Exception): - pass +# Also, HTMLParseError is deprecated and never raised. +class HTMLParseError(Exception): + pass class LinkParser(HTMLParser): @@ -141,7 +135,7 @@ def read_from_url(url, accept_content_type=None): try: response = _urlopen(req, timeout=timeout, context=context) except URLError as err: - raise SpackWebError("Download failed: {ERROR}".format(ERROR=str(err))) + raise SpackWebError(f"Download failed: {str(err)}") if accept_content_type and not is_web_url: content_type = get_header(response.headers, "Content-type") @@ -152,7 +146,7 @@ def read_from_url(url, accept_content_type=None): if reject_content_type: logger.debug( - "ignoring page {0}{1}{2}".format( + "ignoring page {}{}{}".format( url, " with content type " if content_type is not None else "", content_type or "" ) ) @@ -222,9 +216,7 @@ def push_to_url(local_file_path, remote_path, keep_original=True, extra_args=Non os.remove(local_file_path) else: - raise NotImplementedError( - "Unrecognized URL scheme: {SCHEME}".format(SCHEME=remote_url.scheme) - ) + raise NotImplementedError(f"Unrecognized URL scheme: {remote_url.scheme}") def url_exists(url): @@ -367,8 +359,7 @@ def _iter_s3_prefix(client, url, num_entries=1024): while True: contents, key = _list_s3_objects(client, bucket, prefix, num_entries, start_after=key) - for x in contents: - yield x + yield from contents if not key: break @@ -398,7 +389,7 @@ def list_url(url, recursive=False): if recursive: return list(_iter_s3_prefix(s3, url)) - return list(set(key.split("/", 1)[0] for key in _iter_s3_prefix(s3, url))) + return list({key.split("/", 1)[0] for key in _iter_s3_prefix(s3, url)}) elif url.scheme == "gs": gcs = gcs_util.GCSBucket(url) @@ -502,7 +493,7 @@ def _spider(url, collect_nested): logger.debug(f"Error in _spider: {type(e)}:{str(e)}", traceback.format_exc()) finally: - logger.debug("SPIDER: [url={0}]".format(url)) + logger.debug(f"SPIDER: [url={url}]") return pages, links, subcalls @@ -524,7 +515,7 @@ def _spider(url, collect_nested): try: while current_depth <= depth: logger.debug( - "SPIDER: [depth={0}, max_depth={1}, urls={2}]".format( + "SPIDER: [depth={}, max_depth={}, urls={}]".format( current_depth, depth, len(spider_args) ) ) @@ -731,7 +722,5 @@ class NoNetworkConnectionError(SpackWebError): """Raised when an operation can't get an internet connection.""" def __init__(self, message, url): - super(NoNetworkConnectionError, self).__init__( - "No network connection: " + str(message), "URL was: " + str(url) - ) + super().__init__("No network connection: " + str(message), "URL was: " + str(url)) self.url = url diff --git a/lib/ramble/ramble/util/yaml_generation.py b/lib/ramble/ramble/util/yaml_generation.py index e6692abb4..139b85fae 100644 --- a/lib/ramble/ramble/util/yaml_generation.py +++ b/lib/ramble/ramble/util/yaml_generation.py @@ -40,7 +40,7 @@ def read_config_file(conf_path: str): Returns: (dict): Dictionary representation of the data contained in conf_path """ - with open(conf_path, "r") as base_conf: + with open(conf_path) as base_conf: logger.debug(f"Reading config from {conf_path}") try: config_dict = yaml.safe_load(base_conf) diff --git a/lib/ramble/ramble/workload.py b/lib/ramble/ramble/workload.py index 5011dc1ee..20d7afdb2 100644 --- a/lib/ramble/ramble/workload.py +++ b/lib/ramble/ramble/workload.py @@ -12,7 +12,7 @@ import ramble.util.colors as rucolor -class WorkloadVariable(object): +class WorkloadVariable: """Class representing a variable definition""" def __init__( @@ -68,7 +68,7 @@ def copy(self): return copy.deepcopy(self) -class WorkloadEnvironmentVariable(object): +class WorkloadEnvironmentVariable: """Class representing an environment variable in a workload""" def __init__(self, name: str, value=None, description: str = None): @@ -108,7 +108,7 @@ def copy(self): return copy.deepcopy(self) -class Workload(object): +class Workload: """Class representing a single workload""" def __init__( diff --git a/lib/ramble/ramble/workspace/shell.py b/lib/ramble/ramble/workspace/shell.py index 4acad485d..f63feb858 100644 --- a/lib/ramble/ramble/workspace/shell.py +++ b/lib/ramble/ramble/workspace/shell.py @@ -19,7 +19,7 @@ def activate_header(ws, shell, prompt=None): cmds = "" if shell == "csh": # TODO: figure out how to make color work for csh - cmds += "setenv %s %s;\n" % (ramble.workspace.ramble_workspace_var, ws.root) + cmds += f"setenv {ramble.workspace.ramble_workspace_var} {ws.root};\n" if prompt: cmds += "if (! $?RAMBLE_OLD_PROMPT ) " cmds += 'setenv RAMBLE_OLD_PROMPT "${prompt}";\n' @@ -28,7 +28,7 @@ def activate_header(ws, shell, prompt=None): if "color" in os.getenv("TERM", "") and prompt: prompt = colorize("@G{%s} " % prompt, color=True) - cmds += "set -gx %s %s;\n" % (ramble.workspace.ramble_workspace_var, ws.root) + cmds += f"set -gx {ramble.workspace.ramble_workspace_var} {ws.root};\n" # # NOTE: We're not changing the fish_prompt function (which is fish's # solution to the PS1 variable) here. This is a bit fiddly, and easy to @@ -36,13 +36,13 @@ def activate_header(ws, shell, prompt=None): # elif shell == "bat": # TODO: Color - cmds += 'set "%s=%s"\n' % (ramble.workspace.ramble_workspace_var, ws.root) + cmds += f'set "{ramble.workspace.ramble_workspace_var}={ws.root}"\n' # TODO: prompt else: if "color" in os.getenv("TERM", "") and prompt: prompt = colorize("@G{%s}" % prompt, color=True) - cmds += "export %s=%s;\n" % (ramble.workspace.ramble_workspace_var, ws.root) + cmds += f"export {ramble.workspace.ramble_workspace_var}={ws.root};\n" if prompt: cmds += "if [ -z ${RAMBLE_OLD_PS1+x} ]; then\n" cmds += " if [ -z ${PS1+x} ]; then\n" @@ -74,7 +74,7 @@ def deactivate_header(shell): # TODO: prompt else: cmds += "if [ ! -z ${%s+x} ]; then\n" % (ramble.workspace.ramble_workspace_var) - cmds += "unset %s; export %s;\n" % ( + cmds += "unset {}; export {};\n".format( ramble.workspace.ramble_workspace_var, ramble.workspace.ramble_workspace_var, ) diff --git a/lib/ramble/ramble/workspace/workspace.py b/lib/ramble/ramble/workspace/workspace.py index 1058f8494..75cfb0d2a 100644 --- a/lib/ramble/ramble/workspace/workspace.py +++ b/lib/ramble/ramble/workspace/workspace.py @@ -208,7 +208,7 @@ def activate(ws): # Fail early to avoid ending in an invalid state if not isinstance(ws, Workspace): - raise TypeError("`ws` should be of type {0}".format(Workspace.__name__)) + raise TypeError(f"`ws` should be of type {Workspace.__name__}") # Check if we need to reinitialize the store due to pushing the configuration # below. @@ -431,7 +431,7 @@ def get_workspace(args, cmd_name, required=False): ) -class Workspace(object): +class Workspace: """Class representing a working directory for workload experiments @@ -576,13 +576,13 @@ def _read(self): f"template name of {template_name}" + " which is reserved by ramble." ) - with open(template_path, "r") as f: + with open(template_path) as f: self._read_template(template_name, f.read()) if os.path.exists(self.auxiliary_software_dir): for filename in os.listdir(self.auxiliary_software_dir): aux_file_path = os.path.join(self.auxiliary_software_dir, filename) - with open(aux_file_path, "r") as f: + with open(aux_file_path) as f: self._read_auxiliary_software_file(filename, f.read()) if read_default_script: @@ -605,8 +605,8 @@ def _read_all_application_configs(self): for dirpath, _, filenames in os.walk(app_dir): for file in filenames: if file.endswith(".yaml"): - full_path = "%s/%s" % (dirpath, file) - with open(full_path, "r") as f: + full_path = f"{dirpath}/{file}" + with open(full_path) as f: self._read_application_config(full_path, f) def _read_application_config(self, path, f, raw_yaml=None): @@ -1109,7 +1109,9 @@ def dump_results(self, output_formats=["text"], print_results=False): mod = fom["origin"] name = f"{fom['origin_type']}{delim}{mod}{delim}{name}" - output = "%s = %s %s" % (name, fom["value"], fom["units"]) + output = "{} = {} {}".format( + name, fom["value"], fom["units"] + ) f.write(" %s\n" % (output.strip())) else: logger.msg("No results to write") @@ -1140,7 +1142,7 @@ def dump_results(self, output_formats=["text"], print_results=False): logger.all_msg(f" {out_file}") if print_results: - with open(results_written[0], "r") as f: + with open(results_written[0]) as f: # Use tty directly to avoid cluttering the analyze log tty.msg(f"Results from the analysis pipeline:\n{f.read()}") @@ -1353,13 +1355,11 @@ def template_path(self, name): def all_templates(self): """Iterator over each template in the workspace""" - for name, conf in self._templates.items(): - yield name, conf + yield from self._templates.items() def all_auxiliary_software_files(self): """Iterator over each file in $workspace/configs/auxiliary_software_files""" - for name, value in self._auxiliary_software_files.items(): - yield name, value + yield from self._auxiliary_software_files.items() def included_config_scopes(self): """List of included configuration scopes from the environment. @@ -1388,11 +1388,11 @@ def included_config_scopes(self): if os.path.isdir(config_path): # directories are treated as regular ConfigScopes - config_name = "workspace:%s:%s" % (self.name, os.path.basename(config_path)) + config_name = f"workspace:{self.name}:{os.path.basename(config_path)}" scope = ramble.config.ConfigScope(config_name, config_path) elif os.path.exists(config_path): # files are assumed to be SingleFileScopes - config_name = "workspace:%s:%s" % (self.name, config_path) + config_name = f"workspace:{self.name}:{config_path}" scope = ramble.config.SingleFileScope( config_name, config_path, ramble.schema.merged.schema ) @@ -1403,15 +1403,15 @@ def included_config_scopes(self): scopes.append(scope) if missing: - msg = "Detected {0} missing include path(s):".format(len(missing)) - msg += "\n {0}".format("\n ".join(missing)) + msg = f"Detected {len(missing)} missing include path(s):" + msg += "\n {}".format("\n ".join(missing)) logger.die(f"{msg}\nPlease correct and try again.") return scopes def ws_file_config_scope_name(self): """Name of the config scope of this workspace's config file.""" - return "workspace:%s:%s" % (self.name, self.config_dir) + return f"workspace:{self.name}:{self.config_dir}" # return 'ws:%s' % self.name def ws_file_config_scope(self): From 6a2ae0659981e9e63f472806252ac16b68b94430 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Tue, 30 Jul 2024 15:05:04 -0700 Subject: [PATCH 2/2] Remove unused variable --- .../test/end_to_end/analyze_fom_output.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py b/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py index 5b7f965d4..6b2358471 100644 --- a/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py +++ b/lib/ramble/ramble/test/end_to_end/analyze_fom_output.py @@ -27,25 +27,6 @@ workspace = RambleCommand("workspace") -common_test_config = """ -ramble: - variables: - mpi_command: '' - batch_submit: 'batch_submit {execute_experiment}' - processes_per_node: '1' - applications: - hostname: - workloads: - local: - experiments: - test: - variables: - n_nodes: '1' - spack: - packages: {} - environments: {} -""" - def _setup_workspace(ws_name): test_config = """