diff --git a/lib/ramble/ramble/application.py b/lib/ramble/ramble/application.py index 335da2852..b5bddf74e 100644 --- a/lib/ramble/ramble/application.py +++ b/lib/ramble/ramble/application.py @@ -1089,7 +1089,9 @@ def format_context(context_match, context_format): (context, context_name)) active_contexts[context] = context_name - fom_values[context_name] = {} + + if context_name not in fom_values: + fom_values[context_name] = {} for fom in file_conf['foms']: fom_conf = foms[fom] @@ -1119,7 +1121,9 @@ def format_context(context_match, context_format): fom_val = fom_match.group(fom_conf['group']) fom_values[context][fom_name] = { 'value': fom_val, - 'units': fom_conf['units'] + 'units': fom_conf['units'], + 'origin': fom_conf['origin'], + 'origin_type': fom_conf['origin_type'] } # Test all non-file based success criteria @@ -1220,6 +1224,10 @@ def _analysis_dicts(self, criteria_list): # Remap fom / context / file data # Could push this into the language features in the future fom_definitions = self.figures_of_merit.copy() + for fom, fom_def in fom_definitions.items(): + fom_def['origin'] = self.name + fom_def['origin_type'] = 'application' + fom_contexts = self.figure_of_merit_contexts.copy() for mod in self._modifier_instances: fom_contexts.update(mod.figure_of_merit_contexts) @@ -1227,7 +1235,7 @@ def _analysis_dicts(self, criteria_list): mod_vars = mod.modded_variables(self) for fom, fom_def in mod.figures_of_merit.items(): - fom_definitions[fom] = {} + fom_definitions[fom] = {'origin': f'{mod}', 'origin_type': 'modifier'} for attr in fom_def.keys(): if isinstance(fom_def[attr], list): fom_definitions[fom][attr] = fom_def[attr].copy() @@ -1251,7 +1259,9 @@ def _analysis_dicts(self, criteria_list): 'regex': re.compile(r'%s' % conf['regex']), 'contexts': [], 'group': conf['group_name'], - 'units': conf['units'] + 'units': conf['units'], + 'origin': conf['origin'], + 'origin_type': conf['origin_type'] } if conf['contexts']: foms[fom]['contexts'].extend(conf['contexts']) diff --git a/lib/ramble/ramble/language/shared_language.py b/lib/ramble/ramble/language/shared_language.py index 791475368..35e58a2a0 100644 --- a/lib/ramble/ramble/language/shared_language.py +++ b/lib/ramble/ramble/language/shared_language.py @@ -201,7 +201,7 @@ def _execute_success_criteria(obj): 'file': file, 'fom_name': fom_name, 'fom_context': fom_context, - 'formula': formula, + 'formula': formula } return _execute_success_criteria diff --git a/lib/ramble/ramble/test/end_to_end/shared_context.py b/lib/ramble/ramble/test/end_to_end/shared_context.py new file mode 100644 index 000000000..448f801da --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/shared_context.py @@ -0,0 +1,87 @@ +# Copyright 2022-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import glob + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures('mutable_config', + 'mutable_mock_workspace_path', + 'mock_applications', + 'mock_modifiers', + ) + +workspace = RambleCommand('workspace') + + +def test_shared_contexts( + mutable_config, + mutable_mock_workspace_path, + mock_applications, + mock_modifiers): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: 'batch_submit {execute_experiment}' + partition: 'part1' + processes_per_node: '1' + n_threads: '1' + applications: + shared-context: + workloads: + test_wl: + experiments: + simple_test: + modifiers: + - name: test-mod + variables: + n_nodes: 1 + spack: + concretized: true + packages: {} + environments: {} +""" + workspace_name = 'test_shared_context' + with ramble.workspace.create(workspace_name) as ws: + ws.write() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path, 'w+') as f: + f.write(test_config) + ws._re_read() + + workspace('setup', '--dry-run', global_args=['-w', workspace_name]) + + # Create fake figures of merit. + exp_dir = os.path.join(ws.root, 'experiments', 'shared-context', 'test_wl', 'simple_test') + with open(os.path.join(exp_dir, 'simple_test.out'), 'w+') as f: + f.write('fom_context mod_context\n') + f.write('123.4 seconds app_fom\n') + + with open(os.path.join(exp_dir, 'test_analysis.log'), 'w+') as f: + f.write("fom_contextFOM_GOES_HERE") + + workspace('analyze', '-f', 'text', 'json', global_args=['-w', workspace_name]) + + results_files = glob.glob(os.path.join(ws.root, 'results.latest.txt')) + + with open(results_files[0], 'r') as f: + data = f.read() + assert 'matched_shared_context' in data # find the merged context + assert 'test_fom = 123.4' in data # from the app + assert 'shared_context_fom' in data # from the mod diff --git a/lib/ramble/ramble/workspace/workspace.py b/lib/ramble/ramble/workspace/workspace.py index a26f63a13..5259838a7 100644 --- a/lib/ramble/ramble/workspace/workspace.py +++ b/lib/ramble/ramble/workspace/workspace.py @@ -1157,7 +1157,13 @@ def dump_results(self, output_formats=['text']): f.write(' %s figures of merit:\n' % context['name']) for fom in context['foms']: - output = '%s = %s %s' % (fom['name'], + name = fom['name'] + if fom['origin_type'] == 'modifier': + delim = '::' + mod = fom['origin'] + name = f"{fom['origin_type']}{delim}{mod}{delim}{name}" + + output = '%s = %s %s' % (name, fom['value'], fom['units']) f.write(' %s\n' % (output.strip())) diff --git a/var/ramble/repos/builtin.mock/applications/shared-context/application.py b/var/ramble/repos/builtin.mock/applications/shared-context/application.py new file mode 100644 index 000000000..1bc000c4c --- /dev/null +++ b/var/ramble/repos/builtin.mock/applications/shared-context/application.py @@ -0,0 +1,34 @@ +# Copyright 2022-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from ramble.appkit import * + + +class SharedContext(ExecutableApplication): + name = "shared-context" + + executable('foo', 'bar', use_mpi=False) + executable('bar', 'baz', use_mpi=True) + + input_file('input', url='file:///tmp/test_file.log', + description='Not a file', extension='.log') + + workload('test_wl', executable='foo', input='input') + workload('test_wl2', executable='bar', input='input') + + workload_variable('my_var', default='1.0', + description='Example var', + workload='test_wl') + + archive_pattern('{experiment_run_dir}/archive_test.*') + + figure_of_merit('test_fom', + fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', + group_name='test', units='s', contexts=['test_shared_context']) + + figure_of_merit_context('test_shared_context', regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', output_format='matched_shared_context') diff --git a/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py index 960f1e866..186c244d8 100644 --- a/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py @@ -31,8 +31,13 @@ class TestMod(BasicModifier): figure_of_merit('test_mod_fom', fom_regex=fom_regex, group_name='fom', units='', log_file='{analysis_log}', contexts=['test_mod_context']) + figure_of_merit('shared_context_fom', fom_regex=fom_regex, group_name='fom', + units='', log_file='{analysis_log}', contexts=['test_shared_context']) + figure_of_merit_context('test_mod_context', regex=fom_regex, output_format='{context}') + figure_of_merit_context('test_shared_context', regex=fom_regex, output_format='matched_shared_context') + register_builtin('test_builtin', required=True, injection_method='append') def test_builtin(self):