diff --git a/lib/ramble/docs/workspace_config.rst b/lib/ramble/docs/workspace_config.rst index e2d9bef11..300d310a4 100644 --- a/lib/ramble/docs/workspace_config.rst +++ b/lib/ramble/docs/workspace_config.rst @@ -280,7 +280,7 @@ there would be 4 experiments, each defined by a unique Explicit Variable Zips: ^^^^^^^^^^^^^^^^^^^^^^^ -Ramble's worksapce config contains syntax for defining explicit variable zips. +Ramble's workspace config contains syntax for defining explicit variable zips. These zips are named grouping of variables that are related and should be iterated over together when generating experiments. @@ -315,7 +315,7 @@ Below is an example showing how to define explicit zips: partition_defs: - partition - processes_per_node - matrices: + matrix: - partition_defs - n_nodes @@ -323,6 +323,103 @@ Below is an example showing how to define explicit zips: Which would result in eight experiments, crossing the ``n_nodes`` variable with the zip of ``partition`` and ``processes_per_node``. + +.. _ramble-experiment-exclusion: + +^^^^^^^^^^^^^^^^^^^^^ +Experiment Exclusion: +^^^^^^^^^^^^^^^^^^^^^ + +When writing a workspace configuration file, experiments can be explicitly +excluded from the generated set using an ``exclude`` block inside the +experiment definition. This block contains definitions of ``variables``, +``matrices``, ``zips``, and optional mathematical ``where`` statements to +define which experiments should be excluded from the generation process. + +.. code-block::yaml + :linenos: + + ramble: + variables: + mpi_command: 'mpirun -n {n_ranks}' + batch_submit: '{execute_experiment}' + n_ranks: '{n_nodes}*{processes_per_node}' + applications: + hostname: + variables: + n_threads: '1' + workloads: + serial: + variables: + processes_per_node: ['16', '32'] + partition: ['part1', 'part2'] + n_nodes: ['1', '2', '3', '4'] + experiments: + test_exp_{n_nodes}_{processes_per_node}: + variables: + n_ranks: '1' + zips: + partition_defs: + - partition + - processes_per_node + matrices: + - - partition_defs + - n_nodes + exclude: + variables: + n_nodes: ['2', '3'] + matrix: + - partition_defs + - n_nodes + +In the example above, of the eight experiments that would be generated from the +experiment definition, four will be excluded. In the defined ``exclude`` block +experiments with ``n_nodes = 2`` or ``n_nodes = 3`` will be excluded from the +generation process. + +This logic can be replicated in a ``where`` statement as well: + +.. code-block::yaml + :linenos: + + ramble: + variables: + mpi_command: 'mpirun -n {n_ranks}' + batch_submit: '{execute_experiment}' + n_ranks: '{n_nodes}*{processes_per_node}' + applications: + hostname: + variables: + n_threads: '1' + workloads: + serial: + variables: + processes_per_node: ['16', '32'] + partition: ['part1', 'part2'] + n_nodes: ['1', '2', '3', '4'] + experiments: + test_exp_{n_nodes}_{processes_per_node}: + variables: + n_ranks: '1' + zips: + partition_defs: + - partition + - processes_per_node + matrices: + - - partition_defs + - n_nodes + exclude: + where: + - '{n_nodes} == 2' + - '{n_nodes} == 3' + +``where`` statements can contain mathematical operations, but must result in a +boolean value. If any of the ``where`` statements evalaute to ``True`` within +an experiment, that experiment will be excluded from generation. To be more +explicit, all ``where`` statements are joined together with ``or`` operators. +Within any single ``where`` statement, operators can be joined together with +``and`` and ``or`` operators as well. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Environment Variable Control: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/ramble/ramble/test/end_to_end/experiment_excludes.py b/lib/ramble/ramble/test/end_to_end/experiment_excludes.py new file mode 100644 index 000000000..0249d5b32 --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/experiment_excludes.py @@ -0,0 +1,290 @@ +# Copyright 2022-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import glob + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures('mutable_config', + 'mutable_mock_workspace_path') + +workspace = RambleCommand('workspace') + + +def test_wrfv4_exclusions(mutable_config, mutable_mock_workspace_path): + test_config = """ +ramble: + variables: + mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}' + batch_submit: 'batch_submit {execute_experiment}' + partition: ['part1', 'part2'] + processes_per_node: ['16', '36'] + n_ranks: '{processes_per_node}*{n_nodes}' + n_threads: '1' + applications: + wrfv4: + variables: + env_name: ['wrfv4', 'wrfv4-portable'] + workloads: + CONUS_12km: + experiments: + scaling_{n_nodes}_{partition}_{env_name}: + success_criteria: + - name: 'timing' + mode: 'string' + match: '.*Timing for main.*' + file: '{experiment_run_dir}/rsl.out.0000' + env_vars: + set: + OMP_NUM_THREADS: '{n_threads}' + TEST_VAR: '1' + append: + - var-separator: ', ' + vars: + TEST_VAR: 'add_var' + - paths: + TEST_VAR: 'new_path' + prepend: + - paths: + TEST_VAR: 'pre_path' + unset: + - TEST_VAR + variables: + n_nodes: ['1', '2', '4', '8', '16'] + zips: + partitions: + - partition + - processes_per_node + matrix: + - n_nodes + - env_name + - partitions + exclude: + variables: + n_nodes: '1' + zips: + partitions: + - partition + - processes_per_node + matrix: + - env_name + - partitions + where: + - '{n_nodes} == 16' + spack: + concretized: true + packages: + gcc: + spack_spec: gcc@8.5.0 + intel-mpi: + spack_spec: intel-mpi@2018.4.274 + compiler: gcc + wrfv4: + spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem ~pnetcdf + compiler: gcc + wrfv4-portable: + spack_spec: 'wrf@4.2 build_type=dm+sm compile_type=em_real + nesting=basic ~chem ~pnetcdf target=x86_64' + compiler: gcc + environments: + wrfv4: + packages: + - wrfv4 + - intel-mpi + wrfv4-portable: + packages: + - wrfv4-portable + - intel-mpi +""" + + test_licenses = """ +licenses: + wrfv4: + set: + WRF_LICENSE: port@server +""" + + workspace_name = 'test_end_to_end_wrfv4' + with ramble.workspace.create(workspace_name) as ws1: + ws1.write() + + config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name) + license_path = os.path.join(ws1.config_dir, 'licenses.yaml') + + aux_software_path = os.path.join(ws1.config_dir, + ramble.workspace.auxiliary_software_dir_name) + aux_software_files = ['packages.yaml', 'my_test.sh'] + + with open(config_path, 'w+') as f: + f.write(test_config) + + with open(license_path, 'w+') as f: + f.write(test_licenses) + + for file in aux_software_files: + file_path = os.path.join(aux_software_path, file) + with open(file_path, 'w+') as f: + f.write('') + + # Write a command template + with open(os.path.join(ws1.config_dir, 'full_command.tpl'), 'w+') as f: + f.write('{command}') + + ws1._re_read() + + output = workspace('setup', '--dry-run', global_args=['-w', workspace_name]) + assert "Would download https://www2.mmm.ucar.edu/wrf/users/benchmark/v422/v42_bench_conus12km.tar.gz" in output + + # Test software directories + software_dirs = ['wrfv4.CONUS_12km', 'wrfv4-portable.CONUS_12km'] + software_base_dir = os.path.join(ws1.root, ramble.workspace.workspace_software_path) + assert os.path.exists(software_base_dir) + for software_dir in software_dirs: + software_path = os.path.join(software_base_dir, software_dir) + assert os.path.exists(software_path) + + spack_file = os.path.join(software_path, 'spack.yaml') + assert os.path.exists(spack_file) + for file in aux_software_files: + file_path = os.path.join(software_path, file) + assert os.path.exists(file_path) + + expected_experiments = [ + 'scaling_2_part1_wrfv4', + 'scaling_4_part1_wrfv4', + 'scaling_8_part1_wrfv4', + 'scaling_2_part2_wrfv4', + 'scaling_4_part2_wrfv4', + 'scaling_8_part2_wrfv4', + 'scaling_2_part1_wrfv4-portable', + 'scaling_4_part1_wrfv4-portable', + 'scaling_8_part1_wrfv4-portable', + 'scaling_2_part2_wrfv4-portable', + 'scaling_4_part2_wrfv4-portable', + 'scaling_8_part2_wrfv4-portable', + ] + + excluded_experiments = [ + 'scaling_1_part1_wrfv4', + 'scaling_16_part1_wrfv4', + 'scaling_1_part2_wrfv4', + 'scaling_16_part2_wrfv4', + 'scaling_1_part1_wrfv4-portable', + 'scaling_16_part1_wrfv4-portable', + 'scaling_1_part2_wrfv4-portable', + 'scaling_16_part2_wrfv4-portable' + ] + + for exp in excluded_experiments: + exp_dir = os.path.join(ws1.root, 'experiments', 'wrfv4', 'CONUS_12km', exp) + assert not os.path.isdir(exp_dir) + + # Test experiment directories + for exp in expected_experiments: + exp_dir = os.path.join(ws1.root, 'experiments', 'wrfv4', 'CONUS_12km', exp) + assert os.path.isdir(exp_dir) + assert os.path.exists(os.path.join(exp_dir, 'execute_experiment')) + assert os.path.exists(os.path.join(exp_dir, 'full_command')) + + with open(os.path.join(exp_dir, 'full_command'), 'r') as f: + data = f.read() + # Test the license exists + assert "export WRF_LICENSE=port@server" in data + + # Test the required environment variables exist + assert 'export OMP_NUM_THREADS="1"' in data + assert "export TEST_VAR=1" in data + assert 'unset TEST_VAR' in data + + # Test the expected portions of the execution command exist + assert "sed -i -e 's/ start_hour.*/ start_hour" in data + assert "sed -i -e 's/ restart .*/ restart" in data + assert "mpirun" in data + assert "wrf.exe" in data + + # Test the run script has a reference to the experiment log file + assert os.path.join(exp_dir, f'{exp}.out') in data + + with open(os.path.join(exp_dir, 'execute_experiment'), 'r') as f: + data = f.read() + # Test the license exists + assert "export WRF_LICENSE=port@server" in data + + # Test the required environment variables exist + assert 'export OMP_NUM_THREADS="1"' in data + assert "export TEST_VAR=1" in data + assert 'unset TEST_VAR' in data + + # Test the expected portions of the execution command exist + assert "sed -i -e 's/ start_hour.*/ start_hour" in data + assert "sed -i -e 's/ restart .*/ restart" in data + assert "mpirun" in data + assert "wrf.exe" in data + + # Test the run script has a reference to the experiment log file + assert os.path.join(exp_dir, f'{exp}.out') in data + + # Test that spack is used + assert "spack env activate" in data + + # Create fake figures of merit. + with open(os.path.join(exp_dir, 'rsl.out.0000'), 'w+') as f: + for i in range(1, 6): + f.write(f'Timing for main {i}{i}.{i}\n') + f.write('wrf: SUCCESS COMPLETE WRF\n') + + # Create files that match archive patterns + for i in range(0, 5): + new_name = 'rsl.error.000%s' % i + new_file = os.path.join(exp_dir, new_name) + + f = open(new_file, 'w+') + f.close() + + output = workspace('analyze', '-f', 'text', + 'json', 'yaml', global_args=['-w', workspace_name]) + text_simlink_results_files = glob.glob(os.path.join(ws1.root, 'results.latest.txt')) + text_results_files = glob.glob(os.path.join(ws1.root, 'results*.txt')) + json_results_files = glob.glob(os.path.join(ws1.root, 'results*.json')) + yaml_results_files = glob.glob(os.path.join(ws1.root, 'results*.yaml')) + assert len(text_simlink_results_files) == 1 + assert len(text_results_files) == 2 + assert len(json_results_files) == 2 + assert len(yaml_results_files) == 2 + + with open(text_results_files[0], 'r') as f: + data = f.read() + assert 'Average Timestep Time = 3.3 s' in data + assert 'Cumulative Timestep Time = 16.5 s' in data + assert 'Minimum Timestep Time = 1.1 s' in data + assert 'Maximum Timestep Time = 5.5 s' in data + assert 'Avg. Max Ratio Time = 0.6' in data + assert 'Number of timesteps = 5' in data + + output = workspace('archive', global_args=['-w', workspace_name]) + + assert ws1.latest_archive + assert os.path.exists(ws1.latest_archive_path) + + for exp in expected_experiments: + exp_dir = os.path.join(ws1.latest_archive_path, 'experiments', + 'wrfv4', 'CONUS_12km', exp) + assert os.path.isdir(exp_dir) + assert os.path.exists(os.path.join(exp_dir, 'execute_experiment')) + assert os.path.exists(os.path.join(exp_dir, 'full_command')) + assert os.path.exists(os.path.join(exp_dir, 'rsl.out.0000')) + for i in range(0, 5): + assert os.path.exists(os.path.join(exp_dir, f'rsl.error.000{i}')) diff --git a/lib/ramble/ramble/test/experiment_set.py b/lib/ramble/ramble/test/experiment_set.py index 7723c321f..54e1e46db 100644 --- a/lib/ramble/ramble/test/experiment_set.py +++ b/lib/ramble/ramble/test/experiment_set.py @@ -55,7 +55,8 @@ def test_single_experiment_in_set(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -93,7 +94,8 @@ def test_vector_experiment_in_set(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -133,7 +135,8 @@ def test_nonunique_vector_errors(mutable_mock_workspace_path, capsys): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) with pytest.raises(SystemExit): - exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, + None, None, None, None) captured = capsys.readouterr() assert "is not unique." in captured @@ -170,7 +173,8 @@ def test_zipped_vector_experiments(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4_2' in exp_set.experiments.keys() @@ -214,7 +218,7 @@ def test_matrix_experiments(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, - None, None) + None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -258,7 +262,7 @@ def test_matrix_multiplication_experiments(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, - None, None) + None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_2' in exp_set.experiments.keys() @@ -306,7 +310,7 @@ def test_matrix_vector_experiments(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, - None, None) + None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -353,7 +357,7 @@ def test_multi_matrix_experiments(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, - None, None) + None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4_2' in exp_set.experiments.keys() @@ -443,7 +447,7 @@ def test_experiment_names_match(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, - None, None) + None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4_2' in exp_set.experiments.keys() @@ -494,8 +498,10 @@ def test_cross_experiment_variable_references(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, None, None, None) - exp_set.set_experiment_context(exp2_name, exp2_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, + None, None, None, None) + exp_set.set_experiment_context(exp2_name, exp2_vars, None, None, None, + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -538,7 +544,8 @@ def test_cross_experiment_missing_experiment_errors(mutable_mock_workspace_path) exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -587,7 +594,7 @@ def test_n_ranks_correct_defaults(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, - None, None) + None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -630,7 +637,7 @@ def test_n_nodes_correct_defaults(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, - None, None) + None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4_2' in exp_set.experiments.keys() @@ -670,7 +677,8 @@ def test_processes_per_node_correct_defaults(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4_2' in exp_set.experiments.keys() @@ -780,7 +788,8 @@ def test_reserved_keywords_error_in_experiment(mutable_mock_workspace_path, var, exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) with pytest.raises(ramble.experiment_set.RambleVariableDefinitionError): - exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, + None, None, None, None) captured = capsys.readouterr() assert "In experiment basic.test_wl.series1_{n_ranks}_{processes_per_node}" in captured assert f"{var}" in captured @@ -828,7 +837,8 @@ def test_missing_required_keyword_errors(mutable_mock_workspace_path, var, capsy exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) with pytest.raises(ramble.experiment_set.RambleVariableDefinitionError): - exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, + None, None, None, None) captured = capsys.readouterr() assert f'Required key "{var}" is not defined' in captured.err assert 'One or more required keys are not defined within an experiment.' \ @@ -886,9 +896,10 @@ def test_chained_experiments_populate_new_experiments(mutable_mock_workspace_pat exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, + None, None, None, None) exp_set.set_experiment_context(exp2_name, exp2_vars, None, None, None, None, None, - exp2_chains) + exp2_chains, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series2_4' in \ @@ -949,9 +960,10 @@ def test_chained_experiment_has_correct_directory(mutable_mock_workspace_path, c exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, + None, None, None, None) exp_set.set_experiment_context(exp2_name, exp2_vars, None, None, None, None, None, - exp2_chains) + exp2_chains, None) exp_set.build_experiment_chains() parent_name = 'basic.test_wl.series2_4' @@ -1011,9 +1023,10 @@ def test_chained_cycle_errors(mutable_mock_workspace_path, capsys): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, + None, None, None, None) exp_set.set_experiment_context(exp2_name, exp2_vars, None, None, None, None, None, - exp2_chains) + exp2_chains, None) with pytest.raises(ChainCycleDetectedError): exp_set.build_experiment_chains() captured = capsys.readouterr @@ -1063,9 +1076,10 @@ def test_chained_invalid_order_errors(mutable_mock_workspace_path, capsys): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) - exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, None, None, None) + exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, + None, None, None, None) exp_set.set_experiment_context(exp2_name, exp2_vars, None, None, None, None, None, - exp2_chains) + exp2_chains, None) with pytest.raises(InvalidChainError): exp_set.build_experiment_chains() captured = capsys.readouterr @@ -1133,7 +1147,7 @@ def test_modifiers_set_correctly(mutable_mock_workspace_path, capsys): exp_set.set_application_context(app_name, app_vars, None, None, None, None, app_mods) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None, wl_mods) exp_set.set_experiment_context(exp1_name, exp1_vars, None, None, None, None, - None, None, exp1_mods) + None, None, exp1_mods, None) assert 'basic.test_wl.test1' in exp_set.experiments app_inst = exp_set.experiments['basic.test_wl.test1'] @@ -1183,7 +1197,7 @@ def test_explicit_zips_work(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, None, None, - None, None, None) + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -1229,7 +1243,7 @@ def test_explicit_zips_in_matrix(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, exp_matrices, None, - None, None, None) + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4_1' in exp_set.experiments.keys() @@ -1279,7 +1293,7 @@ def test_explicit_zips_unconsumed(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, exp_matrices, None, - None, None, None) + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4_1' in exp_set.experiments.keys() @@ -1327,7 +1341,7 @@ def test_single_var_explicit_zip(mutable_mock_workspace_path): exp_set.set_application_context(app_name, app_vars, None, None, None, None) exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, None, None, - None, None, None) + None, None, None, None) exp_set.build_experiment_chains() assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() @@ -1372,7 +1386,7 @@ def test_zip_undefined_var_errors(mutable_mock_workspace_path, capsys): exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) with pytest.raises(SystemExit): exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, None, None, - None, None, None) + None, None, None, None) captured = capsys.readouterr() assert "An undefined variable foo is defined in zip test_zip" in captured @@ -1416,7 +1430,7 @@ def test_zip_multi_use_var_errors(mutable_mock_workspace_path, capsys): exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) with pytest.raises(SystemExit): exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, None, None, - None, None, None) + None, None, None, None) captured = capsys.readouterr() assert 'Variable n_nodes is used across multiple zips' in captured @@ -1459,7 +1473,7 @@ def test_zip_non_list_var_errors(mutable_mock_workspace_path, capsys): exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) with pytest.raises(SystemExit): exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, None, None, - None, None, None) + None, None, None, None) captured = capsys.readouterr() assert 'Variable exp_var1 in zip test_zip does not refer to a vector' in captured @@ -1502,8 +1516,204 @@ def test_zip_variable_lengths_errors(mutable_mock_workspace_path, capsys): exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) with pytest.raises(SystemExit): exp_set.set_experiment_context(exp_name, exp_vars, None, exp_zips, None, None, - None, None, None) + None, None, None, None) captured = capsys.readouterr() assert 'Variable exp_var2 in zip test_zip' in captured assert 'has a length of 1 which differs from' in captured assert 'the current max of 2' in captured + + +def test_vector_experiment_with_explicit_excludes(mutable_mock_workspace_path): + workspace('create', 'test') + + assert 'test' in workspace('list') + + with ramble.workspace.read('test') as ws: + exp_set = ramble.experiment_set.ExperimentSet(ws) + + app_name = 'basic' + app_vars = { + 'app_var1': '1', + 'app_var2': '2', + 'n_ranks': '{processes_per_node}*{n_nodes}', + 'mpi_command': '', + 'batch_submit': '' + } + + wl_name = 'test_wl' + wl_vars = { + 'wl_var1': '1', + 'wl_var2': '2', + 'processes_per_node': '2' + } + exp_name = 'series1_{n_ranks}' + exp_vars = { + 'exp_var1': '1', + 'exp_var2': '2', + 'n_nodes': ['2', '4'] + } + + exp_exclude = { + 'variables': { + 'n_nodes': ['4'] + } + } + + exp_set.set_application_context(app_name, app_vars, None, None, None, None) + exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, + None, None, None, exp_exclude) + exp_set.build_experiment_chains() + + assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_8' not in exp_set.experiments.keys() + + +def test_matrix_experiments_explicit_excludes(mutable_mock_workspace_path): + workspace('create', 'test') + + assert 'test' in workspace('list') + + with ramble.workspace.read('test') as ws: + exp_set = ramble.experiment_set.ExperimentSet(ws) + + app_name = 'basic' + app_vars = { + 'app_var1': '1', + 'app_var2': '2', + 'n_ranks': '{processes_per_node}*{n_nodes}', + 'mpi_command': '', + 'batch_submit': '' + } + + wl_name = 'test_wl' + wl_vars = { + 'wl_var1': '1', + 'wl_var2': '2', + 'processes_per_node': '2' + } + exp_name = 'series1_{n_ranks}' + exp_vars = { + 'exp_var1': '1', + 'exp_var2': '2', + 'n_nodes': ['2', '3'] + } + + exp_matrices = [ + ['n_nodes'] + ] + + exp_exclude = { + 'variables': { + 'n_nodes': ['3'], + }, + 'matrix': ['n_nodes'] + } + + exp_set.set_application_context(app_name, app_vars, None, None, None, None) + exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, exp_matrices, None, + None, None, None, exp_exclude) + exp_set.build_experiment_chains() + + assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_6' not in exp_set.experiments.keys() + + +def test_vector_experiment_with_where_excludes(mutable_mock_workspace_path): + workspace('create', 'test') + + assert 'test' in workspace('list') + + with ramble.workspace.read('test') as ws: + exp_set = ramble.experiment_set.ExperimentSet(ws) + + app_name = 'basic' + app_vars = { + 'app_var1': '1', + 'app_var2': '2', + 'n_ranks': '{processes_per_node}*{n_nodes}', + 'mpi_command': '', + 'batch_submit': '' + } + + wl_name = 'test_wl' + wl_vars = { + 'wl_var1': '1', + 'wl_var2': '2', + 'processes_per_node': '2' + } + exp_name = 'series1_{n_ranks}' + exp_vars = { + 'exp_var1': '1', + 'exp_var2': '2', + 'n_nodes': ['1', '2', '3', '4', '5'] + } + + exp_exclude = { + 'where': [ + '{n_nodes} > 2 and {n_nodes} < 5' + ] + } + + exp_set.set_application_context(app_name, app_vars, None, None, None, None) + exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, + None, None, None, exp_exclude) + exp_set.build_experiment_chains() + + assert 'basic.test_wl.series1_2' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_6' not in exp_set.experiments.keys() + assert 'basic.test_wl.series1_8' not in exp_set.experiments.keys() + assert 'basic.test_wl.series1_10' in exp_set.experiments.keys() + + +def test_vector_experiment_with_multi_where_excludes(mutable_mock_workspace_path): + workspace('create', 'test') + + assert 'test' in workspace('list') + + with ramble.workspace.read('test') as ws: + exp_set = ramble.experiment_set.ExperimentSet(ws) + + app_name = 'basic' + app_vars = { + 'app_var1': '1', + 'app_var2': '2', + 'n_ranks': '{processes_per_node}*{n_nodes}', + 'mpi_command': '', + 'batch_submit': '' + } + + wl_name = 'test_wl' + wl_vars = { + 'wl_var1': '1', + 'wl_var2': '2', + 'processes_per_node': '2' + } + exp_name = 'series1_{n_ranks}' + exp_vars = { + 'exp_var1': '1', + 'exp_var2': '2', + 'n_nodes': ['1', '2', '3', '4', '5'] + } + + exp_exclude = { + 'where': [ + '{n_nodes} < 2', + '{n_nodes} > 4' + ] + } + + exp_set.set_application_context(app_name, app_vars, None, None, None, None) + exp_set.set_workload_context(wl_name, wl_vars, None, None, None, None) + exp_set.set_experiment_context(exp_name, exp_vars, None, None, None, None, + None, None, None, exp_exclude) + exp_set.build_experiment_chains() + + assert 'basic.test_wl.series1_2' not in exp_set.experiments.keys() + assert 'basic.test_wl.series1_4' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_6' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_8' in exp_set.experiments.keys() + assert 'basic.test_wl.series1_10' not in exp_set.experiments.keys() diff --git a/lib/ramble/ramble/test/software_environment.py b/lib/ramble/ramble/test/software_environment.py index 58a32c2e8..8c81f8eff 100644 --- a/lib/ramble/ramble/test/software_environment.py +++ b/lib/ramble/ramble/test/software_environment.py @@ -631,3 +631,131 @@ def test_environment_warns_with_pkg_compiler(mutable_mock_workspace_path, capsys 'in the package list. These include:' in captured.err assert 'Package: basic, Compiler: test_comp' in captured.err + + +def test_package_vector_expansion_exclusions(mutable_mock_workspace_path): + ws_name = 'test_package_vector_expansion_exclusions' + workspace('create', ws_name) + + assert ws_name in workspace('list') + + with ramble.workspace.read(ws_name) as ws: + spack_dict = ws.get_spack_dict() + + spack_dict['packages'] = {} + spack_dict['packages']['basic-{arch}'] = { + 'spack_spec': 'basic@1.1 target={arch}', + 'variables': { + 'arch': ['x86_64', 'x86_64_v4'] + }, + 'exclude': { + 'variables': { + 'arch': 'x86_64_v4' + } + } + } + spack_dict['environments'] = { + 'basic': { + 'packages': ['basic-x86_64'] + } + } + + software_environments = ramble.software_environments.SoftwareEnvironments(ws) + + assert len(software_environments._packages.keys()) == 1 + assert 'basic-x86_64' in software_environments._packages.keys() + assert 'basic' in software_environments._environments.keys() + assert 'basic-x86_64' in software_environments._environments['basic']['packages'] + assert 'basic-x86_64_v4' not in software_environments._packages.keys() + assert 'basic-x86_64_v4' not in software_environments._environments['basic']['packages'] + + +def test_package_matrix_expansion_exclusions(mutable_mock_workspace_path): + ws_name = 'test_package_matrix_expansion_exclusions' + workspace('create', ws_name) + + assert ws_name in workspace('list') + + with ramble.workspace.read(ws_name) as ws: + spack_dict = ws.get_spack_dict() + + spack_dict['packages'] = {} + spack_dict['packages']['basic-{ver}-{arch}'] = { + 'spack_spec': 'basic@{ver} target={arch}', + 'variables': { + 'arch': ['x86_64', 'x86_64_v4'], + 'ver': ['1.1', '2.0'] + }, + 'matrix': ['arch', 'ver'], + 'exclude': { + 'variables': { + 'arch': ['x86_64_v4'], + 'ver': ['2.0'], + }, + 'matrix': ['arch', 'ver'], + } + } + spack_dict['environments'] = { + 'basic': { + 'packages': [ + 'basic-1.1-x86_64', + 'basic-2.0-x86_64', + 'basic-1.1-x86_64_v4', + ] + } + } + + software_environments = ramble.software_environments.SoftwareEnvironments(ws) + + assert len(software_environments._packages.keys()) == 3 + assert 'basic-1.1-x86_64' in software_environments._packages.keys() + assert 'basic-2.0-x86_64' in software_environments._packages.keys() + assert 'basic-1.1-x86_64_v4' in software_environments._packages.keys() + assert 'basic-2.0-x86_64_v4' not in software_environments._packages.keys() + assert 'basic' in software_environments._environments.keys() + assert 'basic-1.1-x86_64' in software_environments._environments['basic']['packages'] + assert 'basic-2.0-x86_64' in software_environments._environments['basic']['packages'] + assert 'basic-1.1-x86_64_v4' in software_environments._environments['basic']['packages'] + assert 'basic-2.0-x86_64_v4' not in \ + software_environments._environments['basic']['packages'] + + +def test_environment_vector_expansion_exclusion(mutable_mock_workspace_path): + ws_name = 'test_package_vector_expansion_exclusions' + workspace('create', ws_name) + + assert ws_name in workspace('list') + + with ramble.workspace.read(ws_name) as ws: + spack_dict = ws.get_spack_dict() + + spack_dict['packages'] = {} + spack_dict['packages']['basic-{arch}'] = { + 'spack_spec': 'basic@1.1 target={arch}', + 'variables': { + 'arch': ['x86_64', 'x86_64_v4'] + }, + } + spack_dict['environments'] = { + 'basic-{arch}': { + 'packages': ['basic-{arch}'], + 'variables': { + 'arch': ['x86_64', 'x86_64_v4'] + }, + 'exclude': { + 'variables': { + 'arch': 'x86_64_v4' + } + } + } + } + + software_environments = ramble.software_environments.SoftwareEnvironments(ws) + + assert len(software_environments._packages.keys()) == 2 + assert len(software_environments._environments.keys()) == 1 + assert 'basic-x86_64' in software_environments._packages.keys() + assert 'basic-x86_64_v4' in software_environments._packages.keys() + assert 'basic-x86_64' in software_environments._environments.keys() + assert 'basic-x86_64_v4' not in software_environments._environments.keys() + assert 'basic-x86_64' in software_environments._environments['basic-x86_64']['packages']