From 790e3721647e1d481a9dba8e26c48df30fa9571c Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Tue, 21 Nov 2023 16:11:36 -0700 Subject: [PATCH 1/3] Set mw to `None` if omitted from config file + test --- reV/generation/cli_gen.py | 1 + tests/test_gen_config.py | 44 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/reV/generation/cli_gen.py b/reV/generation/cli_gen.py index 3882a4bdf..69c62bc49 100644 --- a/reV/generation/cli_gen.py +++ b/reV/generation/cli_gen.py @@ -43,6 +43,7 @@ def _preprocessor(config, job_name, log_directory, verbose, Updated config file. """ init_cli_logging(job_name, log_directory, verbose) + config.get("execution_control", {}).setdefault("max_workers") analysis_years = format_analysis_years(analysis_years) config["resource_file"] = _parse_res_files(config["resource_file"], diff --git a/tests/test_gen_config.py b/tests/test_gen_config.py index 45e7098f0..d5e06da1c 100644 --- a/tests/test_gen_config.py +++ b/tests/test_gen_config.py @@ -17,6 +17,7 @@ from reV.cli import main from reV.config.project_points import ProjectPoints +from reV.generation.base import BaseGen from reV.generation.generation import Gen from reV import TESTDATADIR from reV.handlers.outputs import Outputs @@ -181,6 +182,49 @@ def test_sam_config(tech): gen_dict.out['cf_profile']), msg +@pytest.mark.parametrize('expected_log_message', + ["Running serial generation for", + "Running parallel generation for"]) +def test_gen_mw_config_input(runner, clear_loggers, expected_log_message): + """Test max_workers input from gen config""" + with tempfile.TemporaryDirectory() as td: + + run_dir = os.path.join(td, 'generation') + os.mkdir(run_dir) + project_points = os.path.join(TESTDATADIR, 'config', '..', + 'config', "wtk_pp_2012_10.csv") + resource_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_{}.h5') + sam_files = {"wind0": + os.path.join(TESTDATADIR, + "SAM/wind_gen_standard_losses_0.json")} + + config = os.path.join(TESTDATADIR, 'config', 'local_wind.json') + config = safe_json_load(config) + if "parallel" in expected_log_message: + config["execution_control"].pop("max_workers") + else: + config["execution_control"]["max_workers"] = 1 + + config['project_points'] = project_points + config['resource_file'] = resource_file + config['sam_files'] = sam_files + config['log_directory'] = run_dir + + config_path = os.path.join(run_dir, 'config.json') + with open(config_path, 'w') as f: + json.dump(config, f) + + result = runner.invoke(main, ['generation', '-c', config_path]) + msg = ('Failed with error {}' + .format(traceback.print_exception(*result.exc_info))) + assert result.exit_code == 0, msg + + log_file = os.path.join(run_dir, 'generation_generation.log') + with open(log_file, "r") as fh: + assert expected_log_message in fh.read() + clear_loggers() + + def execute_pytest(capture='all', flags='-rapP'): """Execute module as pytest with detailed summary report. From 65404331410d9aa8d1c5c8fcba282281a2a00bb8 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Tue, 21 Nov 2023 16:11:54 -0700 Subject: [PATCH 2/3] Updated documentation for new behavior --- reV/generation/generation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reV/generation/generation.py b/reV/generation/generation.py index 42a8a0530..ef45e3f80 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -782,7 +782,10 @@ def run(self, out_fpath=None, max_workers=1, timeout=1800, module name and/or resource data year will get added to the output file name. By default, ``None``. max_workers : int, optional - Number of local workers to run on. By default, ``1``. + Number of local workers to run on. If ``None``, or if + running from the command line and omitting this argument + from your config file completely, this input is set to + ``os.cpu_count()``. Otherwise, the default is ``1``. timeout : int, optional Number of seconds to wait for parallel run iteration to complete before returning zeros. By default, ``1800`` From ec710b639d08351270d9b8b2519f200e57b77350 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Tue, 21 Nov 2023 16:15:14 -0700 Subject: [PATCH 3/3] Remove unused import --- tests/test_gen_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_gen_config.py b/tests/test_gen_config.py index d5e06da1c..364dd43c5 100644 --- a/tests/test_gen_config.py +++ b/tests/test_gen_config.py @@ -17,7 +17,6 @@ from reV.cli import main from reV.config.project_points import ProjectPoints -from reV.generation.base import BaseGen from reV.generation.generation import Gen from reV import TESTDATADIR from reV.handlers.outputs import Outputs