From ec1529ae6ef458d7c174485d88831d4b4ba7a23d Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 23 Nov 2024 12:28:09 -0800 Subject: [PATCH] Change 'salish' submit job command to 'bash' (#80) * Drop scheduler directives from script for `salish` `salish` no longer uses a scheduler. So, its run execution script no longer needs either `#SBATCH` or `#PBS` directives sections. Also refactored `test_salish` to use `monkeypatch.setattr` instead of `patch`. * Refactor tests to use monkeypatch for setting SYSTEM Replaced `with patch` context managers with `monkeypatch.setattr` in all TestModules test cases to improve code readability and consistency. Added new test for "salish" system to ensure future-proofing and full coverage of available systems. * Implement stdout & stderr redirection for `salish` This commit adds redirection of stdout and stderr to files within `${RESULTS_DIR}` for the `salish` system, ensuring logging of output and error streams. New tests were added to verify the functionality for both the `--deflate` and non-deflate cases. Necessary changes were made in the `_execute()` function and corresponding test cases to support this redirection. * Refactor execution call in run.py Moved the _execute() call out of the script string construction for compatibility with Python 3.11. This change assigns the _execute call result to a variable before joining it into the final script. * Change job submission command for 'salish' to 'bash' Updated the job submission command for the 'salish' system from 'qsub' to 'bash'. Added logic to handle job submissions specifically for 'bash' and adjusted unit tests accordingly to ensure compatibility and functionality. --- salishsea_cmd/run.py | 103 +++++++---- tests/test_run.py | 415 +++++++++++++++++++++++++++---------------- 2 files changed, 334 insertions(+), 184 deletions(-) diff --git a/salishsea_cmd/run.py b/salishsea_cmd/run.py index bd1f9b6..493171b 100644 --- a/salishsea_cmd/run.py +++ b/salishsea_cmd/run.py @@ -275,7 +275,7 @@ def run( "graham": "sbatch", "omega": "qsub -q mpi", # optimum.eoas.ubc.ca login node "orcinus": "qsub", - "salish": "qsub", + "salish": "bash", "seawolf1": "qsub", # orcinus.westgrid.ca login node "seawolf2": "qsub", # orcinus.westgrid.ca login node "seawolf3": "qsub", # orcinus.westgrid.ca login node @@ -543,12 +543,23 @@ def _build_tmp_run_dir( def _submit_job(batch_file, queue_job_cmd, waitjob): if waitjob != "0": - depend_opt = ( - "-W depend=afterok" if queue_job_cmd.startswith("qsub") else "-d afterok" - ) + match queue_job_cmd.split(" ")[0]: + case "qsub": + depend_opt = "-W depend=afterok" + case "sbatch": + depend_opt = "-d afterok" + case _: + log.error( + f"dependent jobs are not available for systems that launch jobs with " + f"{queue_job_cmd}" + ) + raise SystemExit(2) cmd = f"{queue_job_cmd} {depend_opt}:{waitjob} {batch_file}" else: cmd = f"{queue_job_cmd} {batch_file}" + if queue_job_cmd == "bash": + subprocess.Popen(shlex.split(cmd), start_new_session=True) + return f"{cmd} started" submit_job_msg = subprocess.run( shlex.split(cmd), check=True, universal_newlines=True, stdout=subprocess.PIPE ).stdout @@ -634,7 +645,10 @@ def _build_batch_script( email = get_run_desc_value(run_desc, ("email",), fatal=False) except KeyError: email = f"{os.getenv('USER')}@eoas.ubc.ca" - if SYSTEM in {"beluga", "cedar", "graham", "sockeye"}: + if SYSTEM == "salish": + # salish doesn't use a scheduler, so no sbatch or PBS directives in its script + pass + elif SYSTEM in {"beluga", "cedar", "graham", "sockeye"}: procs_per_node = { "beluga": 40 if not cores_per_node else int(cores_per_node), "cedar": 48 if not cores_per_node else int(cores_per_node), @@ -654,7 +668,6 @@ def _build_batch_script( "omega": 20 if not cores_per_node else int(cores_per_node), "sigma": 20 if not cores_per_node else int(cores_per_node), "orcinus": 12 if not cores_per_node else int(cores_per_node), - "salish": 0, # solish only has 1 node; 0 gets things right "seawolf1": 12 if not cores_per_node else int(cores_per_node), "seawolf2": 12 if not cores_per_node else int(cores_per_node), "seawolf3": 12 if not cores_per_node else int(cores_per_node), @@ -668,12 +681,21 @@ def _build_batch_script( f"{_pbs_directives(run_desc, nemo_processors + xios_processors, email, results_dir, procs_per_node, cpu_arch, )}\n", ) ) + redirect_stdout_stderr = True if SYSTEM == "salish" else False + execute_section = _execute( + nemo_processors, + xios_processors, + deflate, + max_deflate_jobs, + separate_deflate, + redirect_stdout_stderr, + ) script = "\n".join( ( script, f"{_definitions(run_desc, desc_file, run_dir, results_dir, deflate)}\n" f"{_modules()}\n" - f"{_execute(nemo_processors, xios_processors, deflate, max_deflate_jobs, separate_deflate,)}\n" + f"{execute_section}\n" f"{_fix_permissions()}\n" f"{_cleanup()}", ) @@ -1032,8 +1054,18 @@ def _modules(): def _execute( - nemo_processors, xios_processors, deflate, max_deflate_jobs, separate_deflate + nemo_processors, + xios_processors, + deflate, + max_deflate_jobs, + separate_deflate, + redirect_stdout_stderr, ): + redirect = ( + "" + if not redirect_stdout_stderr + else " >>${RESULTS_DIR}/stdout 2>>${RESULTS_DIR}/stderr" + ) mpirun = { "beluga": "mpirun", "cedar": "mpirun", @@ -1064,34 +1096,35 @@ def _execute( }.get(SYSTEM, f"{mpirun} -np {nemo_processors} ./nemo.exe") if xios_processors: mpirun = { - "beluga": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "cedar": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "delta": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", - "graham": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "omega": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", - "orcinus": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "salish": f"{mpirun} : --bind-to none -np {xios_processors} ./xios_server.exe", - "seawolf1": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "seawolf2": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "seawolf3": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "sigma": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", - "sockeye": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", + "beluga": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "cedar": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "delta": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", + "graham": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "omega": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", + "orcinus": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "salish": f"{mpirun} : --bind-to none -np {xios_processors} ./xios_server.exe{redirect}", + "seawolf1": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "seawolf2": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "seawolf3": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "sigma": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", + "sockeye": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", }.get( SYSTEM, - f"{mpirun} : -np {xios_processors} ./xios_server.exe", + f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", ) + redirect = "" if not redirect_stdout_stderr else " >>${RESULTS_DIR}/stdout" script = textwrap.dedent( f"""\ mkdir -p ${{RESULTS_DIR}} cd ${{WORK_DIR}} - echo "working dir: $(pwd)" + echo "working dir: $(pwd)"{redirect} - echo "Starting run at $(date)" + echo "Starting run at $(date)"{redirect} {mpirun} MPIRUN_EXIT_CODE=$? - echo "Ended run at $(date)" + echo "Ended run at $(date)"{redirect} - echo "Results combining started at $(date)" + echo "Results combining started at $(date)"{redirect} """ ) if SYSTEM in {"delta", "omega", "sigma"}: @@ -1109,16 +1142,16 @@ def _execute( """ ) script += textwrap.dedent( - """\ - ${COMBINE} ${RUN_DESC} --debug - echo "Results combining ended at $(date)" + f"""\ + ${{COMBINE}} ${{RUN_DESC}} --debug + echo "Results combining ended at $(date)"{redirect} """ ) if deflate and not separate_deflate: script += textwrap.dedent( - """\ + f"""\ - echo "Results deflation started at $(date)" + echo "Results deflation started at $(date)"{redirect} """ ) if SYSTEM in {"beluga", "cedar", "graham"}: @@ -1135,15 +1168,15 @@ def _execute( ${{DEFLATE}} *_ptrc_T*.nc *_prod_T*.nc *_carp_T*.nc *_grid_[TUVW]*.nc \\ *_turb_T*.nc *_dia[12n]_T*.nc FVCOM*.nc Slab_[UV]*.nc *_mtrc_T*.nc \\ --jobs {max_deflate_jobs} --debug - echo "Results deflation ended at $(date)" + echo "Results deflation ended at $(date)"{redirect} """ ) script += textwrap.dedent( - """\ + f"""\ - echo "Results gathering started at $(date)" - ${GATHER} ${RESULTS_DIR} --debug - echo "Results gathering ended at $(date)" + echo "Results gathering started at $(date)"{redirect} + ${{GATHER}} ${{RESULTS_DIR}} --debug + echo "Results gathering ended at $(date)"{redirect} """ ) return script diff --git a/tests/test_run.py b/tests/test_run.py index abed1f1..9bbcb7f 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -18,6 +18,7 @@ """SalishSeaCmd run sub-command plug-in unit tests """ +import logging import os import shlex import subprocess @@ -143,8 +144,8 @@ class TestRun: (True, 4, "delta", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "graham", "sbatch", "Submitted batch job 43"), (True, 4, "graham", "sbatch", "Submitted batch job 43"), - (False, 0, "salish", "qsub", "43.master"), - (True, 4, "salish", "qsub", "43.master"), + (False, 0, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), + (True, 4, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), (False, 0, "sigma", "qsub -q mpi", "43.admin.default.domain"), (True, 4, "sigma", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "sockeye", "sbatch", "Submitted batch job 43"), @@ -176,6 +177,7 @@ def test_run_submit( queue_job_cmd, submit_job_msg, tmpdir, + monkeypatch, ): p_run_dir = tmpdir.ensure_dir("run_dir") p_results_dir = tmpdir.ensure_dir("results_dir") @@ -200,10 +202,12 @@ def test_run_submit( Path(str(p_run_dir), "SalishSeaNEMO.sh"), ) m_sj.return_value = submit_job_msg - with patch("salishsea_cmd.run.SYSTEM", system): - submit_job_msg = salishsea_cmd.run.run( - Path("SalishSea.yaml"), Path(str(p_results_dir)) - ) + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + submit_job_msg = salishsea_cmd.run.run( + Path("SalishSea.yaml"), Path(str(p_results_dir)) + ) + m_sj.assert_called_once_with( Path(str(p_run_dir), "SalishSeaNEMO.sh"), queue_job_cmd, waitjob="0" ) @@ -220,8 +224,8 @@ def test_run_submit( (True, 4, "delta", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "graham", "sbatch", "Submitted batch job 43"), (True, 4, "graham", "sbatch", "Submitted batch job 43"), - (False, 0, "salish", "qsub", "43.master"), - (True, 4, "salish", "qsub", "43.master"), + (False, 0, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), + (True, 4, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), (False, 0, "sigma", "qsub -q mpi", "43.admin.default.domain"), (True, 4, "sigma", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "sockeye", "sbatch", "Submitted batch job 43"), @@ -386,8 +390,8 @@ def test_run_no_submit_w_separate_deflate( (True, 4, "delta", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "graham", "sbatch", "Submitted batch job 43"), (True, 4, "graham", "sbatch", "Submitted batch job 43"), - (False, 0, "salish", "qsub", "43.master"), - (True, 4, "salish", "qsub", "43.master"), + (False, 0, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), + (True, 4, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), (False, 0, "sigma", "qsub -q mpi", "43.admin.default.domain"), (True, 4, "sigma", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "sockeye", "sbatch", "Submitted batch job 43"), @@ -520,8 +524,8 @@ def test_run_separate_deflate( ("Submitted batch job 43", "Submitted batch job 44"), "Submitted batch job 43", ), - (False, 0, "salish", "qsub", ("43.master", "44.master"), "43.master"), - (True, 4, "salish", "qsub", ("43.master", "44.master"), "43.master"), + (False, 0, "salish", "bash", ("43.master", "44.master"), "43.master"), + (True, 4, "salish", "bash", ("43.master", "44.master"), "43.master"), ( False, 0, @@ -748,8 +752,8 @@ def test_segmented_run_restart_dirs( ("Submitted batch job 43", "Submitted batch job 44"), "Submitted batch job 43", ), - (False, 0, "salish", "qsub", ("43.master", "44.master"), "43.master"), - (True, 4, "salish", "qsub", ("43.master", "44.master"), "43.master"), + (False, 0, "salish", "bash", ("43.master", "44.master"), "43.master"), + (True, 4, "salish", "bash", ("43.master", "44.master"), "43.master"), ( False, 0, @@ -1824,48 +1828,95 @@ def test_build_tmp_run_dir_separate_deflate( assert batch_file == Path(str(p_run_dir)) / "SalishSeaNEMO.sh" -@patch("salishsea_cmd.run.subprocess.run") -@pytest.mark.parametrize( - "queue_job_cmd, depend_flag, depend_option, submit_job_msg", - [ - ("sbatch", "-d", "afterok", "Submitted batch job 43"), - ("qsub", "-W", "depend=afterok", "43.orca2.ibb"), - ("qsub -q mpi", "-W", "depend=afterok", "43.admin.default.domain"), - ], -) class TestSubmitJob: """Unit tests for _submit_job() function.""" - def test_submit_job( - self, m_run, queue_job_cmd, depend_flag, depend_option, submit_job_msg - ): + @pytest.mark.parametrize( + "queue_job_cmd, msg", + ( + ("bash", "bash run_dir/SalishSeaNEMO.sh started"), # salish + ("sbatch", "Submitted batch job 43"), # sockeye & Alliance HPC clusters + ("qsub", "43.orca2.ibb"), # orcinus + ("qsub -q mpi", "43.admin.default.domain"), # optimum + ), + ) + def test_submit_job(self, queue_job_cmd, msg, monkeypatch): + def mock_subprocess_Popen(cmd, start_new_session): + pass + + monkeypatch.setattr(subprocess, "Popen", mock_subprocess_Popen) + + def mock_subprocess_run(cmd, check, universal_newlines, stdout): + return subprocess.CompletedProcess(cmd, 0, msg) + + monkeypatch.setattr(subprocess, "run", mock_subprocess_run) + submit_job_msg = salishsea_cmd.run._submit_job( Path("run_dir", "SalishSeaNEMO.sh"), queue_job_cmd, "0" ) - m_run.assert_called_once_with( - shlex.split(f"{queue_job_cmd} {Path('run_dir')}/SalishSeaNEMO.sh"), - check=True, - universal_newlines=True, - stdout=subprocess.PIPE, - ) - assert submit_job_msg == submit_job_msg - def test_submit_job_w_waitjob( - self, m_run, queue_job_cmd, depend_flag, depend_option, submit_job_msg + assert submit_job_msg == msg + + @pytest.mark.parametrize( + "queue_job_cmd, depend_flag, depend_option, msg", + [ + # sockeye & Alliance HPC clusters + ( + "sbatch", + "-d", + "afterok", + "Submitted batch job 43", + ), + # sockeye & Alliance HPC clusters + ( + "qsub", + "-W", + "depend=afterok", + "43.orca2.ibb", + ), + # optimum + ( + "qsub -q mpi", + "-W", + "depend=afterok", + "43.admin.default.domain", + ), + ], + ) + def test_submit_job_with_waitjob( + self, + queue_job_cmd, + depend_flag, + depend_option, + msg, + monkeypatch, ): + def mock_subprocess_run(cmd, check, universal_newlines, stdout): + return subprocess.CompletedProcess(cmd, 0, msg) + + monkeypatch.setattr(subprocess, "run", mock_subprocess_run) + submit_job_msg = salishsea_cmd.run._submit_job( Path("run_dir", "SalishSeaNEMO.sh"), queue_job_cmd, 42 ) - m_run.assert_called_once_with( - shlex.split( - f"{queue_job_cmd} {depend_flag} {depend_option}:42 {Path('run_dir')}/SalishSeaNEMO.sh" - ), - check=True, - universal_newlines=True, - stdout=subprocess.PIPE, - ) + assert submit_job_msg == submit_job_msg + def test_no_waitjob_for_bash_submit(self, caplog): + caplog.set_level(logging.DEBUG) + + with pytest.raises(SystemExit) as exc: + salishsea_cmd.run._submit_job( + Path("run_dir", "SalishSeaNEMO.sh"), "bash", "43" + ) + + assert exc.value.code == 2 + assert caplog.records[0].levelname == "ERROR" + expected = ( + "dependent jobs are not available for systems that launch jobs with bash" + ) + assert caplog.records[0].message == expected + @patch("salishsea_cmd.run.log", autospec=True) @patch("salishsea_cmd.run.subprocess.run") @@ -2473,43 +2524,38 @@ def test_orcinus(self, system, deflate): assert script == expected @pytest.mark.parametrize("deflate", [True, False]) - def test_salish(self, deflate): - desc_file = StringIO( - "run_id: foo\n" "walltime: 01:02:03\n" "email: me@example.com" - ) - run_desc = yaml.safe_load(desc_file) - with patch("salishsea_cmd.run.SYSTEM", "salish"): - script = salishsea_cmd.run._build_batch_script( - run_desc, - Path("SalishSea.yaml"), - nemo_processors=7, - xios_processors=1, - max_deflate_jobs=4, - results_dir=Path("results_dir"), - run_dir=Path("tmp_run_dir"), - deflate=deflate, - separate_deflate=False, - cores_per_node="", - cpu_arch="", + def test_salish(self, deflate, monkeypatch): + run_desc = yaml.safe_load( + StringIO( + textwrap.dedent( + """\ + run_id: foo + walltime: 01:02:03 + email: me@example.com + """ + ) ) + ) + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + script = salishsea_cmd.run._build_batch_script( + run_desc, + Path("SalishSea.yaml"), + nemo_processors=7, + xios_processors=1, + max_deflate_jobs=4, + results_dir=Path("results_dir"), + run_dir=Path("tmp_run_dir"), + deflate=deflate, + separate_deflate=False, + cores_per_node="", + cpu_arch="", + ) + expected = textwrap.dedent( """\ #!/bin/bash - #PBS -N foo - #PBS -S /bin/bash - #PBS -l walltime=1:02:03 - # email when the job [b]egins and [e]nds, or is [a]borted - #PBS -m bea - #PBS -M me@example.com - #PBS -l procs=8 - # total memory for job - #PBS -l mem=64gb - # stdout and stderr file paths/names - #PBS -o results_dir/stdout - #PBS -e results_dir/stderr - - RUN_ID="foo" RUN_DESC="tmp_run_dir/SalishSea.yaml" WORK_DIR="tmp_run_dir" @@ -2530,35 +2576,35 @@ def test_salish(self, deflate): mkdir -p ${RESULTS_DIR} cd ${WORK_DIR} - echo "working dir: $(pwd)" + echo "working dir: $(pwd)" >>${RESULTS_DIR}/stdout - echo "Starting run at $(date)" - /usr/bin/mpirun --bind-to none -np 7 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe + echo "Starting run at $(date)" >>${RESULTS_DIR}/stdout + /usr/bin/mpirun --bind-to none -np 7 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe >>${RESULTS_DIR}/stdout 2>>${RESULTS_DIR}/stderr MPIRUN_EXIT_CODE=$? - echo "Ended run at $(date)" + echo "Ended run at $(date)" >>${RESULTS_DIR}/stdout - echo "Results combining started at $(date)" + echo "Results combining started at $(date)" >>${RESULTS_DIR}/stdout ${COMBINE} ${RUN_DESC} --debug - echo "Results combining ended at $(date)" + echo "Results combining ended at $(date)" >>${RESULTS_DIR}/stdout """ ) if deflate: expected += textwrap.dedent( """\ - echo "Results deflation started at $(date)" + echo "Results deflation started at $(date)" >>${RESULTS_DIR}/stdout ${DEFLATE} *_ptrc_T*.nc *_prod_T*.nc *_carp_T*.nc *_grid_[TUVW]*.nc \\ *_turb_T*.nc *_dia[12n]_T*.nc FVCOM*.nc Slab_[UV]*.nc *_mtrc_T*.nc \\ --jobs 4 --debug - echo "Results deflation ended at $(date)" + echo "Results deflation ended at $(date)" >>${RESULTS_DIR}/stdout """ ) expected += textwrap.dedent( """\ - echo "Results gathering started at $(date)" + echo "Results gathering started at $(date)" >>${RESULTS_DIR}/stdout ${GATHER} ${RESULTS_DIR} --debug - echo "Results gathering ended at $(date)" + echo "Results gathering ended at $(date)" >>${RESULTS_DIR}/stdout chmod go+rx ${RESULTS_DIR} chmod g+rw ${RESULTS_DIR}/* @@ -2894,12 +2940,6 @@ class TestPbsDirectives: "", "#PBS -l partition=QDR\n#PBS -l procs=42\n# memory per processor\n#PBS -l pmem=2000mb", ), - ( - "salish", - 0, - "", - "#PBS -l procs=42\n# total memory for job\n#PBS -l mem=64gb", - ), ( "sigma", 20, @@ -2962,12 +3002,6 @@ def test_pbs_directives_run( "", "#PBS -l partition=QDR\n#PBS -l procs=42\n# memory per processor\n#PBS -l pmem=2000mb", ), - ( - "salish", - 0, - "", - "#PBS -l procs=42\n# total memory for job\n#PBS -l mem=64gb", - ), ( "sigma", 20, @@ -3138,10 +3172,19 @@ def test_unknown_system(self): modules = salishsea_cmd.run._modules() assert modules == "" + def test_salish(self, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + modules = salishsea_cmd.run._modules() + + assert modules == "" + @pytest.mark.parametrize("system", ["beluga", "cedar", "graham"]) - def test_beluga_cedar_graham(self, system): - with patch("salishsea_cmd.run.SYSTEM", system): - modules = salishsea_cmd.run._modules() + def test_beluga_cedar_graham(self, system, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load StdEnv/2020 @@ -3151,9 +3194,11 @@ def test_beluga_cedar_graham(self, system): assert modules == expected @pytest.mark.parametrize("system", ("orcinus", "seawolf1", "seawolf2", "seawolf3")) - def test_orcinus(self, system): - with patch("salishsea_cmd.run.SYSTEM", system): - modules = salishsea_cmd.run._modules() + def test_orcinus(self, system, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load intel @@ -3168,9 +3213,11 @@ def test_orcinus(self, system): assert modules == expected @pytest.mark.parametrize("system", ["delta", "sigma", "omega"]) - def test_optimum(self, system): - with patch("salishsea_cmd.run.SYSTEM", system): - modules = salishsea_cmd.run._modules() + def test_optimum(self, system, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load OpenMPI/2.1.6/GCC/SYSTEM @@ -3178,9 +3225,11 @@ def test_optimum(self, system): ) assert modules == expected - def test_sockeye(self): - with patch("salishsea_cmd.run.SYSTEM", "sockeye"): - modules = salishsea_cmd.run._modules() + def test_sockeye(self, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "sockeye") + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load gcc/5.5.0 @@ -3209,10 +3258,6 @@ class TestExecute: "omega", "mpiexec -hostfile $(openmpi_nodefile) --bind-to core -np 42 ./nemo.exe : --bind-to core -np 1 ./xios_server.exe", ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - ), ( "sigma", "mpiexec -hostfile $(openmpi_nodefile) --bind-to core -np 42 ./nemo.exe : --bind-to core -np 1 ./xios_server.exe", @@ -3223,15 +3268,18 @@ class TestExecute: ), ], ) - def test_execute_with_deflate(self, system, mpirun_cmd): - with patch("salishsea_cmd.run.SYSTEM", system): - script = salishsea_cmd.run._execute( - nemo_processors=42, - xios_processors=1, - deflate=True, - max_deflate_jobs=4, - separate_deflate=False, - ) + def test_execute_with_deflate(self, system, mpirun_cmd, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + script = salishsea_cmd.run._execute( + nemo_processors=42, + xios_processors=1, + deflate=True, + max_deflate_jobs=4, + separate_deflate=False, + redirect_stdout_stderr=False, + ) + expected = textwrap.dedent( f"""\ mkdir -p ${{RESULTS_DIR}} @@ -3289,6 +3337,46 @@ def test_execute_with_deflate(self, system, mpirun_cmd): ) assert script == expected + def test_salish_execute_with_deflate(self, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + script = salishsea_cmd.run._execute( + nemo_processors=42, + xios_processors=1, + deflate=True, + max_deflate_jobs=4, + separate_deflate=False, + redirect_stdout_stderr=True, + ) + + expected = textwrap.dedent( + f"""\ + mkdir -p ${{RESULTS_DIR}} + cd ${{WORK_DIR}} + echo "working dir: $(pwd)" >>${{RESULTS_DIR}}/stdout + + echo "Starting run at $(date)" >>${{RESULTS_DIR}}/stdout + /usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe >>${{RESULTS_DIR}}/stdout 2>>${{RESULTS_DIR}}/stderr + MPIRUN_EXIT_CODE=$? + echo "Ended run at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results combining started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{COMBINE}} ${{RUN_DESC}} --debug + echo "Results combining ended at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results deflation started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{DEFLATE}} *_ptrc_T*.nc *_prod_T*.nc *_carp_T*.nc *_grid_[TUVW]*.nc \\ + *_turb_T*.nc *_dia[12n]_T*.nc FVCOM*.nc Slab_[UV]*.nc *_mtrc_T*.nc \\ + --jobs 4 --debug + echo "Results deflation ended at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results gathering started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{GATHER}} ${{RESULTS_DIR}} --debug + echo "Results gathering ended at $(date)" >>${{RESULTS_DIR}}/stdout + """ + ) + assert script == expected + @pytest.mark.parametrize( "system, mpirun_cmd, deflate, separate_deflate", [ @@ -3395,24 +3483,6 @@ def test_execute_with_deflate(self, system, mpirun_cmd): True, True, ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - False, - True, - ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - False, - False, - ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - True, - True, - ), ( "sigma", "mpiexec -hostfile $(openmpi_nodefile) --bind-to core -np 42 ./nemo.exe : --bind-to core -np 1 ./xios_server.exe", @@ -3452,16 +3522,19 @@ def test_execute_with_deflate(self, system, mpirun_cmd): ], ) def test_execute_without_deflate( - self, system, mpirun_cmd, deflate, separate_deflate + self, system, mpirun_cmd, deflate, separate_deflate, monkeypatch ): - with patch("salishsea_cmd.run.SYSTEM", system): - script = salishsea_cmd.run._execute( - nemo_processors=42, - xios_processors=1, - deflate=deflate, - max_deflate_jobs=4, - separate_deflate=separate_deflate, - ) + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + script = salishsea_cmd.run._execute( + nemo_processors=42, + xios_processors=1, + deflate=deflate, + max_deflate_jobs=4, + separate_deflate=separate_deflate, + redirect_stdout_stderr=False, + ) + expected = textwrap.dedent( f"""\ mkdir -p ${{RESULTS_DIR}} @@ -3499,6 +3572,50 @@ def test_execute_without_deflate( ) assert script == expected + @pytest.mark.parametrize( + "deflate, separate_deflate", + [ + (False, True), + (False, False), + (True, True), + ], + ) + def test_salish_execute_without_deflate( + self, deflate, separate_deflate, monkeypatch + ): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + script = salishsea_cmd.run._execute( + nemo_processors=7, + xios_processors=1, + deflate=deflate, + max_deflate_jobs=4, + separate_deflate=separate_deflate, + redirect_stdout_stderr=True, + ) + + expected = textwrap.dedent( + f"""\ + mkdir -p ${{RESULTS_DIR}} + cd ${{WORK_DIR}} + echo "working dir: $(pwd)" >>${{RESULTS_DIR}}/stdout + + echo "Starting run at $(date)" >>${{RESULTS_DIR}}/stdout + /usr/bin/mpirun --bind-to none -np 7 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe >>${{RESULTS_DIR}}/stdout 2>>${{RESULTS_DIR}}/stderr + MPIRUN_EXIT_CODE=$? + echo "Ended run at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results combining started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{COMBINE}} ${{RUN_DESC}} --debug + echo "Results combining ended at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results gathering started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{GATHER}} ${{RESULTS_DIR}} --debug + echo "Results gathering ended at $(date)" >>${{RESULTS_DIR}}/stdout + """ + ) + assert script == expected + class TestCleanup: """Unit test for _cleanup() function."""