diff --git a/salishsea_cmd/run.py b/salishsea_cmd/run.py index bd1f9b6..493171b 100644 --- a/salishsea_cmd/run.py +++ b/salishsea_cmd/run.py @@ -275,7 +275,7 @@ def run( "graham": "sbatch", "omega": "qsub -q mpi", # optimum.eoas.ubc.ca login node "orcinus": "qsub", - "salish": "qsub", + "salish": "bash", "seawolf1": "qsub", # orcinus.westgrid.ca login node "seawolf2": "qsub", # orcinus.westgrid.ca login node "seawolf3": "qsub", # orcinus.westgrid.ca login node @@ -543,12 +543,23 @@ def _build_tmp_run_dir( def _submit_job(batch_file, queue_job_cmd, waitjob): if waitjob != "0": - depend_opt = ( - "-W depend=afterok" if queue_job_cmd.startswith("qsub") else "-d afterok" - ) + match queue_job_cmd.split(" ")[0]: + case "qsub": + depend_opt = "-W depend=afterok" + case "sbatch": + depend_opt = "-d afterok" + case _: + log.error( + f"dependent jobs are not available for systems that launch jobs with " + f"{queue_job_cmd}" + ) + raise SystemExit(2) cmd = f"{queue_job_cmd} {depend_opt}:{waitjob} {batch_file}" else: cmd = f"{queue_job_cmd} {batch_file}" + if queue_job_cmd == "bash": + subprocess.Popen(shlex.split(cmd), start_new_session=True) + return f"{cmd} started" submit_job_msg = subprocess.run( shlex.split(cmd), check=True, universal_newlines=True, stdout=subprocess.PIPE ).stdout @@ -634,7 +645,10 @@ def _build_batch_script( email = get_run_desc_value(run_desc, ("email",), fatal=False) except KeyError: email = f"{os.getenv('USER')}@eoas.ubc.ca" - if SYSTEM in {"beluga", "cedar", "graham", "sockeye"}: + if SYSTEM == "salish": + # salish doesn't use a scheduler, so no sbatch or PBS directives in its script + pass + elif SYSTEM in {"beluga", "cedar", "graham", "sockeye"}: procs_per_node = { "beluga": 40 if not cores_per_node else int(cores_per_node), "cedar": 48 if not cores_per_node else int(cores_per_node), @@ -654,7 +668,6 @@ def _build_batch_script( "omega": 20 if not cores_per_node else int(cores_per_node), "sigma": 20 if not cores_per_node else int(cores_per_node), "orcinus": 12 if not cores_per_node else int(cores_per_node), - "salish": 0, # solish only has 1 node; 0 gets things right "seawolf1": 12 if not cores_per_node else int(cores_per_node), "seawolf2": 12 if not cores_per_node else int(cores_per_node), "seawolf3": 12 if not cores_per_node else int(cores_per_node), @@ -668,12 +681,21 @@ def _build_batch_script( f"{_pbs_directives(run_desc, nemo_processors + xios_processors, email, results_dir, procs_per_node, cpu_arch, )}\n", ) ) + redirect_stdout_stderr = True if SYSTEM == "salish" else False + execute_section = _execute( + nemo_processors, + xios_processors, + deflate, + max_deflate_jobs, + separate_deflate, + redirect_stdout_stderr, + ) script = "\n".join( ( script, f"{_definitions(run_desc, desc_file, run_dir, results_dir, deflate)}\n" f"{_modules()}\n" - f"{_execute(nemo_processors, xios_processors, deflate, max_deflate_jobs, separate_deflate,)}\n" + f"{execute_section}\n" f"{_fix_permissions()}\n" f"{_cleanup()}", ) @@ -1032,8 +1054,18 @@ def _modules(): def _execute( - nemo_processors, xios_processors, deflate, max_deflate_jobs, separate_deflate + nemo_processors, + xios_processors, + deflate, + max_deflate_jobs, + separate_deflate, + redirect_stdout_stderr, ): + redirect = ( + "" + if not redirect_stdout_stderr + else " >>${RESULTS_DIR}/stdout 2>>${RESULTS_DIR}/stderr" + ) mpirun = { "beluga": "mpirun", "cedar": "mpirun", @@ -1064,34 +1096,35 @@ def _execute( }.get(SYSTEM, f"{mpirun} -np {nemo_processors} ./nemo.exe") if xios_processors: mpirun = { - "beluga": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "cedar": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "delta": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", - "graham": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "omega": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", - "orcinus": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "salish": f"{mpirun} : --bind-to none -np {xios_processors} ./xios_server.exe", - "seawolf1": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "seawolf2": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "seawolf3": f"{mpirun} : -np {xios_processors} ./xios_server.exe", - "sigma": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", - "sockeye": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe", + "beluga": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "cedar": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "delta": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", + "graham": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "omega": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", + "orcinus": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "salish": f"{mpirun} : --bind-to none -np {xios_processors} ./xios_server.exe{redirect}", + "seawolf1": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "seawolf2": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "seawolf3": f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", + "sigma": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", + "sockeye": f"{mpirun} : --bind-to core -np {xios_processors} ./xios_server.exe{redirect}", }.get( SYSTEM, - f"{mpirun} : -np {xios_processors} ./xios_server.exe", + f"{mpirun} : -np {xios_processors} ./xios_server.exe{redirect}", ) + redirect = "" if not redirect_stdout_stderr else " >>${RESULTS_DIR}/stdout" script = textwrap.dedent( f"""\ mkdir -p ${{RESULTS_DIR}} cd ${{WORK_DIR}} - echo "working dir: $(pwd)" + echo "working dir: $(pwd)"{redirect} - echo "Starting run at $(date)" + echo "Starting run at $(date)"{redirect} {mpirun} MPIRUN_EXIT_CODE=$? - echo "Ended run at $(date)" + echo "Ended run at $(date)"{redirect} - echo "Results combining started at $(date)" + echo "Results combining started at $(date)"{redirect} """ ) if SYSTEM in {"delta", "omega", "sigma"}: @@ -1109,16 +1142,16 @@ def _execute( """ ) script += textwrap.dedent( - """\ - ${COMBINE} ${RUN_DESC} --debug - echo "Results combining ended at $(date)" + f"""\ + ${{COMBINE}} ${{RUN_DESC}} --debug + echo "Results combining ended at $(date)"{redirect} """ ) if deflate and not separate_deflate: script += textwrap.dedent( - """\ + f"""\ - echo "Results deflation started at $(date)" + echo "Results deflation started at $(date)"{redirect} """ ) if SYSTEM in {"beluga", "cedar", "graham"}: @@ -1135,15 +1168,15 @@ def _execute( ${{DEFLATE}} *_ptrc_T*.nc *_prod_T*.nc *_carp_T*.nc *_grid_[TUVW]*.nc \\ *_turb_T*.nc *_dia[12n]_T*.nc FVCOM*.nc Slab_[UV]*.nc *_mtrc_T*.nc \\ --jobs {max_deflate_jobs} --debug - echo "Results deflation ended at $(date)" + echo "Results deflation ended at $(date)"{redirect} """ ) script += textwrap.dedent( - """\ + f"""\ - echo "Results gathering started at $(date)" - ${GATHER} ${RESULTS_DIR} --debug - echo "Results gathering ended at $(date)" + echo "Results gathering started at $(date)"{redirect} + ${{GATHER}} ${{RESULTS_DIR}} --debug + echo "Results gathering ended at $(date)"{redirect} """ ) return script diff --git a/tests/test_run.py b/tests/test_run.py index abed1f1..9bbcb7f 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -18,6 +18,7 @@ """SalishSeaCmd run sub-command plug-in unit tests """ +import logging import os import shlex import subprocess @@ -143,8 +144,8 @@ class TestRun: (True, 4, "delta", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "graham", "sbatch", "Submitted batch job 43"), (True, 4, "graham", "sbatch", "Submitted batch job 43"), - (False, 0, "salish", "qsub", "43.master"), - (True, 4, "salish", "qsub", "43.master"), + (False, 0, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), + (True, 4, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), (False, 0, "sigma", "qsub -q mpi", "43.admin.default.domain"), (True, 4, "sigma", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "sockeye", "sbatch", "Submitted batch job 43"), @@ -176,6 +177,7 @@ def test_run_submit( queue_job_cmd, submit_job_msg, tmpdir, + monkeypatch, ): p_run_dir = tmpdir.ensure_dir("run_dir") p_results_dir = tmpdir.ensure_dir("results_dir") @@ -200,10 +202,12 @@ def test_run_submit( Path(str(p_run_dir), "SalishSeaNEMO.sh"), ) m_sj.return_value = submit_job_msg - with patch("salishsea_cmd.run.SYSTEM", system): - submit_job_msg = salishsea_cmd.run.run( - Path("SalishSea.yaml"), Path(str(p_results_dir)) - ) + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + submit_job_msg = salishsea_cmd.run.run( + Path("SalishSea.yaml"), Path(str(p_results_dir)) + ) + m_sj.assert_called_once_with( Path(str(p_run_dir), "SalishSeaNEMO.sh"), queue_job_cmd, waitjob="0" ) @@ -220,8 +224,8 @@ def test_run_submit( (True, 4, "delta", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "graham", "sbatch", "Submitted batch job 43"), (True, 4, "graham", "sbatch", "Submitted batch job 43"), - (False, 0, "salish", "qsub", "43.master"), - (True, 4, "salish", "qsub", "43.master"), + (False, 0, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), + (True, 4, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), (False, 0, "sigma", "qsub -q mpi", "43.admin.default.domain"), (True, 4, "sigma", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "sockeye", "sbatch", "Submitted batch job 43"), @@ -386,8 +390,8 @@ def test_run_no_submit_w_separate_deflate( (True, 4, "delta", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "graham", "sbatch", "Submitted batch job 43"), (True, 4, "graham", "sbatch", "Submitted batch job 43"), - (False, 0, "salish", "qsub", "43.master"), - (True, 4, "salish", "qsub", "43.master"), + (False, 0, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), + (True, 4, "salish", "bash", "bash run_dir/SalishSeaNEMO.sh started"), (False, 0, "sigma", "qsub -q mpi", "43.admin.default.domain"), (True, 4, "sigma", "qsub -q mpi", "43.admin.default.domain"), (False, 0, "sockeye", "sbatch", "Submitted batch job 43"), @@ -520,8 +524,8 @@ def test_run_separate_deflate( ("Submitted batch job 43", "Submitted batch job 44"), "Submitted batch job 43", ), - (False, 0, "salish", "qsub", ("43.master", "44.master"), "43.master"), - (True, 4, "salish", "qsub", ("43.master", "44.master"), "43.master"), + (False, 0, "salish", "bash", ("43.master", "44.master"), "43.master"), + (True, 4, "salish", "bash", ("43.master", "44.master"), "43.master"), ( False, 0, @@ -748,8 +752,8 @@ def test_segmented_run_restart_dirs( ("Submitted batch job 43", "Submitted batch job 44"), "Submitted batch job 43", ), - (False, 0, "salish", "qsub", ("43.master", "44.master"), "43.master"), - (True, 4, "salish", "qsub", ("43.master", "44.master"), "43.master"), + (False, 0, "salish", "bash", ("43.master", "44.master"), "43.master"), + (True, 4, "salish", "bash", ("43.master", "44.master"), "43.master"), ( False, 0, @@ -1824,48 +1828,95 @@ def test_build_tmp_run_dir_separate_deflate( assert batch_file == Path(str(p_run_dir)) / "SalishSeaNEMO.sh" -@patch("salishsea_cmd.run.subprocess.run") -@pytest.mark.parametrize( - "queue_job_cmd, depend_flag, depend_option, submit_job_msg", - [ - ("sbatch", "-d", "afterok", "Submitted batch job 43"), - ("qsub", "-W", "depend=afterok", "43.orca2.ibb"), - ("qsub -q mpi", "-W", "depend=afterok", "43.admin.default.domain"), - ], -) class TestSubmitJob: """Unit tests for _submit_job() function.""" - def test_submit_job( - self, m_run, queue_job_cmd, depend_flag, depend_option, submit_job_msg - ): + @pytest.mark.parametrize( + "queue_job_cmd, msg", + ( + ("bash", "bash run_dir/SalishSeaNEMO.sh started"), # salish + ("sbatch", "Submitted batch job 43"), # sockeye & Alliance HPC clusters + ("qsub", "43.orca2.ibb"), # orcinus + ("qsub -q mpi", "43.admin.default.domain"), # optimum + ), + ) + def test_submit_job(self, queue_job_cmd, msg, monkeypatch): + def mock_subprocess_Popen(cmd, start_new_session): + pass + + monkeypatch.setattr(subprocess, "Popen", mock_subprocess_Popen) + + def mock_subprocess_run(cmd, check, universal_newlines, stdout): + return subprocess.CompletedProcess(cmd, 0, msg) + + monkeypatch.setattr(subprocess, "run", mock_subprocess_run) + submit_job_msg = salishsea_cmd.run._submit_job( Path("run_dir", "SalishSeaNEMO.sh"), queue_job_cmd, "0" ) - m_run.assert_called_once_with( - shlex.split(f"{queue_job_cmd} {Path('run_dir')}/SalishSeaNEMO.sh"), - check=True, - universal_newlines=True, - stdout=subprocess.PIPE, - ) - assert submit_job_msg == submit_job_msg - def test_submit_job_w_waitjob( - self, m_run, queue_job_cmd, depend_flag, depend_option, submit_job_msg + assert submit_job_msg == msg + + @pytest.mark.parametrize( + "queue_job_cmd, depend_flag, depend_option, msg", + [ + # sockeye & Alliance HPC clusters + ( + "sbatch", + "-d", + "afterok", + "Submitted batch job 43", + ), + # sockeye & Alliance HPC clusters + ( + "qsub", + "-W", + "depend=afterok", + "43.orca2.ibb", + ), + # optimum + ( + "qsub -q mpi", + "-W", + "depend=afterok", + "43.admin.default.domain", + ), + ], + ) + def test_submit_job_with_waitjob( + self, + queue_job_cmd, + depend_flag, + depend_option, + msg, + monkeypatch, ): + def mock_subprocess_run(cmd, check, universal_newlines, stdout): + return subprocess.CompletedProcess(cmd, 0, msg) + + monkeypatch.setattr(subprocess, "run", mock_subprocess_run) + submit_job_msg = salishsea_cmd.run._submit_job( Path("run_dir", "SalishSeaNEMO.sh"), queue_job_cmd, 42 ) - m_run.assert_called_once_with( - shlex.split( - f"{queue_job_cmd} {depend_flag} {depend_option}:42 {Path('run_dir')}/SalishSeaNEMO.sh" - ), - check=True, - universal_newlines=True, - stdout=subprocess.PIPE, - ) + assert submit_job_msg == submit_job_msg + def test_no_waitjob_for_bash_submit(self, caplog): + caplog.set_level(logging.DEBUG) + + with pytest.raises(SystemExit) as exc: + salishsea_cmd.run._submit_job( + Path("run_dir", "SalishSeaNEMO.sh"), "bash", "43" + ) + + assert exc.value.code == 2 + assert caplog.records[0].levelname == "ERROR" + expected = ( + "dependent jobs are not available for systems that launch jobs with bash" + ) + assert caplog.records[0].message == expected + @patch("salishsea_cmd.run.log", autospec=True) @patch("salishsea_cmd.run.subprocess.run") @@ -2473,43 +2524,38 @@ def test_orcinus(self, system, deflate): assert script == expected @pytest.mark.parametrize("deflate", [True, False]) - def test_salish(self, deflate): - desc_file = StringIO( - "run_id: foo\n" "walltime: 01:02:03\n" "email: me@example.com" - ) - run_desc = yaml.safe_load(desc_file) - with patch("salishsea_cmd.run.SYSTEM", "salish"): - script = salishsea_cmd.run._build_batch_script( - run_desc, - Path("SalishSea.yaml"), - nemo_processors=7, - xios_processors=1, - max_deflate_jobs=4, - results_dir=Path("results_dir"), - run_dir=Path("tmp_run_dir"), - deflate=deflate, - separate_deflate=False, - cores_per_node="", - cpu_arch="", + def test_salish(self, deflate, monkeypatch): + run_desc = yaml.safe_load( + StringIO( + textwrap.dedent( + """\ + run_id: foo + walltime: 01:02:03 + email: me@example.com + """ + ) ) + ) + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + script = salishsea_cmd.run._build_batch_script( + run_desc, + Path("SalishSea.yaml"), + nemo_processors=7, + xios_processors=1, + max_deflate_jobs=4, + results_dir=Path("results_dir"), + run_dir=Path("tmp_run_dir"), + deflate=deflate, + separate_deflate=False, + cores_per_node="", + cpu_arch="", + ) + expected = textwrap.dedent( """\ #!/bin/bash - #PBS -N foo - #PBS -S /bin/bash - #PBS -l walltime=1:02:03 - # email when the job [b]egins and [e]nds, or is [a]borted - #PBS -m bea - #PBS -M me@example.com - #PBS -l procs=8 - # total memory for job - #PBS -l mem=64gb - # stdout and stderr file paths/names - #PBS -o results_dir/stdout - #PBS -e results_dir/stderr - - RUN_ID="foo" RUN_DESC="tmp_run_dir/SalishSea.yaml" WORK_DIR="tmp_run_dir" @@ -2530,35 +2576,35 @@ def test_salish(self, deflate): mkdir -p ${RESULTS_DIR} cd ${WORK_DIR} - echo "working dir: $(pwd)" + echo "working dir: $(pwd)" >>${RESULTS_DIR}/stdout - echo "Starting run at $(date)" - /usr/bin/mpirun --bind-to none -np 7 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe + echo "Starting run at $(date)" >>${RESULTS_DIR}/stdout + /usr/bin/mpirun --bind-to none -np 7 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe >>${RESULTS_DIR}/stdout 2>>${RESULTS_DIR}/stderr MPIRUN_EXIT_CODE=$? - echo "Ended run at $(date)" + echo "Ended run at $(date)" >>${RESULTS_DIR}/stdout - echo "Results combining started at $(date)" + echo "Results combining started at $(date)" >>${RESULTS_DIR}/stdout ${COMBINE} ${RUN_DESC} --debug - echo "Results combining ended at $(date)" + echo "Results combining ended at $(date)" >>${RESULTS_DIR}/stdout """ ) if deflate: expected += textwrap.dedent( """\ - echo "Results deflation started at $(date)" + echo "Results deflation started at $(date)" >>${RESULTS_DIR}/stdout ${DEFLATE} *_ptrc_T*.nc *_prod_T*.nc *_carp_T*.nc *_grid_[TUVW]*.nc \\ *_turb_T*.nc *_dia[12n]_T*.nc FVCOM*.nc Slab_[UV]*.nc *_mtrc_T*.nc \\ --jobs 4 --debug - echo "Results deflation ended at $(date)" + echo "Results deflation ended at $(date)" >>${RESULTS_DIR}/stdout """ ) expected += textwrap.dedent( """\ - echo "Results gathering started at $(date)" + echo "Results gathering started at $(date)" >>${RESULTS_DIR}/stdout ${GATHER} ${RESULTS_DIR} --debug - echo "Results gathering ended at $(date)" + echo "Results gathering ended at $(date)" >>${RESULTS_DIR}/stdout chmod go+rx ${RESULTS_DIR} chmod g+rw ${RESULTS_DIR}/* @@ -2894,12 +2940,6 @@ class TestPbsDirectives: "", "#PBS -l partition=QDR\n#PBS -l procs=42\n# memory per processor\n#PBS -l pmem=2000mb", ), - ( - "salish", - 0, - "", - "#PBS -l procs=42\n# total memory for job\n#PBS -l mem=64gb", - ), ( "sigma", 20, @@ -2962,12 +3002,6 @@ def test_pbs_directives_run( "", "#PBS -l partition=QDR\n#PBS -l procs=42\n# memory per processor\n#PBS -l pmem=2000mb", ), - ( - "salish", - 0, - "", - "#PBS -l procs=42\n# total memory for job\n#PBS -l mem=64gb", - ), ( "sigma", 20, @@ -3138,10 +3172,19 @@ def test_unknown_system(self): modules = salishsea_cmd.run._modules() assert modules == "" + def test_salish(self, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + modules = salishsea_cmd.run._modules() + + assert modules == "" + @pytest.mark.parametrize("system", ["beluga", "cedar", "graham"]) - def test_beluga_cedar_graham(self, system): - with patch("salishsea_cmd.run.SYSTEM", system): - modules = salishsea_cmd.run._modules() + def test_beluga_cedar_graham(self, system, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load StdEnv/2020 @@ -3151,9 +3194,11 @@ def test_beluga_cedar_graham(self, system): assert modules == expected @pytest.mark.parametrize("system", ("orcinus", "seawolf1", "seawolf2", "seawolf3")) - def test_orcinus(self, system): - with patch("salishsea_cmd.run.SYSTEM", system): - modules = salishsea_cmd.run._modules() + def test_orcinus(self, system, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load intel @@ -3168,9 +3213,11 @@ def test_orcinus(self, system): assert modules == expected @pytest.mark.parametrize("system", ["delta", "sigma", "omega"]) - def test_optimum(self, system): - with patch("salishsea_cmd.run.SYSTEM", system): - modules = salishsea_cmd.run._modules() + def test_optimum(self, system, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load OpenMPI/2.1.6/GCC/SYSTEM @@ -3178,9 +3225,11 @@ def test_optimum(self, system): ) assert modules == expected - def test_sockeye(self): - with patch("salishsea_cmd.run.SYSTEM", "sockeye"): - modules = salishsea_cmd.run._modules() + def test_sockeye(self, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "sockeye") + + modules = salishsea_cmd.run._modules() + expected = textwrap.dedent( """\ module load gcc/5.5.0 @@ -3209,10 +3258,6 @@ class TestExecute: "omega", "mpiexec -hostfile $(openmpi_nodefile) --bind-to core -np 42 ./nemo.exe : --bind-to core -np 1 ./xios_server.exe", ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - ), ( "sigma", "mpiexec -hostfile $(openmpi_nodefile) --bind-to core -np 42 ./nemo.exe : --bind-to core -np 1 ./xios_server.exe", @@ -3223,15 +3268,18 @@ class TestExecute: ), ], ) - def test_execute_with_deflate(self, system, mpirun_cmd): - with patch("salishsea_cmd.run.SYSTEM", system): - script = salishsea_cmd.run._execute( - nemo_processors=42, - xios_processors=1, - deflate=True, - max_deflate_jobs=4, - separate_deflate=False, - ) + def test_execute_with_deflate(self, system, mpirun_cmd, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + script = salishsea_cmd.run._execute( + nemo_processors=42, + xios_processors=1, + deflate=True, + max_deflate_jobs=4, + separate_deflate=False, + redirect_stdout_stderr=False, + ) + expected = textwrap.dedent( f"""\ mkdir -p ${{RESULTS_DIR}} @@ -3289,6 +3337,46 @@ def test_execute_with_deflate(self, system, mpirun_cmd): ) assert script == expected + def test_salish_execute_with_deflate(self, monkeypatch): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + script = salishsea_cmd.run._execute( + nemo_processors=42, + xios_processors=1, + deflate=True, + max_deflate_jobs=4, + separate_deflate=False, + redirect_stdout_stderr=True, + ) + + expected = textwrap.dedent( + f"""\ + mkdir -p ${{RESULTS_DIR}} + cd ${{WORK_DIR}} + echo "working dir: $(pwd)" >>${{RESULTS_DIR}}/stdout + + echo "Starting run at $(date)" >>${{RESULTS_DIR}}/stdout + /usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe >>${{RESULTS_DIR}}/stdout 2>>${{RESULTS_DIR}}/stderr + MPIRUN_EXIT_CODE=$? + echo "Ended run at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results combining started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{COMBINE}} ${{RUN_DESC}} --debug + echo "Results combining ended at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results deflation started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{DEFLATE}} *_ptrc_T*.nc *_prod_T*.nc *_carp_T*.nc *_grid_[TUVW]*.nc \\ + *_turb_T*.nc *_dia[12n]_T*.nc FVCOM*.nc Slab_[UV]*.nc *_mtrc_T*.nc \\ + --jobs 4 --debug + echo "Results deflation ended at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results gathering started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{GATHER}} ${{RESULTS_DIR}} --debug + echo "Results gathering ended at $(date)" >>${{RESULTS_DIR}}/stdout + """ + ) + assert script == expected + @pytest.mark.parametrize( "system, mpirun_cmd, deflate, separate_deflate", [ @@ -3395,24 +3483,6 @@ def test_execute_with_deflate(self, system, mpirun_cmd): True, True, ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - False, - True, - ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - False, - False, - ), - ( - "salish", - "/usr/bin/mpirun --bind-to none -np 42 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe", - True, - True, - ), ( "sigma", "mpiexec -hostfile $(openmpi_nodefile) --bind-to core -np 42 ./nemo.exe : --bind-to core -np 1 ./xios_server.exe", @@ -3452,16 +3522,19 @@ def test_execute_with_deflate(self, system, mpirun_cmd): ], ) def test_execute_without_deflate( - self, system, mpirun_cmd, deflate, separate_deflate + self, system, mpirun_cmd, deflate, separate_deflate, monkeypatch ): - with patch("salishsea_cmd.run.SYSTEM", system): - script = salishsea_cmd.run._execute( - nemo_processors=42, - xios_processors=1, - deflate=deflate, - max_deflate_jobs=4, - separate_deflate=separate_deflate, - ) + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", system) + + script = salishsea_cmd.run._execute( + nemo_processors=42, + xios_processors=1, + deflate=deflate, + max_deflate_jobs=4, + separate_deflate=separate_deflate, + redirect_stdout_stderr=False, + ) + expected = textwrap.dedent( f"""\ mkdir -p ${{RESULTS_DIR}} @@ -3499,6 +3572,50 @@ def test_execute_without_deflate( ) assert script == expected + @pytest.mark.parametrize( + "deflate, separate_deflate", + [ + (False, True), + (False, False), + (True, True), + ], + ) + def test_salish_execute_without_deflate( + self, deflate, separate_deflate, monkeypatch + ): + monkeypatch.setattr(salishsea_cmd.run, "SYSTEM", "salish") + + script = salishsea_cmd.run._execute( + nemo_processors=7, + xios_processors=1, + deflate=deflate, + max_deflate_jobs=4, + separate_deflate=separate_deflate, + redirect_stdout_stderr=True, + ) + + expected = textwrap.dedent( + f"""\ + mkdir -p ${{RESULTS_DIR}} + cd ${{WORK_DIR}} + echo "working dir: $(pwd)" >>${{RESULTS_DIR}}/stdout + + echo "Starting run at $(date)" >>${{RESULTS_DIR}}/stdout + /usr/bin/mpirun --bind-to none -np 7 ./nemo.exe : --bind-to none -np 1 ./xios_server.exe >>${{RESULTS_DIR}}/stdout 2>>${{RESULTS_DIR}}/stderr + MPIRUN_EXIT_CODE=$? + echo "Ended run at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results combining started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{COMBINE}} ${{RUN_DESC}} --debug + echo "Results combining ended at $(date)" >>${{RESULTS_DIR}}/stdout + + echo "Results gathering started at $(date)" >>${{RESULTS_DIR}}/stdout + ${{GATHER}} ${{RESULTS_DIR}} --debug + echo "Results gathering ended at $(date)" >>${{RESULTS_DIR}}/stdout + """ + ) + assert script == expected + class TestCleanup: """Unit test for _cleanup() function."""