From 988b23e9ba36fa29c3f55b11c2565ef701d20ad8 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 13 Jul 2024 16:00:47 -0700 Subject: [PATCH 1/5] Add get_vfpa_hadcp after 06 weather collection Collect the previous day's observations from the VFPA HADCP located at the 2nd Narrows railway bridge early in the morning after the 06Z weather forecast products have been downloaded. This restores daily collection of the HADCP obs that were inadvertently stopped when we stopped running the VHFR FVCOM model in Mar-2023. --- nowcast/next_workers.py | 5 +++++ tests/test_next_workers.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/nowcast/next_workers.py b/nowcast/next_workers.py index 56bebd02..6fb0d59f 100644 --- a/nowcast/next_workers.py +++ b/nowcast/next_workers.py @@ -78,6 +78,11 @@ def after_download_weather(msg, config, checklist): next_workers["success 2.5km 06"].append( NextWorker("nowcast.workers.get_onc_ferry", args=[ferry]) ) + next_workers["success 2.5km 06"].append( + NextWorker( + "nowcast.workers.get_vfpa_hadcp", args=["--data-date", data_date] + ) + ) if "forecast2" in config["run types"]: next_workers["success 2.5km 06"].append( NextWorker("nowcast.workers.collect_NeahBay_ssh", args=["00"]), diff --git a/tests/test_next_workers.py b/tests/test_next_workers.py index e535214e..d5589997 100644 --- a/tests/test_next_workers.py +++ b/tests/test_next_workers.py @@ -208,6 +208,11 @@ def mock_now(): NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"), + NextWorker( + "nowcast.workers.get_vfpa_hadcp", + ["--data-date", "2018-12-26"], + host="localhost", + ), NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"), ] assert workers == expected @@ -327,6 +332,11 @@ def mock_now(): NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"), NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"), + NextWorker( + "nowcast.workers.get_vfpa_hadcp", + ["--data-date", "2018-12-26"], + host="localhost", + ), NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"), NextWorker( "nowcast.workers.collect_weather", ["12", "2.5km"], host="localhost" From c17b8923998564105d2767f2e2a7c497e394d77a Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 13 Jul 2024 16:09:54 -0700 Subject: [PATCH 2/5] Update get_vfpa_hadcp main() function docstring Remove non-informative "Set up and run the worker." line at the beginning. re: issue #121 --- nowcast/workers/get_vfpa_hadcp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nowcast/workers/get_vfpa_hadcp.py b/nowcast/workers/get_vfpa_hadcp.py index e2e34188..f74ac255 100644 --- a/nowcast/workers/get_vfpa_hadcp.py +++ b/nowcast/workers/get_vfpa_hadcp.py @@ -37,9 +37,7 @@ def main(): - """Set up and run the worker. - - For command-line usage see: + """For command-line usage see: :command:`python -m nowcast.workers.get_vfpa_hadcp --help` """ From 3dab2896c5fe8cfe1a2c641d835e763d7d3eb798 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 13 Jul 2024 16:19:05 -0700 Subject: [PATCH 3/5] Change NowcastWorker mock to pytest fixture Test suite maintenance. re: issue #81 --- nowcast/workers/get_vfpa_hadcp.py | 1 + tests/workers/test_get_vfpa_hadcp.py | 50 +++++++++++----------------- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/nowcast/workers/get_vfpa_hadcp.py b/nowcast/workers/get_vfpa_hadcp.py index f74ac255..0c6d53ad 100644 --- a/nowcast/workers/get_vfpa_hadcp.py +++ b/nowcast/workers/get_vfpa_hadcp.py @@ -49,6 +49,7 @@ def main(): help="UTC date to get VFPA HADPC data for.", ) worker.run(get_vfpa_hadcp, success, failure) + return worker def success(parsed_args): diff --git a/tests/workers/test_get_vfpa_hadcp.py b/tests/workers/test_get_vfpa_hadcp.py index b91cb87a..b1008d18 100644 --- a/tests/workers/test_get_vfpa_hadcp.py +++ b/tests/workers/test_get_vfpa_hadcp.py @@ -21,7 +21,7 @@ import textwrap from pathlib import Path from types import SimpleNamespace -from unittest.mock import Mock, patch +from unittest.mock import patch import arrow import nemo_nowcast @@ -51,40 +51,30 @@ def config(base_config): return config_ -@patch("nowcast.workers.get_vfpa_hadcp.NowcastWorker", spec=True) +@pytest.fixture +def mock_worker(mock_nowcast_worker, monkeypatch): + monkeypatch.setattr(get_vfpa_hadcp, "NowcastWorker", mock_nowcast_worker) + + class TestMain: """Unit tests for main() function.""" - def test_instantiate_worker(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - args, kwargs = m_worker.call_args - assert args == ("get_vfpa_hadcp",) - assert list(kwargs.keys()) == ["description"] - - def test_init_cli(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - m_worker().init_cli.assert_called_once_with() - - def test_add_data_date_option(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - args, kwargs = m_worker().cli.add_date_option.call_args_list[0] - assert args == ("--data-date",) - assert kwargs["default"] == arrow.now().floor("day") - assert "help" in kwargs - - def test_run_worker(self, m_worker): - m_worker().cli = Mock(name="cli") - get_vfpa_hadcp.main() - args, kwargs = m_worker().run.call_args - assert args == ( - get_vfpa_hadcp.get_vfpa_hadcp, - get_vfpa_hadcp.success, - get_vfpa_hadcp.failure, + def test_instantiate_worker(self, mock_worker): + worker = get_vfpa_hadcp.main() + + assert worker.name == "get_vfpa_hadcp" + assert worker.description.startswith( + "SalishSeaCast worker that processes VFPA HADCP observations from the 2nd Narrows Rail Bridge" ) + def test_add_data_date_option(self, mock_worker): + worker = get_vfpa_hadcp.main() + assert worker.cli.parser._actions[3].dest == "data_date" + expected = nemo_nowcast.cli.CommandLineInterface.arrow_date + assert worker.cli.parser._actions[3].type == expected + assert worker.cli.parser._actions[3].default == arrow.now().floor("day") + assert worker.cli.parser._actions[3].help + class TestConfig: """Unit tests for production YAML config file elements related to worker.""" From 858813abd677c596e0b65a49a5a46444fef5d765 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 13 Jul 2024 16:24:22 -0700 Subject: [PATCH 4/5] Change logging mocks to pytest caplog fixture Replace unittest.mock.patch decorator with pytest caplog fixture for tests of logging. Test suite maintenance re: issue #82. --- tests/workers/test_get_vfpa_hadcp.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/workers/test_get_vfpa_hadcp.py b/tests/workers/test_get_vfpa_hadcp.py index b1008d18..61590899 100644 --- a/tests/workers/test_get_vfpa_hadcp.py +++ b/tests/workers/test_get_vfpa_hadcp.py @@ -18,6 +18,7 @@ """Unit tests for SalishSeaCast get_vfpa_hadcp worker. """ +import logging import textwrap from pathlib import Path from types import SimpleNamespace @@ -97,29 +98,29 @@ def test_observations(self, prod_config): assert hadcp_obs["filepath template"] == "VFPA_2ND_NARROWS_HADCP_2s_{yyyymm}.nc" -@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True) class TestSuccess: """Unit test for success() function.""" - def test_success(self, m_logger): + def test_success(self, caplog): parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01")) + caplog.set_level(logging.DEBUG) msg_type = get_vfpa_hadcp.success(parsed_args) - m_logger.info.assert_called_once_with( - "VFPA HADCP observations added to 2018-10 netcdf file" - ) + assert caplog.records[0].levelname == "INFO" + expected = "VFPA HADCP observations added to 2018-10 netcdf file" + assert caplog.messages[0] == expected assert msg_type == "success" -@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True) class TestFailure: """Unit test for failure() function.""" - def test_failure(self, m_logger): + def test_failure(self, caplog): parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01")) + caplog.set_level(logging.DEBUG) msg_type = get_vfpa_hadcp.failure(parsed_args) - m_logger.critical.assert_called_once_with( - "Addition of VFPA HADCP observations to 2018-10 netcdf file failed" - ) + assert caplog.records[0].levelname == "CRITICAL" + expected = "Addition of VFPA HADCP observations to 2018-10 netcdf file failed" + assert caplog.messages[0] == expected assert msg_type == "failure" From 6e7b34573a29d705621e5e540213eab5da953988 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Tue, 16 Jul 2024 16:10:38 -0700 Subject: [PATCH 5/5] Modernize unit tests for get_vfpa_hadcp worker Modified the unit tests in test_get_vfpa_hadcp.py to use pytest fixtures and monkeypatching for more accurate and isolated tests. This includes new mocks for make_hour_dataset and write_netcdf, as well as changes in logging and managing file paths. Particular attention given was given to ensuring accurate capture of log messages. --- tests/workers/test_get_vfpa_hadcp.py | 131 ++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 23 deletions(-) diff --git a/tests/workers/test_get_vfpa_hadcp.py b/tests/workers/test_get_vfpa_hadcp.py index 61590899..b95ab9fd 100644 --- a/tests/workers/test_get_vfpa_hadcp.py +++ b/tests/workers/test_get_vfpa_hadcp.py @@ -19,14 +19,15 @@ """Unit tests for SalishSeaCast get_vfpa_hadcp worker. """ import logging +import os import textwrap from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch import arrow import nemo_nowcast import pytest +import xarray from nowcast.workers import get_vfpa_hadcp @@ -124,49 +125,133 @@ def test_failure(self, caplog): assert msg_type == "failure" -@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True) -@patch("nowcast.workers.get_vfpa_hadcp._make_hour_dataset", autospec=True) class TestGetVFPA_HADCP: """Unit test for get_vfpa_hadcp() function.""" - def test_checklist_create(self, m_mk_hr_ds, m_logger, config): + @staticmethod + @pytest.fixture + def mock_make_hour_dataset(monkeypatch): + + def _mock_make_hour_dataset(csv_dir, utc_start_hr, place): + return xarray.Dataset() + + monkeypatch.setattr( + get_vfpa_hadcp, "_make_hour_dataset", _mock_make_hour_dataset + ) + + @staticmethod + @pytest.fixture + def mock_write_netcdf(monkeypatch): + def _mock_write_netcdf(ds, nc_filepath): + return + + monkeypatch.setattr(get_vfpa_hadcp, "_write_netcdf", _mock_write_netcdf) + + @pytest.mark.parametrize("nc_file_exists", (True, False)) + def test_log_messages( + self, + nc_file_exists, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, + ): + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) + ) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_202407.nc" + if nc_file_exists: + nc_filepath.write_bytes(b"") + parsed_args = SimpleNamespace(data_date=arrow.get("2024-07-13")) + caplog.set_level(logging.DEBUG) + get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) + assert caplog.records[0].levelname == "INFO" + expected = ( + "processing VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13" + ) + assert caplog.messages[0] == expected + if not nc_file_exists: + assert caplog.records[1].levelname == "INFO" + assert caplog.records[1].message.startswith("created") + assert caplog.messages[1].endswith("VFPA_2ND_NARROWS_HADCP_2s_202407.nc") + for rec_num, hr in zip(range(2, 24), range(1, 23)): + assert caplog.records[rec_num].levelname == "DEBUG" + expected = f"no data for 2024-07-13 {hr:02d}:00 hour" + assert caplog.messages[rec_num] == expected + assert caplog.records[25].levelname == "INFO" + expected = f"added VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13 to {nc_filepath}" + assert caplog.messages[25] == expected + + def test_checklist_create( + self, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, + ): + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) + ) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc" parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01")) + caplog.set_level(logging.DEBUG) checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) expected = { - "created": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc", + "created": f"{nc_filepath}", "UTC date": "2018-10-01", } assert checklist == expected - @patch( - "nowcast.workers.get_vfpa_hadcp.Path.exists", return_value=True, autospec=True - ) - @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True) - def test_checklist_extend(self, m_xarray, m_exists, m_mk_hr_ds, m_logger, config): + def test_checklist_extend( + self, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, + ): + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) + ) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc" + xarray.DataArray().to_netcdf(nc_filepath) parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-21")) + caplog.set_level(logging.DEBUG) checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) expected = { - "extended": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc", + "extended": f"{nc_filepath}", "UTC date": "2018-10-21", } assert checklist == expected - @pytest.mark.parametrize("ds_exists", (True, False)) - @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True) def test_checklist_missing_data( - self, m_xarray, m_mk_hr_ds, m_logger, ds_exists, config + self, + mock_make_hour_dataset, + mock_write_netcdf, + config, + caplog, + tmp_path, + monkeypatch, ): - parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23")) - m_mk_hr_ds.side_effect = ValueError - p_exists = patch( - "nowcast.workers.get_vfpa_hadcp.Path.exists", - return_value=ds_exists, - autospec=True, + dest_dir = tmp_path + monkeypatch.setitem( + config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir) ) - with p_exists: - checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) + nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201812.nc" + nc_filepath.write_bytes(b"") + caplog.set_level(logging.DEBUG) + parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23")) + checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config) expected = { - "missing data": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201812.nc", + "missing data": f"{nc_filepath}", "UTC date": "2018-12-23", } assert checklist == expected