Add get_vfpa_hadcp after 06 weather collection

nowcast/next_workers.py

@@ -78,6 +78,11 @@ def after_download_weather(msg, config, checklist):
                 next_workers["success 2.5km 06"].append(
                     NextWorker("nowcast.workers.get_onc_ferry", args=[ferry])
                 )
+            next_workers["success 2.5km 06"].append(
+                NextWorker(
+                    "nowcast.workers.get_vfpa_hadcp", args=["--data-date", data_date]
+                )
+            )
             if "forecast2" in config["run types"]:
                 next_workers["success 2.5km 06"].append(
                     NextWorker("nowcast.workers.collect_NeahBay_ssh", args=["00"]),

nowcast/workers/get_vfpa_hadcp.py

@@ -37,9 +37,7 @@
 
 
 def main():
-    """Set up and run the worker.
-
-    For command-line usage see:
+    """For command-line usage see:
 
     :command:`python -m nowcast.workers.get_vfpa_hadcp --help`
     """
@@ -51,6 +49,7 @@ def main():
         help="UTC date to get VFPA HADPC data for.",
     )
     worker.run(get_vfpa_hadcp, success, failure)
+    return worker
 
 
 def success(parsed_args):

tests/test_next_workers.py

@@ -208,6 +208,11 @@ def mock_now():
             NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"),
+            NextWorker(
+                "nowcast.workers.get_vfpa_hadcp",
+                ["--data-date", "2018-12-26"],
+                host="localhost",
+            ),
             NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"),
         ]
         assert workers == expected
@@ -327,6 +332,11 @@ def mock_now():
             NextWorker("nowcast.workers.get_onc_ctd", ["SCVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["SEVIP"], host="localhost"),
             NextWorker("nowcast.workers.get_onc_ctd", ["USDDL"], host="localhost"),
+            NextWorker(
+                "nowcast.workers.get_vfpa_hadcp",
+                ["--data-date", "2018-12-26"],
+                host="localhost",
+            ),
             NextWorker("nowcast.workers.collect_NeahBay_ssh", ["00"], host="localhost"),
             NextWorker(
                 "nowcast.workers.collect_weather", ["12", "2.5km"], host="localhost"

tests/workers/test_get_vfpa_hadcp.py

@@ -18,14 +18,16 @@
 
 """Unit tests for SalishSeaCast get_vfpa_hadcp worker.
 """
+import logging
+import os
 import textwrap
 from pathlib import Path
 from types import SimpleNamespace
-from unittest.mock import Mock, patch
 
 import arrow
 import nemo_nowcast
 import pytest
+import xarray
 
 from nowcast.workers import get_vfpa_hadcp
 
@@ -51,40 +53,30 @@ def config(base_config):
     return config_
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.NowcastWorker", spec=True)
+@pytest.fixture
+def mock_worker(mock_nowcast_worker, monkeypatch):
+    monkeypatch.setattr(get_vfpa_hadcp, "NowcastWorker", mock_nowcast_worker)
+
+
 class TestMain:
     """Unit tests for main() function."""
 
-    def test_instantiate_worker(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        args, kwargs = m_worker.call_args
-        assert args == ("get_vfpa_hadcp",)
-        assert list(kwargs.keys()) == ["description"]
-
-    def test_init_cli(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        m_worker().init_cli.assert_called_once_with()
-
-    def test_add_data_date_option(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        args, kwargs = m_worker().cli.add_date_option.call_args_list[0]
-        assert args == ("--data-date",)
-        assert kwargs["default"] == arrow.now().floor("day")
-        assert "help" in kwargs
-
-    def test_run_worker(self, m_worker):
-        m_worker().cli = Mock(name="cli")
-        get_vfpa_hadcp.main()
-        args, kwargs = m_worker().run.call_args
-        assert args == (
-            get_vfpa_hadcp.get_vfpa_hadcp,
-            get_vfpa_hadcp.success,
-            get_vfpa_hadcp.failure,
+    def test_instantiate_worker(self, mock_worker):
+        worker = get_vfpa_hadcp.main()
+
+        assert worker.name == "get_vfpa_hadcp"
+        assert worker.description.startswith(
+            "SalishSeaCast worker that processes VFPA HADCP observations from the 2nd Narrows Rail Bridge"
         )
 
+    def test_add_data_date_option(self, mock_worker):
+        worker = get_vfpa_hadcp.main()
+        assert worker.cli.parser._actions[3].dest == "data_date"
+        expected = nemo_nowcast.cli.CommandLineInterface.arrow_date
+        assert worker.cli.parser._actions[3].type == expected
+        assert worker.cli.parser._actions[3].default == arrow.now().floor("day")
+        assert worker.cli.parser._actions[3].help
+
 
 class TestConfig:
     """Unit tests for production YAML config file elements related to worker."""
@@ -107,75 +99,159 @@ def test_observations(self, prod_config):
         assert hadcp_obs["filepath template"] == "VFPA_2ND_NARROWS_HADCP_2s_{yyyymm}.nc"
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True)
 class TestSuccess:
     """Unit test for success() function."""
 
-    def test_success(self, m_logger):
+    def test_success(self, caplog):
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01"))
+        caplog.set_level(logging.DEBUG)
         msg_type = get_vfpa_hadcp.success(parsed_args)
-        m_logger.info.assert_called_once_with(
-            "VFPA HADCP observations added to 2018-10 netcdf file"
-        )
+        assert caplog.records[0].levelname == "INFO"
+        expected = "VFPA HADCP observations added to 2018-10 netcdf file"
+        assert caplog.messages[0] == expected
         assert msg_type == "success"
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True)
 class TestFailure:
     """Unit test for failure() function."""
 
-    def test_failure(self, m_logger):
+    def test_failure(self, caplog):
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01"))
+        caplog.set_level(logging.DEBUG)
         msg_type = get_vfpa_hadcp.failure(parsed_args)
-        m_logger.critical.assert_called_once_with(
-            "Addition of VFPA HADCP observations to 2018-10 netcdf file failed"
-        )
+        assert caplog.records[0].levelname == "CRITICAL"
+        expected = "Addition of VFPA HADCP observations to 2018-10 netcdf file failed"
+        assert caplog.messages[0] == expected
         assert msg_type == "failure"
 
 
-@patch("nowcast.workers.get_vfpa_hadcp.logger", autospec=True)
-@patch("nowcast.workers.get_vfpa_hadcp._make_hour_dataset", autospec=True)
 class TestGetVFPA_HADCP:
     """Unit test for get_vfpa_hadcp() function."""
 
-    def test_checklist_create(self, m_mk_hr_ds, m_logger, config):
+    @staticmethod
+    @pytest.fixture
+    def mock_make_hour_dataset(monkeypatch):
+
+        def _mock_make_hour_dataset(csv_dir, utc_start_hr, place):
+            return xarray.Dataset()
+
+        monkeypatch.setattr(
+            get_vfpa_hadcp, "_make_hour_dataset", _mock_make_hour_dataset
+        )
+
+    @staticmethod
+    @pytest.fixture
+    def mock_write_netcdf(monkeypatch):
+        def _mock_write_netcdf(ds, nc_filepath):
+            return
+
+        monkeypatch.setattr(get_vfpa_hadcp, "_write_netcdf", _mock_write_netcdf)
+
+    @pytest.mark.parametrize("nc_file_exists", (True, False))
+    def test_log_messages(
+        self,
+        nc_file_exists,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
+    ):
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
+        )
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_202407.nc"
+        if nc_file_exists:
+            nc_filepath.write_bytes(b"")
+        parsed_args = SimpleNamespace(data_date=arrow.get("2024-07-13"))
+        caplog.set_level(logging.DEBUG)
+        get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
+        assert caplog.records[0].levelname == "INFO"
+        expected = (
+            "processing VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13"
+        )
+        assert caplog.messages[0] == expected
+        if not nc_file_exists:
+            assert caplog.records[1].levelname == "INFO"
+            assert caplog.records[1].message.startswith("created")
+            assert caplog.messages[1].endswith("VFPA_2ND_NARROWS_HADCP_2s_202407.nc")
+        for rec_num, hr in zip(range(2, 24), range(1, 23)):
+            assert caplog.records[rec_num].levelname == "DEBUG"
+            expected = f"no data for 2024-07-13 {hr:02d}:00 hour"
+            assert caplog.messages[rec_num] == expected
+        assert caplog.records[25].levelname == "INFO"
+        expected = f"added VFPA HADCP data from 2nd Narrows Rail Bridge for 2024-07-13 to {nc_filepath}"
+        assert caplog.messages[25] == expected
+
+    def test_checklist_create(
+        self,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
+    ):
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
+        )
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc"
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-01"))
+        caplog.set_level(logging.DEBUG)
         checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
         expected = {
-            "created": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc",
+            "created": f"{nc_filepath}",
             "UTC date": "2018-10-01",
         }
         assert checklist == expected
 
-    @patch(
-        "nowcast.workers.get_vfpa_hadcp.Path.exists", return_value=True, autospec=True
-    )
-    @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True)
-    def test_checklist_extend(self, m_xarray, m_exists, m_mk_hr_ds, m_logger, config):
+    def test_checklist_extend(
+        self,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
+    ):
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
+        )
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201810.nc"
+        xarray.DataArray().to_netcdf(nc_filepath)
         parsed_args = SimpleNamespace(data_date=arrow.get("2018-10-21"))
+        caplog.set_level(logging.DEBUG)
         checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
         expected = {
-            "extended": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201810.nc",
+            "extended": f"{nc_filepath}",
             "UTC date": "2018-10-21",
         }
         assert checklist == expected
 
-    @pytest.mark.parametrize("ds_exists", (True, False))
-    @patch("nowcast.workers.get_vfpa_hadcp.xarray", autospec=True)
     def test_checklist_missing_data(
-        self, m_xarray, m_mk_hr_ds, m_logger, ds_exists, config
+        self,
+        mock_make_hour_dataset,
+        mock_write_netcdf,
+        config,
+        caplog,
+        tmp_path,
+        monkeypatch,
     ):
-        parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23"))
-        m_mk_hr_ds.side_effect = ValueError
-        p_exists = patch(
-            "nowcast.workers.get_vfpa_hadcp.Path.exists",
-            return_value=ds_exists,
-            autospec=True,
+        dest_dir = tmp_path
+        monkeypatch.setitem(
+            config["observations"]["hadcp data"], "dest dir", os.fspath(dest_dir)
         )
-        with p_exists:
-            checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
+        nc_filepath = dest_dir / "VFPA_2ND_NARROWS_HADCP_2s_201812.nc"
+        nc_filepath.write_bytes(b"")
+        caplog.set_level(logging.DEBUG)
+        parsed_args = SimpleNamespace(data_date=arrow.get("2018-12-23"))
+        checklist = get_vfpa_hadcp.get_vfpa_hadcp(parsed_args, config)
         expected = {
-            "missing data": "opp/obs/AISDATA/netcdf/VFPA_2ND_NARROWS_HADCP_2s_201812.nc",
+            "missing data": f"{nc_filepath}",
             "UTC date": "2018-12-23",
         }
         assert checklist == expected