ESM1.5 conversion driver, delete inputs upon successful conversion

test/test_conversion_driver_esm1p5.py

@@ -95,6 +95,11 @@ def mock_process(base_mock_process):
     base_mock_process.return_value = None
     return base_mock_process
 
+@pytest.fixture
+def mock_os_remove():
+    patcher = mock.patch("os.remove")
+    yield patcher.start()
+    patcher.stop()
 
 @pytest.fixture
 def mock_process_with_exception(mock_process):
@@ -111,20 +116,43 @@ def _mock_process_with_exception(error_message):
     "input_list", [[], ["fake_file"], [
         "fake_file_1", "fake_file_2", "fake_file_3"]]
 )
-def test_convert_fields_file_list_success(mock_process, input_list):
+def test_convert_fields_file_list_success_delete(mock_process,
+                                                 mock_os_remove,
+                                                 input_list):
+    """
+    Test that process is called for each input and that each input is deleted.
+    """
     input_list_paths = [Path(p) for p in input_list]
+
     succeeded, _ = esm1p5_convert.convert_fields_file_list(
-        input_list_paths, "fake_nc_write_dir")
+        input_list_paths, "fake_nc_write_dir", delete_ff=True)
 
     assert mock_process.call_count == len(input_list)
+    assert mock_os_remove.call_count == len(input_list)
+
+    for path in input_list_paths:
+        assert mock.call(path) in mock_os_remove.call_args_list
 
     successful_input_paths = [successful_path_pair[0] for
                               successful_path_pair in succeeded]
-    
+
     assert input_list_paths == successful_input_paths
 
 
-def test_convert_fields_file_list_fail_excepted(mock_process_with_exception):
+def test_convert_fields_file_list_no_delete(mock_process, mock_os_remove):
+    """
+    Test that files are not deleted when delete_ff is False.
+    """
+    input_list_paths = [Path("fake_file_1"), Path("fake_file_2")]
+
+    succeeded, _ = esm1p5_convert.convert_fields_file_list(
+        input_list_paths, "fake_nc_write_dir", delete_ff=False)
+
+    mock_os_remove.assert_not_called()
+
+
+def test_convert_fields_file_list_fail_excepted(mock_process_with_exception,
+                                                mock_os_remove):
     # Hopefully this test will be unnecessary with um2nc standalone.
     # Test that the "Variable can not be processed" error arising from time
     # series inputs is excepted.
@@ -135,30 +163,37 @@ def test_convert_fields_file_list_fail_excepted(mock_process_with_exception):
     fake_file_path = Path("fake_file")
 
     _, failed = esm1p5_convert.convert_fields_file_list(
-        [fake_file_path], "fake_nc_write_dir")
+        [fake_file_path], "fake_nc_write_dir", delete_ff=True)
 
     assert failed[0][0] == fake_file_path
 
+    # Assert that no files removed
+    mock_os_remove.assert_not_called()
+
     # TODO: Testing the exception part of the reported failures will be easier
     # once um2nc specific exceptions are added.
 
 
-def test_convert_fields_file_list_fail_critical(mock_process_with_exception):
+def test_convert_fields_file_list_fail_critical(mock_process_with_exception, mock_os_remove):
     # Test that critical exceptions which are not allowed by ALLOWED_UM2NC_EXCEPTION_MESSAGES
     # are raised, and hence lead to the conversion crashing.
     generic_error_message = "Test error"
     mock_process_with_exception(generic_error_message)
     with pytest.raises(Exception) as exc_info:
         esm1p5_convert.convert_fields_file_list(
-            ["fake_file"], "fake_nc_write_dir")
+            ["fake_file"], "fake_nc_write_dir", delete_ff=True)
 
     assert str(exc_info.value) == generic_error_message
 
+    # Assert that no files removed
+    mock_os_remove.assert_not_called()
+
 
 def test_convert_esm1p5_output_dir_error():
     with pytest.raises(FileNotFoundError):
         esm1p5_convert.convert_esm1p5_output_dir(
-            "/test_convert_esm1p5_output_dir_error/fake/path/"
+            "/test_convert_esm1p5_output_dir_error/fake/path/",
+            delete_ff=True
         )
 
 
@@ -213,11 +248,11 @@ def test_format_failures_standard_mode():
     try:
         raise exception_2 from exception_1
     except Exception as exc:
-        exc_with_traceback = exc 
+        exc_with_traceback = exc
 
     failed_file = Path("fake_file")
     failed_conversion = [(failed_file, exc_with_traceback)]
-    
+
     formatted_failure_report_list = list(
         esm1p5_convert.format_failures(failed_conversion, quiet=False)
     )

umpost/conversion_driver_esm1p5.py

@@ -162,15 +162,15 @@ def format_successes(succeeded):
     Format reports of successful conversions to be shared with user.
 
     Parameters
-    ---------- 
+    ----------
     succeeded: list of (input, output) tuples of filepaths for successful
                conversions.
 
     Yields
     -------
     success_report: formatted report of successful conversion.
     """
-    
+
     for input_path, output_path in succeeded:
         success_report = f"Successfully converted {input_path} to {output_path}"
         yield success_report
@@ -182,9 +182,9 @@ def format_failures(failed, quiet):
 
     Parameters
     ----------
-    failed: list of tuples of form (filepath, exception) for files which failed 
+    failed: list of tuples of form (filepath, exception) for files which failed
             to convert due to an allowable exception.
-    quiet: boolean. Report only final exception type and message rather than 
+    quiet: boolean. Report only final exception type and message rather than
            full stack trace when true.
 
     Yields
@@ -201,7 +201,7 @@ def format_failures(failed, quiet):
             )
             yield failure_report
     else:
-        
+
         for fields_file_path, exception in failed:
             formatted_traceback = "".join(
                 traceback.format_exception(exception)
@@ -219,8 +219,9 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
 
     Parameters
     ----------
-    esm1p5_output_dir: an "outputXYZ" directory produced by an ESM1.5 simulation. 
-        Fields files in the "atmosphere" subdirectory will be converted to NetCDF.
+    esm1p5_output_dir: an "outputXYZ" directory produced by an ESM1.5 simulation.
+            Fields files in the "atmosphere" subdirectory will be
+            converted to NetCDF.
 
     Returns
     -------
@@ -270,6 +271,14 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
 
     return succeeded, failed
 
+def success_fail_overlap(succeeded, failed):
+    succeeded_inputs = [succeed_path for succeed_path, _ in succeeded]
+    failed_inputs = [fail_path for fail_path, _ in failed]
+
+    overlap = set(succeeded_inputs) & set(failed_inputs)
+
+    return overlap
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -283,6 +292,9 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
                             "Otherwise report full stack trace."
                         )
                         )
+    parser.add_argument("--delete-ff", "-d", action="store_true",
+                        help="Delete fields files upon successful conversion."
+                        )
     args = parser.parse_args()
 
     current_output_dir = args.current_output_dir
@@ -294,3 +306,17 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
         print(success_message)
     for failure_message in format_failures(failures, args.quiet):
         warnings.warn(failure_message)
+
+    if args.delete_ff:
+        # Check that no successful inputs somehow simultaneously failed.
+        overlap = success_fail_overlap(successes, failures)
+        if overlap:
+            msg = (
+                    "Following inputs reported simultaneous successful and "
+                    "failed conversions. Inputs will not be deleted.\n"
+                    f"{overlap}"
+            )
+            raise um2netcdf.PostProcessingError(msg)
+        else:
+            for successful_input_path, _ in successes:
+                os.remove(successful_input_path)