Merge pull request #77 from ACCESS-NRI/49/esm1.5-driver-delete-inputs

ESM1.5 conversion driver, delete inputs upon successful conversion

test/test_conversion_driver_esm1p5.py

@@ -111,16 +111,21 @@ def _mock_process_with_exception(error_message):
     "input_list", [[], ["fake_file"], [
         "fake_file_1", "fake_file_2", "fake_file_3"]]
 )
-def test_convert_fields_file_list_success(mock_process, input_list):
+def test_convert_fields_file_list_success(mock_process,
+                                          input_list):
+    """
+    Test that process is called for each input.
+    """
     input_list_paths = [Path(p) for p in input_list]
+
     succeeded, _ = esm1p5_convert.convert_fields_file_list(
         input_list_paths, "fake_nc_write_dir")
 
     assert mock_process.call_count == len(input_list)
 
     successful_input_paths = [successful_path_pair[0] for
                               successful_path_pair in succeeded]
-    
+
     assert input_list_paths == successful_input_paths
 
 
@@ -213,11 +218,11 @@ def test_format_failures_standard_mode():
     try:
         raise exception_2 from exception_1
     except Exception as exc:
-        exc_with_traceback = exc 
+        exc_with_traceback = exc
 
     failed_file = Path("fake_file")
     failed_conversion = [(failed_file, exc_with_traceback)]
-    
+
     formatted_failure_report_list = list(
         esm1p5_convert.format_failures(failed_conversion, quiet=False)
     )
@@ -228,3 +233,18 @@ def test_format_failures_standard_mode():
 
     assert exception_1.args[0] in formatted_failure_report
     assert exception_2.args[0] in formatted_failure_report
+
+
+def test_success_fail_overlap():
+    # Test that inputs listed as both successes and failures
+    # are removed as candidates for deletion.
+    success_only_path = Path("success_only")
+    success_and_fail_path = Path("success_and_fail")
+    successes = [(success_only_path, Path("success_only.nc")),
+                 (success_and_fail_path, Path("success_and_fail.nc"))]
+    failures = [(success_and_fail_path, "Exception_placeholder")]
+
+    result = esm1p5_convert.safe_removal(successes, failures)
+
+    assert success_and_fail_path not in result
+    assert success_only_path in result

umpost/conversion_driver_esm1p5.py

@@ -162,15 +162,15 @@ def format_successes(succeeded):
     Format reports of successful conversions to be shared with user.
 
     Parameters
-    ---------- 
+    ----------
     succeeded: list of (input, output) tuples of filepaths for successful
                conversions.
 
     Yields
     -------
     success_report: formatted report of successful conversion.
     """
-    
+
     for input_path, output_path in succeeded:
         success_report = f"Successfully converted {input_path} to {output_path}"
         yield success_report
@@ -182,9 +182,9 @@ def format_failures(failed, quiet):
 
     Parameters
     ----------
-    failed: list of tuples of form (filepath, exception) for files which failed 
+    failed: list of tuples of form (filepath, exception) for files which failed
             to convert due to an allowable exception.
-    quiet: boolean. Report only final exception type and message rather than 
+    quiet: boolean. Report only final exception type and message rather than
            full stack trace when true.
 
     Yields
@@ -201,7 +201,7 @@ def format_failures(failed, quiet):
             )
             yield failure_report
     else:
-        
+
         for fields_file_path, exception in failed:
             formatted_traceback = "".join(
                 traceback.format_exception(exception)
@@ -219,8 +219,9 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
 
     Parameters
     ----------
-    esm1p5_output_dir: an "outputXYZ" directory produced by an ESM1.5 simulation. 
-        Fields files in the "atmosphere" subdirectory will be converted to NetCDF.
+    esm1p5_output_dir: an "outputXYZ" directory produced by an ESM1.5 simulation.
+            Fields files in the "atmosphere" subdirectory will be
+            converted to NetCDF.
 
     Returns
     -------
@@ -271,6 +272,29 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
     return succeeded, failed
 
 
+def safe_removal(succeeded, failed):
+    """
+    Check whether any input files were reported as simultaneously
+    successful and failed conversions. Return those that appear
+    only as successes as targets for safe deletion.
+
+    Parameters
+    ----------
+    succeeded: List of (input_file, output_file) tuples of filepaths from
+        successful conversions.
+    failed: List of (input_file, Exception) tuples from failed conversions.
+
+    Returns
+    -------
+    successful_only: set of input filepaths which appear in succeeded but
+        not failed.
+    """
+    succeeded_inputs = {succeed_path for succeed_path, _ in succeeded}
+    failed_inputs = {fail_path for fail_path, _ in failed}
+
+    return succeeded_inputs - failed_inputs
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -283,6 +307,9 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
                             "Otherwise report full stack trace."
                         )
                         )
+    parser.add_argument("--delete-ff", "-d", action="store_true",
+                        help="Delete fields files upon successful conversion."
+                        )
     args = parser.parse_args()
 
     current_output_dir = args.current_output_dir
@@ -294,3 +321,8 @@ def convert_esm1p5_output_dir(esm1p5_output_dir):
         print(success_message)
     for failure_message in format_failures(failures, args.quiet):
         warnings.warn(failure_message)
+
+    if args.delete_ff:
+        # Remove files that appear only as successful conversions
+        for path in safe_removal(successes, failures):
+            os.remove(path)