diff --git a/test/test_conversion_driver_esm1p5.py b/test/test_conversion_driver_esm1p5.py index ef3e669..3f7684d 100644 --- a/test/test_conversion_driver_esm1p5.py +++ b/test/test_conversion_driver_esm1p5.py @@ -111,8 +111,13 @@ def _mock_process_with_exception(error_message): "input_list", [[], ["fake_file"], [ "fake_file_1", "fake_file_2", "fake_file_3"]] ) -def test_convert_fields_file_list_success(mock_process, input_list): +def test_convert_fields_file_list_success(mock_process, + input_list): + """ + Test that process is called for each input. + """ input_list_paths = [Path(p) for p in input_list] + succeeded, _ = esm1p5_convert.convert_fields_file_list( input_list_paths, "fake_nc_write_dir") @@ -120,7 +125,7 @@ def test_convert_fields_file_list_success(mock_process, input_list): successful_input_paths = [successful_path_pair[0] for successful_path_pair in succeeded] - + assert input_list_paths == successful_input_paths @@ -213,11 +218,11 @@ def test_format_failures_standard_mode(): try: raise exception_2 from exception_1 except Exception as exc: - exc_with_traceback = exc + exc_with_traceback = exc failed_file = Path("fake_file") failed_conversion = [(failed_file, exc_with_traceback)] - + formatted_failure_report_list = list( esm1p5_convert.format_failures(failed_conversion, quiet=False) ) @@ -228,3 +233,18 @@ def test_format_failures_standard_mode(): assert exception_1.args[0] in formatted_failure_report assert exception_2.args[0] in formatted_failure_report + + +def test_success_fail_overlap(): + # Test that inputs listed as both successes and failures + # are removed as candidates for deletion. + success_only_path = Path("success_only") + success_and_fail_path = Path("success_and_fail") + successes = [(success_only_path, Path("success_only.nc")), + (success_and_fail_path, Path("success_and_fail.nc"))] + failures = [(success_and_fail_path, "Exception_placeholder")] + + result = esm1p5_convert.safe_removal(successes, failures) + + assert success_and_fail_path not in result + assert success_only_path in result diff --git a/umpost/conversion_driver_esm1p5.py b/umpost/conversion_driver_esm1p5.py index 36413b5..d7cbe9f 100755 --- a/umpost/conversion_driver_esm1p5.py +++ b/umpost/conversion_driver_esm1p5.py @@ -162,7 +162,7 @@ def format_successes(succeeded): Format reports of successful conversions to be shared with user. Parameters - ---------- + ---------- succeeded: list of (input, output) tuples of filepaths for successful conversions. @@ -170,7 +170,7 @@ def format_successes(succeeded): ------- success_report: formatted report of successful conversion. """ - + for input_path, output_path in succeeded: success_report = f"Successfully converted {input_path} to {output_path}" yield success_report @@ -182,9 +182,9 @@ def format_failures(failed, quiet): Parameters ---------- - failed: list of tuples of form (filepath, exception) for files which failed + failed: list of tuples of form (filepath, exception) for files which failed to convert due to an allowable exception. - quiet: boolean. Report only final exception type and message rather than + quiet: boolean. Report only final exception type and message rather than full stack trace when true. Yields @@ -201,7 +201,7 @@ def format_failures(failed, quiet): ) yield failure_report else: - + for fields_file_path, exception in failed: formatted_traceback = "".join( traceback.format_exception(exception) @@ -219,8 +219,9 @@ def convert_esm1p5_output_dir(esm1p5_output_dir): Parameters ---------- - esm1p5_output_dir: an "outputXYZ" directory produced by an ESM1.5 simulation. - Fields files in the "atmosphere" subdirectory will be converted to NetCDF. + esm1p5_output_dir: an "outputXYZ" directory produced by an ESM1.5 simulation. + Fields files in the "atmosphere" subdirectory will be + converted to NetCDF. Returns ------- @@ -271,6 +272,29 @@ def convert_esm1p5_output_dir(esm1p5_output_dir): return succeeded, failed +def safe_removal(succeeded, failed): + """ + Check whether any input files were reported as simultaneously + successful and failed conversions. Return those that appear + only as successes as targets for safe deletion. + + Parameters + ---------- + succeeded: List of (input_file, output_file) tuples of filepaths from + successful conversions. + failed: List of (input_file, Exception) tuples from failed conversions. + + Returns + ------- + successful_only: set of input filepaths which appear in succeeded but + not failed. + """ + succeeded_inputs = {succeed_path for succeed_path, _ in succeeded} + failed_inputs = {fail_path for fail_path, _ in failed} + + return succeeded_inputs - failed_inputs + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( @@ -283,6 +307,9 @@ def convert_esm1p5_output_dir(esm1p5_output_dir): "Otherwise report full stack trace." ) ) + parser.add_argument("--delete-ff", "-d", action="store_true", + help="Delete fields files upon successful conversion." + ) args = parser.parse_args() current_output_dir = args.current_output_dir @@ -294,3 +321,8 @@ def convert_esm1p5_output_dir(esm1p5_output_dir): print(success_message) for failure_message in format_failures(failures, args.quiet): warnings.warn(failure_message) + + if args.delete_ff: + # Remove files that appear only as successful conversions + for path in safe_removal(successes, failures): + os.remove(path)