Merge pull request #104 from nansencenter/hotfix_syntool_converter

fix sentinel 1 syntool converter intermediate files cleanup
nansencenter · Mar 6, 2024 · 2495812 · 2495812
2 parents c05086f + 69ec9e9
commit 2495812
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 10 deletions.
diff --git a/geospaas_processing/converters/syntool/converter.py b/geospaas_processing/converters/syntool/converter.py
@@ -244,7 +244,10 @@ def run(self, in_file, out_dir, **kwargs):
                         '--options-file',
                         self.PARAMETERS_DIR / ingest_config],
                     **kwargs))
-            os.remove(converted_file)
+            try:
+                os.remove(converted_path)
+            except IsADirectoryError:
+                shutil.rmtree(converted_path)
 
         self.post_ingest(results, results_dir, **kwargs)
         return results
@@ -296,7 +299,6 @@ def ingest(self, in_file, out_dir, options, **kwargs):
                     result_dir.replace(final_result_path)
                     results.append(str(final_result_path.relative_to(out_dir)))
                 base_result_path.rmdir()
-        os.rmdir(in_file)
         return results
 
 

diff --git a/tests/converters/test_syntool_converters.py b/tests/converters/test_syntool_converters.py
@@ -213,11 +213,12 @@ def test_run(self):
             converter_type='foo',
             ingest_parameter_files='bar')
         with mock.patch.object(converter, 'convert',
-                               return_value=['conv1.tiff', 'conv2.tiff']) as mock_convert, \
+                               return_value=['conv1.tiff', 'conv2']) as mock_convert, \
              mock.patch.object(converter, 'ingest',
                                side_effect=[['ingested_dir1'], ['ingested_dir2']]) as mock_ingest, \
              mock.patch.object(converter, 'post_ingest') as mock_post_ingest, \
-             mock.patch('os.remove') as mock_remove:
+             mock.patch('os.remove', side_effect=(None, IsADirectoryError)) as mock_remove, \
+             mock.patch('shutil.rmtree') as mock_rmtree:
             converter.run(
                 in_file='in.nc',
                 out_dir='out',
@@ -231,13 +232,16 @@ def test_run(self):
                 ['--config', Path('parameters/3413.ini'),
                  '--options-file', converter.PARAMETERS_DIR / 'bar']),
             mock.call(
-                Path('out', 'conv2.tiff'),
+                Path('out', 'conv2'),
                 'results',
                 ['--config', Path('parameters/3413.ini'),
                  '--options-file', converter.PARAMETERS_DIR / 'bar']),
         ])
         mock_post_ingest.assert_called_once_with(['ingested_dir1', 'ingested_dir2'], 'results')
-        mock_remove.assert_has_calls((mock.call('conv1.tiff'), mock.call('conv2.tiff')))
+        mock_remove.assert_has_calls((
+            mock.call(Path('out', 'conv1.tiff')),
+            mock.call(Path('out', 'conv2'))))
+        mock_rmtree.assert_called_once_with(Path('out', 'conv2'))
 
     def test_run_no_converter(self):
         """If no converter is set, convert() should return the path to
@@ -262,7 +266,7 @@ def test_run_no_converter(self):
             ['--config', Path('parameters/3413.ini'),
              '--options-file', converter.PARAMETERS_DIR / 'bar'])
         mock_post_ingest.assert_called_once_with(['ingested_dir1'], 'results')
-        mock_remove.assert_called_once_with('in.nc')
+        mock_remove.assert_called_once_with(Path('out', 'in.nc'))
 
 
 class Sentinel1SyntoolConverterTestCase(unittest.TestCase):
@@ -308,9 +312,6 @@ def test_convert(self):
             for file_name in file_names:
                 mock_convert.assert_any_call(measurement_dir / file_name, 'out_dir', ['--baz'])
 
-
-
-
     def test_ingest(self):
         """Test that the subdirectories created by ingestion are copied
         to the ingested results folder