Merge pull request #103 from nansencenter/issue102_argo_floats

Syntool conversion for ARGO floats

geospaas_processing/converters/syntool/converter.py

@@ -1,4 +1,5 @@
 """Tools for converting dataset into a format displayable by Syntool"""
+import collections.abc
 import configparser
 import logging
 import os
@@ -10,8 +11,7 @@
 
 from geospaas.catalog.managers import LOCAL_FILE_SERVICE
 
-from ..base import ConversionError, ConversionManager, Converter, ParameterSelector
-from ...ops import crop
+from ..base import ConversionError, ConversionManager, Converter, NoMatch, ParameterSelector
 
 
 logger = logging.getLogger(__name__)
@@ -85,7 +85,6 @@ def ingest(self, in_file, out_dir, options, **kwargs):
                 "syntool-ingestor did not produce any file. "
                 f"stdout: {process.stdout}"
                 f"stderr: {process.stderr}"))
-        os.remove(in_file)
         return results
 
     def post_ingest(self, results, out_dir, **kwargs):
@@ -157,6 +156,11 @@ class BasicSyntoolConverter(SyntoolConverter):
             matches=lambda d: d.entry_id.startswith('asi-AMSR2-'),
             converter_type='amsr_sea_ice_conc',
             ingest_parameter_files='ingest_geotiff_3411_raster'),
+        ParameterSelector(
+            matches=lambda d: d.source.platform.short_name == 'Argo float',
+            converter_type=None,
+            ingest_parameter_files=('ingest_erddap_json_3413_profile',
+                                    'ingest_erddap_json_3413_trajectory')),
     )
 
     def __init__(self, **kwargs):
@@ -168,12 +172,31 @@ def __init__(self, **kwargs):
 
     def find_ingest_config(self, converted_file):
         """Find the right ingestion config for a converted file"""
+        invalid_ingest_parameter_files_error = ConversionError(
+            "'ingest_parameter_files' must be a string, list of strings "
+            "or a list of ParameterSelector objects")
+
         if isinstance(self.ingest_parameter_files, str):
-            return self.ingest_parameter_files
-        for selector in self.ingest_parameter_files:
-            if selector.matches(converted_file):
-                return selector.parameters['ingest_file']
-        raise ConversionError("Ingestor not found")
+            ingest_parameter_files = [self.ingest_parameter_files]
+        elif isinstance(self.ingest_parameter_files, collections.abc.Sequence):
+            ingest_parameter_files = self.ingest_parameter_files
+        else:
+            raise invalid_ingest_parameter_files_error
+
+        results = []
+        for ingest_config in ingest_parameter_files:
+            if isinstance(ingest_config, str):
+                results.append(ingest_config)
+            elif isinstance(ingest_config, ParameterSelector):
+                if ingest_config.matches(converted_file):
+                    results.append(ingest_config.parameters['ingest_file'])
+            else:
+                raise invalid_ingest_parameter_files_error
+
+        if results:
+            return results
+        else:
+            raise ConversionError("Ingestor not found")
 
     def parse_converter_options(self, kwargs):
         """Merges the converter options defined in the Converter class
@@ -202,21 +225,26 @@ def run(self, in_file, out_dir, **kwargs):
         """
         results_dir = kwargs.pop('results_dir')
         # syntool-converter
-        converted_files = self.convert(in_file, out_dir,
-                                       self.parse_converter_args(kwargs),
-                                       **kwargs)
+        if self.converter_type is not None:
+            converted_files = self.convert(in_file, out_dir,
+                                           self.parse_converter_args(kwargs),
+                                           **kwargs)
+        else:
+            converted_files = (in_file,)
 
         # syntool-ingestor
         ingestor_config = Path(kwargs.pop('ingestor_config', 'parameters/3413.ini'))
         results = []
         for converted_file in converted_files:
             converted_path = Path(out_dir, converted_file)
-            results.extend(self.ingest(
-                converted_path, results_dir, [
-                    '--config', ingestor_config,
-                    '--options-file',
-                    self.PARAMETERS_DIR / self.find_ingest_config(converted_path)],
-                **kwargs))
+            for ingest_config in self.find_ingest_config(converted_path):
+                results.extend(self.ingest(
+                    converted_path, results_dir, [
+                        '--config', ingestor_config,
+                        '--options-file',
+                        self.PARAMETERS_DIR / ingest_config],
+                    **kwargs))
+            os.remove(converted_file)
 
         self.post_ingest(results, results_dir, **kwargs)
         return results
@@ -373,3 +401,4 @@ class CustomReaderSyntoolConverter(BasicSyntoolConverter):
 
     def parse_converter_args(self, kwargs):
         return ['-r', self.converter_type, *self.parse_converter_options(kwargs)]
+

geospaas_processing/converters/syntool/parameters/ingest_erddap_json_3413_profile

@@ -0,0 +1,3 @@
+--input-format argo_profile_erddap_json
+--input-options projection=4326
+--output-format argo_profile

geospaas_processing/converters/syntool/parameters/ingest_erddap_json_3413_trajectory

@@ -0,0 +1,5 @@
+--input-format argo_trajectory_erddap_json
+--input-options projection=4326
+--output-format geojson_trajectory
+--output-options with-shape=no
+                 singleFeature=true

geospaas_processing/provider_settings.yml

@@ -37,6 +37,7 @@
   authentication_type: 'oauth2'
   token_url: 'https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token'
   client_id: 'cdse-public'
+  max_parallel_downloads: 4
 'https://podaac-tools.jpl.nasa.gov/drive/files':
   username: !ENV 'PODAAC_DRIVE_USERNAME'
   password: !ENV 'PODAAC_DRIVE_PASSWORD'

tests/converters/test_syntool_converters.py

@@ -53,7 +53,6 @@ def test_ingest(self):
         """Test that the right command is called"""
         converter = syntool_converter.SyntoolConverter()
         with mock.patch('subprocess.run') as mock_run, \
-             mock.patch('os.remove') as mock_remove, \
              mock.patch('tempfile.TemporaryDirectory') as mock_tmp_dir, \
              mock.patch.object(converter, 'move_results',
                                return_value=['3413_foo/1', '3413_foo/2']):
@@ -66,7 +65,6 @@ def test_ingest(self):
                 check=True,
                 capture_output=True,
                 env=None)
-            mock_remove.assert_called_with('/bar/foo.tiff')
             self.assertEqual(result, ['ingested/3413_foo/1', 'ingested/3413_foo/2'])
 
     def test_ingest_subprocess_error(self):
@@ -122,7 +120,7 @@ def test_find_ingest_config_str(self):
         converter = syntool_converter.BasicSyntoolConverter(
             converter_type='foo',
             ingest_parameter_files='bar')
-        self.assertEqual(converter.find_ingest_config('baz'), 'bar')
+        self.assertEqual(converter.find_ingest_config('baz'), ['bar'])
 
     def test_find_ingest_config_list(self):
         """Test getting the ingester parameters file from a list of
@@ -138,7 +136,7 @@ def test_find_ingest_config_list(self):
                     matches=lambda f: f.startswith('a'),
                     ingest_file='qux'),
             ])
-        self.assertEqual(converter.find_ingest_config('baz'), 'bar')
+        self.assertEqual(converter.find_ingest_config('baz'), ['bar'])
 
     def test_find_ingest_config_error(self):
         """An exception must be raised if no config is found"""
@@ -152,6 +150,19 @@ def test_find_ingest_config_error(self):
         with self.assertRaises(converters_base.ConversionError):
             converter.find_ingest_config('foo')
 
+    def test_find_ingest_config_type_error(self):
+        """An exception must be raised if ingest_parameter_files is
+        not a string or ParameterSelector or a list of these
+        """
+        with self.assertRaises(converters_base.ConversionError):
+            syntool_converter.BasicSyntoolConverter(
+                converter_type='foo',
+                ingest_parameter_files=1).find_ingest_config('foo')
+        with self.assertRaises(converters_base.ConversionError):
+            syntool_converter.BasicSyntoolConverter(
+                converter_type='foo',
+                ingest_parameter_files=[1, 2]).find_ingest_config('foo')
+
     def test_parse_converter_options(self):
         """Test parsing and merging converter options"""
         converter = syntool_converter.BasicSyntoolConverter(
@@ -205,7 +216,8 @@ def test_run(self):
                                return_value=['conv1.tiff', 'conv2.tiff']) as mock_convert, \
              mock.patch.object(converter, 'ingest',
                                side_effect=[['ingested_dir1'], ['ingested_dir2']]) as mock_ingest, \
-             mock.patch.object(converter, 'post_ingest') as mock_post_ingest:
+             mock.patch.object(converter, 'post_ingest') as mock_post_ingest, \
+             mock.patch('os.remove') as mock_remove:
             converter.run(
                 in_file='in.nc',
                 out_dir='out',
@@ -225,6 +237,32 @@ def test_run(self):
                  '--options-file', converter.PARAMETERS_DIR / 'bar']),
         ])
         mock_post_ingest.assert_called_once_with(['ingested_dir1', 'ingested_dir2'], 'results')
+        mock_remove.assert_has_calls((mock.call('conv1.tiff'), mock.call('conv2.tiff')))
+
+    def test_run_no_converter(self):
+        """If no converter is set, convert() should return the path to
+        the input file
+        """
+        converter = syntool_converter.BasicSyntoolConverter(
+            converter_type=None,
+            ingest_parameter_files='bar')
+        with mock.patch.object(converter, 'convert',) as mock_convert, \
+             mock.patch.object(converter, 'ingest',
+                               return_value=['ingested_dir1']) as mock_ingest, \
+             mock.patch.object(converter, 'post_ingest') as mock_post_ingest, \
+             mock.patch('os.remove') as mock_remove:
+            converter.run(
+                in_file='in.nc',
+                out_dir='out',
+                results_dir='results')
+        mock_convert.assert_not_called()
+        mock_ingest.assert_called_once_with(
+            Path('out', 'in.nc'),
+            'results',
+            ['--config', Path('parameters/3413.ini'),
+             '--options-file', converter.PARAMETERS_DIR / 'bar'])
+        mock_post_ingest.assert_called_once_with(['ingested_dir1'], 'results')
+        mock_remove.assert_called_once_with('in.nc')
 
 
 class Sentinel1SyntoolConverterTestCase(unittest.TestCase):
@@ -270,6 +308,9 @@ def test_convert(self):
             for file_name in file_names:
                 mock_convert.assert_any_call(measurement_dir / file_name, 'out_dir', ['--baz'])
 
+
+
+
     def test_ingest(self):
         """Test that the subdirectories created by ingestion are copied
         to the ingested results folder