Performance improvement for CoTeDe

cotede_qc/cotede_test.py

@@ -3,71 +3,12 @@
 import json
 import logging
 import numpy as np
+from wodpy.extra import Wod4CoTeDe
 
 '''Runs QC tests from the CoTeDe package.
    CoTeDe (https://github.com/castelao/CoTeDe) is copyright (c) 2011-2015, Guilherme Pimenta Castelao.
 '''
 
-class DummyCNV(object):
-    '''Mimics the CNV object from the Seabird package. 
-       Seabird (https://github.com/castelao/seabird) is copyright (c) 2011-2015, Guilherme Pimenta Castelao.
-    '''
-    def __init__(self, p):
-        self.p    = p
-
-        # Data section.
-        # These are repeated to allow different names.
-        longnames = ['Pressure', 'Temperature', 'Salinity']
-        names = ['PRES', 'TEMP', 'PSAL']
-        self.data    = []
-        for i, data in enumerate([p.z(), p.t(), p.s()]):
-            data.attributes = {'id':i,
-                               'longname':longnames[i],
-                               'name':names[i],
-                               'span':[np.min(data), np.max(data)]}
-            self.data.append(data)
-        self.keylist = names
-
-        # Assign date and location.
-        self.attributes = {}
-        self.get_datetime()
-        self.get_location()
-
-    def keys(self):
-        """ Return the available keys in self.data
-        """
-        return self.keylist
-
-    def __getitem__(self, key):
-        """ Return the key array from self.data.
-        """
-        if key in self.keys():
-            return self.data[np.nonzero(np.array(self.keys()) == key)[0][0]]
-        else:
-            raise KeyError(key + ' is not available in data structure')
-
-    def get_datetime(self):
-        year  = self.p.year()
-        month = self.p.month()
-        day   = self.p.day()
-        if day == 0: day = 15
-        time  = self.p.time()
-        if time is None or time < 0 or time >= 24:
-            hours   = 0
-            minutes = 0
-            seconds = 0
-        else:
-            hours = int(time)
-            minutesf = (time - hours) * 60
-            minutes  = int(minutesf)
-            seconds  = min(int(round((minutesf - minutes) * 60)), 59)
-
-        self.attributes['datetime'] = datetime.datetime(year, month, day, hours, minutes, seconds)
-
-    def get_location(self):
-        self.attributes['LATITUDE']  = self.p.latitude()
-        self.attributes['LONGITUDE'] = self.p.longitude()
-
 def get_qc(p, config, test):
     '''Wrapper for running and returning results of CoTeDe tests.
        Inputs are:
@@ -88,21 +29,40 @@ def get_qc(p, config, test):
     except NameError:
         cotede_results = [-1, '', None]
 
+    var = 'TEMP'
+
     # Check if we need to perform the quality control.
     if (p.uid() != cotede_results[0] or 
             config != cotede_results[1] or
                 p.uid() is None):
-        inputs = DummyCNV(p)
-        if isinstance(config, dict):
-            cfg = config
-        else:
-            with open('cotede_qc/qc_cfg/' + config + '.json') as f:
-                cfg = json.load(f)
-        cotede_results = [p.uid(), 
-                          config, 
-                          ProfileQC(inputs, cfg=cfg)]
-
-    var = 'TEMP'
+        inputs = Wod4CoTeDe(p)
+
+        # If config is a dictionary, use it.
+        if type(config) is not dict:
+            try:
+                # Load config from CoTeDe
+                cfg = load_cfg(config)
+
+                if test == config:
+                    # AutoQC runs only on TEMP, so clean the rest.
+                    for v in cfg.keys():
+                        if v not in ['main', var]:
+                            del(cfg[v])
+                # If is a specific test,
+                elif test != config:
+                    # Load from TEMP,
+                    try:
+                        cfg = {var: {test: cfg[var][test]}}
+                    # otherwise load it from main.
+                    except:
+                        cfg = {'main': {test: cfg['main'][test]}}
+            except:
+                with open('cotede_qc/qc_cfg/' + config + '.json') as f:
+                    cfg = json.load(f)
+
+        pqc = ProfileQC(inputs, cfg=cfg)
+
+        cotede_results = [p.uid(), config, pqc]
 
     # Get the QC results, which use the IOC conventions.
     qc_returned = cotede_results[2].flags[var][test]

qctests/CoTeDe_GTSPP_global_range.py

@@ -3,7 +3,7 @@
 def test(p):
     '''Run the global range QC from the CoTeDe GTSPP config.'''
 
-    config   = {'TEMP': {'global_range':{ 'minval': -2.0, 'maxval': 40}}}
+    config   = 'gtspp'
     testname = 'global_range'
 
     return get_qc(p, config, testname)

qctests/CoTeDe_GTSPP_gradient.py

@@ -3,7 +3,7 @@
 def test(p):
     '''Run the gradient QC from the CoTeDe GTSPP config.'''
 
-    config   = {'TEMP': {'gradient': 10.0}}
+    config   = 'gtspp'
     testname = 'gradient'
 
     return get_qc(p, config, testname)

qctests/CoTeDe_GTSPP_spike_check.py

@@ -3,7 +3,7 @@
 def test(p):
     '''Run the spike QC from the CoTeDe GTSPP config.'''
 
-    config   = {'TEMP': {'spike': 2.0}}
+    config   = 'gtspp'
     testname = 'spike'
 
     return get_qc(p, config, testname)

qctests/CoTeDe_digit_roll_over.py

@@ -3,7 +3,7 @@
 def test(p):
     '''Run the digit roll over QC from the CoTeDe config.'''
 
-    config   = {'TEMP': {'digit_roll_over': 10}}
+    config   = 'cotede'
     testname = 'digit_roll_over'
 
     return get_qc(p, config, testname)

qctests/CoTeDe_rate_of_change.py

@@ -3,7 +3,7 @@
 def test(p):
     '''Run the rate_of_change QC from the CoTeDe config.'''
 
-    config   = {'TEMP': {'rate_of_change': 4}}
+    config   = 'cotede'
     testname = 'rate_of_change'
 
     return get_qc(p, config, testname)

qctests/CoTeDe_tukey53H_norm.py

@@ -3,7 +3,7 @@
 def test(p):
     '''Run the tukey53H norm QC from the CoTeDe config.'''
 
-    config   = {'TEMP': {'tukey53H_norm': {'k': 1.5, 'l': 12}}}
+    config   = 'cotede'
     testname = 'tukey53H_norm'
 
     return get_qc(p, config, testname)

tests/CoTeDe_checks_validation.py

@@ -112,12 +112,12 @@ def test_CoTeDe_location_at_sea_test_land_point():
     qc = qctests.CoTeDe_location_at_sea_test.test(pland)
     assert np.all(qc), 'mismatch between qc results and expected values'
 
-def test_CoTeDe_profile_envelop():
+def test_CoTeDe_GTSPP_profile_envelop():
     '''
     Make sure CoTeDe profile_envelop is working as expected.
     '''
-    import qctests.CoTeDe_profile_envelop
-    qc = qctests.CoTeDe_profile_envelop.test(p)
+    import qctests.CoTeDe_GTSPP_profile_envelop
+    qc = qctests.CoTeDe_GTSPP_profile_envelop.test(p)
     assert np.array_equal(qc, expected_qc_profile_envelop), 'mismatch between qc results and expected values'
 
 def test_CoTeDe_rate_of_change():

util/testingProfile.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 import numpy as np
 
 class fakeProfile:
@@ -87,6 +88,18 @@ def time(self):
         """ Returns the time. """
         return self.primary_header['Time']
 
+    def datetime(self):
+        day = self.primary_header['Day']
+        if day == 0:
+            day = 15
+        time  = self.primary_header['Time']
+        if time is None or time < 0 or time >= 24:
+            time = 0
+
+        d = datetime(self.year(), self.month(), day) + \
+            timedelta(hours=time)
+        return d
+
     def var_data(self, dat):
         """ Returns the data values for a variable given the variable index. """
         data = np.ma.array(np.zeros(len(dat)), mask=True)
@@ -105,4 +118,9 @@ def probe_type(self):
                 pt = item['Value']
         return pt
 
+    def z_level_qc(self):
+        return np.zeros(self.depths.shape).astype('bool')
+
+    def t_qc_mask(self):
+        return np.zeros(self.temperatures.shape).astype('bool')