Merge pull request #174 from NREL/bnb/allow_dask_array

Allow params array for qdm to be a dask array. Also make sure dset is…

rex/resource.py

@@ -2,17 +2,18 @@
 """
 Classes to handle resource data
 """
+import os
 from abc import ABC
+from warnings import warn
+
+import dateutil
 import h5py
 import numpy as np
-import os
 import pandas as pd
-import dateutil
-from warnings import warn
 
 from rex.sam_resource import SAMResource
-from rex.utilities.parse_keys import parse_keys, parse_slice
 from rex.utilities.exceptions import ResourceKeyError, ResourceRuntimeError
+from rex.utilities.parse_keys import parse_keys, parse_slice
 from rex.utilities.utilities import check_tz, get_lat_lon_cols
 
 
@@ -202,8 +203,7 @@ def _check_slice(ds_slice):
                        'broadcast together with shapes {}'
                        .format(['({},)'.format(ln) for ln in list_len]))
                 raise IndexError(msg)
-            else:
-                list_len = list_len[0]
+            list_len = list_len[0]
         else:
             list_len = None
 
@@ -587,6 +587,7 @@ class BaseResource(ABC):
     """
     Abstract Base class to handle resource .h5 files
     """
+
     SCALE_ATTR = 'scale_factor'
     ADD_ATTR = 'add_offset'
     UNIT_ATTR = 'units'
@@ -619,7 +620,7 @@ def __init__(self, h5_file, mode='r', unscale=True, str_decode=True,
         self.h5_file = h5_file
         if hsds:
             if mode != 'r':
-                raise IOError('Cannot write to files accessed vias HSDS!')
+                raise OSError('Cannot write to files accessed vias HSDS!')
 
             import h5pyd
             if hsds_kwargs is None:
@@ -633,7 +634,7 @@ def __init__(self, h5_file, mode='r', unscale=True, str_decode=True,
             except Exception as e:
                 msg = ('Could not open file in mode "{}": "{}"'
                        .format(mode, self.h5_file))
-                raise IOError(msg) from e
+                raise OSError(msg) from e
 
         self._group = group
         self._unscale = unscale
@@ -925,7 +926,7 @@ def attrs(self):
         """
         if self._attrs is None:
             self._attrs = {}
-            for dset in self.datasets:
+            for dset in set(self.datasets).intersection(self.h5):
                 self._attrs[dset] = dict(self.h5[dset].attrs)
 
         return self._attrs
@@ -1572,7 +1573,6 @@ class Resource(BaseResource):
 
     Examples
     --------
-
     Extracting the resource's Datetime Index
 
     >>> file = '$TESTDATADIR/nsrdb/ri_100_nsrdb_2012.h5'
@@ -1741,6 +1741,7 @@ class Resource(BaseResource):
      [15. 15.]
      [15. 15.]]
     """
+
     SCALE_ATTR = 'scale_factor'
     ADD_ATTR = 'add_offset'
     UNIT_ATTR = 'units'

rex/resource_extraction/resource_extraction.py

@@ -3,28 +3,36 @@
 Resource Extraction Tools
 """
 import copy
-import h5py
 import logging
-import numpy as np
 import os
-import pandas as pd
 import pickle
-from scipy.spatial import cKDTree
 from tempfile import TemporaryDirectory
 from warnings import warn
 
-from rex.multi_file_resource import (MultiFileNSRDB, MultiFileResource,
-                                     MultiFileWTK)
+import h5py
+import numpy as np
+import pandas as pd
+from scipy.spatial import cKDTree
+
+from rex.multi_file_resource import (
+    MultiFileNSRDB,
+    MultiFileResource,
+    MultiFileWTK,
+)
 from rex.multi_time_resource import MultiTimeResource
 from rex.multi_year_resource import MultiYearResource
+from rex.renewable_resource import (
+    NSRDB,
+    SolarResource,
+    WaveResource,
+    WindResource,
+)
 from rex.resource import Resource, ResourceDataset
-from rex.renewable_resource import (NSRDB, SolarResource, WaveResource,
-                                    WindResource)
 from rex.temporal_stats.temporal_stats import TemporalStats
 from rex.utilities.exceptions import ResourceValueError, ResourceWarning
 from rex.utilities.execution import SpawnProcessPool
 from rex.utilities.loggers import log_versions
-from rex.utilities.utilities import parse_year, check_tz, res_dist_threshold
+from rex.utilities.utilities import check_tz, parse_year, res_dist_threshold
 
 # pylint: disable=consider-using-with
 TREE_DIR = TemporaryDirectory()
@@ -918,7 +926,7 @@ def get_SAM_gid(self, gid, out_path=None, write_time=True,
             returned
         """
         if isinstance(gid, (int, np.integer)):
-            gid = [gid, ]
+            gid = [gid]
 
         SAM_df = []
         for res_id in gid:
@@ -1675,41 +1683,47 @@ class SolarX(ResourceX):
     """
     Solar Resource extraction class
     """
+
     DEFAULT_RES_CLS = SolarResource
 
 
 class NSRDBX(ResourceX):
     """
     NSRDB extraction class
     """
+
     DEFAULT_RES_CLS = NSRDB
 
 
 class MultiFileNSRDBX(MultiFileResourceX):
     """
     Multi-File NSRDB extraction class
     """
+
     DEFAULT_RES_CLS = MultiFileNSRDB
 
 
 class MultiYearNSRDBX(MultiYearResourceX):
     """
     Multi Year NSRDB extraction class
     """
+
     DEFAULT_RES_CLS = NSRDB
 
 
 class MultiTimeNSRDBX(MultiTimeResourceX):
     """
     NSRDB extraction class for data stored temporaly accross multiple files
     """
+
     DEFAULT_RES_CLS = NSRDB
 
 
 class WindX(ResourceX):
     """
     Wind Resource extraction class
     """
+
     DEFAULT_RES_CLS = WindResource
 
     def get_SAM_gid(self, hub_height, gid, out_path=None, write_time=True,
@@ -1837,13 +1851,15 @@ class MultiFileWindX(MultiFileResourceX):
     """
     Multi-File Wind Resource extraction class
     """
+
     DEFAULT_RES_CLS = MultiFileWTK
 
 
 class MultiYearWindX(MultiYearResourceX):
     """
     Multi Year Wind Resource extraction class
     """
+
     DEFAULT_RES_CLS = WindResource
 
 
@@ -1852,13 +1868,15 @@ class MultiTimeWindX(MultiTimeResourceX):
     Wind resource extraction class for data stored temporaly accross multiple
     files
     """
+
     DEFAULT_RES_CLS = WindResource
 
 
 class WaveX(ResourceX):
     """
     Wave data extraction class
     """
+
     DEFAULT_RES_CLS = WaveResource
 
     def get_gid_ts(self, ds_name, gid):
@@ -1906,7 +1924,7 @@ def get_gid_df(self, ds_name, gid):
             index = pd.MultiIndex.from_product(
                 [self.time_index, self['frequency'], self['direction']],
                 names=['time_index', 'frequency', 'direction'])
-            ax1 = np.product(df.shape[:3])
+            ax1 = np.prod(df.shape[:3])
             ax2 = df.shape[-1] if len(df.shape) == 4 else 1
             df = df.reshape(ax1, ax2)
         else:
@@ -1929,6 +1947,7 @@ class MultiYearWaveX(MultiYearResourceX):
     """
     Multi Year Wave extraction class
     """
+
     DEFAULT_RES_CLS = WaveResource
 
 
@@ -1937,4 +1956,5 @@ class MultiTimeWaveX(MultiTimeResourceX):
     Wave resource extraction class for data stored temporaly accross multiple
     files
     """
+
     DEFAULT_RES_CLS = WaveResource

rex/utilities/bc_utils.py

@@ -2,10 +2,10 @@
 """
 rex bias correction utilities.
 """
-import scipy
-import numpy as np
 import logging
 
+import numpy as np
+import scipy
 
 logger = logging.getLogger(__name__)
 
@@ -192,7 +192,7 @@ def _clean_params(self, params, arr_shape):
         """
 
         msg = f'params must be 2D array but received {type(params)}'
-        assert isinstance(params, np.ndarray), msg
+        assert hasattr(params, 'shape'), msg
 
         if len(params.shape) == 1:
             params = np.expand_dims(params, 0)
@@ -281,9 +281,13 @@ def __call__(self, arr):
         # Changes in Quantiles and Extremes? Journal of Climate 28, 6938–6959
         # (2015).
 
+        logger.debug('Computing CDF on modeled future data')
         q_mf = self.cdf(arr, params_mf)  # Eq.3: Tau_m_p = F_m_p(x_m_p)
+        logger.debug('Computing PPF on observed historical data')
         x_oh = self.ppf(q_mf, params_oh)  # Eq.5: x^_o:m_h:p = F-1_o_h(Tau_m_p)
+        logger.debug('Computing PPF on modeled historical data')
         x_mh_mf = self.ppf(q_mf, params_mh)  # Eq.4 denom: F-1_m_h(Tau_m_p)
+        logger.debug('Finished computing distributions.')
 
         if self.relative:
             x_mh_mf[x_mh_mf == 0] = 0.001  # arbitrary limit to prevent div 0