Merge pull request #439 from NREL/gb/bc

Gb/bc

reV/SAM/SAM.py

@@ -254,15 +254,23 @@ def get(cls, res_file, project_points, module,
             res_file to lr_res_file spatial mapping. For details on this
             argument, see the rex.MultiResolutionResource docstring.
         bias_correct : None | pd.DataFrame
-            None if not provided or extracted DataFrame with wind or solar
-            resource bias correction table. This has columns: gid (can be index
-            name), adder, scalar. The gid field should match the true resource
-            gid regardless of the optional gid_map input. If both adder and
-            scalar are present, the wind or solar resource is corrected by
-            (res*scalar)+adder. If either adder or scalar is not present,
-            scalar defaults to 1 and adder to 0. Only windspeed or GHI+DNI are
-            corrected depending on the technology. GHI and DNI are corrected
-            with the same correction factors.
+            Optional DataFrame or CSV filepath to a wind or solar
+            resource bias correction table. This has columns:
+
+                - ``gid``: GID of site (can be index name of dataframe)
+                - ``method``: function name from ``rex.bias_correction`` module
+
+            The ``gid`` field should match the true resource ``gid`` regardless
+            of the optional ``gid_map`` input. Only ``windspeed`` **or**
+            ``GHI`` + ``DNI`` + ``DHI`` are corrected, depending on the
+            technology (wind for the former, PV or CSP for the latter). See the
+            functions in the ``rex.bias_correction`` module for available
+            inputs for ``method``. Any additional kwargs required for the
+            requested ``method`` can be input as additional columns in the
+            ``bias_correct`` table e.g., for linear bias correction functions
+            you can include ``scalar`` and ``adder`` inputs as columns in the
+            ``bias_correct`` table on a site-by-site basis. If ``None``, no
+            corrections are applied. By default, ``None``.
 
 
         Returns

reV/SAM/generation.py

@@ -431,15 +431,23 @@ def reV_run(cls, points_control, res_file, site_df,
             res_file to lr_res_file spatial mapping. For details on this
             argument, see the rex.MultiResolutionResource docstring.
         bias_correct : None | pd.DataFrame
-            None if not provided or extracted DataFrame with wind or solar
-            resource bias correction table. This has columns: gid (can be index
-            name), adder, scalar. The gid field should match the true resource
-            gid regardless of the optional gid_map input. If both adder and
-            scalar are present, the wind or solar resource is corrected by
-            (res*scalar)+adder. If either adder or scalar is not present,
-            scalar defaults to 1 and adder to 0. Only windspeed or GHI+DNI are
-            corrected depending on the technology. GHI and DNI are corrected
-            with the same correction factors.
+            Optional DataFrame or CSV filepath to a wind or solar
+            resource bias correction table. This has columns:
+
+                - ``gid``: GID of site (can be index name of dataframe)
+                - ``method``: function name from ``rex.bias_correction`` module
+
+            The ``gid`` field should match the true resource ``gid`` regardless
+            of the optional ``gid_map`` input. Only ``windspeed`` **or**
+            ``GHI`` + ``DNI`` + ``DHI`` are corrected, depending on the
+            technology (wind for the former, PV or CSP for the latter). See the
+            functions in the ``rex.bias_correction`` module for available
+            inputs for ``method``. Any additional kwargs required for the
+            requested ``method`` can be input as additional columns in the
+            ``bias_correct`` table e.g., for linear bias correction functions
+            you can include ``scalar`` and ``adder`` inputs as columns in the
+            ``bias_correct`` table on a site-by-site basis. If ``None``, no
+            corrections are applied. By default, ``None``.
 
         Returns
         -------
@@ -540,6 +548,7 @@ def set_resource_data(self, resource, meta):
             location. Should include values for latitude, longitude,
             elevation, and timezone.
         """
+        meta = self._parse_meta(meta)
         self.time_interval = self.get_time_interval(resource.index.values)
         pysam_w_fname = self._create_pysam_wfile(resource, meta)
         self[self.PYSAM_WEATHER_TAG] = pysam_w_fname
@@ -689,6 +698,7 @@ def set_resource_data(self, resource, meta):
             and timezone.
         """
 
+        meta = self._parse_meta(meta)
         time_index = resource.index
         self.time_interval = self.get_time_interval(resource.index.values)
 
@@ -1875,6 +1885,8 @@ def set_resource_data(self, resource, meta):
             and timezone.
         """
 
+        meta = self._parse_meta(meta)
+
         # map resource data names to SAM required data names
         var_map = {'speed': 'windspeed',
                    'direction': 'winddirection',
@@ -1915,17 +1927,17 @@ def set_resource_data(self, resource, meta):
         temp = np.roll(temp, n_roll, axis=0)
         data_dict['data'] = temp.tolist()
 
-        data_dict['lat'] = meta['latitude']
-        data_dict['lon'] = meta['longitude']
-        data_dict['tz'] = meta['timezone']
-        data_dict['elev'] = meta['elevation']
+        data_dict['lat'] = float(meta['latitude'])
+        data_dict['lon'] = float(meta['longitude'])
+        data_dict['tz'] = int(meta['timezone'])
+        data_dict['elev'] = float(meta['elevation'])
 
         time_index = self.ensure_res_len(time_index, time_index)
-        data_dict['minute'] = time_index.minute
-        data_dict['hour'] = time_index.hour
-        data_dict['year'] = time_index.year
-        data_dict['month'] = time_index.month
-        data_dict['day'] = time_index.day
+        data_dict['minute'] = time_index.minute.tolist()
+        data_dict['hour'] = time_index.hour.tolist()
+        data_dict['year'] = time_index.year.tolist()
+        data_dict['month'] = time_index.month.tolist()
+        data_dict['day'] = time_index.day.tolist()
 
         # add resource data to self.data and clear
         self['wind_resource_data'] = data_dict
@@ -2055,6 +2067,8 @@ def set_resource_data(self, resource, meta):
             and timezone.
         """
 
+        meta = self._parse_meta(meta)
+
         # map resource data names to SAM required data names
         var_map = {'significantwaveheight': 'significant_wave_height',
                    'waveheight': 'significant_wave_height',

reV/bespoke/bespoke.py

@@ -2,8 +2,7 @@
 """
 reV bespoke wind plant analysis tools
 """
-# TODO update docstring
-# TODO check on outputs
+from inspect import signature
 import time
 import logging
 import copy
@@ -31,6 +30,7 @@
                                       FileInputError)
 from reV.utilities import log_versions, ModuleName
 
+from rex.utilities.bc_parse_table import parse_bc_table
 from rex.joint_pd.joint_pd import JointPD
 from rex.renewable_resource import WindResource
 from rex.multi_year_resource import MultiYearWindResource
@@ -325,14 +325,24 @@ def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs,
             extract from the resource input). This is useful if you're running
             forecasted resource data (e.g., ECMWF) to complement historical
             meteorology (e.g., WTK).
-        bias_correct : str | pd.DataFrame | None
-            Optional DataFrame or csv filepath to a wind bias correction table.
-            This has columns: gid (can be index name), adder, scalar. If both
-            adder and scalar are present, the wind is corrected by
-            (res*scalar)+adder. If either is not present, scalar defaults to 1
-            and adder to 0. Only windspeed is corrected. Note that if gid_map
-            is provided, the bias_correct gid corresponds to the actual
-            resource data gid and not the techmap gid.
+        bias_correct : str | pd.DataFrame, optional
+            Optional DataFrame or CSV filepath to a wind or solar
+            resource bias correction table. This has columns:
+
+                - ``gid``: GID of site (can be index name of dataframe)
+                - ``method``: function name from ``rex.bias_correction`` module
+
+            The ``gid`` field should match the true resource ``gid`` regardless
+            of the optional ``gid_map`` input. Only ``windspeed`` **or**
+            ``GHI`` + ``DNI`` + ``DHI`` are corrected, depending on the
+            technology (wind for the former, PV or CSP for the latter). See the
+            functions in the ``rex.bias_correction`` module for available
+            inputs for ``method``. Any additional kwargs required for the
+            requested ``method`` can be input as additional columns in the
+            ``bias_correct`` table e.g., for linear bias correction functions
+            you can include ``scalar`` and ``adder`` inputs as columns in the
+            ``bias_correct`` table on a site-by-site basis. If ``None``, no
+            corrections are applied. By default, ``None``.
         pre_loaded_data : BespokeSinglePlantData, optional
             A pre-loaded :class:`BespokeSinglePlantData` object, or
             ``None``. Can be useful to speed up execution on file
@@ -530,10 +540,56 @@ class was initialized with close=False, this will not close any
         handlers."""
         self.sc_point.close()
 
+    def bias_correct_ws(self, ws, dset, h5_gids):
+        """Bias correct windspeed data if the ``bias_correct`` input was
+        provided.
+
+        Parameters
+        ----------
+        ws : np.ndarray
+            Windspeed data in shape (time, space)
+        dset : str
+            Resource dataset name e.g., "windspeed_100m", "temperature_100m",
+            "pressure_100m", or something similar
+        h5_gids : list | np.ndarray
+            Array of integer gids (spatial indices) from the source h5 file.
+            This is used to get the correct bias correction parameters from
+            ``bias_correct`` table based on its ``gid`` column
+
+        Returns
+        -------
+        ws : np.ndarray
+            Bias corrected windspeed data in same shape as input
+        """
+
+        if self._bias_correct is not None and dset.startswith('windspeed_'):
+
+            out = parse_bc_table(self._bias_correct, h5_gids)
+            bc_fun, bc_fun_kwargs, bool_bc = out
+
+            if bool_bc.any():
+                logger.debug('Bias correcting windspeed with function {} '
+                             'for h5 gids: {}'.format(bc_fun, h5_gids))
+
+                bc_fun_kwargs['ws'] = ws[:, bool_bc]
+                sig = signature(bc_fun)
+                bc_fun_kwargs = {k: v for k, v in bc_fun_kwargs.items()
+                                 if k in sig.parameters}
+
+                ws[:, bool_bc] = bc_fun(**bc_fun_kwargs)
+
+        return ws
+
     def get_weighted_res_ts(self, dset):
         """Special method for calculating the exclusion-weighted mean resource
         timeseries data for the BespokeSinglePlant.
 
+        Parameters
+        ----------
+        dset : str
+            Resource dataset name e.g., "windspeed_100m", "temperature_100m",
+            "pressure_100m", or something similar
+
         Returns
         -------
         data : np.ndarray
@@ -550,16 +606,7 @@ def get_weighted_res_ts(self, dset):
         else:
             data = self._pre_loaded_data[dset, :, h5_gids]
 
-        if self._bias_correct is not None and dset.startswith('windspeed_'):
-            missing = [g for g in h5_gids if g not in self._bias_correct.index]
-            for missing_gid in missing:
-                self._bias_correct.loc[missing_gid, 'scalar'] = 1
-                self._bias_correct.loc[missing_gid, 'adder'] = 0
-
-            scalar = self._bias_correct.loc[h5_gids, 'scalar'].values
-            adder = self._bias_correct.loc[h5_gids, 'adder'].values
-            data = data * scalar + adder
-            data = np.maximum(data, 0)
+        data = self.bias_correct_ws(data, dset, h5_gids)
 
         weights = np.zeros(len(gids))
         for i, gid in enumerate(gids):
@@ -1055,9 +1102,9 @@ def run_wind_plant_ts(self):
 
         # copy dataset outputs to meta data for supply curve table summary
         if 'cf_mean-means' in self.outputs:
-            self._meta['mean_cf'] = self.outputs['cf_mean-means']
+            self._meta.loc[:, 'mean_cf'] = self.outputs['cf_mean-means']
         if 'lcoe_fcr-means' in self.outputs:
-            self._meta['mean_lcoe'] = self.outputs['lcoe_fcr-means']
+            self._meta.loc[:, 'mean_lcoe'] = self.outputs['lcoe_fcr-means']
             self.recalc_lcoe()
 
         logger.debug('Timeseries analysis complete!')
@@ -1596,20 +1643,20 @@ def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function,
             Optional DataFrame or CSV filepath to a wind or solar
             resource bias correction table. This has columns:
 
-                - ``gid``: GID of site (can be index name)
-                - ``adder``: Value to add to resource at each site
-                - ``scalar``: Value to scale resource at each site by
-
-            The ``gid`` field should match the true resource ``gid``
-            regardless of the optional ``gid_map`` input. If both
-            ``adder`` and ``scalar`` are present, the wind or solar
-            resource is corrected by :math:`(res*scalar)+adder`. If
-            *either* is missing, ``scalar`` defaults to 1 and ``adder``
-            to 0. Only `windspeed` **or** `GHI` + `DNI` are corrected,
-            depending on the technology (wind for the former, solar
-            for the latter). `GHI` and `DNI` are corrected with the
-            same correction factors. If ``None``, no corrections are
-            applied. By default, ``None``.
+                - ``gid``: GID of site (can be index name of dataframe)
+                - ``method``: function name from ``rex.bias_correction`` module
+
+            The ``gid`` field should match the true resource ``gid`` regardless
+            of the optional ``gid_map`` input. Only ``windspeed`` **or**
+            ``GHI`` + ``DNI`` + ``DHI`` are corrected, depending on the
+            technology (wind for the former, PV or CSP for the latter). See the
+            functions in the ``rex.bias_correction`` module for available
+            inputs for ``method``. Any additional kwargs required for the
+            requested ``method`` can be input as additional columns in the
+            ``bias_correct`` table e.g., for linear bias correction functions
+            you can include ``scalar`` and ``adder`` inputs as columns in the
+            ``bias_correct`` table on a site-by-site basis. If ``None``, no
+            corrections are applied. By default, ``None``.
         pre_load_data : bool, optional
             Option to pre-load resource data. This step can be
             time-consuming up front, but it drastically reduces the
@@ -1842,6 +1889,44 @@ def _pre_loaded_data_for_sc_gid(self, sc_gid):
 
         return self._pre_loaded_data.get_preloaded_data_for_gid(sc_gid)
 
+    def _get_bc_for_gid(self, gid):
+        """Get the bias correction table trimmed down just for the resource
+        pixels corresponding to a single supply curve GID. This can help
+        prevent excess memory usage when doing complex bias correction
+        distributed to parallel workers.
+
+        Parameters
+        ----------
+        gid : int
+            SC point gid for site to pull bias correction data for
+
+        Returns
+        -------
+        out : pd.DataFrame | None
+            If bias_correct was input, this is just the rows from the larger
+            bias correction table that correspond to the SC point gid
+        """
+        out = self._bias_correct
+
+        if self._bias_correct is not None:
+            h5_gids = []
+            try:
+                scp_kwargs = dict(gid=gid, excl=self._excl_fpath,
+                                  tm_dset=self._tm_dset,
+                                  resolution=self._resolution)
+                with SupplyCurvePoint(**scp_kwargs) as scp:
+                    h5_gids = scp.h5_gid_set
+            except EmptySupplyCurvePointError:
+                pass
+
+            if self._gid_map is not None:
+                h5_gids = [self._gid_map[g] for g in h5_gids]
+
+            mask = self._bias_correct.index.isin(h5_gids)
+            out = self._bias_correct[mask]
+
+        return out
+
     @property
     def outputs(self):
         """Saved outputs for the multi wind plant bespoke optimization. Keys
@@ -2216,7 +2301,7 @@ def run_parallel(self, max_workers=None):
                     slice_lookup=copy.deepcopy(self.slice_lookup),
                     prior_meta=self._get_prior_meta(gid),
                     gid_map=self._gid_map,
-                    bias_correct=self._bias_correct,
+                    bias_correct=self._get_bc_for_gid(gid),
                     pre_loaded_data=self._pre_loaded_data_for_sc_gid(gid)))
 
             # gather results
@@ -2272,6 +2357,8 @@ def run(self, out_fpath=None, max_workers=None):
                 pre_loaded_data = self._pre_loaded_data_for_sc_gid(gid)
                 afk = self._area_filter_kernel
                 wlm = self._wake_loss_multiplier
+                i_bc = self._get_bc_for_gid(gid)
+
                 si = self.run_serial(self._excl_fpath,
                                      self._res_fpath,
                                      self._tm_dset,
@@ -2296,7 +2383,7 @@ def run(self, out_fpath=None, max_workers=None):
                                      slice_lookup=slice_lookup,
                                      prior_meta=prior_meta,
                                      gid_map=self._gid_map,
-                                     bias_correct=self._bias_correct,
+                                     bias_correct=i_bc,
                                      gids=gid,
                                      pre_loaded_data=pre_loaded_data)
                 self._outputs.update(si)