Merge pull request #446 from NREL/pp/agg_warning_for_2D_datasets

Supply-curve-aggregation warning for 2D datasets

reV/supply_curve/sc_aggregation.py

@@ -427,7 +427,14 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None,
         h5_dsets : list, optional
             Optional list of additional datasets from the ``reV``
             generation/econ HDF5 output file to aggregate. If ``None``,
-            no extra datasets are aggregated. By default, ``None``.
+            no extra datasets are aggregated.
+
+            .. WARNING:: This input is meant for passing through 1D
+               datasets. If you specify a 2D or higher-dimensional
+               dataset, you may run into memory errors. If you wish to
+               aggregate 2D datasets, see the rep-profiles module.
+
+            By default, ``None``.
         data_layers : dict, optional
             Dictionary of aggregation data layers of the format::
 
@@ -777,6 +784,7 @@ def _get_res_gen_lcoe_data(gen, res_class_dset, res_class_bins,
 
         for i, dset in enumerate(dset_list):
             if dset in gen_dsets:
+                _warn_about_large_datasets(gen, dset)
                 temp[i] = gen[dset]
             elif dset not in gen_dsets and dset is not None:
                 w = ('Could not find "{}" input as "{}" in source files: {}. '
@@ -1355,3 +1363,15 @@ def _format_sc_agg_out_fpath(out_fpath):
     out_fn = out_fn.replace("supply_curve_aggregation",
                             "supply-curve-aggregation")
     return os.path.join(project_dir, out_fn)
+
+
+def _warn_about_large_datasets(gen, dset):
+    """Warn user about multi-dimensional datasets in passthrough datasets"""
+    dset_shape = gen.shapes.get(dset, (1,))
+    if len(dset_shape) > 1:
+        msg = ("Generation dataset {!r} is not 1-dimensional (shape: {})."
+               "You may run into memory errors during aggregation - use "
+               "rep-profiles for aggregating higher-order datasets instead!"
+               .format(dset, dset_shape))
+        logger.warning(msg)
+        warn(msg, UserWarning)

tests/test_supply_curve_sc_aggregation.py

@@ -20,10 +20,11 @@
 
 from reV.cli import main
 from reV.econ.utilities import lcoe_fcr
-from reV.supply_curve.sc_aggregation import SupplyCurveAggregation
+from reV.supply_curve.sc_aggregation import (SupplyCurveAggregation,
+                                             _warn_about_large_datasets)
 from reV.utilities import ModuleName
 from reV import TESTDATADIR
-from rex import Outputs
+from rex import Resource, Outputs
 from rex.utilities.loggers import LOGGERS
 
 
@@ -250,6 +251,22 @@ def test_agg_extra_dsets():
     assert np.allclose(avg.values, summary['mean_lcoe'].values)
 
 
+def test_agg_extra_2D_dsets():
+    """Test that warning is thrown for 2D datasets."""
+    dset = "cf_profile"
+    fp = os.path.join(TESTDATADIR, 'gen_out/pv_gen_2018_node00.h5')
+    with pytest.warns(UserWarning) as records:
+        with Resource(fp) as res:
+            _warn_about_large_datasets(res, dset)
+
+    messages = [r.message.args[0] for r in records]
+    assert any("Generation dataset {!r} is not 1-dimensional (shape: {})"
+               .format(dset, (17520, 50))
+               in msg for msg in messages)
+    assert any("You may run into memory errors during aggregation"
+               in msg for msg in messages)
+
+
 def test_agg_scalar_excl():
     """Test the aggregation summary with exclusions of 0.5"""