From 524f9b121df5a108586f397934f457ebf500dfd3 Mon Sep 17 00:00:00 2001 From: Michael Gleason Date: Wed, 22 Jan 2025 17:10:49 -0700 Subject: [PATCH] setting up cli to run tech-mapping, updating docstrings, argument names, and defaults for consistnecy with supplycurve-aggregation --- examples/bespoke_wind_plants/single_run.py | 4 +- reV/cli.py | 6 +- reV/supply_curve/aggregation.py | 2 +- reV/supply_curve/cli_tech_mapping.py | 66 ++++++++++++++++ reV/supply_curve/tech_mapping.py | 67 +++++++++------- reV/utilities/__init__.py | 1 + tests/test_bespoke.py | 90 +++++++++++++++++----- tests/test_supply_curve_tech_mapping.py | 60 ++++++++++++++- 8 files changed, 243 insertions(+), 53 deletions(-) create mode 100644 reV/supply_curve/cli_tech_mapping.py diff --git a/examples/bespoke_wind_plants/single_run.py b/examples/bespoke_wind_plants/single_run.py index cd45b8bc8..f2c8470e6 100644 --- a/examples/bespoke_wind_plants/single_run.py +++ b/examples/bespoke_wind_plants/single_run.py @@ -70,7 +70,9 @@ shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format('*') - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1 + ) bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, objective_function, cost_function, diff --git a/reV/cli.py b/reV/cli.py index debe12dc1..97a64d2e6 100644 --- a/reV/cli.py +++ b/reV/cli.py @@ -12,6 +12,7 @@ from reV.handlers.cli_multi_year import my_command from reV.supply_curve.cli_sc_aggregation import sc_agg_command from reV.supply_curve.cli_supply_curve import sc_command +from reV.supply_curve.cli_tech_mapping import tm_command from reV.rep_profiles.cli_rep_profiles import rep_profiles_command from reV.hybrids.cli_hybrids import hybrids_command from reV.nrwal.cli_nrwal import nrwal_command @@ -24,8 +25,9 @@ commands = [bespoke_command, gen_command, econ_command, collect_command, - my_command, sc_agg_command, sc_command, rep_profiles_command, - hybrids_command, nrwal_command, qa_qc_command] + my_command, tm_command, sc_agg_command, sc_command, + rep_profiles_command, hybrids_command, nrwal_command, + qa_qc_command] main = make_cli(commands, info={"name": "reV", "version": __version__}) main.add_command(qa_qc_extra) main.add_command(project_points) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 06d6fdf7d..0bc64766e 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -274,7 +274,7 @@ def _validate_tech_mapping(self): ) try: TechMapping.run( - self._excl_fpath, self._res_fpath, dset=self._tm_dset + self._excl_fpath, self._res_fpath, tm_dset=self._tm_dset ) except Exception as e: msg = ( diff --git a/reV/supply_curve/cli_tech_mapping.py b/reV/supply_curve/cli_tech_mapping.py new file mode 100644 index 000000000..b53305798 --- /dev/null +++ b/reV/supply_curve/cli_tech_mapping.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +reV Tech Mapping CLI utility functions. +""" +import logging + +from gaps.cli import as_click_command, CLICommandFromClass + +from reV.supply_curve.tech_mapping import TechMapping +from reV.utilities import ModuleName +from reV.utilities.exceptions import ConfigError +from reV.supply_curve.cli_sc_aggregation import _format_res_fpath + +logger = logging.getLogger(__name__) + + +def _preprocessor(config): + """Preprocess tech mapping config user input. + + Parameters + ---------- + config : dict + User configuration file input as (nested) dict. + + Returns + ------- + dict + Updated config file. + """ + _validate_excl_fpath(config) + config = _format_res_fpath(config) + _validate_tm_dset(config) + + return config + + +def _validate_excl_fpath(config): + paths = config["excl_fpath"] + if isinstance(paths, list): + raise ConfigError( + "Multiple exclusion file paths passed via excl_fpath. " + "Cannot run tech mapping with arbitrary multiple exclusion. " + "Specify a single exclusion file path to write to." + ) + + +def _validate_tm_dset(config): + if config.get("tm_dset") is None: + raise ConfigError( + "tm_dset must be specified to run tech mapping." + ) + + +tm_command = CLICommandFromClass(TechMapping, method="run", + name=str(ModuleName.TECH_MAPPING), + add_collect=False, split_keys=None, + config_preprocessor=_preprocessor) +main = as_click_command(tm_command) + + +if __name__ == '__main__': + try: + main(obj={}) + except Exception: + logger.exception('Error running reV Tech Mapping CLI.') + raise diff --git a/reV/supply_curve/tech_mapping.py b/reV/supply_curve/tech_mapping.py index 3293a0de3..4de5e417f 100644 --- a/reV/supply_curve/tech_mapping.py +++ b/reV/supply_curve/tech_mapping.py @@ -31,7 +31,7 @@ class TechMapping: """Framework to create map between tech layer (exclusions), res, and gen""" def __init__( - self, excl_fpath, res_fpath, sc_resolution=2560, dist_margin=1.05 + self, excl_fpath, res_fpath, resolution=2560, dist_margin=1.05 ): """ Parameters @@ -41,7 +41,7 @@ def __init__( arrays to allow for mapping to resource points res_fpath : str Filepath to .h5 resource file that we're mapping to. - sc_resolution : int | None, optional + resolution : int | None, optional Supply curve resolution, does not affect the exclusion to resource (tech) mapping, but defines how many exclusion pixels are mapped at a time, by default 2560 @@ -58,9 +58,9 @@ def __init__( ) with SupplyCurveExtent( - self._excl_fpath, resolution=sc_resolution + self._excl_fpath, resolution=resolution ) as sc: - self._sc_resolution = sc.resolution + self._resolution = sc.resolution self._gids = np.array(list(range(len(sc))), dtype=np.uint32) self._excl_shape = sc.exclusions.shape self._n_excl = np.product(self._excl_shape) @@ -296,43 +296,43 @@ def map_resource_gids( return ind_out - def initialize_dataset(self, dset, chunks=(128, 128)): + def initialize_dataset(self, tm_dset, chunks=(128, 128)): """ Initialize output dataset in exclusions h5 file. If dataset already exists, a warning will be issued. Parameters ---------- - dset : str + tm_dset : str Name of the dataset in the exclusions H5 file to create. chunks : tuple, optional Chunk size for the dataset, by default (128, 128). """ with h5py.File(self._excl_fpath, "a") as f: - if dset in list(f): + if tm_dset in list(f): wmsg = ( 'TechMap results dataset "{}" already exists ' 'in pre-existing Exclusions TechMapping file "{}"'.format( - dset, self._excl_fpath + tm_dset, self._excl_fpath ) ) logger.warning(wmsg) warn(wmsg, FileInputWarning) else: f.create_dataset( - dset, + tm_dset, shape=self._excl_shape, dtype=np.int32, chunks=chunks, ) - f[dset][:] = -1 + f[tm_dset][:] = -1 if self._dist_thresh: - f[dset].attrs["distance_threshold"] = self._dist_thresh + f[tm_dset].attrs["distance_threshold"] = self._dist_thresh if self._res_fpath: - f[dset].attrs["src_res_fpath"] = self._res_fpath + f[tm_dset].attrs["src_res_fpath"] = self._res_fpath def _check_fout(self): """Check the TechMapping output file for cached data.""" @@ -345,14 +345,14 @@ def _check_fout(self): logger.exception(emsg) raise FileInputError(emsg) - def map_resource(self, dset, max_workers=None, points_per_worker=10): + def map_resource(self, tm_dset, max_workers=None, points_per_worker=10): """ Map all resource gids to exclusion gids. Save results to dset in exclusions h5 file. Parameters ---------- - dset : str, optional + tm_dset : str, optional Name of the output dataset in the exclusions H5 file to which the tech map will be saved. max_workers : int, optional @@ -390,7 +390,7 @@ def map_resource(self, dset, max_workers=None, points_per_worker=10): ] = i with h5py.File(self._excl_fpath, "a") as f: - indices = f[dset] + indices = f[tm_dset] n_finished = 0 for future in as_completed(futures): n_finished += 1 @@ -422,8 +422,8 @@ def run( cls, excl_fpath, res_fpath, - dset, - sc_resolution=2560, + tm_dset, + resolution=64, dist_margin=1.05, max_workers=None, points_per_worker=10, @@ -433,16 +433,25 @@ def run( Parameters ---------- excl_fpath : str - Filepath to exclusions h5 (tech layer). dset will be - created in excl_fpath. + Filepath to exclusions data HDF5 file. This file must must contain + latitude and longitude datasets. res_fpath : str - Filepath to .h5 resource file that we're mapping to. - dset : str, optional - Dataset name in excl_fpath to save mapping results to. - sc_resolution : int | None, optional - Supply curve resolution, does not affect the exclusion to resource - (tech) mapping, but defines how many exclusion pixels are mapped - at a time, by default 2560 + Filepath to HDF5 resource file (e.g. WTK or NSRDB) to which + the exclusions will be mapped. Can refer to a single file (e.g., + "/path/to/nsrdb_2024.h5" or a wild-card e.g., + "/path/to/nsrdb_{}.h5") + tm_dset : str + Dataset name in the `excl_fpath` file to which the the + techmap (exclusions-to-resource mapping data) will be saved. + + .. Important:: If this dataset already exists in the h5 file, + it will be overwritten. + resolution : int | None, optional + Supply Curve resolution. This value defines how many pixels + are in a single side of a supply curve cell. For example, + a value of ``64`` would generate a supply curve where the + side of each supply curve cell is ``64x64`` exclusion + pixels. By default, ``64``. dist_margin : float, optional Extra margin to multiply times the computed distance between neighboring resource points, by default 1.05 @@ -453,11 +462,11 @@ def run( Number of supply curve points to map to resource gids on each worker, by default 10 """ - kwargs = {"dist_margin": dist_margin, "sc_resolution": sc_resolution} + kwargs = {"dist_margin": dist_margin, "resolution": resolution} mapper = cls(excl_fpath, res_fpath, **kwargs) - mapper.initialize_dataset(dset) + mapper.initialize_dataset(tm_dset) mapper.map_resource( max_workers=max_workers, points_per_worker=points_per_worker, - dset=dset + tm_dset=tm_dset ) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 147b313aa..e6800b7db 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -292,6 +292,7 @@ class ModuleName(str, Enum): REP_PROFILES = "rep-profiles" SUPPLY_CURVE = "supply-curve" SUPPLY_CURVE_AGGREGATION = "supply-curve-aggregation" + TECH_MAPPING = "tech-mapping" def __str__(self): return self.value diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index 8639f148f..eb236ab9c 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -136,7 +136,10 @@ def test_turbine_placement(gid=33): sam_sys_inputs["fixed_operating_cost_multiplier"] = 2 sam_sys_inputs["variable_operating_cost_multiplier"] = 5 - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, sam_sys_inputs, OBJECTIVE_FUNCTION, @@ -230,7 +233,10 @@ def test_zero_area(gid=33): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, objective_function, CAP_COST_FUN, @@ -274,7 +280,10 @@ def test_correct_turb_location(gid=33): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, objective_function, CAP_COST_FUN, @@ -318,7 +327,10 @@ def test_packing_algorithm(gid=33): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant( gid, excl_fp, @@ -381,7 +393,10 @@ def test_single(gid=33): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, OBJECTIVE_FUNCTION, CAP_COST_FUN, @@ -470,7 +485,10 @@ def test_extra_outputs(gid=33): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) with pytest.raises(KeyError): bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, @@ -608,7 +626,10 @@ def test_bespoke(): } ) - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) sam_configs = copy.deepcopy(SAM_CONFIGS) sam_configs["default"]["fixed_charge_rate"] = 0.0975 @@ -792,7 +813,10 @@ def test_consistent_eval_namespace(gid=33): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant( gid, excl_fp, @@ -889,7 +913,10 @@ def test_wake_loss_multiplier(wlm): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, OBJECTIVE_FUNCTION, @@ -958,7 +985,10 @@ def test_bespoke_wind_plant_with_power_curve_losses(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, OBJECTIVE_FUNCTION, @@ -1024,7 +1054,10 @@ def test_bespoke_run_with_power_curve_losses(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant( 33, excl_fp, @@ -1091,7 +1124,10 @@ def test_bespoke_run_with_scheduled_losses(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, OBJECTIVE_FUNCTION, CAP_COST_FUN, @@ -1168,7 +1204,10 @@ def test_bespoke_aep_is_zero_if_no_turbines_placed(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, SAM_SYS_INPUTS, objective_function, @@ -1238,7 +1277,10 @@ def test_bespoke_prior_run(): } ) - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) assert not os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) @@ -1354,7 +1396,10 @@ def test_gid_map(): fp_gid_map = os.path.join(td, "gid_map.csv") gid_map.to_csv(fp_gid_map) - TechMapping.run(excl_fp, RES.format(2013), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2013), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) assert not os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) @@ -1455,7 +1500,10 @@ def test_bespoke_bias_correct(): fp_bc = os.path.join(td, "bc.csv") bias_correct.to_csv(fp_bc) - TechMapping.run(excl_fp, RES.format(2013), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2013), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) assert not os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) @@ -1522,7 +1570,10 @@ def test_cli(runner, clear_loggers): shutil.copy(RES.format(2013), res_fp_2.format(2013)) res_fp = [res_fp_1.format(2012), res_fp_2.format("*")] - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) config = { "log_directory": td, @@ -1676,7 +1727,10 @@ def test_bespoke_run_with_icing_cutoff(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") - TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) + TechMapping.run( + excl_fp, RES.format(2012), tm_dset=TM_DSET, max_workers=1, + resolution=2560 + ) bsp = BespokeSinglePlant( 33, excl_fp, diff --git a/tests/test_supply_curve_tech_mapping.py b/tests/test_supply_curve_tech_mapping.py index 0121ce749..334d9ff4a 100644 --- a/tests/test_supply_curve_tech_mapping.py +++ b/tests/test_supply_curve_tech_mapping.py @@ -6,6 +6,8 @@ """ import os import shutil +import json +import traceback import h5py import numpy as np @@ -13,6 +15,8 @@ import pytest from reV import TESTDATADIR +from reV.cli import main +from reV.utilities import ModuleName from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE from reV.handlers.outputs import Outputs from reV.supply_curve.tech_mapping import TechMapping @@ -32,7 +36,59 @@ def test_resource_tech_mapping(tmp_path): shutil.copy(EXCL, excl_fpath) dset = "tm" - TechMapping.run(excl_fpath, RES, dset=dset, max_workers=2) + TechMapping.run( + excl_fpath, RES, tm_dset=dset, max_workers=2, resolution=2560 + ) + + with ExclusionLayers(EXCL) as ex: + ind_truth = ex[TM_DSET] + + with ExclusionLayers(excl_fpath) as out: + assert dset in out, "Techmap dataset was not written to H5" + ind = out[dset] + + msg = 'Tech mapping failed for {} vs. baseline results.' + assert np.allclose(ind, ind_truth), msg.format('index mappings') + + msg = 'Tech mapping didnt find all 100 generation points!' + assert len(set(ind.flatten())) == 101, msg + + +def test_tech_mapping_cli(runner, clear_loggers, tmp_path): + """Test tech-mapping CLI command""" + + excl_fpath = EXCL + excl_fpath = tmp_path.joinpath("excl.h5").as_posix() + shutil.copy(EXCL, excl_fpath) + + dset = "tm" + config = { + "log_directory": tmp_path.as_posix(), + "execution_control": { + "option": "local", + "max_workers": 2, + }, + "log_level": "INFO", + "excl_fpath": excl_fpath, + "tm_dset": "tm", + "res_fpath": RES, + "resolution": 2560, + } + + config_path = tmp_path.joinpath("config.json") + with open(config_path, "w") as f: + json.dump(config, f) + + result = runner.invoke( + main, [ModuleName.TECH_MAPPING, "-c", config_path.as_posix()] + ) + clear_loggers() + + if result.exit_code != 0: + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) + raise RuntimeError(msg) with ExclusionLayers(EXCL) as ex: ind_truth = ex[TM_DSET] @@ -64,7 +120,7 @@ def plot_tech_mapping(dist_margin=1.05): gen_meta = fgen.meta ind_test = TechMapping.run(EXCL, RES, dset=None, max_workers=2, - dist_margin=dist_margin) + dist_margin=dist_margin, resolution=2560) df = pd.DataFrame({LATITUDE: lats, LONGITUDE: lons,