diff --git a/reV/config/execution.py b/reV/config/execution.py index 721edc939..1400891da 100644 --- a/reV/config/execution.py +++ b/reV/config/execution.py @@ -22,11 +22,9 @@ def __init__(self, config_dict): """ super().__init__(config_dict) - self._option = 'local' - self._nodes = 1 - self._max_workers = None - self._sites_per_worker = None - self._mem_util_lim = 0.4 + self._default_option = 'local' + self._default_nodes = 1 + self._default_mem_util_lim = 0.4 @property def option(self): @@ -37,8 +35,7 @@ def option(self): option : str Execution control option, e.g. local, peregrine, eagle... """ - self._option = str(self.get('option', self._option)).lower() - return self._option + return str(self.get('option', self._default_option)).lower() @property def nodes(self): @@ -49,8 +46,7 @@ def nodes(self): nodes : int Number of available nodes. Default is 1 node. """ - self._nodes = int(self.get('nodes', self._nodes)) - return self._nodes + return int(self.get('nodes', self._default_nodes)) @property def max_workers(self): @@ -61,8 +57,7 @@ def max_workers(self): max_workers : int | None Processes per node. Default is None max_workers (all available). """ - self._max_workers = self.get('max_workers', self._max_workers) - return self._max_workers + return self.get('max_workers', None) @property def sites_per_worker(self): @@ -73,9 +68,7 @@ def sites_per_worker(self): sites_per_worker : int | None Number of sites to run per worker in a parallel scheme. """ - self._sites_per_worker = self.get('sites_per_worker', - self._sites_per_worker) - return self._sites_per_worker + return self.get('sites_per_worker', None) @property def memory_utilization_limit(self): @@ -88,29 +81,15 @@ def memory_utilization_limit(self): Memory utilization limit (fractional). Key in the config json is "memory_utilization_limit". """ - self._mem_util_lim = self.get('memory_utilization_limit', - self._mem_util_lim) - return self._mem_util_lim + mem_util_lim = self.get('memory_utilization_limit', + self._default_mem_util_lim) + + return mem_util_lim class HPCConfig(BaseExecutionConfig): """Class to handle HPC configuration inputs.""" - - def __init__(self, config_dict): - """ - Parameters - ---------- - config_dict : str | dict - File path to config json (str), serialized json object (str), - or dictionary with pre-extracted config. - """ - - super().__init__(config_dict) - - self._hpc_alloc = 'rev' - self._feature = None - self._module = None - self._conda_env = None + REQUIREMENTS = ('allocation') @property def allocation(self): @@ -121,8 +100,8 @@ def allocation(self): hpc_alloc : str Name of the HPC allocation account for the specified job. """ - self._hpc_alloc = self.get('allocation', self._hpc_alloc) - return self._hpc_alloc + + return self['allocation'] @property def feature(self): @@ -138,8 +117,7 @@ def feature(self): "feature": "--qos=high" "feature": "--depend=[state:job_id]" """ - self._feature = self.get('feature', self._feature) - return self._feature + return self.get('feature', None) @property def module(self): @@ -151,8 +129,7 @@ def module(self): module : str Module to load on node """ - self._module = self.get('module', self._module) - return self._module + return self.get('module', None) @property def conda_env(self): @@ -164,8 +141,7 @@ def conda_env(self): conda_env : str Conda environment to activate """ - self._conda_env = self.get('conda_env', self._conda_env) - return self._conda_env + return self.get('conda_env', None) class SlurmConfig(HPCConfig): @@ -182,8 +158,7 @@ def __init__(self, config_dict): super().__init__(config_dict) - self._hpc_node_mem = None - self._hpc_walltime = 1 + self._default_hpc_walltime = 1 @property def memory(self): @@ -194,8 +169,7 @@ def memory(self): _hpc_node_mem : int | None Requested node memory in GB. """ - self._hpc_node_mem = self.get('memory', self._hpc_node_mem) - return self._hpc_node_mem + return self.get('memory', None) @property def walltime(self): @@ -206,5 +180,4 @@ def walltime(self): _hpc_walltime : int Requested single node job time in hours. """ - self._hpc_walltime = float(self.get('walltime', self._hpc_walltime)) - return self._hpc_walltime + return float(self.get('walltime', self._default_hpc_walltime)) diff --git a/reV/config/supply_curve_configs.py b/reV/config/supply_curve_configs.py index 5dfab7bba..cf9471411 100644 --- a/reV/config/supply_curve_configs.py +++ b/reV/config/supply_curve_configs.py @@ -33,17 +33,11 @@ def __init__(self, config): """ super().__init__(config) - self._default_res_fpath = None - self._default_econ_fpath = None - self._default_res_class_dset = None - self._default_res_class_bins = None self._default_cf_dset = 'cf_mean-means' self._default_lcoe_dset = 'lcoe_fcr-means' - self._default_data_layers = None self._default_resolution = 64 self._default_area_filter_kernel = 'queen' - self._default_min_area = None - self._default_excl_dict = None + self._default_points_per_worker = 10 self._sc_agg_preflight() @@ -104,7 +98,7 @@ def econ_fpath(self): """Get the econ data filepath. This is an optional argument only used if reV gen and econ outputs are being used from different files.""" - fpath = self.get('econ_fpath', self._default_econ_fpath) + fpath = self.get('econ_fpath', None) if fpath == 'PIPELINE': target_modules = ['multi-year', 'collect', 'econ'] @@ -131,7 +125,7 @@ def econ_fpath(self): @property def res_fpath(self): """Get the resource data filepath""" - res_fpath = self.get('res_fpath', self._default_res_fpath) + res_fpath = self.get('res_fpath', None) if isinstance(res_fpath, str): if '{}' in res_fpath: for year in range(1998, 2018): @@ -150,17 +144,17 @@ def tm_dset(self): @property def excl_dict(self): """Get the exclusions dictionary""" - return self.get('excl_dict', self._default_excl_dict) + return self.get('excl_dict', None) @property def res_class_dset(self): """Get the resource class dataset""" - return self.get('res_class_dset', self._default_res_class_dset) + return self.get('res_class_dset', None) @property def res_class_bins(self): """Get the resource class bins""" - return self.get('res_class_bins', self._default_res_class_bins) + return self.get('res_class_bins', None) @property def cf_dset(self): @@ -180,7 +174,7 @@ def h5_dsets(self): @property def data_layers(self): """Get the data layers dict""" - return self.get('data_layers', self._default_data_layers) + return self.get('data_layers', None) @property def resolution(self): @@ -206,7 +200,7 @@ def area_filter_kernel(self): @property def min_area(self): """Get the minimum area filter minimum area in km2.""" - return self.get('min_area', self._default_min_area) + return self.get('min_area', None) @property def friction_fpath(self): @@ -235,6 +229,16 @@ def cap_cost_scale(self): """ return self.get('cap_cost_scale', None) + @property + def max_workers(self): + """Get the number of workers to use during computation""" + return self.get('max_workers', None) + + @property + def points_per_worker(self): + """Get the number of sc points to summarize on each worker""" + return self.get('max_workers', None) + class SupplyCurveConfig(AnalysisConfig): """SC config.""" @@ -252,8 +256,6 @@ def __init__(self, config): """ super().__init__(config) - self._default_sc_features = None - self._default_transmission_costs = None self._default_sort_on = 'total_lcoe' self._default_n_dirs = 2 @@ -297,12 +299,12 @@ def fixed_charge_rate(self): @property def sc_features(self): """Get the supply curve features input.""" - return self.get('sc_features', self._default_sc_features) + return self.get('sc_features', None) @property def transmission_costs(self): """Get the transmission costs input.""" - return self.get('transmission_costs', self._default_transmission_costs) + return self.get('transmission_costs', None) @property def simple(self): diff --git a/reV/econ/cli_econ.py b/reV/econ/cli_econ.py index c171686c0..dcd40222e 100644 --- a/reV/econ/cli_econ.py +++ b/reV/econ/cli_econ.py @@ -29,6 +29,7 @@ @click.group() @click.option('--name', '-n', default='reV-econ', type=STR, + show_default=True, help='reV Economics job name, by default "reV-econ".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -195,30 +196,39 @@ def submit_from_config(ctx, name, cf_file, year, config, verbose): @click.option('--sam_files', '-sf', required=True, type=SAMFILES, help='SAM config files (required) (str, dict, or list).') @click.option('--cf_file', '-cf', default=None, type=click.Path(exists=True), + show_default=True, help='Single generation results file (str).') @click.option('--year', '-y', default=None, type=INT, + show_default=True, help='Year of generation results to analyze (if multiple years ' 'in cf_file). Default is None (use the only cf_mean dataset in ' 'cf_file).') @click.option('--points', '-p', default=slice(0, 100), type=PROJECTPOINTS, + show_default=True, help=('reV project points to analyze (slice, list, or file ' 'string). Default is slice(0, 100)')) @click.option('--site_data', '-sd', default=None, type=click.Path(exists=True), + show_default=True, help='Site-specific data file for econ calculation. Input ' 'should be a filepath that points to a csv. Rows match sites, ' 'columns are input keys. Needs a "gid" column. Input as None ' 'if no site-specific data.') @click.option('--sites_per_worker', '-spw', default=None, type=INT, + show_default=True, help=('Number of sites to run in series on a single worker. ' 'Default is the resource column chunk size.')) @click.option('--fout', '-fo', default='econ_output.h5', type=STR, + show_default=True, help=('Filename output specification (should be .h5). ' 'Default is "econ_output.h5"')) @click.option('--dirout', '-do', default='./out/econ_out', type=STR, + show_default=True, help='Output directory specification. Default is ./out/econ_out') @click.option('--logdir', '-lo', default='./out/log_econ', type=STR, + show_default=True, help='Econ log file directory. Default is ./out/log_econ') @click.option('-or', '--output_request', type=STRLIST, default=['lcoe_fcr'], + show_default=True, help=('Requested output variable name(s). ' 'Default is ["lcoe_fcr"].')) @click.option('-ap', '--append', is_flag=True, @@ -248,12 +258,15 @@ def direct(ctx, sam_files, cf_file, year, points, site_data, @direct.command() @click.option('--max_workers', '-mw', type=INT, default=None, + show_default=True, help='Number of workers. Use 1 for serial, None for all cores.') @click.option('--timeout', '-to', type=INT, default=1800, + show_default=True, help='Number of seconds to wait for econ parallel run ' 'iterations to complete before returning zeros. ' 'Default is 1800 seconds.') @click.option('--points_range', '-pr', default=None, type=INTLIST, + show_default=True, help='Optional range list to run a subset of sites.') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging.') @@ -451,27 +464,34 @@ def get_node_cmd(name, sam_files, cf_file, year=None, site_data=None, @direct.command() +@click.option('--alloc', '-a', required=True, type=STR, + help='SLURM allocation account name.') @click.option('--nodes', '-no', default=1, type=INT, + show_default=True, help='Number of SLURM nodes for econ job. Default is 1.') -@click.option('--alloc', '-a', default='rev', type=STR, - help='SLURM allocation account name. Default is "rev".') @click.option('--memory', '-mem', default=None, type=INT, + show_default=True, help='SLURM node memory request in GB. Default is None') @click.option('--walltime', '-wt', default=0.5, type=float, + show_default=True, help='SLURM walltime request in hours. Default is 0.5') @click.option('--feature', '-l', default=None, type=STR, + show_default=True, help=('Additional flags for SLURM job. Format is "--qos=high" ' 'or "--depend=[state:job_id]". Default is None.')) @click.option('--module', '-mod', default=None, type=STR, + show_default=True, help='Module to load') @click.option('--conda_env', '-env', default=None, type=STR, + show_default=True, help='Conda env to activate') @click.option('--stdout_path', '-sout', default='./out/stdout', type=STR, + show_default=True, help='Subprocess standard output path. Default is ./out/stdout') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @click.pass_context -def slurm(ctx, nodes, alloc, memory, walltime, feature, module, conda_env, +def slurm(ctx, alloc, nodes, memory, walltime, feature, module, conda_env, stdout_path, verbose): """Run econ on HPC via SLURM job submission.""" diff --git a/reV/generation/cli_gen.py b/reV/generation/cli_gen.py index 5a9f927bb..3b8199051 100644 --- a/reV/generation/cli_gen.py +++ b/reV/generation/cli_gen.py @@ -30,6 +30,7 @@ @click.group() @click.option('--name', '-n', default='reV-gen', type=STR, + show_default=True, help='reV generation job name, by default "reV-gen".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -237,44 +238,55 @@ def make_fout(name, year): help='Filepath to single resource file, multi-h5 directory, ' 'or /h5_dir/prefix*suffix.') @click.option('--points', '-p', default=None, type=PROJECTPOINTS, + show_default=True, help=('reV project points to analyze ' '(slice, list, or file string). ' 'Default is slice(0, 100)')) @click.option('--lat_lon_fpath', '-llf', type=click.Path(exists=True), - default=None, + default=None, show_default=True, help=('File path to .csv or .json containing latitude, ' 'longitude coordinates of interest')) @click.option('--lat_lon_coords', '--llc', nargs=2, type=float, default=None, + show_default=True, help='(lat, lon) coordinates of interest') @click.option('--regions', '-regs', type=STR, default=None, + show_default=True, help='File path to .json containing regions of interest') @click.option('--region', '-r', type=STR, default=None, + show_default=True, help='Region to extract') @click.option('--region_col', '-col', type=STR, default='state', + show_default=True, help='Meta column to search for region') @click.option('--sites_per_worker', '-spw', default=None, type=INT, + show_default=True, help=('Number of sites to run in series on a single worker. ' 'Default is the resource column chunk size.')) @click.option('--fout', '-fo', default='gen_output.h5', type=STR, + show_default=True, help=('Filename output specification (should be .h5). ' 'Default is "gen_output.h5"')) @click.option('--dirout', '-do', default='./out/gen_out', type=STR, + show_default=True, help='Output directory specification. Default is ./out/gen_out') @click.option('--logdir', '-lo', default='./out/log_gen', type=STR, help='Generation log file directory. Default is ./out/log_gen') @click.option('-or', '--output_request', type=STRLIST, default=['cf_mean'], + show_default=True, help=('List of requested output variable names. ' 'Default is ["cf_mean"].')) @click.option('--site_data', '-sd', default=None, type=click.Path(exists=True), + show_default=True, help='Site-specific data file for gen calculation. Input ' 'should be a filepath that points to a csv. Rows match sites, ' 'columns are input keys. Needs a "gid" column. Input as None ' 'if no site-specific data.') @click.option('-mem', '--mem_util_lim', type=float, default=0.4, + show_default=True, help='Fractional node memory utilization limit. Default is 0.4 ' 'to account for numpy memory spikes and memory bloat.') @click.option('-curt', '--curtailment', type=click.Path(exists=True), - default=None, + default=None, show_default=True, help=('JSON file with curtailment inputs parameters. ' 'Default is None (no curtailment).')) @click.option('-v', '--verbose', is_flag=True, @@ -370,12 +382,15 @@ def _parse_points(ctx): @direct.command() @click.option('--max_workers', '-mw', type=INT, default=None, + show_default=True, help='Number of workers. Use 1 for serial, None for all cores.') @click.option('--timeout', '-to', type=INT, default=1800, + show_default=True, help='Number of seconds to wait for parallel generation run ' 'iterations to complete before returning zeros. ' 'Default is 1800 seconds.') @click.option('--points_range', '-pr', default=None, type=INTLIST, + show_default=True, help='Optional range list to run a subset of sites.') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging.') @@ -634,27 +649,34 @@ def get_node_cmd(name, tech, sam_files, res_file, points=slice(0, 100), @direct.command() +@click.option('--alloc', '-a', required=True, type=STR, + help='SLURM allocation account name.') @click.option('--nodes', '-no', default=1, type=INT, + show_default=True, help='Number of SLURM nodes for gen job. Default is 1.') -@click.option('--alloc', '-a', default='rev', type=STR, - help='SLURM allocation account name. Default is "rev".') @click.option('--memory', '-mem', default=None, type=INT, + show_default=True, help='Single node memory request in GB. Default is None') @click.option('--walltime', '-wt', default=1.0, type=float, + show_default=True, help='SLURM walltime request in hours. Default is 1.0') @click.option('--feature', '-l', default=None, type=STR, + show_default=True, help=('Additional flags for SLURM job. Format is "--qos=high" ' 'or "--depend=[state:job_id]". Default is None.')) @click.option('--conda_env', '-env', default=None, type=STR, + show_default=True, help='Conda env to activate') @click.option('--module', '-mod', default=None, type=STR, + show_default=True, help='Module to load') @click.option('--stdout_path', '-sout', default='./out/stdout', type=STR, + show_default=True, help='Subprocess standard output path. Default is ./out/stdout') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @click.pass_context -def slurm(ctx, nodes, alloc, memory, walltime, feature, conda_env, module, +def slurm(ctx, alloc, nodes, memory, walltime, feature, conda_env, module, stdout_path, verbose): """Run generation on HPC via SLURM job submission.""" diff --git a/reV/handlers/cli_collect.py b/reV/handlers/cli_collect.py index d70f6bbe0..1b56d6eb3 100644 --- a/reV/handlers/cli_collect.py +++ b/reV/handlers/cli_collect.py @@ -24,6 +24,7 @@ @click.group() @click.option('--name', '-n', default='reV_collect', type=str, + show_default=True, help='Collection job name. Default is "reV_collect".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -134,14 +135,16 @@ def from_config(ctx, config_file, verbose): help='H5 file to be collected into.') @click.option('--h5_dir', '-d', required=True, type=click.Path(exists=True), help='Directory containing h5 files to collect.') -@click.option('--project_points', '-pp', type=STR, +@click.option('--project_points', '-pp', type=STR, required=True, help='Project points file representing the full ' 'collection scope.') @click.option('--dsets', '-ds', required=True, type=STRLIST, help='Dataset names to be collected.') @click.option('--file_prefix', '-fp', type=STR, default=None, + show_default=True, help='File prefix found in the h5 file names to be collected.') @click.option('--log_dir', '-ld', type=STR, default='./logs', + show_default=True, help='Directory to put log files.') @click.option('-p', '--purge_chunks', is_flag=True, help='Flag to delete chunked files after collection.') @@ -276,20 +279,25 @@ def get_node_cmd(name, h5_file, h5_dir, project_points, dsets, @direct.command() -@click.option('--alloc', '-a', default='rev', type=str, - help='SLURM allocation account name. Default is "rev".') +@click.option('--alloc', '-a', required=True, type=STR, + help='SLURM allocation account name.') @click.option('--memory', '-mem', default=None, type=INT, + show_default=True, help='SLURM node memory request in GB. Default is None') @click.option('--walltime', '-wt', default=1.0, type=float, + show_default=True, help='SLURM walltime request in hours. Default is 1.0') @click.option('--feature', '-l', default=None, type=STR, + show_default=True, help=('Additional flags for SLURM job. Format is "--qos=high" ' 'or "--depend=[state:job_id]". Default is None.')) @click.option('--conda_env', '-env', default=None, type=STR, + show_default=True, help='Conda env to activate') -@click.option('--module', '-mod', default=None, type=STR, +@click.option('--module', '-mod', default=None, type=STR, show_default=True, help='Module to load') @click.option('--stdout_path', '-sout', default='./out/stdout', type=str, + show_default=True, help='Subprocess standard output path. Default is ./out/stdout') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') diff --git a/reV/handlers/cli_multi_year.py b/reV/handlers/cli_multi_year.py index c7a3c975b..5e5fc5c67 100644 --- a/reV/handlers/cli_multi_year.py +++ b/reV/handlers/cli_multi_year.py @@ -24,6 +24,7 @@ @click.group() @click.option('--name', '-n', default='reV_multi-year', type=str, + show_default=True, help='Multi-year job name. Default is "reV_multi-year".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -129,15 +130,17 @@ def direct(ctx, my_file, verbose): @direct.command() +@click.option('--source_files', '-sf', required=True, type=PATHLIST, + help='List of files to collect from.') @click.option('--group', '-g', type=STR, default=None, + show_default=True, help='Group to collect into. Useful for collecting multiple ' 'scenarios into a single file.') -@click.option('--source_files', '-sf', required=True, type=PATHLIST, - help='List of files to collect from.') @click.option('--dsets', '-ds', required=True, type=STRLIST, help='Dataset names to be collected. If means, multi-year ' 'means will be computed.') @click.option('--pass_through_dsets', '-pt', default=None, type=STRLIST, + show_default=True, help='Optional list of datasets that are identical in the ' 'multi-year files (e.g. input datasets that dont vary ' 'from year to year) that should be copied to the output ' @@ -146,7 +149,7 @@ def direct(ctx, my_file, verbose): @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging.') @click.pass_context -def multi_year(ctx, group, source_files, dsets, pass_through_dsets, verbose): +def multi_year(ctx, source_files, group, dsets, pass_through_dsets, verbose): """Run multi year collection and means on local worker.""" name = ctx.obj['NAME'] @@ -299,30 +302,35 @@ def get_slurm_cmd(name, my_file, group_params, verbose=False): @direct.command() -@click.option('--alloc', '-a', default='rev', type=str, - help='SLURM allocation account name. Default is "rev".') +@click.option('--group_params', '-gp', required=True, type=str, + help='Stringified dictionary of collection groups and their ' + 'parameters, e.g.: ' + '{group1: {group: null, source_files: [], dsets: []}}') +@click.option('--alloc', '-a', required=True, type=STR, + help='SLURM allocation account name.') @click.option('--walltime', '-wt', default=4.0, type=float, + show_default=True, help='SLURM walltime request in hours. Default is 1.0') @click.option('--feature', '-l', default=None, type=STR, + show_default=True, help='Additional flags for SLURM job. Format is "--qos=high" ' 'or "--depend=[state:job_id]". Default is None.') @click.option('--memory', '-mem', default=None, type=INT, + show_default=True, help='SLURM node memory request in GB. Default is None') @click.option('--conda_env', '-env', default=None, type=STR, + show_default=True, help='Conda env to activate') -@click.option('--module', '-mod', default=None, type=STR, +@click.option('--module', '-mod', default=None, type=STR, show_default=True, help='Module to load') @click.option('--stdout_path', '-sout', default='./out/stdout', type=str, + show_default=True, help='Subprocess standard output path. Default is ./out/stdout') -@click.option('--group_params', '-gp', required=True, type=str, - help='Stringified dictionary of collection groups and their ' - 'parameters, e.g.: ' - '{group1: {group: null, source_files: [], dsets: []}}') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @click.pass_context -def multi_year_slurm(ctx, alloc, walltime, feature, memory, conda_env, - module, stdout_path, group_params, verbose): +def multi_year_slurm(ctx, group_params, alloc, walltime, feature, memory, + conda_env, module, stdout_path, verbose): """ Run multi year collection and means on HPC via SLURM job submission. """ diff --git a/reV/pipeline/cli_pipeline.py b/reV/pipeline/cli_pipeline.py index 0edab6fb6..ac1dc639d 100644 --- a/reV/pipeline/cli_pipeline.py +++ b/reV/pipeline/cli_pipeline.py @@ -16,6 +16,7 @@ @click.group() @click.option('--name', '-n', default='reV-pipeline', type=STR, + show_default=True, help='reV pipeline name, by default "reV-pipeline".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') diff --git a/reV/qa_qc/cli_qa_qc.py b/reV/qa_qc/cli_qa_qc.py index 2607c8a62..322257441 100644 --- a/reV/qa_qc/cli_qa_qc.py +++ b/reV/qa_qc/cli_qa_qc.py @@ -26,6 +26,7 @@ @click.group() @click.option('--name', '-n', default='reV-QA_QC', type=STR, + show_default=True, help='reV QA/QC name, by default "reV-QA/QC".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -57,6 +58,7 @@ def valid_config_keys(): @click.option('--out_dir', '-o', type=click.Path(), required=True, help="Directory path to save summary tables and plots too") @click.option('--log_file', '-log', type=click.Path(), default=None, + show_default=True, help='File to log to, by default None') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging.') @@ -78,13 +80,17 @@ def summarize(ctx, out_dir, log_file, verbose): @click.option('--h5_file', '-h5', type=click.Path(exists=True), required=True, help='Path to .h5 file to summarize') @click.option('--dsets', '-ds', type=STRLIST, default=None, + show_default=True, help='Datasets to summarize, by default None') @click.option('--group', '-grp', type=STR, default=None, + show_default=True, help=('Group within h5_file to summarize datasets for, by ' 'default None')) @click.option('--process_size', '-ps', type=INT, default=None, + show_default=True, help='Number of sites to process at a time, by default None') @click.option('--max_workers', '-w', type=INT, default=None, + show_default=True, help=('Number of workers to use when summarizing 2D datasets,' ' by default None')) @click.pass_context @@ -99,9 +105,11 @@ def h5(ctx, h5_file, dsets, group, process_size, max_workers): @summarize.command() @click.option('--plot_type', '-plt', default='plotly', type=click.Choice(['plot', 'plotly'], case_sensitive=False), + show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) @click.option('--cmap', '-cmap', type=str, default='viridis', + show_default=True, help="Colormap name, by default 'viridis'") @click.pass_context def scatter_plots(ctx, plot_type, cmap): @@ -115,6 +123,7 @@ def scatter_plots(ctx, plot_type, cmap): @click.option('--sc_table', '-sct', type=click.Path(exists=True), required=True, help='Path to .csv containing Supply Curve table') @click.option('--columns', '-cols', type=STRLIST, default=None, + show_default=True, help=('Column(s) to summarize, if None summarize all numeric ' 'columns, by default None')) @click.pass_context @@ -128,10 +137,12 @@ def supply_curve_table(ctx, sc_table, columns): @summarize.command() @click.option('--sc_table', '-sct', type=click.Path(exists=True), default=None, + show_default=True, help=("Path to .csv containing Supply Curve table, can be " "supplied in 'supply-curve-table'")) @click.option('--plot_type', '-plt', default='plotly', type=click.Choice(['plot', 'plotly'], case_sensitive=False), + show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) @click.option('--lcoe', '-lcoe', type=STR, default='mean_lcoe', @@ -154,11 +165,14 @@ def supply_curve_plot(ctx, sc_table, plot_type, lcoe): help='Path to .npy file containing final exclusions mask') @click.option('--plot_type', '-plt', default='plotly', type=click.Choice(['plot', 'plotly'], case_sensitive=False), + show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) @click.option('--cmap', '-cmap', type=str, default='viridis', + show_default=True, help="Colormap name, by default 'viridis'") @click.option('--plot_step', '-step', type=int, default=100, + show_default=True, help="Step between points to plot") @click.pass_context def exclusions_mask(ctx, excl_mask, plot_type, cmap, plot_step): @@ -179,22 +193,29 @@ def exclusions_mask(ctx, excl_mask, plot_type, cmap, plot_step): @click.option('--sub_dir', '-sd', type=STR, required=True, help="Sub directory to save summary tables and plots too") @click.option('--dsets', '-ds', type=STRLIST, default=None, + show_default=True, help='Datasets to summarize, by default None') @click.option('--group', '-grp', type=STR, default=None, + show_default=True, help=('Group within h5_file to summarize datasets for, by ' 'default None')) @click.option('--process_size', '-ps', type=INT, default=None, + show_default=True, help='Number of sites to process at a time, by default None') @click.option('--max_workers', '-w', type=INT, default=None, + show_default=True, help=('Number of workers to use when summarizing 2D datasets,' ' by default None')) @click.option('--plot_type', '-plt', default='plotly', type=click.Choice(['plot', 'plotly'], case_sensitive=False), + show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) @click.option('--cmap', '-cmap', type=str, default='viridis', + show_default=True, help="Colormap name, by default 'viridis'") @click.option('--log_file', '-log', type=click.Path(), default=None, + show_default=True, help='File to log to, by default None') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging.') @@ -237,17 +258,22 @@ def reV_h5(ctx, h5_file, out_dir, sub_dir, dsets, group, process_size, @click.option('--sub_dir', '-sd', type=STR, required=True, help="Sub directory to save summary tables and plots too") @click.option('--columns', '-cols', type=STRLIST, default=None, + show_default=True, help=('Column(s) to summarize, if None summarize all numeric ' 'columns, by default None')) @click.option('--plot_type', '-plt', default='plotly', type=click.Choice(['plot', 'plotly'], case_sensitive=False), + show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) @click.option('--cmap', '-cmap', type=str, default='viridis', + show_default=True, help="Colormap name, by default 'viridis'") @click.option('--lcoe', '-lcoe', type=STR, default='mean_lcoe', + show_default=True, help="LCOE column label to plot, by default 'mean_lcoe'") @click.option('--log_file', '-log', type=click.Path(), default=None, + show_default=True, help='File to log to, by default None') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging.') @@ -290,25 +316,32 @@ def supply_curve(ctx, sc_table, out_dir, sub_dir, columns, plot_type, cmap, @click.option('--sub_dir', '-sd', type=STR, required=True, help="Sub directory to save summary tables and plots too") @click.option('--excl_dict', '-exd', type=STR, default=None, + show_default=True, help='String representation of a dictionary of exclusions ' 'LayerMask arguments {layer: {kwarg: value}} where layer is a ' 'dataset in excl_fpath and kwarg can be "inclusion_range", ' '"exclude_values", "include_values", "use_as_weights", ' '"exclude_nodata", and/or "weight".') @click.option('--area_filter_kernel', '-afk', type=STR, default='queen', + show_default=True, help='Contiguous area filter kernel name ("queen", "rook").') @click.option('--min_area', '-ma', type=FLOAT, default=None, + show_default=True, help='Contiguous area filter minimum area, default is None ' '(No minimum area filter).') @click.option('--plot_type', '-plt', default='plotly', type=click.Choice(['plot', 'plotly'], case_sensitive=False), + show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) @click.option('--cmap', '-cmap', type=str, default='viridis', + show_default=True, help="Colormap name, by default 'viridis'") @click.option('--plot_step', '-step', type=int, default=100, + show_default=True, help="Step between points to plot") @click.option('--log_file', '-log', type=click.Path(), default=None, + show_default=True, help='File to log to, by default None') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging.') diff --git a/reV/rep_profiles/cli_rep_profiles.py b/reV/rep_profiles/cli_rep_profiles.py index 4c8549066..ff981ab90 100644 --- a/reV/rep_profiles/cli_rep_profiles.py +++ b/reV/rep_profiles/cli_rep_profiles.py @@ -24,6 +24,7 @@ @click.group() @click.option('--name', '-n', default='reV-rep_profiles', type=STR, + show_default=True, help='Job name. Default is "reV-rep_profiles".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -157,27 +158,36 @@ def from_config(ctx, config_file, verbose): @click.option('--rev_summary', '-r', type=click.Path(exists=True), required=True, help='Filepath to reV SC summary (agg) file.') @click.option('--reg_cols', '-rc', type=STRLIST, default=None, + show_default=True, help='List of column rev summary column labels to define ' 'regions to get rep profiles for.') @click.option('--cf_dset', '-cf', type=str, default='cf_profile', + show_default=True, help='Capacity factor dataset in gen_fpath to get profiles from') @click.option('--rep_method', '-rm', type=STR, default='meanoid', + show_default=True, help='String identifier for representative method ' '(e.g. meanoid, medianoid).') @click.option('--err_method', '-em', type=STR, default='rmse', + show_default=True, help='String identifier for error method ' '(e.g. rmse, mae, mbe).') @click.option('--weight', '-w', type=STR, default='gid_counts', + show_default=True, help='The supply curve column to use for a weighted average in ' 'the representative profile meanoid algorithm. ' 'Default weighting factor is "gid_counts".') @click.option('--n_profiles', '-np', type=INT, default=1, + show_default=True, help='Number of representative profiles to save.') @click.option('--out_dir', '-od', type=STR, default='./', + show_default=True, help='Directory to save rep profile output h5.') @click.option('--log_dir', '-ld', type=STR, default='./logs/', + show_default=True, help='Directory to save rep profile logs.') @click.option('--max_workers', '-mw', type=INT, default=None, + show_default=True, help='Number of parallel workers. 1 will run in serial. ' 'None will use all available.') @click.option('-agg', '--aggregate_profiles', is_flag=True, @@ -272,18 +282,24 @@ def get_node_cmd(name, gen_fpath, rev_summary, reg_cols, cf_dset, rep_method, @click.option('--alloc', '-a', required=True, type=STR, help='SLURM allocation account name.') @click.option('--memory', '-mem', default=None, type=INT, + show_default=True, help='SLURM node memory request in GB. Default is None') @click.option('--walltime', '-wt', default=1.0, type=float, + show_default=True, help='SLURM walltime request in hours for single year ' 'rep_profiles run. Default is 1.0') @click.option('--feature', '-l', default=None, type=STR, + show_default=True, help=('Additional flags for SLURM job. Format is "--qos=high" ' 'or "--depend=[state:job_id]". Default is None.')) @click.option('--conda_env', '-env', default=None, type=STR, + show_default=True, help='Conda env to activate') @click.option('--module', '-mod', default=None, type=STR, + show_default=True, help='Module to load') @click.option('--stdout_path', '-sout', default=None, type=STR, + show_default=True, help='Subprocess standard output path. Default is in out_dir.') @click.pass_context def slurm(ctx, alloc, memory, walltime, feature, conda_env, module, diff --git a/reV/supply_curve/cli_sc_aggregation.py b/reV/supply_curve/cli_sc_aggregation.py index 6cf05abd2..0f24277e5 100644 --- a/reV/supply_curve/cli_sc_aggregation.py +++ b/reV/supply_curve/cli_sc_aggregation.py @@ -27,6 +27,7 @@ @click.group() @click.option('--name', '-n', default='reV-agg', type=STR, + show_default=True, help='Job name. Default is "reV-agg".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -121,6 +122,8 @@ def from_config(ctx, config_file, verbose): friction_dset=config.friction_dset, cap_cost_scale=config.cap_cost_scale, out_dir=config.dirout, + max_workers=config.max_workers, + points_per_worker=config.points_per_worker, log_dir=config.logdir, verbose=verbose) @@ -149,6 +152,8 @@ def from_config(ctx, config_file, verbose): ctx.obj['FRICTION_DSET'] = config.friction_dset ctx.obj['CAP_COST_SCALE'] = config.cap_cost_scale ctx.obj['OUT_DIR'] = config.dirout + ctx.obj['MAX_WORKERS'] = config.max_workers + ctx.obj['POINTS_PER_WORKER'] = config.points_per_worker ctx.obj['LOG_DIR'] = config.logdir ctx.obj['VERBOSE'] = verbose @@ -166,16 +171,19 @@ def from_config(ctx, config_file, verbose): help='Exclusions file (.h5).') @click.option('--gen_fpath', '-gf', type=STR, required=True, help='reV generation/econ output file.') +@click.option('--tm_dset', '-tm', type=STR, required=True, + help='Dataset in the exclusions file that maps the exclusions ' + 'to the resource being analyzed.') @click.option('--econ_fpath', '-ef', type=STR, default=None, + show_default=True, help='reV econ output file (optional argument that can be ' 'included if reV gen and econ data are being used from ' 'different files.') @click.option('--res_fpath', '-rf', type=STR, default=None, + show_default=True, help='Resource file, required if techmap dset is to be created.') -@click.option('--tm_dset', '-tm', type=STR, required=True, - help='Dataset in the exclusions file that maps the exclusions ' - 'to the resource being analyzed.') @click.option('--excl_dict', '-exd', type=STR, default=None, + show_default=True, help=('String representation of a dictionary of exclusion ' 'LayerMask arguments {layer: {kwarg: value}} where layer ' 'is a dataset in excl_fpath and kwarg can be ' @@ -186,29 +194,38 @@ def from_config(ctx, config_file, verbose): help=('run a pre-flight check on each exclusion layer to ' 'ensure they contain un-excluded values')) @click.option('--res_class_dset', '-cd', type=STR, default=None, + show_default=True, help='Dataset to determine the resource class ' '(must be in gen_fpath).') @click.option('--res_class_bins', '-cb', type=FLOATLIST, default=None, + show_default=True, help='List of resource class bin edges.') @click.option('--cf_dset', '-cf', type=STR, default='cf_mean-means', + show_default=True, help='Dataset containing capacity factor values to aggregate.') @click.option('--lcoe_dset', '-lc', type=STR, default='lcoe_fcr-means', + show_default=True, help='Dataset containing lcoe values to aggregate.') @click.option('--h5_dsets', '-hd', type=STRLIST, default=None, + show_default=True, help='Additional datasets from the source gen/econ h5 files to ' 'aggregate.') @click.option('--data_layers', '-d', type=STR, default=None, + show_default=True, help='String representation of a dictionary of additional data ' 'layers to include in the aggregation e.g. ' '{"slope": {"dset": "srtm_slope", "method": "mean"}}') @click.option('--resolution', '-r', type=INT, default=64, + show_default=True, help='Number of exclusion points along a squares edge to ' 'include in an aggregated supply curve point.') @click.option('--excl_area', '-ea', type=FLOAT, default=0.0081, + show_default=True, help='Area of an exclusion pixel in km2. None will try to ' 'infer the area from the profile transform attribute in ' 'excl_fpath.') @click.option('--power_density', '-pd', type=STRFLOAT, default=None, + show_default=True, help='Power density in MW/km2 or filepath to variable power ' 'density csv file. None will attempt to infer a constant ' 'power density from the generation meta data technology. ' @@ -217,19 +234,24 @@ def from_config(ctx, config_file, verbose): '(typically wtk or nsrdb gid). and the power_density column ' 'is in MW/km2.') @click.option('--area_filter_kernel', '-afk', type=STR, default='queen', + show_default=True, help='Contiguous area filter kernel name ("queen", "rook").') @click.option('--min_area', '-ma', type=FLOAT, default=None, + show_default=True, help='Contiguous area filter minimum area, default is None ' '(No minimum area filter).') @click.option('--friction_fpath', '-ff', type=STR, default=None, + show_default=True, help='Optional h5 filepath to friction surface data. ' 'Must match the exclusion shape/resolution and be ' 'paired with the --friction_dset input arg. The friction data ' 'creates a "mean_lcoe_friction" output which is the nominal ' 'LCOE multiplied by the friction data.') @click.option('--friction_dset', '-fd', type=STR, default=None, + show_default=True, help='Optional friction surface dataset in friction_fpath.') @click.option('--cap_cost_scale', '-cs', type=STR, default=None, + show_default=True, help='Optional LCOE scaling equation to implement "economies ' 'of scale". Equations must be in python string format and ' 'return a scalar value to multiply the capital cost by. ' @@ -237,17 +259,27 @@ def from_config(ctx, config_file, verbose): 'of the columns in the reV supply curve aggregation table. ' 'This will not affect offshore wind LCOE.') @click.option('--out_dir', '-o', type=STR, default='./', + show_default=True, help='Directory to save aggregation summary output.') +@click.option('--max_workers', '-mw', type=INT, default=None, + show_default=True, + help=("Number of cores to run summary on. None is all " + "available cpus")) +@click.option('--points_per_worker', '-ppw', type=int, default=10, + show_default=True, + help="Number of sc_points to summarize on each worker") @click.option('--log_dir', '-ld', type=STR, default='./logs/', + show_default=True, help='Directory to save aggregation logs.') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @click.pass_context -def direct(ctx, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, +def direct(ctx, excl_fpath, gen_fpath, tm_dset, econ_fpath, res_fpath, excl_dict, check_excl_layers, res_class_dset, res_class_bins, cf_dset, lcoe_dset, h5_dsets, data_layers, resolution, excl_area, power_density, area_filter_kernel, min_area, friction_fpath, - friction_dset, cap_cost_scale, out_dir, log_dir, verbose): + friction_dset, cap_cost_scale, out_dir, max_workers, + points_per_worker, log_dir, verbose): """reV Supply Curve Aggregation Summary CLI.""" name = ctx.obj['NAME'] @@ -273,6 +305,8 @@ def direct(ctx, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, ctx.obj['FRICTION_DSET'] = friction_dset ctx.obj['CAP_COST_SCALE'] = cap_cost_scale ctx.obj['OUT_DIR'] = out_dir + ctx.obj['MAX_WORKERS'] = max_workers + ctx.obj['POINTS_PER_WORKER'] = points_per_worker ctx.obj['LOG_DIR'] = log_dir ctx.obj['VERBOSE'] = verbose @@ -316,7 +350,9 @@ def direct(ctx, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, friction_fpath=friction_fpath, friction_dset=friction_dset, check_excl_layers=check_excl_layers, - cap_cost_scale=cap_cost_scale) + cap_cost_scale=cap_cost_scale, + max_workers=max_workers, + points_per_worker=points_per_worker) except Exception as e: logger.exception('Supply curve Aggregation failed. Received the ' @@ -354,7 +390,7 @@ def get_node_cmd(name, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, cf_dset, lcoe_dset, h5_dsets, data_layers, resolution, excl_area, power_density, area_filter_kernel, min_area, friction_fpath, friction_dset, cap_cost_scale, - out_dir, log_dir, verbose): + out_dir, max_workers, points_per_worker, log_dir, verbose): """Get a CLI call command for the SC aggregation cli.""" args = ['-exf {}'.format(SLURM.s(excl_fpath)), @@ -378,6 +414,8 @@ def get_node_cmd(name, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, '-fd {}'.format(SLURM.s(friction_dset)), '-cs {}'.format(SLURM.s(cap_cost_scale)), '-o {}'.format(SLURM.s(out_dir)), + '-mw {}'.format(SLURM.s(max_workers)), + '-ppw {}'.format(SLURM.s(points_per_worker)), '-ld {}'.format(SLURM.s(log_dir)), ] @@ -398,17 +436,23 @@ def get_node_cmd(name, excl_fpath, gen_fpath, econ_fpath, res_fpath, tm_dset, @click.option('--alloc', '-a', required=True, type=STR, help='SLURM allocation account name.') @click.option('--walltime', '-wt', default=1.0, type=float, + show_default=True, help='SLURM walltime request in hours. Default is 1.0') @click.option('--feature', '-l', default=None, type=STR, + show_default=True, help=('Additional flags for SLURM job. Format is "--qos=high" ' 'or "--depend=[state:job_id]". Default is None.')) @click.option('--memory', '-mem', default=None, type=INT, + show_default=True, help='SLURM node memory request in GB. Default is None') @click.option('--module', '-mod', default=None, type=STR, + show_default=True, help='Module to load') @click.option('--conda_env', '-env', default=None, type=STR, + show_default=True, help='Conda env to activate') @click.option('--stdout_path', '-sout', default=None, type=STR, + show_default=True, help='Subprocess standard output path. Default is in out_dir.') @click.pass_context def slurm(ctx, alloc, walltime, feature, memory, module, conda_env, @@ -437,6 +481,8 @@ def slurm(ctx, alloc, walltime, feature, memory, module, conda_env, friction_dset = ctx.obj['FRICTION_DSET'] cap_cost_scale = ctx.obj['CAP_COST_SCALE'] out_dir = ctx.obj['OUT_DIR'] + max_workers = ctx.obj['MAX_WORKERS'] + points_per_worker = ctx.obj['POINTS_PER_WORKER'] log_dir = ctx.obj['LOG_DIR'] verbose = ctx.obj['VERBOSE'] @@ -450,7 +496,8 @@ def slurm(ctx, alloc, walltime, feature, memory, module, conda_env, resolution, excl_area, power_density, area_filter_kernel, min_area, friction_fpath, friction_dset, cap_cost_scale, - out_dir, log_dir, verbose) + out_dir, max_workers, points_per_worker, log_dir, + verbose) slurm_manager = ctx.obj.get('SLURM_MANAGER', None) if slurm_manager is None: diff --git a/reV/supply_curve/cli_supply_curve.py b/reV/supply_curve/cli_supply_curve.py index fdd43a461..4514f38fb 100644 --- a/reV/supply_curve/cli_supply_curve.py +++ b/reV/supply_curve/cli_supply_curve.py @@ -24,6 +24,7 @@ @click.group() @click.option('--name', '-n', default='reV-sc', type=STR, + show_default=True, help='Job name. Default is "reV-sc".') @click.option('-v', '--verbose', is_flag=True, help='Flag to turn on debug logging. Default is not verbose.') @@ -149,21 +150,27 @@ def from_config(ctx, config_file, verbose): @click.option('--fixed_charge_rate', '-fcr', type=float, required=True, help='Fixed charge rate used to compute LCOT') @click.option('--sc_features', '-scf', type=STR, default=None, + show_default=True, help='Table containing additional supply curve features ' '(.csv or .json)') @click.option('--transmission_costs', '-tc', type=STR, default=None, + show_default=True, help='Table or serialized dict of transmission cost inputs.') @click.option('--sort_on', '-so', type=str, default='total_lcoe', + show_default=True, help='The supply curve table column label to sort on. ' 'This determines the ordering of the SC buildout algorithm.') @click.option('--offshore_trans_table', '-ott', type=STR, default=None, + show_default=True, help=('Path to offshore transmission table, if None offshore sc ' 'points will not be included, by default None')) @click.option('--wind_dirs', '-wd', type=click.Path(exists=True), default=None, + show_default=True, help=('Path to .csv containing reVX.wind_dirs.wind_dirs.WindDirs' ' output with the neighboring supply curve point gids and ' 'power-rose value at each cardinal direction')) @click.option('--n_dirs', '-dirs', type=int, default=2, + show_default=True, help='Number of prominent directions to use') @click.option('--downwind', '-dw', is_flag=True, help=('Flag to remove downwind neighbors as well as upwind ' @@ -172,11 +179,14 @@ def from_config(ctx, config_file, verbose): help=('Flag as to whether offshore farms should be included ' 'during CompetitiveWindFarms, by default False')) @click.option('--max_workers', '-mw', type=INT, default=None, + show_default=True, help=('Number of workers to use to compute lcot, if > 1 run in ' 'parallel. None uses all available cpus.')) @click.option('--out_dir', '-o', type=STR, default='./', + show_default=True, help='Directory to save aggregation summary output.') @click.option('--log_dir', '-ld', type=STR, default='./logs/', + show_default=True, help='Directory to save aggregation logs.') @click.option('-s', '--simple', is_flag=True, help='Flag to turn on simple supply curve calculation.') @@ -316,18 +326,24 @@ def get_node_cmd(name, sc_points, trans_table, fixed_charge_rate, sc_features, @direct.command() @click.option('--alloc', '-a', required=True, type=STR, help='SLURM allocation account name.') -@click.option('--memory', '-mem', default=None, type=INT, help='SLURM node ' - 'memory request in GB. Default is None') +@click.option('--memory', '-mem', default=None, type=INT, + show_default=True, + help='SLURM node memory request in GB. Default is None') @click.option('--walltime', '-wt', default=1.0, type=float, + show_default=True, help='SLURM walltime request in hours. Default is 1.0') @click.option('--feature', '-l', default=None, type=STR, + show_default=True, help=('Additional flags for SLURM job. Format is "--qos=high" ' 'or "--depend=[state:job_id]". Default is None.')) @click.option('--module', '-mod', default=None, type=STR, + show_default=True, help='Module to load') @click.option('--conda_env', '-env', default=None, type=STR, + show_default=True, help='Conda env to activate') @click.option('--stdout_path', '-sout', default=None, type=STR, + show_default=True, help='Subprocess standard output path. Default is in out_dir.') @click.pass_context def slurm(ctx, alloc, memory, walltime, feature, module, conda_env, diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 372d5acc2..b4069a99d 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -1136,7 +1136,8 @@ def run_serial(cls, excl_fpath, gen_fpath, tm_dset, gen_index, return summary - def run_parallel(self, args=None, excl_area=0.0081, max_workers=None): + def run_parallel(self, args=None, excl_area=0.0081, max_workers=None, + points_per_worker=10): """Get the supply curve points aggregation summary using futures. Parameters @@ -1144,20 +1145,21 @@ def run_parallel(self, args=None, excl_area=0.0081, max_workers=None): args : tuple | list | None List of summary arguments to include. None defaults to all available args defined in the class attr. - excl_area : float - Area of an exclusion cell (square km). - max_workers : int | None + excl_area : float, optional + Area of an exclusion cell (square km), by default 0.0081 + max_workers : int | None, optional Number of cores to run summary on. None is all - available cpus. + available cpus, by default None + points_per_worker : int + Number of sc_points to summarize on each worker, by default 10 Returns ------- summary : list List of dictionaries, each being an SC point summary. """ - - chunks = np.array_split(self._gids, - int(np.ceil(len(self._gids) / 1000))) + chunks = len(self._gids) // points_per_worker + chunks = np.array_split(self._gids, chunks) logger.info('Running supply curve point aggregation for ' 'points {} through {} at a resolution of {} ' @@ -1320,7 +1322,7 @@ def _summary_to_df(summary): return summary - def summarize(self, args=None, max_workers=None, + def summarize(self, args=None, max_workers=None, points_per_worker=10, offshore_capacity=600, offshore_gid_counts=494, offshore_pixel_area=4, offshore_meta_cols=None): """ @@ -1331,9 +1333,11 @@ def summarize(self, args=None, max_workers=None, args : tuple | list | None List of summary arguments to include. None defaults to all available args defined in the class attr. - max_workers : int | None + max_workers : int | None, optional Number of cores to run summary on. None is all - available cpus. + available cpus, by default None + points_per_worker : int + Number of sc_points to summarize on each worker, by default 10 offshore_capacity : int | float Offshore resource pixel generation capacity in MW. offshore_gid_counts : int @@ -1380,7 +1384,8 @@ def summarize(self, args=None, max_workers=None, cap_cost_scale=self._cap_cost_scale) else: summary = self.run_parallel(args=args, excl_area=self._excl_area, - max_workers=max_workers) + max_workers=max_workers, + points_per_worker=points_per_worker) summary = self.run_offshore(summary, offshore_capacity=offshore_capacity, @@ -1409,6 +1414,7 @@ def summary(cls, excl_fpath, gen_fpath, tm_dset, econ_fpath=None, h5_dsets=None, data_layers=None, power_density=None, friction_fpath=None, friction_dset=None, args=None, excl_area=None, max_workers=None, + points_per_worker=10, cap_cost_scale=None, offshore_capacity=600, offshore_gid_counts=494, offshore_pixel_area=4, offshore_meta_cols=None): @@ -1481,9 +1487,11 @@ def summary(cls, excl_fpath, gen_fpath, tm_dset, econ_fpath=None, excl_area : float | None Area of an exclusion pixel in km2. None will try to infer the area from the profile transform attribute in excl_fpath. - max_workers : int | None + max_workers : int | None, optional Number of cores to run summary on. None is all - available cpus. + available cpus, by default None + points_per_worker : int + Number of sc_points to summarize on each worker, by default 10 cap_cost_scale : str | None Optional LCOE scaling equation to implement "economies of scale". Equations must be in python string format and return a scalar @@ -1532,6 +1540,7 @@ def summary(cls, excl_fpath, gen_fpath, tm_dset, econ_fpath=None, summary = agg.summarize(args=args, max_workers=max_workers, + points_per_worker=points_per_worker, offshore_capacity=offshore_capacity, offshore_gid_counts=offshore_gid_counts, offshore_pixel_area=offshore_pixel_area,