Merge pull request #268 from NREL/sc_agg_workers

SC_Agg parallel update

reV/config/execution.py

@@ -22,11 +22,9 @@ def __init__(self, config_dict):
         """
         super().__init__(config_dict)
 
-        self._option = 'local'
-        self._nodes = 1
-        self._max_workers = None
-        self._sites_per_worker = None
-        self._mem_util_lim = 0.4
+        self._default_option = 'local'
+        self._default_nodes = 1
+        self._default_mem_util_lim = 0.4
 
     @property
     def option(self):
@@ -37,8 +35,7 @@ def option(self):
         option : str
             Execution control option, e.g. local, peregrine, eagle...
         """
-        self._option = str(self.get('option', self._option)).lower()
-        return self._option
+        return str(self.get('option', self._default_option)).lower()
 
     @property
     def nodes(self):
@@ -49,8 +46,7 @@ def nodes(self):
         nodes : int
             Number of available nodes. Default is 1 node.
         """
-        self._nodes = int(self.get('nodes', self._nodes))
-        return self._nodes
+        return int(self.get('nodes', self._default_nodes))
 
     @property
     def max_workers(self):
@@ -61,8 +57,7 @@ def max_workers(self):
         max_workers : int | None
             Processes per node. Default is None max_workers (all available).
         """
-        self._max_workers = self.get('max_workers', self._max_workers)
-        return self._max_workers
+        return self.get('max_workers', None)
 
     @property
     def sites_per_worker(self):
@@ -73,9 +68,7 @@ def sites_per_worker(self):
         sites_per_worker : int | None
             Number of sites to run per worker in a parallel scheme.
         """
-        self._sites_per_worker = self.get('sites_per_worker',
-                                          self._sites_per_worker)
-        return self._sites_per_worker
+        return self.get('sites_per_worker', None)
 
     @property
     def memory_utilization_limit(self):
@@ -88,29 +81,15 @@ def memory_utilization_limit(self):
             Memory utilization limit (fractional). Key in the config json is
             "memory_utilization_limit".
         """
-        self._mem_util_lim = self.get('memory_utilization_limit',
-                                      self._mem_util_lim)
-        return self._mem_util_lim
+        mem_util_lim = self.get('memory_utilization_limit',
+                                self._default_mem_util_lim)
+
+        return mem_util_lim
 
 
 class HPCConfig(BaseExecutionConfig):
     """Class to handle HPC configuration inputs."""
-
-    def __init__(self, config_dict):
-        """
-        Parameters
-        ----------
-        config_dict : str | dict
-            File path to config json (str), serialized json object (str),
-            or dictionary with pre-extracted config.
-        """
-
-        super().__init__(config_dict)
-
-        self._hpc_alloc = 'rev'
-        self._feature = None
-        self._module = None
-        self._conda_env = None
+    REQUIREMENTS = ('allocation')
 
     @property
     def allocation(self):
@@ -121,8 +100,8 @@ def allocation(self):
         hpc_alloc : str
             Name of the HPC allocation account for the specified job.
         """
-        self._hpc_alloc = self.get('allocation', self._hpc_alloc)
-        return self._hpc_alloc
+
+        return self['allocation']
 
     @property
     def feature(self):
@@ -138,8 +117,7 @@ def feature(self):
                 "feature": "--qos=high"
                 "feature": "--depend=[state:job_id]"
         """
-        self._feature = self.get('feature', self._feature)
-        return self._feature
+        return self.get('feature', None)
 
     @property
     def module(self):
@@ -151,8 +129,7 @@ def module(self):
         module : str
             Module to load on node
         """
-        self._module = self.get('module', self._module)
-        return self._module
+        return self.get('module', None)
 
     @property
     def conda_env(self):
@@ -164,8 +141,7 @@ def conda_env(self):
         conda_env : str
             Conda environment to activate
         """
-        self._conda_env = self.get('conda_env', self._conda_env)
-        return self._conda_env
+        return self.get('conda_env', None)
 
 
 class SlurmConfig(HPCConfig):
@@ -182,8 +158,7 @@ def __init__(self, config_dict):
 
         super().__init__(config_dict)
 
-        self._hpc_node_mem = None
-        self._hpc_walltime = 1
+        self._default_hpc_walltime = 1
 
     @property
     def memory(self):
@@ -194,8 +169,7 @@ def memory(self):
         _hpc_node_mem : int | None
             Requested node memory in GB.
         """
-        self._hpc_node_mem = self.get('memory', self._hpc_node_mem)
-        return self._hpc_node_mem
+        return self.get('memory', None)
 
     @property
     def walltime(self):
@@ -206,5 +180,4 @@ def walltime(self):
         _hpc_walltime : int
             Requested single node job time in hours.
         """
-        self._hpc_walltime = float(self.get('walltime', self._hpc_walltime))
-        return self._hpc_walltime
+        return float(self.get('walltime', self._default_hpc_walltime))

reV/config/supply_curve_configs.py

@@ -33,17 +33,11 @@ def __init__(self, config):
         """
         super().__init__(config)
 
-        self._default_res_fpath = None
-        self._default_econ_fpath = None
-        self._default_res_class_dset = None
-        self._default_res_class_bins = None
         self._default_cf_dset = 'cf_mean-means'
         self._default_lcoe_dset = 'lcoe_fcr-means'
-        self._default_data_layers = None
         self._default_resolution = 64
         self._default_area_filter_kernel = 'queen'
-        self._default_min_area = None
-        self._default_excl_dict = None
+        self._default_points_per_worker = 10
 
         self._sc_agg_preflight()
 
@@ -104,7 +98,7 @@ def econ_fpath(self):
         """Get the econ data filepath. This is an optional argument only used
         if reV gen and econ outputs are being used from different files."""
 
-        fpath = self.get('econ_fpath', self._default_econ_fpath)
+        fpath = self.get('econ_fpath', None)
 
         if fpath == 'PIPELINE':
             target_modules = ['multi-year', 'collect', 'econ']
@@ -131,7 +125,7 @@ def econ_fpath(self):
     @property
     def res_fpath(self):
         """Get the resource data filepath"""
-        res_fpath = self.get('res_fpath', self._default_res_fpath)
+        res_fpath = self.get('res_fpath', None)
         if isinstance(res_fpath, str):
             if '{}' in res_fpath:
                 for year in range(1998, 2018):
@@ -150,17 +144,17 @@ def tm_dset(self):
     @property
     def excl_dict(self):
         """Get the exclusions dictionary"""
-        return self.get('excl_dict', self._default_excl_dict)
+        return self.get('excl_dict', None)
 
     @property
     def res_class_dset(self):
         """Get the resource class dataset"""
-        return self.get('res_class_dset', self._default_res_class_dset)
+        return self.get('res_class_dset', None)
 
     @property
     def res_class_bins(self):
         """Get the resource class bins"""
-        return self.get('res_class_bins', self._default_res_class_bins)
+        return self.get('res_class_bins', None)
 
     @property
     def cf_dset(self):
@@ -180,7 +174,7 @@ def h5_dsets(self):
     @property
     def data_layers(self):
         """Get the data layers dict"""
-        return self.get('data_layers', self._default_data_layers)
+        return self.get('data_layers', None)
 
     @property
     def resolution(self):
@@ -206,7 +200,7 @@ def area_filter_kernel(self):
     @property
     def min_area(self):
         """Get the minimum area filter minimum area in km2."""
-        return self.get('min_area', self._default_min_area)
+        return self.get('min_area', None)
 
     @property
     def friction_fpath(self):
@@ -235,6 +229,16 @@ def cap_cost_scale(self):
         """
         return self.get('cap_cost_scale', None)
 
+    @property
+    def max_workers(self):
+        """Get the number of workers to use during computation"""
+        return self.get('max_workers', None)
+
+    @property
+    def points_per_worker(self):
+        """Get the number of sc points to summarize on each worker"""
+        return self.get('max_workers', None)
+
 
 class SupplyCurveConfig(AnalysisConfig):
     """SC config."""
@@ -252,8 +256,6 @@ def __init__(self, config):
         """
         super().__init__(config)
 
-        self._default_sc_features = None
-        self._default_transmission_costs = None
         self._default_sort_on = 'total_lcoe'
         self._default_n_dirs = 2
 
@@ -297,12 +299,12 @@ def fixed_charge_rate(self):
     @property
     def sc_features(self):
         """Get the supply curve features input."""
-        return self.get('sc_features', self._default_sc_features)
+        return self.get('sc_features', None)
 
     @property
     def transmission_costs(self):
         """Get the transmission costs input."""
-        return self.get('transmission_costs', self._default_transmission_costs)
+        return self.get('transmission_costs', None)
 
     @property
     def simple(self):

reV/econ/cli_econ.py

@@ -29,6 +29,7 @@
 
 @click.group()
 @click.option('--name', '-n', default='reV-econ', type=STR,
+              show_default=True,
               help='reV Economics job name, by default "reV-econ".')
 @click.option('-v', '--verbose', is_flag=True,
               help='Flag to turn on debug logging. Default is not verbose.')
@@ -195,30 +196,39 @@ def submit_from_config(ctx, name, cf_file, year, config, verbose):
 @click.option('--sam_files', '-sf', required=True, type=SAMFILES,
               help='SAM config files (required) (str, dict, or list).')
 @click.option('--cf_file', '-cf', default=None, type=click.Path(exists=True),
+              show_default=True,
               help='Single generation results file (str).')
 @click.option('--year', '-y', default=None, type=INT,
+              show_default=True,
               help='Year of generation results to analyze (if multiple years '
               'in cf_file). Default is None (use the only cf_mean dataset in '
               'cf_file).')
 @click.option('--points', '-p', default=slice(0, 100), type=PROJECTPOINTS,
+              show_default=True,
               help=('reV project points to analyze (slice, list, or file '
                     'string). Default is slice(0, 100)'))
 @click.option('--site_data', '-sd', default=None, type=click.Path(exists=True),
+              show_default=True,
               help='Site-specific data file for econ calculation. Input '
               'should be a filepath that points to a csv. Rows match sites, '
               'columns are input keys. Needs a "gid" column. Input as None '
               'if no site-specific data.')
 @click.option('--sites_per_worker', '-spw', default=None, type=INT,
+              show_default=True,
               help=('Number of sites to run in series on a single worker. '
                     'Default is the resource column chunk size.'))
 @click.option('--fout', '-fo', default='econ_output.h5', type=STR,
+              show_default=True,
               help=('Filename output specification (should be .h5). '
                     'Default is "econ_output.h5"'))
 @click.option('--dirout', '-do', default='./out/econ_out', type=STR,
+              show_default=True,
               help='Output directory specification. Default is ./out/econ_out')
 @click.option('--logdir', '-lo', default='./out/log_econ', type=STR,
+              show_default=True,
               help='Econ log file directory. Default is ./out/log_econ')
 @click.option('-or', '--output_request', type=STRLIST, default=['lcoe_fcr'],
+              show_default=True,
               help=('Requested output variable name(s). '
                     'Default is ["lcoe_fcr"].'))
 @click.option('-ap', '--append', is_flag=True,
@@ -248,12 +258,15 @@ def direct(ctx, sam_files, cf_file, year, points, site_data,
 
 @direct.command()
 @click.option('--max_workers', '-mw', type=INT, default=None,
+              show_default=True,
               help='Number of workers. Use 1 for serial, None for all cores.')
 @click.option('--timeout', '-to', type=INT, default=1800,
+              show_default=True,
               help='Number of seconds to wait for econ parallel run '
               'iterations to complete before returning zeros. '
               'Default is 1800 seconds.')
 @click.option('--points_range', '-pr', default=None, type=INTLIST,
+              show_default=True,
               help='Optional range list to run a subset of sites.')
 @click.option('-v', '--verbose', is_flag=True,
               help='Flag to turn on debug logging.')
@@ -451,27 +464,34 @@ def get_node_cmd(name, sam_files, cf_file, year=None, site_data=None,
 
 
 @direct.command()
+@click.option('--alloc', '-a', required=True, type=STR,
+              help='SLURM allocation account name.')
 @click.option('--nodes', '-no', default=1, type=INT,
+              show_default=True,
               help='Number of SLURM nodes for econ job. Default is 1.')
-@click.option('--alloc', '-a', default='rev', type=STR,
-              help='SLURM allocation account name. Default is "rev".')
 @click.option('--memory', '-mem', default=None, type=INT,
+              show_default=True,
               help='SLURM node memory request in GB. Default is None')
 @click.option('--walltime', '-wt', default=0.5, type=float,
+              show_default=True,
               help='SLURM walltime request in hours. Default is 0.5')
 @click.option('--feature', '-l', default=None, type=STR,
+              show_default=True,
               help=('Additional flags for SLURM job. Format is "--qos=high" '
                     'or "--depend=[state:job_id]". Default is None.'))
 @click.option('--module', '-mod', default=None, type=STR,
+              show_default=True,
               help='Module to load')
 @click.option('--conda_env', '-env', default=None, type=STR,
+              show_default=True,
               help='Conda env to activate')
 @click.option('--stdout_path', '-sout', default='./out/stdout', type=STR,
+              show_default=True,
               help='Subprocess standard output path. Default is ./out/stdout')
 @click.option('-v', '--verbose', is_flag=True,
               help='Flag to turn on debug logging. Default is not verbose.')
 @click.pass_context
-def slurm(ctx, nodes, alloc, memory, walltime, feature, module, conda_env,
+def slurm(ctx, alloc, nodes, memory, walltime, feature, module, conda_env,
           stdout_path, verbose):
     """Run econ on HPC via SLURM job submission."""