diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b85463c0..54cfa7e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: - synchronize - opened jobs: - tests: + full_tests: runs-on: ubuntu-latest strategy: fail-fast: false @@ -85,3 +85,43 @@ jobs: with: name: documentation path: buildstockbatch/docs/_build/html/ + pip_install_test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + with: + repository: NREL/resstock + path: resstock + ref: develop + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Download weather + run: | + mkdir weather + cd weather + wget --quiet https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip + - name: Download and Install OpenStudio + run: | + sudo snap install yq + export OS_VER=`yq '.os_version' resstock/project_national/national_baseline.yml` + export OS_SHA=`yq '.os_sha' resstock/project_national/national_baseline.yml` + export OS_INSTALLER_FILENAME="OpenStudio-${OS_VER}+${OS_SHA}-Ubuntu-22.04-x86_64.deb" + wget -q "https://github.com/NREL/OpenStudio/releases/download/v${OS_VER}/${OS_INSTALLER_FILENAME}" + sudo apt install -y "./${OS_INSTALLER_FILENAME}" + openstudio openstudio_version + which openstudio + - name: Install buildstockbatch + run: | + python -m pip install --progress-bar off --upgrade pip + pip install git+https://github.com/NREL/buildstockbatch.git@${{ github.head_ref }} + - name: Run local validation of project national yml + run: buildstock_local --validateonly resstock/project_national/national_baseline.yml + - name: Validate upgrades + run: buildstock_local --validateonly resstock/project_national/national_upgrades.yml + - name: Validate testing projects + run: buildstock_local --validateonly resstock/project_testing/testing_baseline.yml + - name: Validate testing upgrades + run: buildstock_local --validateonly resstock/project_testing/testing_upgrades.yml diff --git a/buildstockbatch/aws/aws.py b/buildstockbatch/aws/aws.py index fe300f07..2bd93038 100644 --- a/buildstockbatch/aws/aws.py +++ b/buildstockbatch/aws/aws.py @@ -1360,7 +1360,7 @@ def main(): # validate the project, and in case of the --validateonly flag return True if validation passes AwsBatch.validate_project(os.path.abspath(args.project_filename)) if args.validateonly: - return True + return batch = AwsBatch(args.project_filename) if args.clean: diff --git a/buildstockbatch/gcp/gcp.py b/buildstockbatch/gcp/gcp.py index e0d173e8..18a40137 100644 --- a/buildstockbatch/gcp/gcp.py +++ b/buildstockbatch/gcp/gcp.py @@ -1151,7 +1151,7 @@ def main(): # validate the project, and if --validateonly flag is set, return True if validation passes GcpBatch.validate_project(os.path.abspath(args.project_filename)) if args.validateonly: - return True + return batch = GcpBatch(args.project_filename, args.job_identifier, missing_only=args.missingonly) if args.clean: diff --git a/buildstockbatch/hpc.py b/buildstockbatch/hpc.py index 32793fab..96e917a0 100644 --- a/buildstockbatch/hpc.py +++ b/buildstockbatch/hpc.py @@ -890,18 +890,18 @@ def user_cli(Batch: SlurmBatch, argv: list): # validate the project, and in case of the --validateonly flag return True if validation passes Batch.validate_project(project_filename) if args.validateonly: - return True + return # if the project has already been run, simply queue the correct post-processing step if args.postprocessonly or args.uploadonly: batch = Batch(project_filename) batch.queue_post_processing(upload_only=args.uploadonly, hipri=args.hipri) - return True + return if args.rerun_failed: batch = Batch(project_filename) batch.rerun_failed_jobs(hipri=args.hipri) - return True + return # otherwise, queue up the whole buildstockbatch process # the main work of the first job is to run the sampling script ... diff --git a/buildstockbatch/local.py b/buildstockbatch/local.py index dec2cfdf..5aa1402b 100644 --- a/buildstockbatch/local.py +++ b/buildstockbatch/local.py @@ -409,7 +409,7 @@ def main(): # Validate the project, and in case of the --validateonly flag return True if validation passes LocalBatch.validate_project(args.project_filename) if args.validateonly: - return True + return batch = LocalBatch(args.project_filename) if not (args.postprocessonly or args.uploadonly or args.validateonly): batch.run_batch( diff --git a/buildstockbatch/workflow_generator/residential/residential_hpxml.py b/buildstockbatch/workflow_generator/residential/residential_hpxml.py index 529c755f..69ae48a3 100644 --- a/buildstockbatch/workflow_generator/residential/residential_hpxml.py +++ b/buildstockbatch/workflow_generator/residential/residential_hpxml.py @@ -36,7 +36,7 @@ def __init__(self, cfg, n_datapoints): self.measures_dir = os.path.join(self.buildstock_dir, "measures") self.workflow_args = self.cfg["workflow_generator"].get("args", {}) self.default_args = copy.deepcopy(DEFAULT_MEASURE_ARGS) - self.arg_map = copy.deepcopy(ARG_MAP) + self.all_arg_map = copy.deepcopy(ARG_MAP) def validate(self): """Validate arguments @@ -112,22 +112,10 @@ def create_osw(self, sim_id, building_id, upgrade_idx): :param upgrade_idx: integer index of the upgrade scenario to apply, None if baseline """ logger.debug("Generating OSW, sim_id={}".format(sim_id)) - workflow_args = copy.deepcopy(self.workflow_args) - - measure_args = self._get_mapped_args(workflow_args) # start with the mapped arguments - - measure_args["BuildExistingModel"].update( - { - "building_id": building_id, - "sample_weight": self.cfg["baseline"]["n_buildings_represented"] / self.n_datapoints, - } - ) - debug = workflow_args.get("debug", False) - - measure_args_mapping = { + workflow_key_to_measure_names = { # This is the order the osw steps will be in "build_existing_model": "BuildExistingModel", - "hpxml_to_openstudio": "HPXMLtoOpenStudio", + "hpxml_to_openstudio": "HPXMLtoOpenStudio", # Non-existing Workflow Key is fine "simulation_output_report": "ReportSimulationOutput", "report_hpxml_output": "ReportHPXMLOutput", "report_utility_bills": "ReportUtilityBills", @@ -136,20 +124,41 @@ def create_osw(self, sim_id, building_id, upgrade_idx): } steps = [] - for key, measure_dir_name in measure_args_mapping.items(): - if measure_dir_name not in measure_args: - measure_args[measure_dir_name] = {} + measure_args = {} + debug = workflow_args.get("debug", False) - measure_args[measure_dir_name].update( - self._get_measure_args(workflow_args.get(key, {}), measure_dir_name, debug) - ) + # start with defaults + for workflow_key, measure_name in workflow_key_to_measure_names.items(): + measure_args[measure_name] = self.default_args.get(measure_name, {}).copy() + + # update with mapped args + for workflow_key, measure_name in workflow_key_to_measure_names.items(): + measure_args[measure_name].update(self._get_mapped_args(workflow_args, measure_name)) + + # update with workflow block args + for workflow_key, measure_name in workflow_key_to_measure_names.items(): + measure_args[measure_name].update(workflow_args.get(workflow_key, {}).copy()) + + # Verify the arguments and add to steps + for workflow_key, measure_name in workflow_key_to_measure_names.items(): + xml_args = self.get_measure_arguments_from_xml(self.buildstock_dir, measure_name) + self._validate_against_xml_args(measure_args[measure_name], measure_name, xml_args) + if "debug" in xml_args: + measure_args[measure_name]["debug"] = debug steps.append( { - "measure_dir_name": measure_dir_name, - "arguments": measure_args[measure_dir_name], + "measure_dir_name": measure_name, + "arguments": measure_args[measure_name], } ) + measure_args["BuildExistingModel"].update( + { + "building_id": building_id, + "sample_weight": self.cfg["baseline"]["n_buildings_represented"] / self.n_datapoints, + } + ) + osw = { "id": sim_id, "steps": steps, @@ -202,37 +211,37 @@ def add_upgrade_step_to_osw(self, upgrade_idx, osw): ) osw["steps"].insert(1, apply_upgrade_measure) # right after BuildExistingModel - def _get_measure_args(self, workflow_block_args, measure_dir_name, debug): + def _validate_against_xml_args(self, measure_args, measure_dir_name, xml_args): """ - Get the arguments to the measure from the workflow_args and defaults. The arguments are filtered based - on the measure's measure.xml file. If an argument is not found in the measure.xml file, it is not - passed to the measure and a warning is logged. + Check if the arguments in the measure_args are valid for the measure_dir_name + based on the measure.xml file in the measure directory. + Optionally add the debug argument if it is present in the measure.xml file. """ xml_args = self.get_measure_arguments_from_xml(self.buildstock_dir, measure_dir_name) - measure_args = self.default_args.get(measure_dir_name, {}).copy() - measure_args.update(workflow_block_args) for key in list(measure_args.keys()): if key not in xml_args: - location = "workflow_generator" if key in workflow_block_args else "defaults" logger.warning( - f"'{key}' in {location} not found in '{measure_dir_name}'. This key will not be passed" + f"'{key}' not found in '{measure_dir_name}'. This key will not be passed" " to the measure. This warning is expected if you are using older version of ResStock." ) del measure_args[key] - if "debug" in xml_args: - measure_args["debug"] = debug - return measure_args - def _get_mapped_args(self, workflow_args): + def _get_mapped_args( + self, + workflow_args, + measure_dir_name, + ): """ - Get the arguments to various measures from the workflow_args. The mapping is defined in the ARG_MAP + Get the arguments to the measures from the workflow_args using the mapping in self.all_arg_map """ measure_args = {} - for yaml_blockname, arg_maps in self.arg_map.items(): + for yaml_blockname, arg_map in self.all_arg_map.get(measure_dir_name, {}).items(): if yaml_blockname not in workflow_args: continue yaml_block = workflow_args[yaml_blockname] - self.recursive_dict_update(measure_args, self._get_mapped_args_from_block(yaml_block, arg_maps)) + measure_args.update( + self._get_mapped_args_from_block(yaml_block, arg_map, self.default_args.get(measure_dir_name, {})) + ) return measure_args @staticmethod @@ -253,19 +262,6 @@ def get_measure_arguments_from_xml(buildstock_dir, measure_dir_name: str): arguments.add(name) return arguments - @staticmethod - def recursive_dict_update(base_dict, new_dict): - """ - Fully update a dictionary with another dictionary, traversing nested dictionaries - """ - for key, value in new_dict.items(): - if isinstance(value, dict): - base_dict.setdefault(key, {}) - ResidentialHpxmlWorkflowGenerator.recursive_dict_update(base_dict[key], value) - else: - base_dict[key] = value - return True - @staticmethod def _get_condensed_block(yaml_block): """ @@ -308,7 +304,7 @@ def _get_condensed_block(yaml_block): return condensed_block @staticmethod - def _get_mapped_args_from_block(block, arg_maps: Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, Any]]: + def _get_mapped_args_from_block(block, arg_map: Dict[str, str], default_args) -> Dict[str, Any]: """ Get the arguments to meaures using the ARG_MAP for the given block. The block is either a dict or a list of dicts. If it is a list of dicts, it is @@ -322,46 +318,53 @@ def _get_mapped_args_from_block(block, arg_maps: Dict[str, Dict[str, str]]) -> D If a value is a list, it is joined into a comma separated string. If a value is a list of dicts, then the "name" key is used to join into a comma separated string. Otherwise, the value is passed as is. - Example Input: - { - "utility_bills": [ + Example Input1: + + block = [ {"scenario_name": "scenario1", "simple_filepath": "file1"}, {"scenario_name": "scenario2", "simple_filepath": "file2"} - ], - ... - "report_simulation_output": { + ] + arg_map = { + "scenario_name": "utility_bill_scenario_names", + "simple_filepath": "utility_bill_simple_filepaths" + } + Example output: + output = {"utility_bill_scenario_names": "scenario1,scenario2"} + Example Input2: + block: { + "normal_arg1": 1, "output_variables": [ {"name": "var1"}, {"name": "var2"} ] } - } + arg_map = {"output_variables": "user_output_variables"} + Example output: { - "BuildExistingModel": { - "utility_bill_scenario_names": "scenario1,scenario2", - ... - }, + output = {"normal_arg1", 1, "user_output_variables": "var1,var2"} "ReportSimulationOutput": { "user_output_variables": "var1,va2", } """ block_count = len(block) if isinstance(block, list) else 1 block = ResidentialHpxmlWorkflowGenerator._get_condensed_block(block) - measure_args = {} - for measure_dir_name, arg_map in arg_maps.items(): - mapped_args = measure_args.setdefault(measure_dir_name, {}) - for source_arg, dest_arg in arg_map.items(): - if source_arg in block: - # Use pop to remove the key from the block since it is already consumed - if isinstance(block[source_arg], list): - if isinstance(block[source_arg][0], dict): - mapped_args[dest_arg] = ",".join(str(v.get("name", "")) for v in block.pop(source_arg)) - else: - mapped_args[dest_arg] = ",".join(str(v) for v in block.pop(source_arg)) + mapped_args = {} + + for source_arg, dest_arg in arg_map.items(): + if source_arg in block: + # Use pop to remove the key from the block since it is already consumed + if isinstance(block[source_arg], list): + if isinstance(block[source_arg][0], dict): + mapped_args[dest_arg] = ",".join(str(v.get("name", "")) for v in block.pop(source_arg)) else: - mapped_args[dest_arg] = block.pop(source_arg) + mapped_args[dest_arg] = ",".join(str(v) for v in block.pop(source_arg)) else: - mapped_args[dest_arg] = ",".join([""] * block_count) + mapped_args[dest_arg] = block.pop(source_arg) + else: + if block_count > 1: + mapped_args[dest_arg] = ",".join([str(default_args.get(dest_arg, ""))] * block_count) + else: + mapped_args[dest_arg] = default_args.get(dest_arg, "") - return measure_args + return mapped_args diff --git a/buildstockbatch/workflow_generator/residential/residential_hpxml_arg_mapping.py b/buildstockbatch/workflow_generator/residential/residential_hpxml_arg_mapping.py index 114a2a4c..5e6452f3 100644 --- a/buildstockbatch/workflow_generator/residential/residential_hpxml_arg_mapping.py +++ b/buildstockbatch/workflow_generator/residential/residential_hpxml_arg_mapping.py @@ -10,8 +10,8 @@ Structure of ARG_MAP: ARG_MAP = { - "workflow_generator_arg_group_name": { <-- Source - "MeasureDirName": { <-- Destination + "MeasureDirName": { <-- Destination + "workflow_generator_arg_group_name": { <-- Source "workflow_generator_arg_name": "measure_arg_name", <-- Source: Destination ... }, @@ -22,11 +22,13 @@ """ ARG_MAP = { - "build_existing_model": { - "HPXMLtoOpenStudio": {"add_component_loads": "add_component_loads"}, + "HPXMLtoOpenStudio": { + "build_existing_model": { + "add_component_loads": "add_component_loads", + }, }, - "emissions": { - "BuildExistingModel": { + "BuildExistingModel": { + "emissions": { "scenario_name": "emissions_scenario_names", "type": "emissions_types", "elec_folder": "emissions_electricity_folders", @@ -35,9 +37,7 @@ "oil_value": "emissions_fuel_oil_values", "wood_value": "emissions_wood_values", }, - }, - "utility_bills": { - "BuildExistingModel": { + "utility_bills": { "scenario_name": "utility_bill_scenario_names", "simple_filepath": "utility_bill_simple_filepaths", "detailed_filepath": "utility_bill_detailed_filepaths", @@ -57,10 +57,10 @@ "pv_feed_in_tariff_rate": "utility_bill_pv_feed_in_tariff_rates", "pv_monthly_grid_connection_fee_units": "utility_bill_pv_monthly_grid_connection_fee_units", "pv_monthly_grid_connection_fee": "utility_bill_pv_monthly_grid_connection_fees", - } + }, }, - "simulation_output_report": { - "ReportSimulationOutput": { + "ReportSimulationOutput": { + "simulation_output_report": { "output_variables": "user_output_variables", }, }, diff --git a/buildstockbatch/workflow_generator/residential/test_residential_workflow_generator.py b/buildstockbatch/workflow_generator/residential/test_residential_workflow_generator.py index e1b7c395..ad542c53 100644 --- a/buildstockbatch/workflow_generator/residential/test_residential_workflow_generator.py +++ b/buildstockbatch/workflow_generator/residential/test_residential_workflow_generator.py @@ -8,138 +8,261 @@ import os import yamale import logging +import copy +import itertools +import pytest - -def test_residential_hpxml(mocker): - sim_id = "bldb1up1" - building_id = 1 - cfg = { - "buildstock_directory": resstock_directory, - "baseline": {"n_buildings_represented": 100}, - "workflow_generator": { - "type": "residential_hpxml", - "args": { - "build_existing_model": { - "simulation_control_run_period_begin_month": 2, - "simulation_control_run_period_begin_day_of_month": 1, - "simulation_control_run_period_end_month": 2, - "simulation_control_run_period_end_day_of_month": 28, - "simulation_control_run_period_calendar_year": 2010, - "add_component_loads": True, +test_cfg = { + "buildstock_directory": resstock_directory, + "baseline": {"n_buildings_represented": 100}, + "workflow_generator": { + "type": "residential_hpxml", + "args": { + "debug": True, + "build_existing_model": { + "simulation_control_timestep": 15, + "simulation_control_run_period_begin_month": 2, + "simulation_control_run_period_begin_day_of_month": 1, + "simulation_control_run_period_end_month": 2, + "simulation_control_run_period_end_day_of_month": 28, + "simulation_control_run_period_calendar_year": 2010, + "add_component_loads": True, + }, + "emissions": [ + { + "scenario_name": "LRMER_MidCase_15", + "type": "CO2e", + "elec_folder": "data/emissions/cambium/2022/LRMER_MidCase_15", + "gas_value": 147.3, + "propane_value": 177.8, + "oil_value": 195.9, + "wood_value": 200.0, }, - "emissions": [ - { - "scenario_name": "LRMER_MidCase_15", - "type": "CO2e", - "elec_folder": "data/emissions/cambium/2022/LRMER_MidCase_15", - "gas_value": 147.3, - "propane_value": 177.8, - "oil_value": 195.9, - "wood_value": 200.0, - }, - { - "scenario_name": "LRMER_HighCase_15", - "type": "CO2e", - "elec_folder": "data/emissions/cambium/2022/LRMER_HighCase_15", - "gas_value": 187.3, - "propane_value": 187.8, - "oil_value": 199.9, - "wood_value": 250.0, - }, - ], - "utility_bills": [ - {"scenario_name": "Bills", "elc_fixed_charge": 10.0, "elc_marginal_rate": 0.12}, - {"scenario_name": "Bills2", "gas_fixed_charge": 12.0, "gas_marginal_rate": 0.15}, - ], - "simulation_output_report": { - "timeseries_frequency": "hourly", - "include_timeseries_total_consumptions": True, - "include_timeseries_end_use_consumptions": True, - "include_timeseries_total_loads": True, - "include_timeseries_zone_temperatures": False, - "output_variables": [ - {"name": "Zone Mean Air Temperature"}, - {"name": "Zone People Occupant Count"}, - ], + { + "scenario_name": "LRMER_HighCase_15", + "type": "CO2e", + "elec_folder": "data/emissions/cambium/2022/LRMER_HighCase_15", + "gas_value": 187.3, + "propane_value": 187.8, + "oil_value": 199.9, + "wood_value": 250.0, }, - "reporting_measures": [ - { - "measure_dir_name": "TestReportingMeasure1", - "arguments": { - "TestReportingMeasure1_arg1": "TestReportingMeasure1_val1", - "TestReportingMeasure1_arg2": "TestReportingMeasure1_val2", - }, + ], + "utility_bills": [ + {"scenario_name": "Bills", "elc_fixed_charge": 10.0, "elc_marginal_rate": 0.12}, + {"scenario_name": "Bills2", "gas_fixed_charge": 12.0, "gas_marginal_rate": 0.15}, + ], + "simulation_output_report": { + "timeseries_frequency": "hourly", + "include_timeseries_total_consumptions": True, + "include_timeseries_end_use_consumptions": True, + "include_timeseries_total_loads": True, + "include_timeseries_zone_temperatures": True, + "output_variables": [ + {"name": "Zone Mean Air Temperature"}, + {"name": "Zone People Occupant Count"}, + ], + }, + "server_directory_cleanup": {"retain_in_osm": True, "retain_eplusout_msgpack": True}, + "reporting_measures": [ + { + "measure_dir_name": "TestReportingMeasure1", + "arguments": { + "TestReportingMeasure1_arg1": "TestReportingMeasure1_val1", + "TestReportingMeasure1_arg2": "TestReportingMeasure1_val2", }, - { - "measure_dir_name": "TestReportingMeasure2", - "arguments": { - "TestReportingMeasure2_arg1": "TestReportingMeasure2_val1", - "TestReportingMeasure2_arg2": "TestReportingMeasure2_val2", - }, + }, + { + "measure_dir_name": "TestReportingMeasure2", + "arguments": { + "TestReportingMeasure2_arg1": "TestReportingMeasure2_val1", + "TestReportingMeasure2_arg2": "TestReportingMeasure2_val2", }, - ], - "measures": [ - { - "measure_dir_name": "TestMeasure1", - "arguments": { - "TestMeasure1_arg1": 1, - "TestMeasure1_arg2": 2, - }, + }, + ], + "measures": [ + { + "measure_dir_name": "TestMeasure1", + "arguments": { + "TestMeasure1_arg1": 1, + "TestMeasure1_arg2": 2, }, - {"measure_dir_name": "TestMeasure2"}, - ], - }, + }, + {"measure_dir_name": "TestMeasure2"}, + ], }, - "upgrades": [ - { - "upgrade_name": "Upgrade 1", - "options": [ - { - "option": "Parameter|Option", - } - ], - } - ], - } + }, + "upgrades": [ + { + "upgrade_name": "Upgrade 1", + "options": [ + { + "option": "Parameter|Option", + } + ], + } + ], +} + + +def pytest_generate_tests(metafunc): + # Generate various combinations of blocks in the configuration file + # because the yaml file will not always contain all the blocks - it can be any subset of the blocks + if "dynamic_cfg" in metafunc.fixturenames: + arg_blocks = [ + "build_existing_model", + "emissions", + "utility_bills", + "measures", + "reporting_measures", + "simulation_output_report", + "server_directory_cleanup", + ] + blocks_to_remove = [] + for i in range(0, len(arg_blocks) + 1): + blocks_to_remove.extend(list(itertools.combinations(arg_blocks, i))) + + cfg_variants = [] + for blocks in blocks_to_remove: + cfg = copy.deepcopy(test_cfg) + for block_name in blocks: + del cfg["workflow_generator"]["args"][block_name] + cfg_variants.append(cfg) + + # Add a variant without the add_component_loads key + if "build_existing_model" not in blocks: + cfg = copy.deepcopy(cfg) + del cfg["workflow_generator"]["args"]["build_existing_model"]["add_component_loads"] + cfg_variants.append(cfg) + + # Add a variant with only one emissions scenario + if "emissions" not in blocks: + cfg = copy.deepcopy(cfg) + cfg["workflow_generator"]["args"]["emissions"].pop() + cfg_variants.append(cfg) + + # Add a variant with only one utility bill scenario + if "utility_bills" not in blocks: + cfg = copy.deepcopy(cfg) + cfg["workflow_generator"]["args"]["utility_bills"].pop() + cfg_variants.append(cfg) + + # Add a variant with only one output_variable, and no output_variables key + if "simulation_output_report" not in blocks: + cfg = copy.deepcopy(cfg) + cfg["workflow_generator"]["args"]["simulation_output_report"]["output_variables"].pop() + cfg_variants.append(cfg) + cfg = copy.deepcopy(cfg) + del cfg["workflow_generator"]["args"]["simulation_output_report"]["output_variables"] + cfg_variants.append(cfg) + + metafunc.parametrize("dynamic_cfg", cfg_variants) + + +@pytest.mark.parametrize("upgrade", [0, None]) +def test_residential_hpxml(upgrade, dynamic_cfg): + sim_id = "bldb1up1" + building_id = 13 n_datapoints = 10 - osw_gen = ResidentialHpxmlWorkflowGenerator(cfg, n_datapoints) + cfg = copy.deepcopy(dynamic_cfg) - osw = osw_gen.create_osw(sim_id, building_id, 0) - assert len(osw["steps"]) == 12 + osw_gen = ResidentialHpxmlWorkflowGenerator(cfg, n_datapoints) + osw = osw_gen.create_osw(sim_id, building_id, upgrade) - apply_upgrade_step = osw["steps"][1] - assert apply_upgrade_step["measure_dir_name"] == "ApplyUpgrade" - assert apply_upgrade_step["arguments"]["upgrade_name"] == "Upgrade 1" - assert apply_upgrade_step["arguments"]["run_measure"] == 1 - assert apply_upgrade_step["arguments"]["option_1"] == "Parameter|Option" + index = 0 - build_existing_model_step = osw["steps"][0] + build_existing_model_step = osw["steps"][index] assert build_existing_model_step["measure_dir_name"] == "BuildExistingModel" - assert build_existing_model_step["arguments"]["simulation_control_run_period_begin_month"] == 2 - assert build_existing_model_step["arguments"]["simulation_control_run_period_begin_day_of_month"] == 1 - assert build_existing_model_step["arguments"]["simulation_control_run_period_end_month"] == 2 - assert build_existing_model_step["arguments"]["simulation_control_run_period_end_day_of_month"] == 28 - assert build_existing_model_step["arguments"]["simulation_control_run_period_calendar_year"] == 2010 + assert build_existing_model_step["arguments"]["building_id"] == building_id - assert build_existing_model_step["arguments"]["emissions_scenario_names"] == "LRMER_MidCase_15,LRMER_HighCase_15" - assert build_existing_model_step["arguments"]["emissions_natural_gas_values"] == "147.3,187.3" + workflow_args = cfg["workflow_generator"].get("args", {}) - assert build_existing_model_step["arguments"]["utility_bill_scenario_names"] == "Bills,Bills2" - assert build_existing_model_step["arguments"]["utility_bill_natural_gas_fixed_charges"] == ",12.0" - assert build_existing_model_step["arguments"]["utility_bill_simple_filepaths"] == "," + if "build_existing_model" in workflow_args: + assert build_existing_model_step["arguments"]["simulation_control_timestep"] == 15 + assert build_existing_model_step["arguments"]["simulation_control_run_period_begin_month"] == 2 + assert build_existing_model_step["arguments"]["simulation_control_run_period_begin_day_of_month"] == 1 + assert build_existing_model_step["arguments"]["simulation_control_run_period_end_month"] == 2 + assert build_existing_model_step["arguments"]["simulation_control_run_period_end_day_of_month"] == 28 + assert build_existing_model_step["arguments"]["simulation_control_run_period_calendar_year"] == 2010 + else: + # Defaults + assert build_existing_model_step["arguments"]["simulation_control_run_period_begin_month"] == 1 + assert build_existing_model_step["arguments"]["simulation_control_run_period_begin_day_of_month"] == 1 + assert build_existing_model_step["arguments"]["simulation_control_run_period_end_month"] == 12 + assert build_existing_model_step["arguments"]["simulation_control_run_period_end_day_of_month"] == 31 + assert build_existing_model_step["arguments"]["simulation_control_run_period_calendar_year"] == 2007 + assert build_existing_model_step["arguments"]["simulation_control_timestep"] == 60 - hpxml_to_os_step = osw["steps"][2] + if "emissions" in workflow_args: + + assert build_existing_model_step["arguments"]["emissions_scenario_names"] == ",".join( + e["scenario_name"] for e in workflow_args["emissions"] + ) + assert build_existing_model_step["arguments"]["emissions_natural_gas_values"] == ",".join( + str(e["gas_value"]) for e in workflow_args["emissions"] + ) + + if "utility_bills" in workflow_args: + assert build_existing_model_step["arguments"]["utility_bill_scenario_names"] == ",".join( + u["scenario_name"] for u in workflow_args["utility_bills"] + ) + assert build_existing_model_step["arguments"]["utility_bill_natural_gas_fixed_charges"] == ",".join( + str(u.get("gas_fixed_charge", "")) for u in workflow_args["utility_bills"] + ) + assert build_existing_model_step["arguments"]["utility_bill_simple_filepaths"] == ",".join( + u.get("simple_filepath", "") for u in workflow_args["utility_bills"] + ) + index += 1 + + if upgrade is not None: + apply_upgrade_step = osw["steps"][index] + assert apply_upgrade_step["measure_dir_name"] == "ApplyUpgrade" + assert apply_upgrade_step["arguments"]["upgrade_name"] == "Upgrade 1" + assert apply_upgrade_step["arguments"]["run_measure"] == 1 + assert apply_upgrade_step["arguments"]["option_1"] == "Parameter|Option" + index += 1 + + hpxml_to_os_step = osw["steps"][index] assert hpxml_to_os_step["measure_dir_name"] == "HPXMLtoOpenStudio" + if "build_existing_model" in workflow_args: + assert hpxml_to_os_step["arguments"]["add_component_loads"] == workflow_args["build_existing_model"].get( + "add_component_loads", False + ) + else: + assert hpxml_to_os_step["arguments"]["add_component_loads"] is False + assert hpxml_to_os_step["arguments"]["debug"] is True + index += 1 - assert osw["steps"][3]["measure_dir_name"] == "TestMeasure1" - assert osw["steps"][3]["arguments"]["TestMeasure1_arg1"] == 1 - assert osw["steps"][3]["arguments"]["TestMeasure1_arg2"] == 2 - assert osw["steps"][4]["measure_dir_name"] == "TestMeasure2" - assert osw["steps"][4].get("arguments") is None + if "measures" in workflow_args: + assert osw["steps"][index]["measure_dir_name"] == "TestMeasure1" + assert osw["steps"][index]["arguments"]["TestMeasure1_arg1"] == 1 + assert osw["steps"][index]["arguments"]["TestMeasure1_arg2"] == 2 + index += 1 - simulation_output_step = osw["steps"][5] + assert osw["steps"][index]["measure_dir_name"] == "TestMeasure2" + assert osw["steps"][index].get("arguments") is None + index += 1 + + simulation_output_step = osw["steps"][index] assert simulation_output_step["measure_dir_name"] == "ReportSimulationOutput" - assert simulation_output_step["arguments"]["timeseries_frequency"] == "hourly" + if "simulation_output_report" in workflow_args: + assert simulation_output_step["arguments"]["timeseries_frequency"] == "hourly" + assert simulation_output_step["arguments"]["include_timeseries_total_consumptions"] is True + assert simulation_output_step["arguments"]["include_timeseries_end_use_consumptions"] is True + assert simulation_output_step["arguments"]["include_timeseries_total_loads"] is True + assert simulation_output_step["arguments"]["include_timeseries_zone_temperatures"] is True + if "output_variables" in workflow_args["simulation_output_report"]: + assert simulation_output_step["arguments"]["user_output_variables"] == ",".join( + v["name"] for v in workflow_args["simulation_output_report"]["output_variables"] + ) + else: # Defaults + assert simulation_output_step["arguments"]["timeseries_frequency"] == "none" + assert simulation_output_step["arguments"]["include_timeseries_total_consumptions"] is False + assert simulation_output_step["arguments"]["include_timeseries_end_use_consumptions"] is True + assert simulation_output_step["arguments"]["include_timeseries_total_loads"] is True + assert simulation_output_step["arguments"]["include_timeseries_zone_temperatures"] is False + assert simulation_output_step["arguments"]["user_output_variables"] == "" + assert simulation_output_step["arguments"]["include_annual_total_consumptions"] is True assert simulation_output_step["arguments"]["include_annual_fuel_consumptions"] is True assert simulation_output_step["arguments"]["include_annual_end_use_consumptions"] is True @@ -155,18 +278,14 @@ def test_residential_hpxml(mocker): assert simulation_output_step["arguments"]["include_annual_hot_water_uses"] is True assert simulation_output_step["arguments"]["include_annual_hvac_summary"] is True assert simulation_output_step["arguments"]["include_annual_resilience"] is True - assert simulation_output_step["arguments"]["include_timeseries_total_consumptions"] is True assert simulation_output_step["arguments"]["include_timeseries_fuel_consumptions"] is False - assert simulation_output_step["arguments"]["include_timeseries_end_use_consumptions"] is True assert simulation_output_step["arguments"]["include_timeseries_system_use_consumptions"] is False assert simulation_output_step["arguments"]["include_timeseries_emissions"] is False assert simulation_output_step["arguments"]["include_timeseries_emission_fuels"] is False assert simulation_output_step["arguments"]["include_timeseries_emission_end_uses"] is False assert simulation_output_step["arguments"]["include_timeseries_hot_water_uses"] is False - assert simulation_output_step["arguments"]["include_timeseries_total_loads"] is True assert simulation_output_step["arguments"]["include_timeseries_component_loads"] is False assert simulation_output_step["arguments"]["include_timeseries_unmet_hours"] is False - assert simulation_output_step["arguments"]["include_timeseries_zone_temperatures"] is False assert simulation_output_step["arguments"]["include_timeseries_airflows"] is False assert simulation_output_step["arguments"]["include_timeseries_weather"] is False assert simulation_output_step["arguments"]["include_timeseries_resilience"] is False @@ -175,29 +294,47 @@ def test_residential_hpxml(mocker): assert simulation_output_step["arguments"]["add_timeseries_dst_column"] is True assert simulation_output_step["arguments"]["add_timeseries_utc_column"] is True - hpxml_output_step = osw["steps"][6] + index += 1 + + hpxml_output_step = osw["steps"][index] assert hpxml_output_step["measure_dir_name"] == "ReportHPXMLOutput" + index += 1 - utility_bills_step = osw["steps"][7] + utility_bills_step = osw["steps"][index] assert utility_bills_step["measure_dir_name"] == "ReportUtilityBills" assert utility_bills_step["arguments"]["include_annual_bills"] is True assert utility_bills_step["arguments"]["include_monthly_bills"] is False + index += 1 - upgrade_costs_step = osw["steps"][8] + upgrade_costs_step = osw["steps"][index] assert upgrade_costs_step["measure_dir_name"] == "UpgradeCosts" + assert upgrade_costs_step["arguments"]["debug"] is True + index += 1 + + if "reporting_measures" in workflow_args: + assert osw["steps"][index]["measure_dir_name"] == "TestReportingMeasure1" + assert osw["steps"][index]["arguments"]["TestReportingMeasure1_arg1"] == "TestReportingMeasure1_val1" + assert osw["steps"][index]["arguments"]["TestReportingMeasure1_arg2"] == "TestReportingMeasure1_val2" + index += 1 - assert osw["steps"][9]["measure_dir_name"] == "TestReportingMeasure1" - assert osw["steps"][9]["arguments"]["TestReportingMeasure1_arg1"] == "TestReportingMeasure1_val1" - assert osw["steps"][9]["arguments"]["TestReportingMeasure1_arg2"] == "TestReportingMeasure1_val2" - assert osw["steps"][10]["measure_dir_name"] == "TestReportingMeasure2" - assert osw["steps"][10]["arguments"]["TestReportingMeasure2_arg1"] == "TestReportingMeasure2_val1" - assert osw["steps"][10]["arguments"]["TestReportingMeasure2_arg2"] == "TestReportingMeasure2_val2" + assert osw["steps"][index]["measure_dir_name"] == "TestReportingMeasure2" + assert osw["steps"][index]["arguments"]["TestReportingMeasure2_arg1"] == "TestReportingMeasure2_val1" + assert osw["steps"][index]["arguments"]["TestReportingMeasure2_arg2"] == "TestReportingMeasure2_val2" + index += 1 - server_dir_cleanup_step = osw["steps"][11] + server_dir_cleanup_step = osw["steps"][index] assert server_dir_cleanup_step["measure_dir_name"] == "ServerDirectoryCleanup" + assert server_dir_cleanup_step["arguments"]["debug"] is True + if "server_directory_cleanup" in workflow_args: + assert server_dir_cleanup_step["arguments"]["retain_in_osm"] is True + assert server_dir_cleanup_step["arguments"]["retain_eplusout_msgpack"] is True + else: # Defaults + assert server_dir_cleanup_step["arguments"]["retain_in_osm"] is False + assert server_dir_cleanup_step["arguments"]["retain_eplusout_msgpack"] is False + index += 1 -def test_old_resstock(mocker): +def test_missing_arg_warning(): """ Some keys defined in schema can be unavailable in the measure. This test verifies that such keys are not passed to the measure, but warnings are raised. @@ -210,6 +347,7 @@ def test_old_resstock(mocker): "args": { "build_existing_model": { "simulation_control_run_period_begin_month": 2, + "add_component_loads": True, "new_key1": "test_value", # simulate old resstock by adding a new key not in the measure }, }, @@ -220,16 +358,20 @@ def test_old_resstock(mocker): osw_gen.default_args["BuildExistingModel"]["new_key2"] = "test_value" with LogCapture(level=logging.INFO) as log: - measure_args = osw_gen._get_measure_args( - cfg["workflow_generator"]["args"]["build_existing_model"], "BuildExistingModel", debug=False - ) + measure_args = osw_gen.create_osw("bldb1up1", 13, None) assert len(log.records) == 2 all_msg = "\n".join([record.msg for record in log.records]) - assert "'new_key1' in workflow_generator not found in 'BuildExistingModel'" in all_msg - assert "'new_key2' in defaults not found in 'BuildExistingModel'" in all_msg + assert "'new_key1' not found in 'BuildExistingModel'" in all_msg + assert "'new_key2' not found in 'BuildExistingModel'" in all_msg assert "new_key1" not in measure_args assert "new_key2" not in measure_args + del cfg["workflow_generator"]["args"]["build_existing_model"]["new_key1"] + osw_gen = ResidentialHpxmlWorkflowGenerator(cfg, n_datapoints) + with LogCapture(level=logging.INFO) as log: + measure_args = osw_gen.create_osw("bldb1up1", 13, None) + assert len(log.records) == 0 + def test_hpmxl_schema_defaults_and_mapping(): """ @@ -274,7 +416,7 @@ def assert_valid_keys(measure_dir_name, yaml_block_name, exclude_keys): def test_block_compression_and_argmap(): test_wf_arg = { - "block1": {"key1": "v1", "key2": "v2", "key3": ["v3", "v4"]}, + "block1": {"key1": "v1", "key2": "v2", "key3": ["v3", "v4"], "key4": "v4"}, "block2": [ { "key1": "val1", @@ -291,40 +433,48 @@ def test_block_compression_and_argmap(): compressed_block = ResidentialHpxmlWorkflowGenerator._get_condensed_block(test_wf_arg["block2"]) assert compressed_block == {"key1": ["val1", "val11"], "key2": ["val2", ""], "key3": ["", "val33"]} arg_map = { - "block1": { - "measure1": { - # "key1": "arg1", # key1 is not to be passed + "measure1": { + "block1": { + # "key1": "arg1", # key1 is passed to measure 2 "key2": "arg2", "key3": "arg3", - }, + "key_not_in_block": "arg5", + "key_not_in_block2": "arg6", + } }, - "block2": { - "measure2": { + "measure2": { + "block2": { "key1": "arg1", "key2": "arg2", "key3": "arg3", }, + "block1": {"key1": "arg4"}, }, } measure_args = ResidentialHpxmlWorkflowGenerator._get_mapped_args_from_block( - test_wf_arg["block1"], arg_map["block1"] + test_wf_arg["block1"], arg_map["measure1"]["block1"], {"arg6": False} ) assert measure_args == { - "measure1": { - "arg2": "v2", - "arg3": "v3,v4", - } + "arg2": "v2", + "arg3": "v3,v4", + "arg5": "", + "arg6": False, } measure_args = ResidentialHpxmlWorkflowGenerator._get_mapped_args_from_block( - test_wf_arg["block2"], arg_map["block2"] + test_wf_arg["block1"], arg_map["measure2"]["block1"], {} ) assert measure_args == { - "measure2": { - "arg1": "val1,val11", - "arg2": "val2,", - "arg3": ",val33", - } + "arg4": "v1", + } + + measure_args = ResidentialHpxmlWorkflowGenerator._get_mapped_args_from_block( + test_wf_arg["block2"], arg_map["measure2"]["block2"], {} + ) + assert measure_args == { + "arg1": "val1,val11", + "arg2": "val2,", + "arg3": ",val33", } - # Only key1 should be remaining since the other two is already mapped to measure - assert test_wf_arg["block1"] == {"key1": "v1"} + # Only key4 should be remaining since the other three is already mapped to measure + assert test_wf_arg["block1"] == {"key4": "v4"} diff --git a/setup.py b/setup.py index 4aa11f83..798947d9 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,10 @@ url=metadata["__url__"], packages=setuptools.find_packages(), python_requires=">=3.8", - package_data={"buildstockbatch": ["*.sh", "schemas/*.yaml"], "": ["LICENSE"]}, + package_data={ + "buildstockbatch": ["*.sh", "schemas/*.yaml", "workflow_generator/residential/*.yml"], + "": ["LICENSE"], + }, install_requires=[ "pyyaml", "requests",