Refactor workflow generator

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
       - name: Run PyTest and Coverage
         run: |
           cd buildstockbatch
-          pytest --junitxml=coverage/junit.xml --cov=buildstockbatch --cov-report=xml:coverage/coverage.xml --cov-report=html:coverage/htmlreport
+          pytest -vv --junitxml=coverage/junit.xml --cov=buildstockbatch --cov-report=xml:coverage/coverage.xml --cov-report=html:coverage/htmlreport
       - name: Test Report
         uses: mikepenz/[email protected]
         if: ${{ matrix.python-version == '3.11' }}

buildstockbatch/aws/aws.py

@@ -1085,7 +1085,7 @@ def start_batch_job(self, batch_info):
         """Implements :func:`DockerBase.start_batch_job`"""
         # Create the output directories
         fs = S3FileSystem()
-        for upgrade_id in range(len(self.cfg.get("upgrades", [])) + 1):
+        for upgrade_id in range(self.num_upgrades + 1):
             fs.makedirs(
                 f"{self.cfg['aws']['s3']['bucket']}/{self.cfg['aws']['s3']['prefix']}/results/simulation_output/"
                 f"timeseries/up{upgrade_id:02d}"

buildstockbatch/base.py

@@ -71,6 +71,10 @@ def __init__(self, project_filename):
         self.os_sha = self.cfg["os_sha"]
         logger.debug(f"Using OpenStudio version: {self.os_version} with SHA: {self.os_sha}")
 
+    @property
+    def num_upgrades(self):
+        return len(self.cfg.get("upgrades", []))
+
     @staticmethod
     def get_sampler_class(sampler_name):
         sampler_class_name = "".join(x.capitalize() for x in sampler_name.strip().split("_")) + "Sampler"
@@ -367,7 +371,7 @@ def validate_buildstock_csv(project_file, buildstock_df):
     def validate_workflow_generator(cls, project_file):
         cfg = get_project_configuration(project_file)
         WorkflowGenerator = cls.get_workflow_generator_class(cfg["workflow_generator"]["type"])
-        return WorkflowGenerator.validate(cfg)
+        return WorkflowGenerator(cfg, 1).validate()
 
     @staticmethod
     def validate_project_schema(project_file):

buildstockbatch/cloud/docker_base.py

@@ -423,17 +423,17 @@ def _prep_jobs_for_batch(self, tmppath):
         building_ids = df.index.tolist()
         n_datapoints = len(building_ids)
         if self.skip_baseline_sims:
-            n_sims = n_datapoints * len(self.cfg.get("upgrades", []))
+            n_sims = n_datapoints * self.num_upgrades
         else:
-            n_sims = n_datapoints * (len(self.cfg.get("upgrades", [])) + 1)
+            n_sims = n_datapoints * (self.num_upgrades + 1)
         logger.debug("Total number of simulations = {}".format(n_sims))
 
         n_sims_per_job = math.ceil(n_sims / self.batch_array_size)
         n_sims_per_job = max(n_sims_per_job, 2)
         logger.debug("Number of simulations per array job = {}".format(n_sims_per_job))
 
         # Create list of (building ID, upgrade to apply) pairs for all simulations to run.
-        upgrade_sims = itertools.product(building_ids, range(len(self.cfg.get("upgrades", []))))
+        upgrade_sims = itertools.product(building_ids, range(self.num_upgrades))
         if not self.skip_baseline_sims:
             baseline_sims = zip(building_ids, itertools.repeat(None))
             all_sims = list(itertools.chain(baseline_sims, upgrade_sims))

buildstockbatch/hpc.py

@@ -132,7 +132,7 @@ def run_batch(self, sampling_only=False):
         # Create simulation_output dir
         sim_out_ts_dir = pathlib.Path(self.output_dir) / "results" / "simulation_output" / "timeseries"
         os.makedirs(sim_out_ts_dir, exist_ok=True)
-        for i in range(0, len(self.cfg.get("upgrades", [])) + 1):
+        for i in range(0, self.num_upgrades + 1):
             os.makedirs(sim_out_ts_dir / f"up{i:02d}")
 
         # create destination_dir and copy housing_characteristics into it
@@ -161,7 +161,7 @@ def run_batch(self, sampling_only=False):
         building_ids = df.index.tolist()
         n_datapoints = len(building_ids)
         # number of simulations is number of buildings * number of upgrades
-        n_sims = n_datapoints * (len(self.cfg.get("upgrades", [])) + 1)
+        n_sims = n_datapoints * (self.num_upgrades + 1)
 
         # this is the number of simulations defined for this run as a "full job"
         #     number of simulations per job if we believe the .yml file n_jobs
@@ -170,7 +170,7 @@ def run_batch(self, sampling_only=False):
         #     larger than we need, now that we know n_sims
         n_sims_per_job = max(n_sims_per_job, self.MIN_SIMS_PER_JOB)
 
-        upgrade_sims = itertools.product(building_ids, range(len(self.cfg.get("upgrades", []))))
+        upgrade_sims = itertools.product(building_ids, range(self.num_upgrades))
         if not self.skip_baseline_sims:
             # create batches of simulations
             baseline_sims = zip(building_ids, itertools.repeat(None))
@@ -213,11 +213,6 @@ def run_job_batch(self, job_array_number):
             pathlib.Path(self.buildstock_dir) / "measures",
             self.local_buildstock_dir / "measures",
         )
-        if os.path.exists(pathlib.Path(self.buildstock_dir) / "resources/hpxml-measures"):
-            self.clear_and_copy_dir(
-                pathlib.Path(self.buildstock_dir) / "resources/hpxml-measures",
-                self.local_buildstock_dir / "resources/hpxml-measures",
-            )
         self.clear_and_copy_dir(self.weather_dir, self.local_weather_dir)
         self.clear_and_copy_dir(
             pathlib.Path(self.output_dir) / "housing_characteristics",

buildstockbatch/local.py

@@ -48,7 +48,7 @@ def __init__(self, project_filename):
         # Create simulation_output dir
         sim_out_ts_dir = os.path.join(self.results_dir, "simulation_output", "timeseries")
         os.makedirs(sim_out_ts_dir, exist_ok=True)
-        for i in range(0, len(self.cfg.get("upgrades", [])) + 1):
+        for i in range(0, self.num_upgrades + 1):
             os.makedirs(os.path.join(sim_out_ts_dir, f"up{i:02d}"), exist_ok=True)
 
         # Install custom gems to a volume that will be used by all workers
@@ -290,7 +290,7 @@ def run_batch(self, n_jobs=None, measures_only=False, sampling_only=False):
             self.cfg,
         )
         upgrade_sims = []
-        for i in range(len(self.cfg.get("upgrades", []))):
+        for i in range(self.num_upgrades):
             upgrade_sims.append(map(functools.partial(run_building_d, upgrade_idx=i), building_ids))
         if not self.skip_baseline_sims:
             baseline_sims = map(run_building_d, building_ids)

buildstockbatch/test/test_inputs/enforce-validate-measures-good-2-with-anchors.yml

@@ -46,7 +46,7 @@ workflow_generator:
       include_timeseries_weather: true
 
     reporting_measures:
-      - measure_dir_name: QOIReport
+      - measure_dir_name: TimeseriesCSVExport
 
     server_directory_cleanup:
       retain_in_idf: false

buildstockbatch/test/test_inputs/enforce-validate-measures-good-2.yml

@@ -46,7 +46,7 @@ workflow_generator:
       include_timeseries_weather: true
 
     reporting_measures:
-      - measure_dir_name: QOIReport
+      - measure_dir_name: TimeseriesCSVExport
 
     server_directory_cleanup:
       retain_in_idf: false

buildstockbatch/test/test_inputs/enforce-validate-measures-missing-dir.yml

@@ -0,0 +1,77 @@
+schema_version: '0.3'
+buildstock_directory: test_openstudio_buildstock
+project_directory: project_singlefamilydetached
+weather_files_url: https://fake-url
+baseline:
+  n_buildings_represented: 81221016
+
+sampler:
+  type: residential_quota
+  args:
+    n_datapoints: 30
+
+workflow_generator:
+  type: residential_hpxml
+  args:
+    build_existing_model:
+      simulation_control_timestep: 60
+      simulation_control_run_period_begin_month: 1
+      simulation_control_run_period_begin_day_of_month: 1
+      simulation_control_run_period_end_month: 12
+      simulation_control_run_period_end_day_of_month: 31
+      simulation_control_run_period_calendar_year: 2007
+
+    emissions:
+      - scenario_name: LRMER_MidCase_15
+        type: CO2e
+        elec_folder: data/cambium/LRMER_MidCase_15
+
+    utility_bills:
+      - scenario_name: Bills
+
+    simulation_output_report:
+      timeseries_frequency: hourly
+      include_timeseries_total_consumptions: true
+      include_timeseries_fuel_consumptions: true
+      include_timeseries_end_use_consumptions: true
+      include_timeseries_emissions: true
+      include_timeseries_emission_fuels: true
+      include_timeseries_emission_end_uses: true
+      include_timeseries_hot_water_uses: true
+      include_timeseries_total_loads: true
+      include_timeseries_component_loads: true
+      include_timeseries_unmet_hours: true
+      include_timeseries_zone_temperatures: true
+      include_timeseries_airflows: true
+      include_timeseries_weather: true
+
+    reporting_measures:
+      - measure_dir_name: QOIReport
+
+    server_directory_cleanup:
+      retain_in_idf: false
+      retain_schedules_csv: false
+
+upgrades:
+  - upgrade_name: cool upgrade
+    options:
+      - option: Vintage<1940
+  - upgrade_name: good upgrade
+    options:
+      - option: Vintage|<1940
+        apply_logic:
+          - or:
+            - Insulation Slab|Good Option
+            - Insulation Slab|None
+          - not: Insulation Wall|Good Option
+          - and:
+              - Vintage|1960s||Vintage|1960s
+              - Vintage|1980s
+        costs:
+          - value: 0.9
+            multiplier: Fixed (1)
+      - option: Insulation Finished Basement|Good Option
+        apply_logic:
+          - Insulation Unfinished Basement|Extra Argument
+    package_apply_logic: Vintage|1960s||Vintage|1940s
+    reference_scenario: cool upgrade

buildstockbatch/test/test_inputs/test_openstudio_buildstock/measures/ReportHPXMLOutput/measure.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<measure>
+  <schema_version>3.1</schema_version>
+  <name>report_hpxml_output</name>
+  <uid>9561a0d7-60ad-48c5-8337-2461df044d80</uid>
+  <version_id>222e94cd-c586-4745-8cde-55e933a00392</version_id>
+  <version_modified>2024-01-26T00:28:04Z</version_modified>
+  <xml_checksum>9BF1E6AC</xml_checksum>
+  <class_name>ReportHPXMLOutput</class_name>
+  <display_name>HPXML Output Report</display_name>
+  <description>Reports HPXML outputs for residential HPXML-based models.</description>
+  <modeler_description>Parses the HPXML file and reports pre-defined outputs.</modeler_description>
+  <arguments>
+    <argument>
+      <name>output_format</name>
+      <display_name>Output Format</display_name>
+      <description>The file format of the annual (and timeseries, if requested) outputs.</description>
+      <type>Choice</type>
+      <required>false</required>
+      <model_dependent>false</model_dependent>
+      <default_value>csv</default_value>
+      <choices>
+        <choice>
+          <value>csv</value>
+          <display_name>csv</display_name>
+        </choice>
+        <choice>
+          <value>json</value>
+          <display_name>json</display_name>
+        </choice>
+        <choice>
+          <value>msgpack</value>
+          <display_name>msgpack</display_name>
+        </choice>
+      </choices>
+    </argument>
+  </arguments>
+  <outputs />
+  <provenances />
+  <tags>
+    <tag>Reporting.QAQC</tag>
+  </tags>
+  <attributes>
+    <attribute>
+      <name>Measure Type</name>
+      <value>ReportingMeasure</value>
+      <datatype>string</datatype>
+    </attribute>
+    <attribute>
+      <name>Intended Software Tool</name>
+      <value>OpenStudio Application</value>
+      <datatype>string</datatype>
+    </attribute>
+    <attribute>
+      <name>Intended Software Tool</name>
+      <value>Parametric Analysis Tool</value>
+      <datatype>string</datatype>
+    </attribute>
+  </attributes>
+  <files>
+    <file>
+      <filename>README.md</filename>
+      <filetype>md</filetype>
+      <usage_type>readme</usage_type>
+      <checksum>0BB92B1A</checksum>
+    </file>
+    <file>
+      <filename>README.md.erb</filename>
+      <filetype>erb</filetype>
+      <usage_type>readmeerb</usage_type>
+      <checksum>513F28E9</checksum>
+    </file>
+    <file>
+      <version>
+        <software_program>OpenStudio</software_program>
+        <identifier>3.2.0</identifier>
+        <min_compatible>3.2.0</min_compatible>
+      </version>
+      <filename>measure.rb</filename>
+      <filetype>rb</filetype>
+      <usage_type>script</usage_type>
+      <checksum>BA1983C8</checksum>
+    </file>
+    <file>
+      <filename>constants.rb</filename>
+      <filetype>rb</filetype>
+      <usage_type>resource</usage_type>
+      <checksum>64289999</checksum>
+    </file>
+  </files>
+</measure>