Merge pull request #450 from NREL/os380

OpenStudio version required in projects, using version in ResStock develop for CI
NREL · May 16, 2024 · 1509ad2 · 1509ad2
2 parents 670ad58 + c896135
commit 1509ad2
Show file tree

Hide file tree

Showing 12 changed files with 78 additions and 57 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,8 +34,12 @@ jobs:
           wget --quiet https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip
       - name: Download and Install OpenStudio
         run: |
-          wget -q https://github.com/NREL/OpenStudio/releases/download/v3.7.0/OpenStudio-3.7.0+d5269793f1-Ubuntu-22.04-x86_64.deb
-          sudo apt install -y ./OpenStudio-3.7.0+d5269793f1-Ubuntu-22.04-x86_64.deb
+          sudo snap install yq
+          export OS_VER=`yq '.os_version' resstock/project_national/national_baseline.yml`
+          export OS_SHA=`yq '.os_sha' resstock/project_national/national_baseline.yml`
+          export OS_INSTALLER_FILENAME="OpenStudio-${OS_VER}+${OS_SHA}-Ubuntu-22.04-x86_64.deb"
+          wget -q "https://github.com/NREL/OpenStudio/releases/download/v${OS_VER}/${OS_INSTALLER_FILENAME}"
+          sudo apt install -y "./${OS_INSTALLER_FILENAME}"
           openstudio openstudio_version
           which openstudio
       - name: Install buildstockbatch

diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py
@@ -42,8 +42,6 @@
 
 class BuildStockBatchBase(object):
     # http://openstudio-builds.s3-website-us-east-1.amazonaws.com
-    DEFAULT_OS_VERSION = "3.6.1"
-    DEFAULT_OS_SHA = "bb9481519e"
     CONTAINER_RUNTIME = None
     LOGO = """
      _ __         _     __,              _ __
@@ -69,8 +67,8 @@ def __init__(self, project_filename):
 
         # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML,
         # otherwise use defaults specified here.
-        self.os_version = self.cfg.get("os_version", self.DEFAULT_OS_VERSION)
-        self.os_sha = self.cfg.get("os_sha", self.DEFAULT_OS_SHA)
+        self.os_version = self.cfg["os_version"]
+        self.os_sha = self.cfg["os_sha"]
         logger.debug(f"Using OpenStudio version: {self.os_version} with SHA: {self.os_sha}")
 
     @staticmethod
@@ -298,8 +296,8 @@ def get_buildstock_dir(project_file, cfg):
     @classmethod
     def validate_openstudio_path(cls, project_file):
         cfg = get_project_configuration(project_file)
-        os_version = cfg.get("os_version", cls.DEFAULT_OS_VERSION)
-        os_sha = cfg.get("os_sha", cls.DEFAULT_OS_SHA)
+        os_version = cfg["os_version"]
+        os_sha = cfg["os_sha"]
         try:
             proc_out = subprocess.run(
                 [cls.openstudio_exe(), "openstudio_version"],
@@ -873,7 +871,7 @@ def validate_openstudio_version(project_file):
         """
         cfg = get_project_configuration(project_file)
 
-        os_version = cfg.get("os_version", BuildStockBatchBase.DEFAULT_OS_VERSION)
+        os_version = cfg["os_version"]
         version_path = "resources/hpxml-measures/HPXMLtoOpenStudio/resources/version.rb"
         version_rb = os.path.join(cfg["buildstock_directory"], version_path)
         if os.path.exists(version_rb):

diff --git a/buildstockbatch/hpc.py b/buildstockbatch/hpc.py
@@ -87,8 +87,8 @@ def validate_apptainer_image_hpc(cls, project_file):
         try:
             cls.get_apptainer_image(
                 cfg,
-                cfg.get("os_version", cls.DEFAULT_OS_VERSION),
-                cfg.get("os_sha", cls.DEFAULT_OS_SHA),
+                cfg["os_version"],
+                cfg["os_sha"],
             )
         except RuntimeError as err:
             raise ValidationError(str(err))

diff --git a/buildstockbatch/schemas/v0.3.yaml b/buildstockbatch/schemas/v0.3.yaml
@@ -12,8 +12,8 @@ aws: include('aws-spec', required=False)
 output_directory: regex('^(.*\/)?[a-z][a-z0-9_]*\/?$', required=True)
 sys_image_dir: str(required=False)
 baseline: include('sim-spec', required=True)
-os_version: str(required=False)
-os_sha: str(required=False)
+os_version: str(required=True)
+os_sha: str(required=True)
 max_minutes_per_sim: int(max=1440, required=False)
 upgrades: list(include('upgrade-spec'), required=False)
 postprocessing: include('postprocessing-spec', required=False)

diff --git a/buildstockbatch/test/conftest.py b/buildstockbatch/test/conftest.py
@@ -51,6 +51,8 @@ def _basic_residential_project_file(update_args={}, raw=False, hpc_name="eagle")
                 "weather.zip",
             )
             cfg = {
+                "os_version": "asdf",
+                "os_sha": "asdf",
                 "buildstock_directory": buildstock_directory,
                 "project_directory": project_directory,
                 "output_directory": output_directory,

diff --git a/buildstockbatch/test/test_aws.py b/buildstockbatch/test/test_aws.py
@@ -10,25 +10,26 @@
 
 
 def test_custom_gem_install(basic_residential_project_file):
-    project_filename, results_dir = basic_residential_project_file()
+    project_filename, results_dir = basic_residential_project_file(
+        update_args={
+            "os_version": "3.7.0",
+            "baseline": {
+                "custom_gems": True,
+                "n_buildings_represented": 80000000,
+            },
+            "aws": {
+                "job_identifier": "testaws",
+                "s3": {"bucket": "resbldg-datasets", "prefix": "testing/external_demo_project"},
+                "region": "us-west-2",
+                "use_spot": True,
+                "batch_array_size": 100,
+                "notifications_email": "[email protected]",
+            },
+        }
+    )
 
-    # Add aws and custom_gems to the project file
     with open(project_filename, "r") as f:
         cfg = yaml.safe_load(f)
-    # custom_gems
-    cfg["baseline"]["custom_gems"] = True
-    # AWS
-    cfg["aws"] = {}
-    cfg["aws"]["job_identifier"] = "testaws"
-    cfg["aws"]["s3"] = {}
-    cfg["aws"]["s3"]["bucket"] = "resbldg-datasets"
-    cfg["aws"]["s3"]["prefix"] = "testing/external_demo_project"
-    cfg["aws"]["region"] = "us-west-2"
-    cfg["aws"]["use_spot"] = True
-    cfg["aws"]["batch_array_size"] = 100
-    cfg["aws"]["notifications_email"] = "[email protected]"
-    with open(project_filename, "w") as f:
-        yaml.dump(cfg, f)
 
     buildstock_directory = cfg["buildstock_directory"]
 
@@ -49,23 +50,23 @@ def test_custom_gem_install(basic_residential_project_file):
 
 
 def test_no_custom_gem_install(basic_residential_project_file):
-    project_filename, results_dir = basic_residential_project_file()
+    project_filename, results_dir = basic_residential_project_file(
+        update_args={
+            "os_version": "3.7.0",
+            "aws": {
+                "job_identifier": "testaws",
+                "s3": {"bucket": "resbldg-datasets", "prefix": "testing/external_demo_project"},
+                "region": "us-west-2",
+                "use_spot": True,
+                "batch_array_size": 100,
+                "notifications_email": "[email protected]",
+            },
+        }
+    )
 
     # Add aws to the project file
     with open(project_filename, "r") as f:
         cfg = yaml.safe_load(f)
-    # AWS
-    cfg["aws"] = {}
-    cfg["aws"]["job_identifier"] = "testaws"
-    cfg["aws"]["s3"] = {}
-    cfg["aws"]["s3"]["bucket"] = "resbldg-datasets"
-    cfg["aws"]["s3"]["prefix"] = "testing/external_demo_project"
-    cfg["aws"]["region"] = "us-west-2"
-    cfg["aws"]["use_spot"] = True
-    cfg["aws"]["batch_array_size"] = 100
-    cfg["aws"]["notifications_email"] = "[email protected]"
-    with open(project_filename, "w") as f:
-        yaml.dump(cfg, f)
 
     buildstock_directory = cfg["buildstock_directory"]
 

diff --git a/buildstockbatch/test/test_inputs/complete-schema.yml b/buildstockbatch/test/test_inputs/complete-schema.yml
@@ -1,3 +1,5 @@
+os_version: asdf
+os_sha: asdf
 buildstock_directory: /home/epresent/OpenStudio-BuildStock-master
 project_directory: project_singlefamilydetached
 output_directory: /scratch/epresent/yaml1s

diff --git a/buildstockbatch/test/test_inputs/enforce-schema-xor.yml b/buildstockbatch/test/test_inputs/enforce-schema-xor.yml
@@ -1,3 +1,5 @@
+os_version: asdf
+os_sha: asdf
 buildstock_directory: /home/epresent/OpenStudio-BuildStock-master
 project_directory: project_singlefamilydetached
 output_directory: /scratch/epresent/yaml1s

diff --git a/buildstockbatch/test/test_inputs/minimal-schema.yml b/buildstockbatch/test/test_inputs/minimal-schema.yml
@@ -1,3 +1,5 @@
+os_version: 3.8.0-rc2
+os_sha: af997f4fef
 buildstock_directory: here
 project_directory: there
 weather_files_path: over-yonder

diff --git a/buildstockbatch/test/test_validation.py b/buildstockbatch/test/test_validation.py
@@ -440,9 +440,7 @@ def test_validate_apptainer_image():
         with pytest.raises(ValidationError, match=r"Could not find apptainer image: .+\.sif or .+\.simg"):
             SlurmBatch.validate_apptainer_image_hpc(str(temp_yml))
         for ext in ["Apptainer.sif", "Singularity.simg"]:
-            filename = pathlib.Path(
-                tmpdir, f"OpenStudio-{SlurmBatch.DEFAULT_OS_VERSION}.{SlurmBatch.DEFAULT_OS_SHA}-{ext}"
-            )
+            filename = pathlib.Path(tmpdir, f"OpenStudio-{cfg['os_version']}.{cfg['os_sha']}-{ext}")
             filename.touch()
             SlurmBatch.validate_apptainer_image_hpc(str(temp_yml))
             filename.unlink()

diff --git a/docs/changelog/changelog_dev.rst b/docs/changelog/changelog_dev.rst
@@ -66,3 +66,9 @@ Development Changelog
         :pullreq: 423
 
         Add GCP implementation.
+
+    .. change::
+        :tags: bugfix, schema
+        :pullreq: 450
+
+        Requires ``os_version`` and ``os_sha`` in the project file.
diff --git a/docs/project_defn.rst b/docs/project_defn.rst
@@ -91,23 +91,29 @@ Information about baseline simulations are listed under the ``baseline`` key.
 - ``skip_sims``: Include this key to control whether the set of baseline simulations are run. The default (i.e., when
   this key is not included) is to run all the baseline simulations. No results csv table with baseline characteristics
   will be provided when the baseline simulations are skipped.
-- ``custom_gems``: true or false. **ONLY WORKS ON EAGLE, KESTREL, AND LOCAL
-  DOCKER** When true, buildstockbatch will call the OpenStudio CLI commands with
-  the  ``bundle`` and ``bundle_path`` options. These options tell the CLI to
-  load a custom set of gems rather than those included in the OpenStudio CLI.
-  For both Eagle, Kestrel, and local Docker runs, these gems are first specified in the
-  ``buildstock\resources\Gemfile``. For Eagle, Kestrel, when the apptainer image is
-  built, these gems are added to the image. For local Docker, when the
+- ``custom_gems``: true or false. **ONLY WORKS ON EAGLE, KESTREL, AND LOCAL**
+  When true, buildstockbatch will call the OpenStudio CLI commands with the
+  ``bundle`` and ``bundle_path`` options. These options tell the CLI to load a
+  custom set of gems rather than those included in the OpenStudio CLI. For both
+  Eagle, Kestrel, and local Docker runs, these gems are first specified in the
+  ``buildstock\resources\Gemfile``. For Eagle, Kestrel, when the apptainer image
+  is built, these gems are added to the image. For local Docker, when the
   containers are started, the gems specified in the Gemfile are installed into a
   Docker volume on the local computer. This volume is mounted by each container
   as models are run, so each run uses the custom gems.
 
-OpenStudio Version Overrides
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+OpenStudio Version
+~~~~~~~~~~~~~~~~~~
+
+The following two top level keys are required:
+
+- ``os_version``: The version of OpenStudio required for the run (e.g. "3.7.0").
+  BuildStockBatch will verify that a suitable version of OpenStudio is available
+  and return an error if not.
+- ``os_sha``: The sha hash of the required OpenStudio version (e.g.
+  "06d9d975e1"). This must match the sha of the matching `OpenStudio release`_.
 
-This is a feature only used by ComStock at the moment. Please refer to the ComStock HPC documentation for additional
-details on the correct configuration. This is noted here to explain the presence of two keys in the version ``0.2``
-schema: ``os_version`` and ``os_sha``.
+.. _OpenStudio release: https://github.com/NREL/OpenStudio/releases
 
 Upgrade Scenarios
 ~~~~~~~~~~~~~~~~~