From a7c929953419714e024f6bf459960b1240053f2d Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 13 May 2024 10:02:50 -0600 Subject: [PATCH 1/6] updating to OpenStudio 3.8 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59159e8a..be1d842f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,8 @@ jobs: wget --quiet https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip - name: Download and Install OpenStudio run: | - wget -q https://github.com/NREL/OpenStudio/releases/download/v3.7.0/OpenStudio-3.7.0+d5269793f1-Ubuntu-22.04-x86_64.deb - sudo apt install -y ./OpenStudio-3.7.0+d5269793f1-Ubuntu-22.04-x86_64.deb + wget -q https://github.com/NREL/OpenStudio/releases/download/v3.8.0-rc2/OpenStudio-3.8.0-rc2+af997f4fef-Ubuntu-22.04-x86_64.deb + sudo apt install -y ./OpenStudio-3.8.0-rc2+af997f4fef-Ubuntu-22.04-x86_64.deb openstudio openstudio_version which openstudio - name: Install buildstockbatch From 8fa11094fdd91e9c2f97729bc7169365c9763401 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 13 May 2024 10:24:21 -0600 Subject: [PATCH 2/6] pulling OpenStudio version from resstock yaml file --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be1d842f..435c8c2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,12 @@ jobs: wget --quiet https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip - name: Download and Install OpenStudio run: | - wget -q https://github.com/NREL/OpenStudio/releases/download/v3.8.0-rc2/OpenStudio-3.8.0-rc2+af997f4fef-Ubuntu-22.04-x86_64.deb - sudo apt install -y ./OpenStudio-3.8.0-rc2+af997f4fef-Ubuntu-22.04-x86_64.deb + sudo apt update && sudo apt install -y yq + export OS_VER=`yq '.os_version' resstock/project_national/national_baseline.yml` + export OS_SHA=`yq '.os_sha' resstock/project_national/national_baseline.yml` + export OS_INSTALLER_FILENAME="OpenStudio-${OS_VER}+${OS_SHA}-Ubuntu-22.04-x86_64.deb" + wget -q "https://github.com/NREL/OpenStudio/releases/download/v${OS_VER}/${OS_INSTALLER_FILENAME}" + sudo apt install -y "./${OS_INSTALLER_FILENAME}" openstudio openstudio_version which openstudio - name: Install buildstockbatch From c5a029e22061ce02afd0e1321537a9b0310479b3 Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 13 May 2024 10:43:31 -0600 Subject: [PATCH 3/6] trying snap to install yq --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 435c8c2e..a827e288 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: wget --quiet https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip - name: Download and Install OpenStudio run: | - sudo apt update && sudo apt install -y yq + sudo snap install -y yq export OS_VER=`yq '.os_version' resstock/project_national/national_baseline.yml` export OS_SHA=`yq '.os_sha' resstock/project_national/national_baseline.yml` export OS_INSTALLER_FILENAME="OpenStudio-${OS_VER}+${OS_SHA}-Ubuntu-22.04-x86_64.deb" From 7291d897f11ebfda2d52792a2dc9da9bf8e93f5c Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 13 May 2024 10:51:44 -0600 Subject: [PATCH 4/6] removing the -y --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a827e288..a1572ef0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: wget --quiet https://data.nrel.gov/system/files/156/BuildStock_TMY3_FIPS.zip - name: Download and Install OpenStudio run: | - sudo snap install -y yq + sudo snap install yq export OS_VER=`yq '.os_version' resstock/project_national/national_baseline.yml` export OS_SHA=`yq '.os_sha' resstock/project_national/national_baseline.yml` export OS_INSTALLER_FILENAME="OpenStudio-${OS_VER}+${OS_SHA}-Ubuntu-22.04-x86_64.deb" From 7badf67c42511498f98905e581f8e2130164a1ab Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 13 May 2024 13:48:32 -0600 Subject: [PATCH 5/6] requiring os_version, os_sha in project file --- buildstockbatch/base.py | 12 ++-- buildstockbatch/hpc.py | 4 +- buildstockbatch/schemas/v0.3.yaml | 4 +- buildstockbatch/test/conftest.py | 2 + buildstockbatch/test/test_aws.py | 59 ++++++++++--------- .../test/test_inputs/complete-schema.yml | 2 + .../test/test_inputs/enforce-schema-xor.yml | 2 + .../test/test_inputs/minimal-schema.yml | 2 + buildstockbatch/test/test_validation.py | 4 +- 9 files changed, 48 insertions(+), 43 deletions(-) diff --git a/buildstockbatch/base.py b/buildstockbatch/base.py index 397ef7fd..e58e06b7 100644 --- a/buildstockbatch/base.py +++ b/buildstockbatch/base.py @@ -42,8 +42,6 @@ class BuildStockBatchBase(object): # http://openstudio-builds.s3-website-us-east-1.amazonaws.com - DEFAULT_OS_VERSION = "3.6.1" - DEFAULT_OS_SHA = "bb9481519e" CONTAINER_RUNTIME = None LOGO = """ _ __ _ __, _ __ @@ -69,8 +67,8 @@ def __init__(self, project_filename): # Load in OS_VERSION and OS_SHA arguments if they exist in the YAML, # otherwise use defaults specified here. - self.os_version = self.cfg.get("os_version", self.DEFAULT_OS_VERSION) - self.os_sha = self.cfg.get("os_sha", self.DEFAULT_OS_SHA) + self.os_version = self.cfg["os_version"] + self.os_sha = self.cfg["os_sha"] logger.debug(f"Using OpenStudio version: {self.os_version} with SHA: {self.os_sha}") @staticmethod @@ -298,8 +296,8 @@ def get_buildstock_dir(project_file, cfg): @classmethod def validate_openstudio_path(cls, project_file): cfg = get_project_configuration(project_file) - os_version = cfg.get("os_version", cls.DEFAULT_OS_VERSION) - os_sha = cfg.get("os_sha", cls.DEFAULT_OS_SHA) + os_version = cfg["os_version"] + os_sha = cfg["os_sha"] try: proc_out = subprocess.run( [cls.openstudio_exe(), "openstudio_version"], @@ -873,7 +871,7 @@ def validate_openstudio_version(project_file): """ cfg = get_project_configuration(project_file) - os_version = cfg.get("os_version", BuildStockBatchBase.DEFAULT_OS_VERSION) + os_version = cfg["os_version"] version_path = "resources/hpxml-measures/HPXMLtoOpenStudio/resources/version.rb" version_rb = os.path.join(cfg["buildstock_directory"], version_path) if os.path.exists(version_rb): diff --git a/buildstockbatch/hpc.py b/buildstockbatch/hpc.py index 26b487b2..ed2755f8 100644 --- a/buildstockbatch/hpc.py +++ b/buildstockbatch/hpc.py @@ -87,8 +87,8 @@ def validate_apptainer_image_hpc(cls, project_file): try: cls.get_apptainer_image( cfg, - cfg.get("os_version", cls.DEFAULT_OS_VERSION), - cfg.get("os_sha", cls.DEFAULT_OS_SHA), + cfg["os_version"], + cfg["os_sha"], ) except RuntimeError as err: raise ValidationError(str(err)) diff --git a/buildstockbatch/schemas/v0.3.yaml b/buildstockbatch/schemas/v0.3.yaml index baed43f5..7bb42fae 100644 --- a/buildstockbatch/schemas/v0.3.yaml +++ b/buildstockbatch/schemas/v0.3.yaml @@ -12,8 +12,8 @@ aws: include('aws-spec', required=False) output_directory: regex('^(.*\/)?[a-z][a-z0-9_]*\/?$', required=True) sys_image_dir: str(required=False) baseline: include('sim-spec', required=True) -os_version: str(required=False) -os_sha: str(required=False) +os_version: str(required=True) +os_sha: str(required=True) max_minutes_per_sim: int(max=1440, required=False) upgrades: list(include('upgrade-spec'), required=False) postprocessing: include('postprocessing-spec', required=False) diff --git a/buildstockbatch/test/conftest.py b/buildstockbatch/test/conftest.py index 6a33e507..97f61f26 100644 --- a/buildstockbatch/test/conftest.py +++ b/buildstockbatch/test/conftest.py @@ -51,6 +51,8 @@ def _basic_residential_project_file(update_args={}, raw=False, hpc_name="eagle") "weather.zip", ) cfg = { + "os_version": "asdf", + "os_sha": "asdf", "buildstock_directory": buildstock_directory, "project_directory": project_directory, "output_directory": output_directory, diff --git a/buildstockbatch/test/test_aws.py b/buildstockbatch/test/test_aws.py index 9fb48060..35610215 100644 --- a/buildstockbatch/test/test_aws.py +++ b/buildstockbatch/test/test_aws.py @@ -10,25 +10,26 @@ def test_custom_gem_install(basic_residential_project_file): - project_filename, results_dir = basic_residential_project_file() + project_filename, results_dir = basic_residential_project_file( + update_args={ + "os_version": "3.7.0", + "baseline": { + "custom_gems": True, + "n_buildings_represented": 80000000, + }, + "aws": { + "job_identifier": "testaws", + "s3": {"bucket": "resbldg-datasets", "prefix": "testing/external_demo_project"}, + "region": "us-west-2", + "use_spot": True, + "batch_array_size": 100, + "notifications_email": "user@example.com", + }, + } + ) - # Add aws and custom_gems to the project file with open(project_filename, "r") as f: cfg = yaml.safe_load(f) - # custom_gems - cfg["baseline"]["custom_gems"] = True - # AWS - cfg["aws"] = {} - cfg["aws"]["job_identifier"] = "testaws" - cfg["aws"]["s3"] = {} - cfg["aws"]["s3"]["bucket"] = "resbldg-datasets" - cfg["aws"]["s3"]["prefix"] = "testing/external_demo_project" - cfg["aws"]["region"] = "us-west-2" - cfg["aws"]["use_spot"] = True - cfg["aws"]["batch_array_size"] = 100 - cfg["aws"]["notifications_email"] = "user@example.com" - with open(project_filename, "w") as f: - yaml.dump(cfg, f) buildstock_directory = cfg["buildstock_directory"] @@ -49,23 +50,23 @@ def test_custom_gem_install(basic_residential_project_file): def test_no_custom_gem_install(basic_residential_project_file): - project_filename, results_dir = basic_residential_project_file() + project_filename, results_dir = basic_residential_project_file( + update_args={ + "os_version": "3.7.0", + "aws": { + "job_identifier": "testaws", + "s3": {"bucket": "resbldg-datasets", "prefix": "testing/external_demo_project"}, + "region": "us-west-2", + "use_spot": True, + "batch_array_size": 100, + "notifications_email": "user@example.com", + }, + } + ) # Add aws to the project file with open(project_filename, "r") as f: cfg = yaml.safe_load(f) - # AWS - cfg["aws"] = {} - cfg["aws"]["job_identifier"] = "testaws" - cfg["aws"]["s3"] = {} - cfg["aws"]["s3"]["bucket"] = "resbldg-datasets" - cfg["aws"]["s3"]["prefix"] = "testing/external_demo_project" - cfg["aws"]["region"] = "us-west-2" - cfg["aws"]["use_spot"] = True - cfg["aws"]["batch_array_size"] = 100 - cfg["aws"]["notifications_email"] = "user@example.com" - with open(project_filename, "w") as f: - yaml.dump(cfg, f) buildstock_directory = cfg["buildstock_directory"] diff --git a/buildstockbatch/test/test_inputs/complete-schema.yml b/buildstockbatch/test/test_inputs/complete-schema.yml index ac1a050c..816192c8 100644 --- a/buildstockbatch/test/test_inputs/complete-schema.yml +++ b/buildstockbatch/test/test_inputs/complete-schema.yml @@ -1,3 +1,5 @@ +os_version: asdf +os_sha: asdf buildstock_directory: /home/epresent/OpenStudio-BuildStock-master project_directory: project_singlefamilydetached output_directory: /scratch/epresent/yaml1s diff --git a/buildstockbatch/test/test_inputs/enforce-schema-xor.yml b/buildstockbatch/test/test_inputs/enforce-schema-xor.yml index 626f3658..6a1007f2 100644 --- a/buildstockbatch/test/test_inputs/enforce-schema-xor.yml +++ b/buildstockbatch/test/test_inputs/enforce-schema-xor.yml @@ -1,3 +1,5 @@ +os_version: asdf +os_sha: asdf buildstock_directory: /home/epresent/OpenStudio-BuildStock-master project_directory: project_singlefamilydetached output_directory: /scratch/epresent/yaml1s diff --git a/buildstockbatch/test/test_inputs/minimal-schema.yml b/buildstockbatch/test/test_inputs/minimal-schema.yml index 31e808d3..5232c663 100644 --- a/buildstockbatch/test/test_inputs/minimal-schema.yml +++ b/buildstockbatch/test/test_inputs/minimal-schema.yml @@ -1,3 +1,5 @@ +os_version: 3.8.0-rc2 +os_sha: af997f4fef buildstock_directory: here project_directory: there weather_files_path: over-yonder diff --git a/buildstockbatch/test/test_validation.py b/buildstockbatch/test/test_validation.py index ee915940..ba8f79d4 100644 --- a/buildstockbatch/test/test_validation.py +++ b/buildstockbatch/test/test_validation.py @@ -440,9 +440,7 @@ def test_validate_apptainer_image(): with pytest.raises(ValidationError, match=r"Could not find apptainer image: .+\.sif or .+\.simg"): SlurmBatch.validate_apptainer_image_hpc(str(temp_yml)) for ext in ["Apptainer.sif", "Singularity.simg"]: - filename = pathlib.Path( - tmpdir, f"OpenStudio-{SlurmBatch.DEFAULT_OS_VERSION}.{SlurmBatch.DEFAULT_OS_SHA}-{ext}" - ) + filename = pathlib.Path(tmpdir, f"OpenStudio-{cfg['os_version']}.{cfg['os_sha']}-{ext}") filename.touch() SlurmBatch.validate_apptainer_image_hpc(str(temp_yml)) filename.unlink() From c896135dffa422d83567b76596be507f37c8230f Mon Sep 17 00:00:00 2001 From: Noel Merket Date: Mon, 13 May 2024 13:58:52 -0600 Subject: [PATCH 6/6] updating docs --- docs/changelog/changelog_dev.rst | 6 ++++++ docs/project_defn.rst | 30 ++++++++++++++++++------------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/changelog/changelog_dev.rst b/docs/changelog/changelog_dev.rst index fe553ac1..ab903ca4 100644 --- a/docs/changelog/changelog_dev.rst +++ b/docs/changelog/changelog_dev.rst @@ -66,3 +66,9 @@ Development Changelog :pullreq: 423 Add GCP implementation. + + .. change:: + :tags: bugfix, schema + :pullreq: 450 + + Requires ``os_version`` and ``os_sha`` in the project file. diff --git a/docs/project_defn.rst b/docs/project_defn.rst index e1f7b339..d57aa7da 100644 --- a/docs/project_defn.rst +++ b/docs/project_defn.rst @@ -91,23 +91,29 @@ Information about baseline simulations are listed under the ``baseline`` key. - ``skip_sims``: Include this key to control whether the set of baseline simulations are run. The default (i.e., when this key is not included) is to run all the baseline simulations. No results csv table with baseline characteristics will be provided when the baseline simulations are skipped. -- ``custom_gems``: true or false. **ONLY WORKS ON EAGLE, KESTREL, AND LOCAL - DOCKER** When true, buildstockbatch will call the OpenStudio CLI commands with - the ``bundle`` and ``bundle_path`` options. These options tell the CLI to - load a custom set of gems rather than those included in the OpenStudio CLI. - For both Eagle, Kestrel, and local Docker runs, these gems are first specified in the - ``buildstock\resources\Gemfile``. For Eagle, Kestrel, when the apptainer image is - built, these gems are added to the image. For local Docker, when the +- ``custom_gems``: true or false. **ONLY WORKS ON EAGLE, KESTREL, AND LOCAL** + When true, buildstockbatch will call the OpenStudio CLI commands with the + ``bundle`` and ``bundle_path`` options. These options tell the CLI to load a + custom set of gems rather than those included in the OpenStudio CLI. For both + Eagle, Kestrel, and local Docker runs, these gems are first specified in the + ``buildstock\resources\Gemfile``. For Eagle, Kestrel, when the apptainer image + is built, these gems are added to the image. For local Docker, when the containers are started, the gems specified in the Gemfile are installed into a Docker volume on the local computer. This volume is mounted by each container as models are run, so each run uses the custom gems. -OpenStudio Version Overrides -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +OpenStudio Version +~~~~~~~~~~~~~~~~~~ + +The following two top level keys are required: + +- ``os_version``: The version of OpenStudio required for the run (e.g. "3.7.0"). + BuildStockBatch will verify that a suitable version of OpenStudio is available + and return an error if not. +- ``os_sha``: The sha hash of the required OpenStudio version (e.g. + "06d9d975e1"). This must match the sha of the matching `OpenStudio release`_. -This is a feature only used by ComStock at the moment. Please refer to the ComStock HPC documentation for additional -details on the correct configuration. This is noted here to explain the presence of two keys in the version ``0.2`` -schema: ``os_version`` and ``os_sha``. +.. _OpenStudio release: https://github.com/NREL/OpenStudio/releases Upgrade Scenarios ~~~~~~~~~~~~~~~~~