From 0416a97ff1faaf61d7ebf8f82d37561f2786e0f5 Mon Sep 17 00:00:00 2001 From: utkarsh gupta Date: Tue, 2 Jul 2024 10:51:57 +1000 Subject: [PATCH 1/5] added spack_config and spack_package data --- tools/release_provenance/models.py | 20 ++++++++++++++-- tools/release_provenance/save_release.py | 24 +++++++++++++++++++ .../release_provenance/test_release_data.json | 10 +++++++- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/tools/release_provenance/models.py b/tools/release_provenance/models.py index e3b4b84..f1cdf4e 100644 --- a/tools/release_provenance/models.py +++ b/tools/release_provenance/models.py @@ -33,6 +33,8 @@ class ModelBuild(Base): spack_hash = Column(String, primary_key=True, index=True) spec = Column(String, nullable=False) spack_version = Column(String, ForeignKey("spack_version.commit")) + spack_package = Column(String, ForeignKey("spack_package.commit")) + spack_config = Column(String, ForeignKey("spack_config.commit")) created_at = Column(DateTime, nullable=False) release_url = Column(Text, nullable=False, unique=True) component_build = relationship('ComponentBuild', secondary="model_component", back_populates='model_build') @@ -44,12 +46,26 @@ class SpackVersion(Base): version = Column(String, nullable=False) model_build = relationship('ModelBuild') +class SpackConfig(Base): + __tablename__ = "spack_config" + + commit = Column(String, primary_key=True, index=True) + version = Column(String, nullable=False) + model_build = relationship('ModelBuild') + +class SpackPackage(Base): + __tablename__ = "spack_package" + + commit = Column(String, primary_key=True, index=True) + version = Column(String, nullable=False) + model_build = relationship('ModelBuild') + model_component_association = Table( "model_component", Base.metadata, Column("id", Integer, primary_key=True, autoincrement=True), - Column("model_build", ForeignKey(ModelBuild.spack_hash)), - Column("component_build", ForeignKey(ComponentBuild.spack_hash)), + Column("model_build", ForeignKey(ModelBuild.spack_hash, ondelete='CASCADE')), + Column("component_build", ForeignKey(ComponentBuild.spack_hash, ondelete='CASCADE')), UniqueConstraint('model_build', 'component_build', name='uix_1') ) diff --git a/tools/release_provenance/save_release.py b/tools/release_provenance/save_release.py index aecd14f..2f0fb6d 100644 --- a/tools/release_provenance/save_release.py +++ b/tools/release_provenance/save_release.py @@ -33,6 +33,8 @@ def get_model_build(model_build_data): if model_build == None: model_build = ModelBuild() model_build.spack_version = get_spack_version(model_build_data["spack_version"]) + model_build.spack_config = get_spack_config(model_build_data["spack_config"]) + model_build.spack_package = get_spack_package(model_build_data["spack_package"]) model_build.spack_hash = model_build_data["spack_hash"] model_build.spec = model_build_data["spec"] model_build.release_url = model_build_data["release_url"] @@ -51,6 +53,28 @@ def get_spack_version(spack_version_data): return spack_version.commit +def get_spack_package(spack_package_data): + spack_package = session.query(SpackPackage).get(spack_package_data["commit"]) + + if spack_package is None: + spack_package = SpackPackage() + spack_package.commit = spack_package_data["commit"] + spack_package.version = spack_package_data["version"] + session.add(spack_package) + + return spack_package.commit + +def get_spack_config(spack_config_data): + spack_config = session.query(SpackConfig).get(spack_config_data["commit"]) + + if spack_config is None: + spack_config = SpackConfig() + spack_config.commit = spack_config_data["commit"] + spack_config.version = spack_config_data["version"] + session.add(spack_config) + + return spack_config.commit + def main(): release_data_filename = sys.argv[1] release_data = read_release_data(release_data_filename) diff --git a/tools/release_provenance/test_release_data.json b/tools/release_provenance/test_release_data.json index c918951..6776e75 100644 --- a/tools/release_provenance/test_release_data.json +++ b/tools/release_provenance/test_release_data.json @@ -12,13 +12,21 @@ } ], "model_build": { - "spack_hash": "f4f2qe5b3c22q4dtzzqs5o2iygy435rh", + "spack_hash": "f4f2qe5b3c22q4dtzzqs5o2iygy435r", "spec": "access-om2@git.2023.11.23=2023.11.23", "created_at": "2024/02/13 11:49:00", "release_url": "https://github.com/ACCESS-NRI/ACCESS-OM3/releases", "spack_version": { "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8", "version": "0.20.3" + }, + "spack_package": { + "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8", + "version": "0.20.3" + }, + "spack_config": { + "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8", + "version": "0.20.3" } } } \ No newline at end of file From 51b3ba0f97077dcab1492e141bc35902ca6b3962 Mon Sep 17 00:00:00 2001 From: utkarsh gupta Date: Wed, 3 Jul 2024 13:11:45 +1000 Subject: [PATCH 2/5] removed spack config/package tables --- tools/release_provenance/models.py | 19 ++------------ tools/release_provenance/save_release.py | 26 ++----------------- .../release_provenance/test_release_data.json | 10 ++----- 3 files changed, 6 insertions(+), 49 deletions(-) diff --git a/tools/release_provenance/models.py b/tools/release_provenance/models.py index f1cdf4e..3734a9a 100644 --- a/tools/release_provenance/models.py +++ b/tools/release_provenance/models.py @@ -33,8 +33,8 @@ class ModelBuild(Base): spack_hash = Column(String, primary_key=True, index=True) spec = Column(String, nullable=False) spack_version = Column(String, ForeignKey("spack_version.commit")) - spack_package = Column(String, ForeignKey("spack_package.commit")) - spack_config = Column(String, ForeignKey("spack_config.commit")) + spack_package = Column(String) + spack_config = Column(String) created_at = Column(DateTime, nullable=False) release_url = Column(Text, nullable=False, unique=True) component_build = relationship('ComponentBuild', secondary="model_component", back_populates='model_build') @@ -46,21 +46,6 @@ class SpackVersion(Base): version = Column(String, nullable=False) model_build = relationship('ModelBuild') -class SpackConfig(Base): - __tablename__ = "spack_config" - - commit = Column(String, primary_key=True, index=True) - version = Column(String, nullable=False) - model_build = relationship('ModelBuild') - -class SpackPackage(Base): - __tablename__ = "spack_package" - - commit = Column(String, primary_key=True, index=True) - version = Column(String, nullable=False) - model_build = relationship('ModelBuild') - - model_component_association = Table( "model_component", Base.metadata, diff --git a/tools/release_provenance/save_release.py b/tools/release_provenance/save_release.py index 2f0fb6d..f466315 100644 --- a/tools/release_provenance/save_release.py +++ b/tools/release_provenance/save_release.py @@ -33,8 +33,8 @@ def get_model_build(model_build_data): if model_build == None: model_build = ModelBuild() model_build.spack_version = get_spack_version(model_build_data["spack_version"]) - model_build.spack_config = get_spack_config(model_build_data["spack_config"]) - model_build.spack_package = get_spack_package(model_build_data["spack_package"]) + model_build.spack_config = model_build_data["spack_config"] + model_build.spack_package = model_build_data["spack_package"] model_build.spack_hash = model_build_data["spack_hash"] model_build.spec = model_build_data["spec"] model_build.release_url = model_build_data["release_url"] @@ -53,28 +53,6 @@ def get_spack_version(spack_version_data): return spack_version.commit -def get_spack_package(spack_package_data): - spack_package = session.query(SpackPackage).get(spack_package_data["commit"]) - - if spack_package is None: - spack_package = SpackPackage() - spack_package.commit = spack_package_data["commit"] - spack_package.version = spack_package_data["version"] - session.add(spack_package) - - return spack_package.commit - -def get_spack_config(spack_config_data): - spack_config = session.query(SpackConfig).get(spack_config_data["commit"]) - - if spack_config is None: - spack_config = SpackConfig() - spack_config.commit = spack_config_data["commit"] - spack_config.version = spack_config_data["version"] - session.add(spack_config) - - return spack_config.commit - def main(): release_data_filename = sys.argv[1] release_data = read_release_data(release_data_filename) diff --git a/tools/release_provenance/test_release_data.json b/tools/release_provenance/test_release_data.json index 6776e75..c70987e 100644 --- a/tools/release_provenance/test_release_data.json +++ b/tools/release_provenance/test_release_data.json @@ -16,17 +16,11 @@ "spec": "access-om2@git.2023.11.23=2023.11.23", "created_at": "2024/02/13 11:49:00", "release_url": "https://github.com/ACCESS-NRI/ACCESS-OM3/releases", + "spack_package": "0.20.3", + "spack_config": "0.20.3", "spack_version": { "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8", "version": "0.20.3" - }, - "spack_package": { - "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8", - "version": "0.20.3" - }, - "spack_config": { - "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8", - "version": "0.20.3" } } } \ No newline at end of file From ad195084b02896c62a56a1d97c65b513bb461c4a Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 3 Jul 2024 13:37:31 +1000 Subject: [PATCH 3/5] Added spack-packages, spack-config version args to generate-build-metadata script --- .github/workflows/deploy-2-start.yml | 3 ++- scripts/generate-build-metadata.bash | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 7188b33..7d83d37 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -167,6 +167,7 @@ jobs: name: Build DB Metadata Upload if: inputs.type == 'release' needs: + - deploy-to-environment - release runs-on: ubuntu-latest steps: @@ -195,7 +196,7 @@ jobs: BUILD_DB_CONNECTION_STR: ${{ secrets.BUILD_DB_CONNECTION_STR }} OUTPUT_PATH: ./metadata_output run: | - ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} + ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ needs.deploy-to-environment.outputs.packages-version }} ${{ needs.deploy-to-environment.outputs.config-version }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} echo "Attempting upload of build_metadata.json" python ./tools/release_provenance/save_release.py "${{ env.OUTPUT_PATH }}/build_metadata.json" diff --git a/scripts/generate-build-metadata.bash b/scripts/generate-build-metadata.bash index cb81a85..262885b 100755 --- a/scripts/generate-build-metadata.bash +++ b/scripts/generate-build-metadata.bash @@ -7,16 +7,20 @@ set -e release_url=$1 # Timestamp for the creation of the $release_url. release_time=$2 +# ACCESS-NRI/spack-packages version used +spack_packages_version=$3 +# ACCESS-NRI/spack-config version used +spack_config_version=$4 # Path to the dir containing the spack.{lock,location.json}. -json_dir=$3 +json_dir=$5 # directory that contains the .json files -output_dir=$4 +output_dir=$6 # Name of the model (or root-sbd) (eg. access-om2) -model_name=$5 +model_name=$7 # the rest of the model components for the given $model_name # (eg. for access-om2 there would be mom5, cice5, etc...) -shift 5 +shift 7 packages=( "$@" ) ### SCRIPT ### @@ -32,6 +36,8 @@ model=$(jq \ --arg model "$model_name" \ --arg release_url "$release_url" \ --arg release_time "$release_time" \ + --arg spack_packages_version "$spack_packages_version" \ + --arg spack_config_version "$spack_config_version" \ --argjson spack "$spack" \ '.concrete_specs | to_entries[] | select(.value.name == $model) | { @@ -39,6 +45,8 @@ model=$(jq \ spec: (.value.name + "@" + .value.version), created_at: $release_time, release_url: $release_url, + spack_package: $spack_packages_version, + spack_config: $spack_config_version, spack_version: $spack }' "$json_dir/spack.lock" ) From c7992071bf176e63fc9502b241c94a4fce74a2c3 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 3 Jul 2024 13:38:02 +1000 Subject: [PATCH 4/5] Updated tests to reference new spack-packages, spack-config version args --- .../inputs/spack.location.json | 16 ++++------------ .../test-generate-build-metadata.bash | 6 ++++-- .../valid/build_metadata.json | 2 ++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/scripts/generate-build-metadata/inputs/spack.location.json b/tests/scripts/generate-build-metadata/inputs/spack.location.json index 4fcdb8c..13ca815 100644 --- a/tests/scripts/generate-build-metadata/inputs/spack.location.json +++ b/tests/scripts/generate-build-metadata/inputs/spack.location.json @@ -1,14 +1,6 @@ { - "ucmkt2gy72vvikhstgvoft65wnskr7rd": { - "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/access-om2-git.2024.03.0=2024.03.0-ucmkt2gy72vvikhstgvoft65wnskr7rd" - }, - "v3zncpqjj2gyseudbwiudolcjq3k3leo": { - "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo" - }, - "qji4nlmr6utrribaiyhewe4je6mifguz": { - "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz" - }, - "i3inxzaihefr3rqljoovhyevwai6bsff": { - "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/netcdf-c-4.7.4-i3inxzaihefr3rqljoovhyevwai6bsff" - } + "ucmkt2gy72vvikhstgvoft65wnskr7rd": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/access-om2-git.2024.03.0=2024.03.0-ucmkt2gy72vvikhstgvoft65wnskr7rd", + "v3zncpqjj2gyseudbwiudolcjq3k3leo": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo", + "qji4nlmr6utrribaiyhewe4je6mifguz": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz", + "i3inxzaihefr3rqljoovhyevwai6bsff": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/netcdf-c-4.7.4-i3inxzaihefr3rqljoovhyevwai6bsff" } diff --git a/tests/scripts/generate-build-metadata/test-generate-build-metadata.bash b/tests/scripts/generate-build-metadata/test-generate-build-metadata.bash index 6ed213a..a0ba6a2 100755 --- a/tests/scripts/generate-build-metadata/test-generate-build-metadata.bash +++ b/tests/scripts/generate-build-metadata/test-generate-build-metadata.bash @@ -2,10 +2,12 @@ set -e -# Remember, script is in the form `generate-build-metadata.bash ...` +# Remember, script is in the form `generate-build-metadata.bash ...` # Where: # release url: is the URL of GitHub release of the entire model, # release created at: is the time the above release was created, +# spack-packages version: is the git tag of ACCESS-NRI/spack-packages used +# spack-config version: is the git tag of ACCESS-NRI/spack-config used # metadata dir: is the path to the directory containing the spack.lock and spack.location.json, # output dir: is the path to the directory that will contain the output, # root sbd: is the overarching SBD of the model, usually the model name, @@ -16,6 +18,6 @@ if [[ $(basename "$PWD") != "generate-build-metadata" ]]; then exit 1 fi -./../../../scripts/generate-build-metadata.bash http://example.org/releases 2024-03-27T05:29:36Z ./inputs ./outputs access-om2 mom5 cice5 +./../../../scripts/generate-build-metadata.bash http://example.org/releases 2024-03-27T05:29:36Z 2023.11.23 2024.01.01 ./inputs ./outputs access-om2 mom5 cice5 cmp ./outputs/build_metadata.json ./valid/build_metadata.json diff --git a/tests/scripts/generate-build-metadata/valid/build_metadata.json b/tests/scripts/generate-build-metadata/valid/build_metadata.json index 05ee23e..7016a76 100644 --- a/tests/scripts/generate-build-metadata/valid/build_metadata.json +++ b/tests/scripts/generate-build-metadata/valid/build_metadata.json @@ -4,6 +4,8 @@ "spec": "access-om2@git.2024.03.0=2024.03.0", "created_at": "2024-03-27T05:29:36Z", "release_url": "http://example.org/releases", + "spack_package": "2023.11.23", + "spack_config": "2024.01.01", "spack_version": { "version": "0.20.3", "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb7" From 118591f2dcdea6007ca3df55fd564bad42b51928 Mon Sep 17 00:00:00 2001 From: utkarsh gupta Date: Fri, 5 Jul 2024 14:38:38 +1000 Subject: [PATCH 5/5] renaming spack_package to spack_packaged --- scripts/generate-build-metadata.bash | 2 +- tests/scripts/generate-build-metadata/valid/build_metadata.json | 2 +- tools/release_provenance/models.py | 2 +- tools/release_provenance/save_release.py | 2 +- tools/release_provenance/test_release_data.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/generate-build-metadata.bash b/scripts/generate-build-metadata.bash index 262885b..38f9a81 100755 --- a/scripts/generate-build-metadata.bash +++ b/scripts/generate-build-metadata.bash @@ -45,7 +45,7 @@ model=$(jq \ spec: (.value.name + "@" + .value.version), created_at: $release_time, release_url: $release_url, - spack_package: $spack_packages_version, + spack_packages: $spack_packages_version, spack_config: $spack_config_version, spack_version: $spack }' "$json_dir/spack.lock" diff --git a/tests/scripts/generate-build-metadata/valid/build_metadata.json b/tests/scripts/generate-build-metadata/valid/build_metadata.json index 7016a76..537c3f8 100644 --- a/tests/scripts/generate-build-metadata/valid/build_metadata.json +++ b/tests/scripts/generate-build-metadata/valid/build_metadata.json @@ -4,7 +4,7 @@ "spec": "access-om2@git.2024.03.0=2024.03.0", "created_at": "2024-03-27T05:29:36Z", "release_url": "http://example.org/releases", - "spack_package": "2023.11.23", + "spack_packages": "2023.11.23", "spack_config": "2024.01.01", "spack_version": { "version": "0.20.3", diff --git a/tools/release_provenance/models.py b/tools/release_provenance/models.py index 3734a9a..d030418 100644 --- a/tools/release_provenance/models.py +++ b/tools/release_provenance/models.py @@ -33,7 +33,7 @@ class ModelBuild(Base): spack_hash = Column(String, primary_key=True, index=True) spec = Column(String, nullable=False) spack_version = Column(String, ForeignKey("spack_version.commit")) - spack_package = Column(String) + spack_packages = Column(String) spack_config = Column(String) created_at = Column(DateTime, nullable=False) release_url = Column(Text, nullable=False, unique=True) diff --git a/tools/release_provenance/save_release.py b/tools/release_provenance/save_release.py index f466315..43338ff 100644 --- a/tools/release_provenance/save_release.py +++ b/tools/release_provenance/save_release.py @@ -34,7 +34,7 @@ def get_model_build(model_build_data): model_build = ModelBuild() model_build.spack_version = get_spack_version(model_build_data["spack_version"]) model_build.spack_config = model_build_data["spack_config"] - model_build.spack_package = model_build_data["spack_package"] + model_build.spack_packages = model_build_data["spack_packages"] model_build.spack_hash = model_build_data["spack_hash"] model_build.spec = model_build_data["spec"] model_build.release_url = model_build_data["release_url"] diff --git a/tools/release_provenance/test_release_data.json b/tools/release_provenance/test_release_data.json index c70987e..84599d5 100644 --- a/tools/release_provenance/test_release_data.json +++ b/tools/release_provenance/test_release_data.json @@ -16,7 +16,7 @@ "spec": "access-om2@git.2023.11.23=2023.11.23", "created_at": "2024/02/13 11:49:00", "release_url": "https://github.com/ACCESS-NRI/ACCESS-OM3/releases", - "spack_package": "0.20.3", + "spack_packages": "0.20.3", "spack_config": "0.20.3", "spack_version": { "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8",