Merge pull request #104 from ACCESS-NRI/spack-provenance

added spack_config and spack_package data
ACCESS-NRI · Jul 8, 2024 · e34fda7 · e34fda7
2 parents e10cd06 + 118591f
commit e34fda7
Show file tree

Hide file tree

Showing 8 changed files with 33 additions and 23 deletions.
diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml
@@ -181,6 +181,7 @@ jobs:
     name: Build DB Metadata Upload
     if: inputs.type == 'release'
     needs:
+      - deploy-to-environment
       - release
     runs-on: ubuntu-latest
     steps:
@@ -209,7 +210,7 @@ jobs:
           BUILD_DB_CONNECTION_STR: ${{ secrets.BUILD_DB_CONNECTION_STR }}
           OUTPUT_PATH: ./metadata_output
         run: |
-          ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }}
+          ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ needs.deploy-to-environment.outputs.packages-version }} ${{ needs.deploy-to-environment.outputs.config-version }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }}
 
           echo "Attempting upload of build_metadata.json"
           python ./tools/release_provenance/save_release.py "${{ env.OUTPUT_PATH }}/build_metadata.json"
diff --git a/scripts/generate-build-metadata.bash b/scripts/generate-build-metadata.bash
@@ -7,16 +7,20 @@ set -e
 release_url=$1
 # Timestamp for the creation of the $release_url.
 release_time=$2
+# ACCESS-NRI/spack-packages version used
+spack_packages_version=$3
+# ACCESS-NRI/spack-config version used
+spack_config_version=$4
 # Path to the dir containing the spack.{lock,location.json}.
-json_dir=$3
+json_dir=$5
 # directory that contains the <package>.json files
-output_dir=$4
+output_dir=$6
 # Name of the model (or root-sbd) (eg. access-om2)
-model_name=$5
+model_name=$7
 
 # the rest of the model components for the given $model_name
 # (eg. for access-om2 there would be mom5, cice5, etc...)
-shift 5
+shift 7
 packages=( "$@" )
 
 ### SCRIPT ###
@@ -32,13 +36,17 @@ model=$(jq \
   --arg model "$model_name" \
   --arg release_url "$release_url" \
   --arg release_time "$release_time" \
+  --arg spack_packages_version "$spack_packages_version" \
+  --arg spack_config_version "$spack_config_version" \
   --argjson spack "$spack" \
   '.concrete_specs | to_entries[] | select(.value.name == $model)
   | {
       spack_hash: .key,
       spec: (.value.name + "@" + .value.version),
       created_at: $release_time,
       release_url: $release_url,
+      spack_packages: $spack_packages_version,
+      spack_config: $spack_config_version,
       spack_version: $spack
   }' "$json_dir/spack.lock"
 )

diff --git a/tests/scripts/generate-build-metadata/inputs/spack.location.json b/tests/scripts/generate-build-metadata/inputs/spack.location.json
@@ -1,14 +1,6 @@
 {
-    "ucmkt2gy72vvikhstgvoft65wnskr7rd": {
-        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/access-om2-git.2024.03.0=2024.03.0-ucmkt2gy72vvikhstgvoft65wnskr7rd"
-    },
-    "v3zncpqjj2gyseudbwiudolcjq3k3leo": {
-        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo"
-    },
-    "qji4nlmr6utrribaiyhewe4je6mifguz": {
-        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz"
-    },
-    "i3inxzaihefr3rqljoovhyevwai6bsff": {
-        "path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/netcdf-c-4.7.4-i3inxzaihefr3rqljoovhyevwai6bsff"
-    }
+    "ucmkt2gy72vvikhstgvoft65wnskr7rd": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/access-om2-git.2024.03.0=2024.03.0-ucmkt2gy72vvikhstgvoft65wnskr7rd",
+    "v3zncpqjj2gyseudbwiudolcjq3k3leo": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo",
+    "qji4nlmr6utrribaiyhewe4je6mifguz": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz",
+    "i3inxzaihefr3rqljoovhyevwai6bsff": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/netcdf-c-4.7.4-i3inxzaihefr3rqljoovhyevwai6bsff"
 }
diff --git a/tests/scripts/generate-build-metadata/test-generate-build-metadata.bash b/tests/scripts/generate-build-metadata/test-generate-build-metadata.bash
@@ -2,10 +2,12 @@
 
 set -e
 
-# Remember, script is in the form `generate-build-metadata.bash <release url> <release created at> <metadata dir> <output dir> <root sbd> <packages>...`
+# Remember, script is in the form `generate-build-metadata.bash <release url> <release created at> <spack-packages version> <spack-config version> <metadata dir> <output dir> <root sbd> <packages>...`
 # Where:
 #   release url: is the URL of GitHub release of the entire model,
 #   release created at: is the time the above release was created,
+#   spack-packages version: is the git tag of ACCESS-NRI/spack-packages used
+#   spack-config version: is the git tag of ACCESS-NRI/spack-config used
 #   metadata dir: is the path to the directory containing the spack.lock and spack.location.json,
 #   output dir: is the path to the directory that will contain the output,
 #   root sbd: is the overarching SBD of the model, usually the model name,
@@ -16,6 +18,6 @@ if [[ $(basename "$PWD") != "generate-build-metadata" ]]; then
   exit 1
 fi
 
-./../../../scripts/generate-build-metadata.bash http://example.org/releases 2024-03-27T05:29:36Z ./inputs ./outputs access-om2 mom5 cice5
+./../../../scripts/generate-build-metadata.bash http://example.org/releases 2024-03-27T05:29:36Z 2023.11.23 2024.01.01 ./inputs ./outputs access-om2 mom5 cice5
 
 cmp ./outputs/build_metadata.json ./valid/build_metadata.json
diff --git a/tests/scripts/generate-build-metadata/valid/build_metadata.json b/tests/scripts/generate-build-metadata/valid/build_metadata.json
@@ -4,6 +4,8 @@
     "spec": "[email protected]=2024.03.0",
     "created_at": "2024-03-27T05:29:36Z",
     "release_url": "http://example.org/releases",
+    "spack_packages": "2023.11.23",
+    "spack_config": "2024.01.01",
     "spack_version": {
       "version": "0.20.3",
       "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb7"

diff --git a/tools/release_provenance/models.py b/tools/release_provenance/models.py
@@ -33,6 +33,8 @@ class ModelBuild(Base):
     spack_hash = Column(String, primary_key=True, index=True)
     spec = Column(String, nullable=False)
     spack_version = Column(String, ForeignKey("spack_version.commit"))
+    spack_packages = Column(String)
+    spack_config = Column(String)
     created_at = Column(DateTime, nullable=False)
     release_url = Column(Text, nullable=False, unique=True)
     component_build = relationship('ComponentBuild', secondary="model_component", back_populates='model_build')
@@ -44,12 +46,11 @@ class SpackVersion(Base):
     version = Column(String, nullable=False)
     model_build = relationship('ModelBuild')
 
-
 model_component_association = Table(
     "model_component",
     Base.metadata,
     Column("id", Integer, primary_key=True, autoincrement=True),
-    Column("model_build", ForeignKey(ModelBuild.spack_hash)),
-    Column("component_build", ForeignKey(ComponentBuild.spack_hash)),
+    Column("model_build", ForeignKey(ModelBuild.spack_hash, ondelete='CASCADE')),
+    Column("component_build", ForeignKey(ComponentBuild.spack_hash, ondelete='CASCADE')),
     UniqueConstraint('model_build', 'component_build', name='uix_1')
 )
diff --git a/tools/release_provenance/save_release.py b/tools/release_provenance/save_release.py
@@ -33,6 +33,8 @@ def get_model_build(model_build_data):
     if model_build == None:
         model_build = ModelBuild()
         model_build.spack_version = get_spack_version(model_build_data["spack_version"])
+        model_build.spack_config = model_build_data["spack_config"]
+        model_build.spack_packages = model_build_data["spack_packages"]
         model_build.spack_hash = model_build_data["spack_hash"]
         model_build.spec = model_build_data["spec"]
         model_build.release_url = model_build_data["release_url"]

diff --git a/tools/release_provenance/test_release_data.json b/tools/release_provenance/test_release_data.json
@@ -12,10 +12,12 @@
       }
     ],
     "model_build": {
-      "spack_hash": "f4f2qe5b3c22q4dtzzqs5o2iygy435rh",
+      "spack_hash": "f4f2qe5b3c22q4dtzzqs5o2iygy435r",
       "spec": "[email protected]=2023.11.23",
       "created_at": "2024/02/13 11:49:00",
       "release_url": "https://github.com/ACCESS-NRI/ACCESS-OM3/releases",
+      "spack_packages": "0.20.3",
+      "spack_config": "0.20.3",
       "spack_version": {
         "commit": "6812713cf470b473a607f0de0e8e1cf53f804fb8",
         "version": "0.20.3"