From a65fd13e12937dfd6d0dc4b672b9862c87597c8c Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Wed, 19 Jun 2024 12:48:57 +0200
Subject: [PATCH 01/27] fix(ci): fix silent error when revision duplicates

---
 src/shared/release_info.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/shared/release_info.py b/src/shared/release_info.py
index b879d68d..86a8d02d 100755
--- a/src/shared/release_info.py
+++ b/src/shared/release_info.py
@@ -71,13 +71,14 @@ def get_revision_to_track(all_revisions_tags: list) -> dict:
     revision_track = {}
     for track_revision in all_revisions_tags:
         track, revision = track_revision.rsplit("_", 1)
+        revision = int(revision)
         if revision in revision_track:
             msg = (
                 "Each revision can only have 1 canonical tag, "
                 f"but revision {revision} is associated with tracks "
-                f"{track} and {revision_track['revision']}!"
+                f"{track} and {revision_track[revision]}!"
             )
             raise BadChannel(msg)
 
-        revision_track[int(revision)] = track
+        revision_track[revision] = track
     return revision_track

From 0d3e588c1551b3fb3dbaccb69f280bbad0d31644 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Wed, 19 Jun 2024 12:41:59 +0200
Subject: [PATCH 02/27] feat(ci): add validation for unique trigger upload
 entry

---
 .github/workflows/Image.yaml                  |  5 ++
 .../prepare_single_image_build_matrix.py      |  5 ++
 src/image/requirements.txt                    |  4 +-
 ...st_validate_unique_trigger_upload_entry.py | 69 +++++++++++++++++++
 .../validate_unique_trigger_upload_entry.py   | 40 +++++++++++
 5 files changed, 122 insertions(+), 1 deletion(-)
 create mode 100644 src/image/test_validate_unique_trigger_upload_entry.py
 create mode 100644 src/image/validate_unique_trigger_upload_entry.py

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 2bd21039..a7d716bc 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -105,6 +105,11 @@ jobs:
 
       - run: pip install -r src/image/requirements.txt
 
+      - name: Validate unique trigger upload entry
+        run: |
+          ./src/image/validate_unique_trigger_upload_entry.py \
+            --image-trigger ${{ steps.validate-image.outputs.img-path }}/image.yaml
+
       - name: Get next revision number
         id: get-next-revision
         env:
diff --git a/src/image/prepare_single_image_build_matrix.py b/src/image/prepare_single_image_build_matrix.py
index a3857838..3a8b1290 100755
--- a/src/image/prepare_single_image_build_matrix.py
+++ b/src/image/prepare_single_image_build_matrix.py
@@ -64,9 +64,14 @@ def validate_image_trigger(data: dict) -> None:
         with open(
             f"{args.revision_data_dir}/{builds[img_number]['revision']}",
             "w",
+            encoding="UTF-8"
         ) as data_file:
             json.dump(builds[img_number], data_file)
 
+        # Add dir_identifier to assemble the cache key and artefact path
+        # No need to write it to rev data file since it's only used in matrix
+        builds[img_number]["dir_identifier"] = builds[img_number]["directory"].rstrip("/").replace("/", "_")
+        
         # set an output as a marker for later knowing if we need to release
         if "release" in builds[img_number]:
             release_to = "true"
diff --git a/src/image/requirements.txt b/src/image/requirements.txt
index 57743760..c699d2f2 100644
--- a/src/image/requirements.txt
+++ b/src/image/requirements.txt
@@ -1,4 +1,6 @@
 PyYAML
 pydantic==1.9.0
 python-swiftclient
-python-keystoneclient
\ No newline at end of file
+python-keystoneclient
+
+pytest
diff --git a/src/image/test_validate_unique_trigger_upload_entry.py b/src/image/test_validate_unique_trigger_upload_entry.py
new file mode 100644
index 00000000..71a1096c
--- /dev/null
+++ b/src/image/test_validate_unique_trigger_upload_entry.py
@@ -0,0 +1,69 @@
+import argparse
+import os
+import pytest
+import yaml
+
+from tempfile import TemporaryDirectory as tempdir
+
+from validate_unique_trigger_upload_entry import *
+
+
+def test_validate_unique_trigger_entry():
+    # Create a temporary image trigger file for testing
+    image_trigger_data = {
+        "version": 1,
+        "upload": [
+            {
+                "source": "source1",
+                "commit": "commit1",
+                "directory": "directory1"
+            },
+            {
+                "source": "source2",
+                "commit": "commit2",
+                "directory": "directory2"
+            }
+        ]
+    }
+
+    with tempdir() as tmp_dir:
+        image_trigger_file = os.path.join(tmp_dir, "trigger.yaml")
+        with open(image_trigger_file, "w") as f:
+            yaml.dump(image_trigger_data, f)
+
+        args = argparse.Namespace(image_trigger=image_trigger_file)
+        main(args)
+
+
+def test_validate_unique_trigger_entry_non_unique():
+    # Create a temporary image trigger file for testing
+    image_trigger_data = {
+        "version": 1,
+        "upload": [
+            {
+                "source": "source1",
+                "commit": "commit1",
+                "directory": "directory1"
+            },
+            {
+                "source": "source2",
+                "commit": "commit2",
+                "directory": "directory2"
+            },
+            {
+                "source": "source1",
+                "commit": "commit1",
+                "directory": "directory1"
+            }
+        ]
+    }
+    with tempdir() as tmp_dir:
+        image_trigger_file = os.path.join(tmp_dir, "trigger.yaml")
+        with open(image_trigger_file, "w") as f:
+            yaml.dump(image_trigger_data, f)
+
+        # Call the main function and assert that it raises a ValueError]
+        args = argparse.Namespace(image_trigger=image_trigger_file)
+        with pytest.raises(ValueError) as e:
+            main(args)
+            assert str(e.value) == "Image trigger source1_commit1_directory1 is not unique."
diff --git a/src/image/validate_unique_trigger_upload_entry.py b/src/image/validate_unique_trigger_upload_entry.py
new file mode 100644
index 00000000..88f3ed20
--- /dev/null
+++ b/src/image/validate_unique_trigger_upload_entry.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+"""
+This script validates that the image trigger entry is unique in the
+given image.yaml file. This is necessary to avoid having multiple
+entries for the same image trigger, which would lead to ambiguous
+results when using the combination of {source}_{commit}_{directory}
+as the cache key for the Image workflow runs.
+"""
+
+import argparse
+import yaml
+
+from utils.schema.triggers import ImageSchema
+
+
+def main(args):
+    print(f"Validating unique upload entry for {args.image_trigger}")
+    with open(args.image_trigger, encoding="UTF-8") as trigger:
+        image_trigger = yaml.load(trigger, Loader=yaml.BaseLoader)
+
+    schema = ImageSchema(**image_trigger)
+    unique_triggers = set()
+    for upload in  schema.upload:
+        trigger = f"{upload.source}_{upload.commit}_{upload.directory}"
+        if trigger in unique_triggers:
+            raise ValueError(f"Image trigger {trigger} is not unique.")
+        unique_triggers.add(trigger)
+    print("OK")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--image-trigger",
+        help="Path to the image trigger file.",
+        required=True,
+    )
+    args = parser.parse_args()
+    
+    main(args)

From 2b80f6475cd76dc1bbea89e17319c0061f7b7768 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Wed, 19 Jun 2024 17:47:45 +0200
Subject: [PATCH 03/27] fix(ci): add prepare-upload as critical section update
 swift

---
 .github/workflows/Image.yaml                  | 108 +++++++++++++-----
 .../prepare_single_image_build_matrix.py      |   1 +
 .../validate_unique_trigger_upload_entry.py   |   0
 src/uploads/upload_dummy_to_swift.sh          |  21 ++++
 src/uploads/upload_to_swift.sh                |   4 +-
 5 files changed, 105 insertions(+), 29 deletions(-)
 mode change 100644 => 100755 src/image/validate_unique_trigger_upload_entry.py
 create mode 100755 src/uploads/upload_dummy_to_swift.sh

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index a7d716bc..30da6da6 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -53,7 +53,6 @@ jobs:
       release-to: ${{ steps.prepare-matrix.outputs.release-to }}
       oci-img-path: ${{ steps.validate-image.outputs.img-path }}
       oci-img-name: ${{ steps.validate-image.outputs.img-name }}
-      revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
     steps:
       - name: ${{ inputs.external_ref_id }} #(2)
         if: ${{ github.event_name == 'workflow_dispatch' }}
@@ -110,18 +109,6 @@ jobs:
           ./src/image/validate_unique_trigger_upload_entry.py \
             --image-trigger ${{ steps.validate-image.outputs.img-path }}/image.yaml
 
-      - name: Get next revision number
-        id: get-next-revision
-        env:
-          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
-          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
-          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
-          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
-          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
-          IMAGE_NAME: ${{ steps.validate-image.outputs.img-name }}
-          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
-        run: ./src/image/define_image_revision.sh
-
       - name: Validate and prepare builds matrix
         id: prepare-matrix
         env:
@@ -132,15 +119,10 @@ jobs:
           ./src/image/prepare_single_image_build_matrix.py \
             --oci-path ${{ steps.validate-image.outputs.img-path }} \
             --revision-data-dir ${{ env.DATA_DIR }} \
-            --next-revision ${{ steps.get-next-revision.outputs.revision }}
+            --next-revision 0
 
           echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"
 
-      - uses: actions/cache/save@v4
-        with:
-          path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
-          key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
-
   run-build:
     needs: [prepare-build]
     strategy:
@@ -148,7 +130,7 @@ jobs:
       matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
     uses: ./.github/workflows/Build-Rock.yaml
     with:
-      oci-archive-name: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}
+      oci-archive-name: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
       oci-factory-path: ${{ matrix.path }}
       rock-name: ${{ matrix.name }}
       rock-repo: ${{ matrix.source }}
@@ -164,22 +146,92 @@ jobs:
       matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
     uses: ./.github/workflows/Tests.yaml
     with:
-      oci-image-name: "${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}"
+      oci-image-name: "${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}"
       oci-image-path: "oci/${{ matrix.name }}"
       test-from: "cache"
-      cache-key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}        
+      cache-key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}        
     secrets: inherit
 
-  upload:
+  
+  prepare-upload:
     runs-on: ubuntu-22.04
     needs: [prepare-build, run-build, test]
+    name: Prepare upload
+    outputs:
+      build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
+      revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Use custom image trigger
+        if: ${{ inputs.b64-image-trigger != '' }}
+        run: echo ${{ inputs.b64-image-trigger }} | base64 -d > ${{ needs.prepare-build.outputs.oci-img-path }}/image.yaml
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - run: |
+          ./src/uploads/requirements.sh
+          pip install -r src/image/requirements.txt -r src/uploads/requirements.txt
+
+      - name: Get next revision number
+        id: get-next-revision
+        env:
+          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
+          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
+          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
+          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
+          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
+          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
+          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
+        run: ./src/image/define_image_revision.sh
+
+      # This is a critical section, have to be executed in sequence outside of matrix
+      - name: Prepare builds matrix with revision and preempt swift slots
+        id: prepare-matrix
+        env:
+          DATA_DIR: "revision-data"
+          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
+          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
+          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
+          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
+          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
+          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
+          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
+        run: |
+          set -ex
+          mkdir ${{ env.DATA_DIR }}
+
+          ./src/image/prepare_single_image_build_matrix.py \
+            --oci-path ${{ needs.prepare-build.outputs.oci-img-path }} \
+            --revision-data-dir ${{ env.DATA_DIR }} \
+            --next-revision ${{ steps.get-next-revision.outputs.revision }}
+
+          for revision_file in `ls ${{ env.DATA_DIR }}`; do
+            ./src/uploads/upload_dummy_to_swift.sh \
+              ${{ needs.prepare-build.outputs.oci-img-name }} \
+              $(jq -r '.release | keys[0]' < ${{ env.DATA_DIR }}/$revision_file) \
+              $revision_file
+          done
+
+          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/cache/save@v4
+        with:
+          path: ${{ steps.prepare-matrix.outputs.revision-data-dir }}
+          key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
+
+  upload:
+    runs-on: ubuntu-22.04
+    needs: [prepare-build, prepare-upload]
     name: Upload
     if: ${{ inputs.upload || (github.ref_name == 'main' && github.event_name == 'push') }}
     strategy:
       fail-fast: true
-      matrix: ${{ fromJSON(needs.prepare-build.outputs.build-matrix) }}
+      matrix: ${{ fromJSON(needs.prepare-upload.outputs.build-matrix) }}
     env:
-      OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}
+      OCI_ARCHIVE_NAME: ${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
     outputs:
       artefacts-hashes: ${{ steps.artefacts-hashes.outputs.hashes }}
     steps:
@@ -221,7 +273,7 @@ jobs:
       - uses: actions/cache/restore@v4
         with:
           path: ${{ env.OCI_ARCHIVE_NAME }}
-          key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}
+          key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
           fail-on-cache-miss: true
 
       - name: Infer track name
@@ -385,7 +437,7 @@ jobs:
   # and commit the _releases.json file in a single commit, outside a matrix job
   prepare-releases:
     name: Prepare releases
-    needs: [prepare-build, upload]
+    needs: [prepare-build, prepare-upload, upload]
     runs-on: ubuntu-22.04
     if: ${{ needs.prepare-build.outputs.release-to != '' }}
     concurrency:
@@ -409,7 +461,7 @@ jobs:
       - uses: actions/cache/restore@v4
         with:
           path: ${{ env.REVISION_DATA_DIR }}
-          key: ${{ needs.prepare-build.outputs.revision-data-cache-key }}
+          key: ${{ needs.prepare-upload.outputs.revision-data-cache-key }}
           fail-on-cache-miss: true
 
       - run: pip install -r src/image/requirements.txt
diff --git a/src/image/prepare_single_image_build_matrix.py b/src/image/prepare_single_image_build_matrix.py
index 3a8b1290..92843f6b 100755
--- a/src/image/prepare_single_image_build_matrix.py
+++ b/src/image/prepare_single_image_build_matrix.py
@@ -57,6 +57,7 @@ def validate_image_trigger(data: dict) -> None:
     for img_number, _ in enumerate(builds):
         builds[img_number]["name"] = args.oci_path.rstrip("/").split("/")[-1]
         builds[img_number]["path"] = args.oci_path
+        builds[img_number]["dir_identifier"] = builds[img_number]["directory"].rstrip("/").replace("/", "_")
         # make sure every build of this image has a unique identifier
         # within the execution of the workflow - use revision number
         builds[img_number]["revision"] = img_number + int(args.next_revision)
diff --git a/src/image/validate_unique_trigger_upload_entry.py b/src/image/validate_unique_trigger_upload_entry.py
old mode 100644
new mode 100755
diff --git a/src/uploads/upload_dummy_to_swift.sh b/src/uploads/upload_dummy_to_swift.sh
new file mode 100755
index 00000000..e892e95f
--- /dev/null
+++ b/src/uploads/upload_dummy_to_swift.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -e
+
+# Source Swift config
+source $(dirname $0)/../configs/swift.public.novarc
+
+set -x
+
+IMAGE_NAME=$1
+TRACK=$2
+REVISION=$3
+
+staging_area=$(mktemp -d)
+
+mkdir -p "${staging_area}/${IMAGE_NAME}/${TRACK}/${REVISION}"
+
+touch "${staging_area}/${IMAGE_NAME}/${TRACK}/${REVISION}/dummy.txt"
+
+pushd "${staging_area}"
+
+# SWIFT_CONTAINER_NAME comes from env
+swift upload "$SWIFT_CONTAINER_NAME" "${IMAGE_NAME}" 
diff --git a/src/uploads/upload_to_swift.sh b/src/uploads/upload_to_swift.sh
index e713e992..2f83c2e3 100755
--- a/src/uploads/upload_to_swift.sh
+++ b/src/uploads/upload_to_swift.sh
@@ -22,4 +22,6 @@ cp "$BUILD_METADATA_FILE" "$SBOM_FILE" "$VULN_REPORT_FILE" \
 pushd "${staging_area}"
 
 # SWIFT_CONTAINER_NAME comes from env
-swift upload "$SWIFT_CONTAINER_NAME" "${IMAGE_NAME}" 
+swift upload "$SWIFT_CONTAINER_NAME" "${IMAGE_NAME}"
+# Remove the dummy file uploaded by `upload_dummy_to_swift.sh` in `prepare-upload`
+swift delete "$SWIFT_CONTAINER_NAME" "${IMAGE_NAME}/${TRACK}/${REVISION}/dummy.txt" 

From 00f03a568cffda736b7b50fe420d013f8c612a25 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Wed, 19 Jun 2024 23:38:25 +0200
Subject: [PATCH 04/27] feat(ci): add infer image track to prepare upload

---
 .github/workflows/Image.yaml                  |  5 +-
 .../prepare_single_image_build_matrix.py      | 23 ++++++
 src/image/requirements.txt                    |  1 +
 src/uploads/infer_image_track.py              | 71 ++++++++++---------
 4 files changed, 66 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 30da6da6..55f701b5 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -206,12 +206,13 @@ jobs:
           ./src/image/prepare_single_image_build_matrix.py \
             --oci-path ${{ needs.prepare-build.outputs.oci-img-path }} \
             --revision-data-dir ${{ env.DATA_DIR }} \
-            --next-revision ${{ steps.get-next-revision.outputs.revision }}
+            --next-revision ${{ steps.get-next-revision.outputs.revision }} \
+            --infer-image-track
 
           for revision_file in `ls ${{ env.DATA_DIR }}`; do
             ./src/uploads/upload_dummy_to_swift.sh \
               ${{ needs.prepare-build.outputs.oci-img-name }} \
-              $(jq -r '.release | keys[0]' < ${{ env.DATA_DIR }}/$revision_file) \
+              $(jq -r '.track' < ${{ env.DATA_DIR }}/$revision_file) \
               $revision_file
           done
 
diff --git a/src/image/prepare_single_image_build_matrix.py b/src/image/prepare_single_image_build_matrix.py
index 92843f6b..e8c60204 100755
--- a/src/image/prepare_single_image_build_matrix.py
+++ b/src/image/prepare_single_image_build_matrix.py
@@ -37,6 +37,12 @@ def validate_image_trigger(data: dict) -> None:
         help="Next revision number",
         required=True,
     )
+    parser.add_argument(
+        "--infer-image-track",
+        help="Infer the track corresponding to the releases",
+        action="store_true",
+        default=False,
+    )
 
     args = parser.parse_args()
 
@@ -62,6 +68,23 @@ def validate_image_trigger(data: dict) -> None:
         # within the execution of the workflow - use revision number
         builds[img_number]["revision"] = img_number + int(args.next_revision)
 
+        if args.infer_image_track:
+            import sys
+            sys.path.append("src/")
+            from git import Repo
+            from tempfile import TemporaryDirectory as tempdir
+            from uploads.infer_image_track import get_base_and_track
+            with tempdir() as d:
+                url = f"https://github.com/{builds[img_number]['source']}.git"
+                repo = Repo.clone_from(url, d)
+                repo.git.checkout(builds[img_number]["commit"])
+                # get the base image from the rockcraft.yaml file
+                with open(f"{d}/{builds[img_number]['directory']}/rockcraft.yaml", encoding="UTF-8") as rockcraft_file:
+                    rockcraft_yaml = yaml.load(rockcraft_file, Loader=yaml.BaseLoader)
+
+            _, track = get_base_and_track(rockcraft_yaml)
+            builds[img_number]["track"] = track
+
         with open(
             f"{args.revision_data_dir}/{builds[img_number]['revision']}",
             "w",
diff --git a/src/image/requirements.txt b/src/image/requirements.txt
index c699d2f2..ad81c055 100644
--- a/src/image/requirements.txt
+++ b/src/image/requirements.txt
@@ -1,3 +1,4 @@
+GitPython
 PyYAML
 pydantic==1.9.0
 python-swiftclient
diff --git a/src/uploads/infer_image_track.py b/src/uploads/infer_image_track.py
index e74101ad..bf277e63 100755
--- a/src/uploads/infer_image_track.py
+++ b/src/uploads/infer_image_track.py
@@ -21,40 +21,47 @@ def get_release_from_codename(codename: str) -> str:
     ].split()[1]
 
 
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "--recipe-dirname",
-    help="Path to the directory where rockcraft.yaml is",
-    required=True,
-)
-args = parser.parse_args()
-
-with open(
-    f"{args.recipe_dirname.rstrip('/')}/rockcraft.yaml", encoding="UTF-8"
-) as rockcraft_file:
-    rockcraft_yaml = yaml.load(rockcraft_file, Loader=yaml.BaseLoader)
-
-rock_base = (
-    rockcraft_yaml["base"]
-    if rockcraft_yaml["base"] != "bare"
-    else rockcraft_yaml["build-base"]
-)
-
-try:
-    base_release = float(rock_base.replace(":", "@").split("@")[-1])
-except ValueError:
-    logging.warning(
-        f"Could not infer ROCK's base release from {rock_base}. Trying with codename."
+def get_base_and_track(rockcraft_yaml) -> tuple[str, str]:
+    rock_base = (
+        rockcraft_yaml["base"]
+        if rockcraft_yaml["base"] != "bare"
+        else rockcraft_yaml["build-base"]
     )
-    base_release = float(
-        get_release_from_codename(rock_base.replace(":", "@").split("@")[-1])
+
+    try:
+        base_release = float(rock_base.replace(":", "@").split("@")[-1])
+    except ValueError:
+        logging.warning(
+            f"Could not infer ROCK's base release from {rock_base}. Trying with codename."
+        )
+        base_release = float(
+            get_release_from_codename(rock_base.replace(":", "@").split("@")[-1])
+        )
+
+    version = rockcraft_yaml["version"]
+
+    return base_release, f"{version}-{base_release}"
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--recipe-dirname",
+        help="Path to the directory where rockcraft.yaml is",
+        required=True,
     )
+    args = parser.parse_args()
+
+    with open(
+        f"{args.recipe_dirname.rstrip('/')}/rockcraft.yaml", encoding="UTF-8"
+    ) as rockcraft_file:
+        rockcraft_yaml = yaml.load(rockcraft_file, Loader=yaml.BaseLoader)
+
 
-version = rockcraft_yaml["version"]
+    base_release, track = get_base_and_track(rockcraft_yaml)
 
-track = f"{version}-{base_release}"
-print(f"rock track: {track}")
+    print(f"rock track: {track}")
 
-with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
-    print(f"track={track}", file=gh_out)
-    print(f"base=ubuntu:{base_release}", file=gh_out)
+    with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
+        print(f"track={track}", file=gh_out)
+        print(f"base=ubuntu:{base_release}", file=gh_out)

From 0139dc92dc2139c3d9852281f333280f33f47cfc Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 20 Jun 2024 00:18:47 +0200
Subject: [PATCH 05/27] fix(ci): sanitise revision files in pre release

---
 .github/workflows/Image.yaml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 55f701b5..d1bdd41d 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -465,6 +465,17 @@ jobs:
           key: ${{ needs.prepare-upload.outputs.revision-data-cache-key }}
           fail-on-cache-miss: true
 
+      # The revision files have to be sanitised before merging,
+      # since the `track` and `dir_identifier` fields should not be present
+      - name: Sanitise revision files
+        run: |
+          set -ex
+          for revision_file in `ls ${{ env.REVISION_DATA_DIR }}`
+          do
+            jq 'del(.track)' ${{ env.REVISION_DATA_DIR }}/$revision_file > ${{ env.REVISION_DATA_DIR }}/$revision_file.tmp
+            mv ${{ env.REVISION_DATA_DIR }}/$revision_file.tmp ${{ env.REVISION_DATA_DIR }}/$revision_file
+          done
+
       - run: pip install -r src/image/requirements.txt
 
       - name: Merge release requests

From 9eccbaaae296d0f198b1a7e243b190807b9a7a09 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 20 Jun 2024 10:22:04 +0200
Subject: [PATCH 06/27] refactor(ci): compartment steps in prepare-upload

* Constraint the usage of secrest into its minimal capacity
* Move sanitisation to prepare-upload to leave the prepare-release job
  untouched
---
 .github/workflows/Image.yaml       | 54 +++++++++++++++++-------------
 src/image/define_image_revision.sh |  2 +-
 2 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index d1bdd41d..dccddd78 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -121,8 +121,6 @@ jobs:
             --revision-data-dir ${{ env.DATA_DIR }} \
             --next-revision 0
 
-          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"
-
   run-build:
     needs: [prepare-build]
     strategy:
@@ -175,6 +173,7 @@ jobs:
           ./src/uploads/requirements.sh
           pip install -r src/image/requirements.txt -r src/uploads/requirements.txt
 
+      # Here starts the critical section, have to be executed in sequence outside of matrix.
       - name: Get next revision number
         id: get-next-revision
         env:
@@ -187,20 +186,13 @@ jobs:
           SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: ./src/image/define_image_revision.sh
 
-      # This is a critical section, have to be executed in sequence outside of matrix
-      - name: Prepare builds matrix with revision and preempt swift slots
+      - name: Prepare builds matrix for upload
         id: prepare-matrix
         env:
           DATA_DIR: "revision-data"
-          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
-          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
-          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
-          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
-          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
-          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
-          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: |
           set -ex
+
           mkdir ${{ env.DATA_DIR }}
 
           ./src/image/prepare_single_image_build_matrix.py \
@@ -209,14 +201,39 @@ jobs:
             --next-revision ${{ steps.get-next-revision.outputs.revision }} \
             --infer-image-track
 
+          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"
+
+      - name: Preempt Swift slot
+        env:
+          DATA_DIR: "revision-data"
+          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
+          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
+          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
+          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
+          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
+          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
+          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
+        run: |
           for revision_file in `ls ${{ env.DATA_DIR }}`; do
             ./src/uploads/upload_dummy_to_swift.sh \
-              ${{ needs.prepare-build.outputs.oci-img-name }} \
+              ${{ env.IMAGE_NAME }} \
               $(jq -r '.track' < ${{ env.DATA_DIR }}/$revision_file) \
               $revision_file
           done
 
-          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"
+      # Here leaves the critical section.
+      # The revision files have to be sanitised before merging,
+      # since the `track` field should not be present.
+      - name: Sanitise revision files
+        env:
+          DATA_DIR: "revision-data"
+        run: |
+          set -ex
+          for revision_file in `ls ${{ env.DATA_DIR }}`
+          do
+            jq 'del(.track, .dir_identifier)' ${{ env.DATA_DIR }}/$revision_file > ${{ env.DATA_DIR }}/$revision_file.tmp
+            mv ${{ env.DATA_DIR }}/$revision_file.tmp ${{ env.DATA_DIR }}/$revision_file
+          done
 
       - uses: actions/cache/save@v4
         with:
@@ -465,17 +482,6 @@ jobs:
           key: ${{ needs.prepare-upload.outputs.revision-data-cache-key }}
           fail-on-cache-miss: true
 
-      # The revision files have to be sanitised before merging,
-      # since the `track` and `dir_identifier` fields should not be present
-      - name: Sanitise revision files
-        run: |
-          set -ex
-          for revision_file in `ls ${{ env.REVISION_DATA_DIR }}`
-          do
-            jq 'del(.track)' ${{ env.REVISION_DATA_DIR }}/$revision_file > ${{ env.REVISION_DATA_DIR }}/$revision_file.tmp
-            mv ${{ env.REVISION_DATA_DIR }}/$revision_file.tmp ${{ env.REVISION_DATA_DIR }}/$revision_file
-          done
-
       - run: pip install -r src/image/requirements.txt
 
       - name: Merge release requests
diff --git a/src/image/define_image_revision.sh b/src/image/define_image_revision.sh
index b6a966be..00dad985 100755
--- a/src/image/define_image_revision.sh
+++ b/src/image/define_image_revision.sh
@@ -17,4 +17,4 @@ highest_revision=$(swift list $SWIFT_CONTAINER_NAME -p $IMAGE_NAME \
                     | awk -F'/' '{print $3}')
 
 REVISION=$(( $highest_revision + 1 ))
-echo "revision=${REVISION}" >> "$GITHUB_OUTPUT"
\ No newline at end of file
+echo "revision=${REVISION}" >> "$GITHUB_OUTPUT"

From ed6d2e63d29ea594044b346bb23dca1d3e212d32 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 20 Jun 2024 21:59:57 +0200
Subject: [PATCH 07/27] feat(ci): add lock for Swift slot preemption

---
 .github/workflows/Image.yaml         | 34 +++++++++++++++++++++++++++
 src/uploads/swift_lockfile_lock.sh   | 35 ++++++++++++++++++++++++++++
 src/uploads/swift_lockfile_unlock.sh | 17 ++++++++++++++
 3 files changed, 86 insertions(+)
 create mode 100755 src/uploads/swift_lockfile_lock.sh
 create mode 100755 src/uploads/swift_lockfile_unlock.sh

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index dccddd78..a9ecb8b0 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -173,6 +173,20 @@ jobs:
           ./src/uploads/requirements.sh
           pip install -r src/image/requirements.txt -r src/uploads/requirements.txt
 
+      - name: Upload the lockfile for the image
+        id: swift-lock
+        env:
+          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
+          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
+          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
+          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
+          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
+          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
+          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
+        run: |
+          ./src/uploads/swift_lockfile_lock.sh \
+            ${{ needs.prepare-build.outputs.oci-img-name }}
+
       # Here starts the critical section, have to be executed in sequence outside of matrix.
       - name: Get next revision number
         id: get-next-revision
@@ -222,6 +236,26 @@ jobs:
           done
 
       # Here leaves the critical section.
+      # The lock will be removed even the steps above failed,
+      # or the workflow is cancelled.
+      - name: Remove the lockfile for the image
+        # Runs if the previous steps failed, or the workflow is cancelled.
+        # However, failing to lock the swift container can mean there are
+        # multiple workflows trying to upload the same image at the same time.
+        # Therefore we should not remove the lockfile if the swift lock failed.
+        if: ${{ always() && steps.swift-lock.outcome != 'failure' }}
+        env:
+          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
+          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
+          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
+          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
+          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
+          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
+          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
+        run: |
+          ./src/uploads/swift_lockfile_unlock.sh \
+            ${{ needs.prepare-build.outputs.oci-img-name }}
+
       # The revision files have to be sanitised before merging,
       # since the `track` field should not be present.
       - name: Sanitise revision files
diff --git a/src/uploads/swift_lockfile_lock.sh b/src/uploads/swift_lockfile_lock.sh
new file mode 100755
index 00000000..b455a5cf
--- /dev/null
+++ b/src/uploads/swift_lockfile_lock.sh
@@ -0,0 +1,35 @@
+#!/bin/bash -e
+
+# Source Swift config
+source $(dirname $0)/../configs/swift.public.novarc
+
+set -x
+
+IMAGE_NAME=$1
+# Timeout and sleep time in seconds
+TIMEOUT=${2:-300}
+SLEEP_TIME=5
+
+staging_area=$(mktemp -d)
+
+mkdir -p "${staging_area}/${IMAGE_NAME}"
+
+touch "${staging_area}/${IMAGE_NAME}/lockfile.lock"
+
+pushd "${staging_area}"
+
+# check if the ${IMAGE_NAME}/lockfile.lock exists in the swift container
+# if it does, wait until the timeout is reached and emit an error
+# if it does not, upload the lockfile.lock to the swift container
+# and exit
+while [ $TIMEOUT -gt 0 ]; do
+    swift list $SWIFT_CONTAINER_NAME -p $IMAGE_NAME | grep "lockfile.lock" && sleep $SLEEP_TIME || break
+    TIMEOUT=$(( $TIMEOUT - $SLEEP_TIME ))
+    if [ $TIMEOUT -eq 0 ]; then
+        echo "Timeout reached while waiting for lockfile.lock to be removed from the swift container"
+        exit 1
+    fi
+done
+
+# SWIFT_CONTAINER_NAME comes from env
+swift upload "$SWIFT_CONTAINER_NAME" "${IMAGE_NAME}"
diff --git a/src/uploads/swift_lockfile_unlock.sh b/src/uploads/swift_lockfile_unlock.sh
new file mode 100755
index 00000000..629c1f62
--- /dev/null
+++ b/src/uploads/swift_lockfile_unlock.sh
@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+# Source Swift config
+source $(dirname $0)/../configs/swift.public.novarc
+
+set -x
+
+IMAGE_NAME=$1
+
+# check if the ${IMAGE_NAME}/lockfile.lock exists in the swift container
+# if it does, remove it
+# if it does not, emit an error
+# SWIFT_CONTAINER_NAME comes from env
+LOCKFILE="${IMAGE_NAME}/lockfile.lock"
+swift list $SWIFT_CONTAINER_NAME -p $IMAGE_NAME | grep "$LOCKFILE" && \
+    (swift delete $SWIFT_CONTAINER_NAME "$LOCKFILE" && echo "Lock file removed successfully.") || \
+    echo "Lock file does not exist."

From 9b12c2423b3773cd5eef1fbd4a2321d5359f62ad Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Fri, 21 Jun 2024 11:25:30 +0200
Subject: [PATCH 08/27] fix(ci): add timestamp to cache key for run tries

---
 .github/workflows/Image.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index a9ecb8b0..42625b91 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -215,7 +215,7 @@ jobs:
             --next-revision ${{ steps.get-next-revision.outputs.revision }} \
             --infer-image-track
 
-          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}" >> "$GITHUB_OUTPUT"
+          echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}-$(date +%s)" >> "$GITHUB_OUTPUT"
 
       - name: Preempt Swift slot
         env:

From d677212e3c1897a48e01943ade11d9d799f1e0f9 Mon Sep 17 00:00:00 2001
From: zhijie-yang <zhijie-yang@localhost>
Date: Fri, 21 Jun 2024 09:45:55 +0000
Subject: [PATCH 09/27] ci: automatically update oci/mock-rock/_releases.json,
 from https://github.com/canonical/oci-factory/actions/runs/9611409828

---
 oci/mock-rock/_releases.json | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
index 9e312931..edba677d 100644
--- a/oci/mock-rock/_releases.json
+++ b/oci/mock-rock/_releases.json
@@ -1,68 +1,68 @@
 {
     "latest": {
         "candidate": {
-            "target": "283"
+            "target": "1.0-22.04_candidate"
         },
         "beta": {
-            "target": "283"
+            "target": "latest_candidate"
         },
         "edge": {
-            "target": "283"
+            "target": "latest_beta"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "283"
+            "target": "313"
         },
         "beta": {
-            "target": "283"
+            "target": "313"
         },
         "edge": {
-            "target": "283"
+            "target": "313"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
     "test": {
         "beta": {
-            "target": "283"
+            "target": "1.0-22.04_beta"
         },
         "edge": {
-            "target": "283"
+            "target": "test_beta"
         },
         "end-of-life": "2026-05-01T00:00:00Z"
     },
     "1.1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "284"
+            "target": "314"
         },
         "beta": {
-            "target": "284"
+            "target": "314"
         },
         "edge": {
-            "target": "284"
+            "target": "314"
         }
     },
     "1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "284"
+            "target": "314"
         },
         "beta": {
-            "target": "284"
+            "target": "314"
         },
         "edge": {
-            "target": "284"
+            "target": "314"
         }
     },
     "1.2-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "beta": {
-            "target": "285"
+            "target": "315"
         },
         "edge": {
-            "target": "285"
+            "target": "1.2-22.04_beta"
         }
     }
 }
\ No newline at end of file

From f49b0866d04e89b3a88a8bb16b60270addac0c9a Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 27 Jun 2024 15:10:36 +0200
Subject: [PATCH 10/27] chore(ci): rename preempt script and lock timeout
 output

---
 .github/workflows/Image.yaml                                    | 2 +-
 src/uploads/{upload_dummy_to_swift.sh => preempt_swift_slot.sh} | 0
 src/uploads/swift_lockfile_lock.sh                              | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename src/uploads/{upload_dummy_to_swift.sh => preempt_swift_slot.sh} (100%)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 42625b91..a76314e0 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -229,7 +229,7 @@ jobs:
           SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: |
           for revision_file in `ls ${{ env.DATA_DIR }}`; do
-            ./src/uploads/upload_dummy_to_swift.sh \
+            ./src/uploads/preempt_swift_slots.sh \
               ${{ env.IMAGE_NAME }} \
               $(jq -r '.track' < ${{ env.DATA_DIR }}/$revision_file) \
               $revision_file
diff --git a/src/uploads/upload_dummy_to_swift.sh b/src/uploads/preempt_swift_slot.sh
similarity index 100%
rename from src/uploads/upload_dummy_to_swift.sh
rename to src/uploads/preempt_swift_slot.sh
diff --git a/src/uploads/swift_lockfile_lock.sh b/src/uploads/swift_lockfile_lock.sh
index b455a5cf..1b251913 100755
--- a/src/uploads/swift_lockfile_lock.sh
+++ b/src/uploads/swift_lockfile_lock.sh
@@ -26,7 +26,7 @@ while [ $TIMEOUT -gt 0 ]; do
     swift list $SWIFT_CONTAINER_NAME -p $IMAGE_NAME | grep "lockfile.lock" && sleep $SLEEP_TIME || break
     TIMEOUT=$(( $TIMEOUT - $SLEEP_TIME ))
     if [ $TIMEOUT -eq 0 ]; then
-        echo "Timeout reached while waiting for lockfile.lock to be removed from the swift container"
+        echo "Timeout reached while waiting to write lockfile into the Swift container for ${IMAGE_NAME}."
         exit 1
     fi
 done

From 8ce4c0d99e7892c398c0ccd5e40d9ec32c596405 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 27 Jun 2024 15:15:57 +0200
Subject: [PATCH 11/27] feat(ci): move env from steps to job preupload

---
 .github/workflows/Image.yaml | 39 ++++++++----------------------------
 1 file changed, 8 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index a76314e0..cf2823fe 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -155,6 +155,14 @@ jobs:
     runs-on: ubuntu-22.04
     needs: [prepare-build, run-build, test]
     name: Prepare upload
+    env:
+      OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
+      OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
+      OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
+      OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
+      OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
+      IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
+      SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
     outputs:
       build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
       revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
@@ -175,14 +183,6 @@ jobs:
 
       - name: Upload the lockfile for the image
         id: swift-lock
-        env:
-          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
-          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
-          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
-          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
-          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
-          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
-          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: |
           ./src/uploads/swift_lockfile_lock.sh \
             ${{ needs.prepare-build.outputs.oci-img-name }}
@@ -190,14 +190,6 @@ jobs:
       # Here starts the critical section, have to be executed in sequence outside of matrix.
       - name: Get next revision number
         id: get-next-revision
-        env:
-          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
-          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
-          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
-          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
-          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
-          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
-          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: ./src/image/define_image_revision.sh
 
       - name: Prepare builds matrix for upload
@@ -220,13 +212,6 @@ jobs:
       - name: Preempt Swift slot
         env:
           DATA_DIR: "revision-data"
-          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
-          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
-          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
-          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
-          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
-          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
-          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: |
           for revision_file in `ls ${{ env.DATA_DIR }}`; do
             ./src/uploads/preempt_swift_slots.sh \
@@ -244,14 +229,6 @@ jobs:
         # multiple workflows trying to upload the same image at the same time.
         # Therefore we should not remove the lockfile if the swift lock failed.
         if: ${{ always() && steps.swift-lock.outcome != 'failure' }}
-        env:
-          OS_USERNAME: ${{ secrets.SWIFT_OS_USERNAME }}
-          OS_TENANT_NAME: ${{ secrets.SWIFT_OS_TENANT_NAME }}
-          OS_PASSWORD: ${{ secrets.SWIFT_OS_PASSWORD }}
-          OS_REGION_NAME: ${{ secrets.SWIFT_OS_REGION_NAME }}
-          OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
-          IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
-          SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: |
           ./src/uploads/swift_lockfile_unlock.sh \
             ${{ needs.prepare-build.outputs.oci-img-name }}

From 92e27a65e9c43bb7e9ffe0fe84f3536551a9b701 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 27 Jun 2024 16:24:32 +0200
Subject: [PATCH 12/27] debug(ci): temporarily disable ppc64el for mock-rock

---
 examples/mock-rock/1.2/rockcraft.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/mock-rock/1.2/rockcraft.yaml b/examples/mock-rock/1.2/rockcraft.yaml
index 4045518f..7dae89a1 100644
--- a/examples/mock-rock/1.2/rockcraft.yaml
+++ b/examples/mock-rock/1.2/rockcraft.yaml
@@ -6,11 +6,11 @@ base: bare
 build-base: ubuntu@22.04
 license: Apache-2.0
 platforms:
-  ppc64el:
+  # ppc64el:
   amd64:
 
 parts:
   hello:
     plugin: nil
     stage-packages:
-      - hello
\ No newline at end of file
+      - hello

From e3649157196189905809ef9614d1569d1f474e38 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 27 Jun 2024 16:28:06 +0200
Subject: [PATCH 13/27] debug(ci): temporarily disable ppc64el for mock-rock -2

---
 oci/mock-rock/image.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oci/mock-rock/image.yaml b/oci/mock-rock/image.yaml
index 72a34809..9711e57e 100644
--- a/oci/mock-rock/image.yaml
+++ b/oci/mock-rock/image.yaml
@@ -36,7 +36,7 @@ upload:
           - edge
           - beta
   - source: "canonical/oci-factory"
-    commit: 486799e24328410678c3534381a5473a46b07d06
+    commit: 92e27a65e9c43bb7e9ffe0fe84f3536551a9b701
     directory: examples/mock-rock/1.2
     release:
       1.2-22.04:

From d9f2f73d0313cd0c1c5c3a306ce4405b1d688ddb Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 27 Jun 2024 16:25:37 +0200
Subject: [PATCH 14/27] feat(ci): remove infer track from upload

* Remove dir-identifer and track from revision data file and put into
  matrix

Signed-off-by: Zhijie Yang <zhijie.yang@canonical.com>
---
 .github/workflows/Image.yaml                  | 30 +++----------------
 .../prepare_single_image_build_matrix.py      |  8 ++---
 2 files changed, 7 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index cf2823fe..52ef7e3c 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -119,7 +119,6 @@ jobs:
           ./src/image/prepare_single_image_build_matrix.py \
             --oci-path ${{ steps.validate-image.outputs.img-path }} \
             --revision-data-dir ${{ env.DATA_DIR }} \
-            --next-revision 0
 
   run-build:
     needs: [prepare-build]
@@ -242,7 +241,7 @@ jobs:
           set -ex
           for revision_file in `ls ${{ env.DATA_DIR }}`
           do
-            jq 'del(.track, .dir_identifier)' ${{ env.DATA_DIR }}/$revision_file > ${{ env.DATA_DIR }}/$revision_file.tmp
+            jq 'del(.track, .base)' ${{ env.DATA_DIR }}/$revision_file > ${{ env.DATA_DIR }}/$revision_file.tmp
             mv ${{ env.DATA_DIR }}/$revision_file.tmp ${{ env.DATA_DIR }}/$revision_file
           done
 
@@ -283,38 +282,17 @@ jobs:
           ./src/uploads/requirements.sh
           pip install -r src/uploads/requirements.txt -r src/image/requirements.txt
 
-      - name: Clone GitHub image repository
-        uses: actions/checkout@v4
-        id: clone-image-repo
-        continue-on-error: true
-        with:
-          repository: ${{ matrix.source }}
-          fetch-depth: 0
-          path: source
-
-      - name: Clone generic image repository
-        if: ${{ steps.clone-image-repo.outcome == 'failure' }}
-        run: |
-          git clone ${{ matrix.source }} source
-
-      - run: cd source && git checkout ${{ matrix.commit }}
-
       - uses: actions/cache/restore@v4
         with:
           path: ${{ env.OCI_ARCHIVE_NAME }}
           key: ${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.dir_identifier }}
           fail-on-cache-miss: true
 
-      - name: Infer track name
-        id: get-track
-        run: |
-          ./src/uploads/infer_image_track.py --recipe-dirname source/${{ matrix.directory }}
-
       - name: Name output artefact
         id: rename-oci-archive
         run: |
           # Rename the OCI archive tarball
-          canonical_tag="${{ steps.get-track.outputs.track }}_${{ matrix.revision }}"
+          canonical_tag="${{ matrix.track }}_${{ matrix.revision }}"
           name="${{ matrix.name }}_${canonical_tag}"
           mv ${{ env.OCI_ARCHIVE_NAME }} $name
 
@@ -442,12 +420,12 @@ jobs:
           IMAGE_NAME: ${{ matrix.name }}
           SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
         run: |
-          jq --arg base "${{ steps.get-track.outputs.base }}" \
+          jq --arg base "${{ matrix.base }}" \
             --arg digest "${{ steps.upload-image.outputs.digest }}" \
             '. + {base: $base, digest: $digest}' <<< '${{ toJSON(matrix) }}' > build_metadata.json
           ./src/uploads/upload_to_swift.sh \
             ${{ matrix.name }} \
-            ${{ steps.get-track.outputs.track }} \
+            ${{ matrix.track }} \
             ${{ matrix.revision }} \
             build_metadata.json \
             ${{ steps.generate-sboms.outputs.sboms }} \
diff --git a/src/image/prepare_single_image_build_matrix.py b/src/image/prepare_single_image_build_matrix.py
index e8c60204..2c44b9fb 100755
--- a/src/image/prepare_single_image_build_matrix.py
+++ b/src/image/prepare_single_image_build_matrix.py
@@ -35,7 +35,7 @@ def validate_image_trigger(data: dict) -> None:
     parser.add_argument(
         "--next-revision",
         help="Next revision number",
-        required=True,
+        default=1,
     )
     parser.add_argument(
         "--infer-image-track",
@@ -63,9 +63,6 @@ def validate_image_trigger(data: dict) -> None:
     for img_number, _ in enumerate(builds):
         builds[img_number]["name"] = args.oci_path.rstrip("/").split("/")[-1]
         builds[img_number]["path"] = args.oci_path
-        builds[img_number]["dir_identifier"] = builds[img_number]["directory"].rstrip("/").replace("/", "_")
-        # make sure every build of this image has a unique identifier
-        # within the execution of the workflow - use revision number
         builds[img_number]["revision"] = img_number + int(args.next_revision)
 
         if args.infer_image_track:
@@ -82,8 +79,9 @@ def validate_image_trigger(data: dict) -> None:
                 with open(f"{d}/{builds[img_number]['directory']}/rockcraft.yaml", encoding="UTF-8") as rockcraft_file:
                     rockcraft_yaml = yaml.load(rockcraft_file, Loader=yaml.BaseLoader)
 
-            _, track = get_base_and_track(rockcraft_yaml)
+            base_release, track = get_base_and_track(rockcraft_yaml)
             builds[img_number]["track"] = track
+            builds[img_number]["base"] = f"ubuntu:{base_release}"
 
         with open(
             f"{args.revision_data_dir}/{builds[img_number]['revision']}",

From 3a466b4f1a5f2f87aa2173d71badd6ecf209298b Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 27 Jun 2024 16:48:43 +0200
Subject: [PATCH 15/27] fix(ci): typo in preempt_swift_slot.sh

---
 .github/workflows/Image.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 52ef7e3c..793a9fbf 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -213,7 +213,7 @@ jobs:
           DATA_DIR: "revision-data"
         run: |
           for revision_file in `ls ${{ env.DATA_DIR }}`; do
-            ./src/uploads/preempt_swift_slots.sh \
+            ./src/uploads/preempt_swift_slot.sh \
               ${{ env.IMAGE_NAME }} \
               $(jq -r '.track' < ${{ env.DATA_DIR }}/$revision_file) \
               $revision_file

From 6ce5b1df86f3b82c51a4fc6279819da815f5c7f1 Mon Sep 17 00:00:00 2001
From: zhijie-yang <zhijie-yang@localhost>
Date: Thu, 27 Jun 2024 16:17:38 +0000
Subject: [PATCH 16/27] ci: automatically update oci/mock-rock/_releases.json,
 from https://github.com/canonical/oci-factory/actions/runs/9699914359

---
 oci/mock-rock/_releases.json | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
index edba677d..74f47c7d 100644
--- a/oci/mock-rock/_releases.json
+++ b/oci/mock-rock/_releases.json
@@ -13,13 +13,13 @@
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "313"
+            "target": "325"
         },
         "beta": {
-            "target": "313"
+            "target": "325"
         },
         "edge": {
-            "target": "313"
+            "target": "325"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
@@ -35,31 +35,31 @@
     "1.1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "314"
+            "target": "326"
         },
         "beta": {
-            "target": "314"
+            "target": "326"
         },
         "edge": {
-            "target": "314"
+            "target": "326"
         }
     },
     "1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "314"
+            "target": "326"
         },
         "beta": {
-            "target": "314"
+            "target": "326"
         },
         "edge": {
-            "target": "314"
+            "target": "326"
         }
     },
     "1.2-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "beta": {
-            "target": "315"
+            "target": "327"
         },
         "edge": {
             "target": "1.2-22.04_beta"

From b2c8e7bb32218ab76f2da883b41a6884519833c6 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Fri, 28 Jun 2024 16:02:48 +0200
Subject: [PATCH 17/27] feat(ci): reduce preemption swift update times to 1

---
 .github/workflows/Image.yaml       |  7 +------
 src/uploads/preempt_swift_slot.sh  | 21 ---------------------
 src/uploads/preempt_swift_slots.sh | 22 ++++++++++++++++++++++
 3 files changed, 23 insertions(+), 27 deletions(-)
 delete mode 100755 src/uploads/preempt_swift_slot.sh
 create mode 100755 src/uploads/preempt_swift_slots.sh

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 793a9fbf..168d9c32 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -212,12 +212,7 @@ jobs:
         env:
           DATA_DIR: "revision-data"
         run: |
-          for revision_file in `ls ${{ env.DATA_DIR }}`; do
-            ./src/uploads/preempt_swift_slot.sh \
-              ${{ env.IMAGE_NAME }} \
-              $(jq -r '.track' < ${{ env.DATA_DIR }}/$revision_file) \
-              $revision_file
-          done
+          ./src/uploads/preempt_swift_slots.sh ${{ env.DATA_DIR }}
 
       # Here leaves the critical section.
       # The lock will be removed even the steps above failed,
diff --git a/src/uploads/preempt_swift_slot.sh b/src/uploads/preempt_swift_slot.sh
deleted file mode 100755
index e892e95f..00000000
--- a/src/uploads/preempt_swift_slot.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash -e
-
-# Source Swift config
-source $(dirname $0)/../configs/swift.public.novarc
-
-set -x
-
-IMAGE_NAME=$1
-TRACK=$2
-REVISION=$3
-
-staging_area=$(mktemp -d)
-
-mkdir -p "${staging_area}/${IMAGE_NAME}/${TRACK}/${REVISION}"
-
-touch "${staging_area}/${IMAGE_NAME}/${TRACK}/${REVISION}/dummy.txt"
-
-pushd "${staging_area}"
-
-# SWIFT_CONTAINER_NAME comes from env
-swift upload "$SWIFT_CONTAINER_NAME" "${IMAGE_NAME}" 
diff --git a/src/uploads/preempt_swift_slots.sh b/src/uploads/preempt_swift_slots.sh
new file mode 100755
index 00000000..87226e72
--- /dev/null
+++ b/src/uploads/preempt_swift_slots.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -e
+
+# Source Swift config
+source $(dirname $0)/../configs/swift.public.novarc
+
+set -x
+
+REVISION_DATA_DIR=$1
+
+staging_area=$(mktemp -d)
+
+for revision in $(ls ${REVISION_DATA_DIR}); do
+    image_name=$(jq -r '.name' "${REVISION_DATA_DIR}/${revision}")
+    track=$(jq -r '.track' "${REVISION_DATA_DIR}/${revision}")
+    mkdir -p "${staging_area}/${image_name}/${track}/${revision}"
+    touch "${staging_area}/${image_name}/${track}/${revision}/dummy.txt"
+done
+
+pushd "${staging_area}"
+
+# SWIFT_CONTAINER_NAME comes from env
+swift upload "$SWIFT_CONTAINER_NAME" "${IMAGE_NAME}"

From a866f38125844c55f21b0cfc5f7ac2a26b2b64e2 Mon Sep 17 00:00:00 2001
From: zhijie-yang <zhijie-yang@localhost>
Date: Fri, 28 Jun 2024 14:24:57 +0000
Subject: [PATCH 18/27] ci: automatically update oci/mock-rock/_releases.json,
 from https://github.com/canonical/oci-factory/actions/runs/9714206734

---
 oci/mock-rock/_releases.json | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
index 74f47c7d..7c0c783a 100644
--- a/oci/mock-rock/_releases.json
+++ b/oci/mock-rock/_releases.json
@@ -13,13 +13,13 @@
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "325"
+            "target": "334"
         },
         "beta": {
-            "target": "325"
+            "target": "334"
         },
         "edge": {
-            "target": "325"
+            "target": "334"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
@@ -35,31 +35,31 @@
     "1.1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "326"
+            "target": "335"
         },
         "beta": {
-            "target": "326"
+            "target": "335"
         },
         "edge": {
-            "target": "326"
+            "target": "335"
         }
     },
     "1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "326"
+            "target": "335"
         },
         "beta": {
-            "target": "326"
+            "target": "335"
         },
         "edge": {
-            "target": "326"
+            "target": "335"
         }
     },
     "1.2-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "beta": {
-            "target": "327"
+            "target": "336"
         },
         "edge": {
             "target": "1.2-22.04_beta"

From ed50adb2bf061e062fac30360ddf90e0e24619c8 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Fri, 28 Jun 2024 17:59:42 +0200
Subject: [PATCH 19/27] fix(ci): fix accord. to code review

---
 src/image/requirements.txt         | 3 +--
 src/uploads/swift_lockfile_lock.sh | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/image/requirements.txt b/src/image/requirements.txt
index ad81c055..6ea9d920 100644
--- a/src/image/requirements.txt
+++ b/src/image/requirements.txt
@@ -1,7 +1,6 @@
 GitPython
 PyYAML
 pydantic==1.9.0
+pytest
 python-swiftclient
 python-keystoneclient
-
-pytest
diff --git a/src/uploads/swift_lockfile_lock.sh b/src/uploads/swift_lockfile_lock.sh
index 1b251913..68d0efd6 100755
--- a/src/uploads/swift_lockfile_lock.sh
+++ b/src/uploads/swift_lockfile_lock.sh
@@ -25,7 +25,7 @@ pushd "${staging_area}"
 while [ $TIMEOUT -gt 0 ]; do
     swift list $SWIFT_CONTAINER_NAME -p $IMAGE_NAME | grep "lockfile.lock" && sleep $SLEEP_TIME || break
     TIMEOUT=$(( $TIMEOUT - $SLEEP_TIME ))
-    if [ $TIMEOUT -eq 0 ]; then
+    if [ $TIMEOUT -lt 1 ]; then
         echo "Timeout reached while waiting to write lockfile into the Swift container for ${IMAGE_NAME}."
         exit 1
     fi

From 50101de4184a2b39a82f0a4216713500ed7cdf63 Mon Sep 17 00:00:00 2001
From: zhijie-yang <zhijie-yang@localhost>
Date: Fri, 28 Jun 2024 16:11:35 +0000
Subject: [PATCH 20/27] ci: automatically update oci/mock-rock/_releases.json,
 from https://github.com/canonical/oci-factory/actions/runs/9715580509

---
 oci/mock-rock/_releases.json | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
index 7c0c783a..e5248c3f 100644
--- a/oci/mock-rock/_releases.json
+++ b/oci/mock-rock/_releases.json
@@ -13,13 +13,13 @@
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "334"
+            "target": "337"
         },
         "beta": {
-            "target": "334"
+            "target": "337"
         },
         "edge": {
-            "target": "334"
+            "target": "337"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
@@ -35,31 +35,31 @@
     "1.1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "335"
+            "target": "338"
         },
         "beta": {
-            "target": "335"
+            "target": "338"
         },
         "edge": {
-            "target": "335"
+            "target": "338"
         }
     },
     "1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "335"
+            "target": "338"
         },
         "beta": {
-            "target": "335"
+            "target": "338"
         },
         "edge": {
-            "target": "335"
+            "target": "338"
         }
     },
     "1.2-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "beta": {
-            "target": "336"
+            "target": "339"
         },
         "edge": {
             "target": "1.2-22.04_beta"

From c613d315ebf1ca7fdcc95a1e9574fe0cdd64fa05 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 4 Jul 2024 10:40:23 +0200
Subject: [PATCH 21/27] fix(ci): changes according to code review

---
 .github/workflows/Image.yaml       | 14 ++++----------
 src/uploads/swift_lockfile_lock.sh |  7 +++++--
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 168d9c32..171d4927 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -162,6 +162,7 @@ jobs:
       OS_STORAGE_URL: ${{ secrets.SWIFT_OS_STORAGE_URL }}
       IMAGE_NAME: ${{ needs.prepare-build.outputs.oci-img-name }}
       SWIFT_CONTAINER_NAME: ${{ vars.SWIFT_CONTAINER_NAME }}
+      DATA_DIR: "revision-data"
     outputs:
       build-matrix: ${{ steps.prepare-matrix.outputs.build-matrix }}
       revision-data-cache-key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
@@ -193,8 +194,6 @@ jobs:
 
       - name: Prepare builds matrix for upload
         id: prepare-matrix
-        env:
-          DATA_DIR: "revision-data"
         run: |
           set -ex
 
@@ -209,18 +208,15 @@ jobs:
           echo "revision-data-cache-key=${{ github.run_id }}-${{ env.DATA_DIR }}-$(date +%s)" >> "$GITHUB_OUTPUT"
 
       - name: Preempt Swift slot
-        env:
-          DATA_DIR: "revision-data"
         run: |
           ./src/uploads/preempt_swift_slots.sh ${{ env.DATA_DIR }}
 
       # Here leaves the critical section.
-      # The lock will be removed even the steps above failed,
+      # The lock will be removed even if the steps above fail,
       # or the workflow is cancelled.
       - name: Remove the lockfile for the image
-        # Runs if the previous steps failed, or the workflow is cancelled.
-        # However, failing to lock the swift container can mean there are
-        # multiple workflows trying to upload the same image at the same time.
+        # Failing to lock the swift container can mean there are multiple
+        # workflows trying to upload the same image at the same time.
         # Therefore we should not remove the lockfile if the swift lock failed.
         if: ${{ always() && steps.swift-lock.outcome != 'failure' }}
         run: |
@@ -230,8 +226,6 @@ jobs:
       # The revision files have to be sanitised before merging,
       # since the `track` field should not be present.
       - name: Sanitise revision files
-        env:
-          DATA_DIR: "revision-data"
         run: |
           set -ex
           for revision_file in `ls ${{ env.DATA_DIR }}`
diff --git a/src/uploads/swift_lockfile_lock.sh b/src/uploads/swift_lockfile_lock.sh
index 68d0efd6..e3da5d4f 100755
--- a/src/uploads/swift_lockfile_lock.sh
+++ b/src/uploads/swift_lockfile_lock.sh
@@ -18,10 +18,13 @@ touch "${staging_area}/${IMAGE_NAME}/lockfile.lock"
 
 pushd "${staging_area}"
 
-# check if the ${IMAGE_NAME}/lockfile.lock exists in the swift container
+# Check if the ${IMAGE_NAME}/lockfile.lock exists in the swift container
 # if it does, wait until the timeout is reached and emit an error
 # if it does not, upload the lockfile.lock to the swift container
-# and exit
+# and exit.
+# There's still the unlikely corner case where 2 concurrent jobs 
+# are waiting for the lockfile to get removed, and they may exit 
+# the while loop at the same time, getting into a race condition.
 while [ $TIMEOUT -gt 0 ]; do
     swift list $SWIFT_CONTAINER_NAME -p $IMAGE_NAME | grep "lockfile.lock" && sleep $SLEEP_TIME || break
     TIMEOUT=$(( $TIMEOUT - $SLEEP_TIME ))

From c8a049750c84ffffad7f4c8360b9e9a19e8fabd5 Mon Sep 17 00:00:00 2001
From: zhijie-yang <zhijie-yang@localhost>
Date: Thu, 4 Jul 2024 08:54:09 +0000
Subject: [PATCH 22/27] ci: automatically update oci/mock-rock/_releases.json,
 from https://github.com/canonical/oci-factory/actions/runs/9791204886

---
 oci/mock-rock/_releases.json | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
index e5248c3f..82786458 100644
--- a/oci/mock-rock/_releases.json
+++ b/oci/mock-rock/_releases.json
@@ -13,13 +13,13 @@
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "337"
+            "target": "343"
         },
         "beta": {
-            "target": "337"
+            "target": "343"
         },
         "edge": {
-            "target": "337"
+            "target": "343"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
@@ -35,31 +35,31 @@
     "1.1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "338"
+            "target": "344"
         },
         "beta": {
-            "target": "338"
+            "target": "344"
         },
         "edge": {
-            "target": "338"
+            "target": "344"
         }
     },
     "1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "338"
+            "target": "344"
         },
         "beta": {
-            "target": "338"
+            "target": "344"
         },
         "edge": {
-            "target": "338"
+            "target": "344"
         }
     },
     "1.2-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "beta": {
-            "target": "339"
+            "target": "345"
         },
         "edge": {
             "target": "1.2-22.04_beta"

From 076d2dc63c205a4489af33dde4b170e4d39c89fb Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Thu, 4 Jul 2024 14:00:26 +0200
Subject: [PATCH 23/27] feat(ci): combine dir validator into ImageSchema

---
 .github/workflows/Image.yaml                  |  5 --
 ...st_validate_unique_trigger_upload_entry.py | 69 -------------------
 src/image/utils/schema/triggers.py            | 17 +++++
 .../validate_unique_trigger_upload_entry.py   | 40 -----------
 4 files changed, 17 insertions(+), 114 deletions(-)
 delete mode 100644 src/image/test_validate_unique_trigger_upload_entry.py
 delete mode 100755 src/image/validate_unique_trigger_upload_entry.py

diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml
index 171d4927..656dd5d4 100644
--- a/.github/workflows/Image.yaml
+++ b/.github/workflows/Image.yaml
@@ -104,11 +104,6 @@ jobs:
 
       - run: pip install -r src/image/requirements.txt
 
-      - name: Validate unique trigger upload entry
-        run: |
-          ./src/image/validate_unique_trigger_upload_entry.py \
-            --image-trigger ${{ steps.validate-image.outputs.img-path }}/image.yaml
-
       - name: Validate and prepare builds matrix
         id: prepare-matrix
         env:
diff --git a/src/image/test_validate_unique_trigger_upload_entry.py b/src/image/test_validate_unique_trigger_upload_entry.py
deleted file mode 100644
index 71a1096c..00000000
--- a/src/image/test_validate_unique_trigger_upload_entry.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import argparse
-import os
-import pytest
-import yaml
-
-from tempfile import TemporaryDirectory as tempdir
-
-from validate_unique_trigger_upload_entry import *
-
-
-def test_validate_unique_trigger_entry():
-    # Create a temporary image trigger file for testing
-    image_trigger_data = {
-        "version": 1,
-        "upload": [
-            {
-                "source": "source1",
-                "commit": "commit1",
-                "directory": "directory1"
-            },
-            {
-                "source": "source2",
-                "commit": "commit2",
-                "directory": "directory2"
-            }
-        ]
-    }
-
-    with tempdir() as tmp_dir:
-        image_trigger_file = os.path.join(tmp_dir, "trigger.yaml")
-        with open(image_trigger_file, "w") as f:
-            yaml.dump(image_trigger_data, f)
-
-        args = argparse.Namespace(image_trigger=image_trigger_file)
-        main(args)
-
-
-def test_validate_unique_trigger_entry_non_unique():
-    # Create a temporary image trigger file for testing
-    image_trigger_data = {
-        "version": 1,
-        "upload": [
-            {
-                "source": "source1",
-                "commit": "commit1",
-                "directory": "directory1"
-            },
-            {
-                "source": "source2",
-                "commit": "commit2",
-                "directory": "directory2"
-            },
-            {
-                "source": "source1",
-                "commit": "commit1",
-                "directory": "directory1"
-            }
-        ]
-    }
-    with tempdir() as tmp_dir:
-        image_trigger_file = os.path.join(tmp_dir, "trigger.yaml")
-        with open(image_trigger_file, "w") as f:
-            yaml.dump(image_trigger_data, f)
-
-        # Call the main function and assert that it raises a ValueError]
-        args = argparse.Namespace(image_trigger=image_trigger_file)
-        with pytest.raises(ValueError) as e:
-            main(args)
-            assert str(e.value) == "Image trigger source1_commit1_directory1 is not unique."
diff --git a/src/image/utils/schema/triggers.py b/src/image/utils/schema/triggers.py
index 221c9ce0..49e928ae 100644
--- a/src/image/utils/schema/triggers.py
+++ b/src/image/utils/schema/triggers.py
@@ -83,3 +83,20 @@ class ImageSchema(pydantic.BaseModel):
 
     class Config:
         extra = pydantic.Extra.forbid
+
+    @pydantic.validator("upload")
+    def ensure_unique_triggers(
+        cls, v: Optional[List[ImageUploadSchema]]
+    ) -> Optional[List[ImageUploadSchema]]:
+        """Ensure that the triggers are unique."""
+        if not v:
+            return v
+        unique_triggers = set()
+        for upload in v:
+            trigger = f"{upload.source}_{upload.commit}_{upload.directory}"
+            if trigger in unique_triggers:
+                raise ImageTriggerValidationError(
+                    f"Image trigger {trigger} is not unique."
+                )
+            unique_triggers.add(trigger)
+        return v
diff --git a/src/image/validate_unique_trigger_upload_entry.py b/src/image/validate_unique_trigger_upload_entry.py
deleted file mode 100755
index 88f3ed20..00000000
--- a/src/image/validate_unique_trigger_upload_entry.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-This script validates that the image trigger entry is unique in the
-given image.yaml file. This is necessary to avoid having multiple
-entries for the same image trigger, which would lead to ambiguous
-results when using the combination of {source}_{commit}_{directory}
-as the cache key for the Image workflow runs.
-"""
-
-import argparse
-import yaml
-
-from utils.schema.triggers import ImageSchema
-
-
-def main(args):
-    print(f"Validating unique upload entry for {args.image_trigger}")
-    with open(args.image_trigger, encoding="UTF-8") as trigger:
-        image_trigger = yaml.load(trigger, Loader=yaml.BaseLoader)
-
-    schema = ImageSchema(**image_trigger)
-    unique_triggers = set()
-    for upload in  schema.upload:
-        trigger = f"{upload.source}_{upload.commit}_{upload.directory}"
-        if trigger in unique_triggers:
-            raise ValueError(f"Image trigger {trigger} is not unique.")
-        unique_triggers.add(trigger)
-    print("OK")
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--image-trigger",
-        help="Path to the image trigger file.",
-        required=True,
-    )
-    args = parser.parse_args()
-    
-    main(args)

From f043633878422ebf4dc4c865d30028b06ec1cb10 Mon Sep 17 00:00:00 2001
From: zhijie-yang <zhijie-yang@localhost>
Date: Thu, 4 Jul 2024 12:14:46 +0000
Subject: [PATCH 24/27] ci: automatically update oci/mock-rock/_releases.json,
 from https://github.com/canonical/oci-factory/actions/runs/9793913592

---
 oci/mock-rock/_releases.json | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
index 82786458..14df0df5 100644
--- a/oci/mock-rock/_releases.json
+++ b/oci/mock-rock/_releases.json
@@ -13,13 +13,13 @@
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "343"
+            "target": "352"
         },
         "beta": {
-            "target": "343"
+            "target": "352"
         },
         "edge": {
-            "target": "343"
+            "target": "352"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
@@ -35,31 +35,31 @@
     "1.1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "344"
+            "target": "353"
         },
         "beta": {
-            "target": "344"
+            "target": "353"
         },
         "edge": {
-            "target": "344"
+            "target": "353"
         }
     },
     "1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "344"
+            "target": "353"
         },
         "beta": {
-            "target": "344"
+            "target": "353"
         },
         "edge": {
-            "target": "344"
+            "target": "353"
         }
     },
     "1.2-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "beta": {
-            "target": "345"
+            "target": "354"
         },
         "edge": {
             "target": "1.2-22.04_beta"

From b7f477951c812d37f0318039960ada94788aabf3 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Tue, 16 Jul 2024 10:27:33 +0200
Subject: [PATCH 25/27] Revert "debug(ci): temporarily disable ppc64el for
 mock-rock"

This reverts commit 92e27a65e9c43bb7e9ffe0fe84f3536551a9b701.
---
 examples/mock-rock/1.2/rockcraft.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/mock-rock/1.2/rockcraft.yaml b/examples/mock-rock/1.2/rockcraft.yaml
index 7dae89a1..4045518f 100644
--- a/examples/mock-rock/1.2/rockcraft.yaml
+++ b/examples/mock-rock/1.2/rockcraft.yaml
@@ -6,11 +6,11 @@ base: bare
 build-base: ubuntu@22.04
 license: Apache-2.0
 platforms:
-  # ppc64el:
+  ppc64el:
   amd64:
 
 parts:
   hello:
     plugin: nil
     stage-packages:
-      - hello
+      - hello
\ No newline at end of file

From 105f1bf764fdc7b777938c68b849da806c1d4726 Mon Sep 17 00:00:00 2001
From: Zhijie Yang <zhijie.yang@canonical.com>
Date: Tue, 16 Jul 2024 10:27:50 +0200
Subject: [PATCH 26/27] Revert "debug(ci): temporarily disable ppc64el for
 mock-rock -2"

This reverts commit e3649157196189905809ef9614d1569d1f474e38.
---
 oci/mock-rock/image.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oci/mock-rock/image.yaml b/oci/mock-rock/image.yaml
index 9711e57e..72a34809 100644
--- a/oci/mock-rock/image.yaml
+++ b/oci/mock-rock/image.yaml
@@ -36,7 +36,7 @@ upload:
           - edge
           - beta
   - source: "canonical/oci-factory"
-    commit: 92e27a65e9c43bb7e9ffe0fe84f3536551a9b701
+    commit: 486799e24328410678c3534381a5473a46b07d06
     directory: examples/mock-rock/1.2
     release:
       1.2-22.04:

From e79a19bb6370f1d316649b509e759f0cf95ce88a Mon Sep 17 00:00:00 2001
From: zhijie-yang <zhijie-yang@localhost>
Date: Tue, 16 Jul 2024 09:39:47 +0000
Subject: [PATCH 27/27] ci: automatically update oci/mock-rock/_releases.json,
 from https://github.com/canonical/oci-factory/actions/runs/9954192147

---
 oci/mock-rock/_releases.json | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/oci/mock-rock/_releases.json b/oci/mock-rock/_releases.json
index df1262c1..a5f29baa 100644
--- a/oci/mock-rock/_releases.json
+++ b/oci/mock-rock/_releases.json
@@ -13,13 +13,13 @@
     },
     "1.0-22.04": {
         "candidate": {
-            "target": "370"
+            "target": "379"
         },
         "beta": {
-            "target": "370"
+            "target": "379"
         },
         "edge": {
-            "target": "370"
+            "target": "379"
         },
         "end-of-life": "2025-05-01T00:00:00Z"
     },
@@ -35,31 +35,31 @@
     "1.1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "371"
+            "target": "380"
         },
         "beta": {
-            "target": "371"
+            "target": "380"
         },
         "edge": {
-            "target": "371"
+            "target": "380"
         }
     },
     "1-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "candidate": {
-            "target": "371"
+            "target": "380"
         },
         "beta": {
-            "target": "371"
+            "target": "380"
         },
         "edge": {
-            "target": "371"
+            "target": "380"
         }
     },
     "1.2-22.04": {
         "end-of-life": "2025-05-01T00:00:00Z",
         "beta": {
-            "target": "372"
+            "target": "381"
         },
         "edge": {
             "target": "1.2-22.04_beta"