From c231f22e4555d2d2c802e5e97b1a0cef30e31f39 Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Fri, 26 Jul 2024 13:10:57 -0300
Subject: [PATCH 01/22] workflows: make podvm_mkosi callable

So that the podvm_mkosi.yaml workflow can be reused in podvm.yaml
to build mkosi artifacts.

Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 .github/workflows/podvm_mkosi.yaml | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml
index 91a073c25..c4953bca5 100644
--- a/.github/workflows/podvm_mkosi.yaml
+++ b/.github/workflows/podvm_mkosi.yaml
@@ -7,6 +7,13 @@ on:
         description: "Prebuild fedora binaries image, as produced by this workflow under ghcr.io/confidential-containers/cloud-api-adaptor/podvm/binaries-fedora"
         required: false
 
+  workflow_call:
+    inputs:
+      binaries-image:
+        description: "Prebuild fedora binaries image, as produced by this workflow under ghcr.io/confidential-containers/cloud-api-adaptor/podvm/binaries-fedora"
+        required: false
+        type: string
+
 defaults:
   run:
     working-directory: src/cloud-api-adaptor
@@ -14,7 +21,7 @@ defaults:
 jobs:
   build-binaries:
     name: Build binaries
-    if : ${{ github.event.inputs.binaries-image == '' }}
+    if : ${{ inputs.binaries-image == '' }}
     runs-on: ubuntu-24.04
     permissions:
       contents: read
@@ -58,13 +65,13 @@ jobs:
             "BUILDER_IMG=ghcr.io/${{ github.repository }}/podvm/builder-fedora:${{ github.sha }}"
 
   build-image:
-    name: Build image
+    name: Build mkosi image
     needs: [build-binaries]
     if: |
       always() && (
           needs.build-binaries.result == 'success' || (
               needs.build-binaries.result == 'skipped' &&
-              github.event.inputs.binaries-image != ''
+              inputs.binaries-image != ''
           )
       )
     runs-on: ubuntu-24.04
@@ -102,10 +109,10 @@ jobs:
       - name: Decide on image to use
         id: binaries-image
         run: |
-          if [ -z "${{ github.event.inputs.binaries-image }}" ]; then
+          if [ -z "${{ inputs.binaries-image }}" ]; then
             echo "image=ghcr.io/${{ github.repository }}/podvm/binaries-fedora:${{ github.sha }}" | tee -a "$GITHUB_OUTPUT"
           else
-            echo "image=${{ github.event.inputs.binaries-image }}" | tee -a "$GITHUB_OUTPUT"
+            echo "image=${{ inputs.binaries-image }}" | tee -a "$GITHUB_OUTPUT"
           fi
 
       - name: Download binaries and unpack into binaries-tree
@@ -117,7 +124,7 @@ jobs:
           tar xf podvm-binaries.tar -C podvm-mkosi/resources/binaries-tree
           rm podvm-binaries.tar
 
-      - name: Build image
+      - name: Build mkosi image
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: make image-debug
 

From 56cc5154efd47c566747d4735f7fb2e228c02c46 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Fri, 6 Sep 2024 11:26:33 +0100
Subject: [PATCH 02/22] workflows: podvm_mkosi: Multi-arch support

- Remove seperate s390x workflow
- Pin runner version
- Add inputs required to plug this into the e2e workflows
- Add check for if builder and binaries image already exist
- Add s390x mkosi install
- bump github actions to latest versions

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/podvm_mkosi.yaml       | 211 +++++++++++++++--------
 .github/workflows/podvm_mkosi_image.yaml |  60 -------
 2 files changed, 140 insertions(+), 131 deletions(-)
 delete mode 100644 .github/workflows/podvm_mkosi_image.yaml

diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml
index c4953bca5..a975fb020 100644
--- a/.github/workflows/podvm_mkosi.yaml
+++ b/.github/workflows/podvm_mkosi.yaml
@@ -3,14 +3,41 @@ name: Create a Pod VM image with mkosi
 on:
   workflow_dispatch:
     inputs:
-      binaries-image:
-        description: "Prebuild fedora binaries image, as produced by this workflow under ghcr.io/confidential-containers/cloud-api-adaptor/podvm/binaries-fedora"
+      registry:
+        default: 'quay.io/confidential-containers'
         required: false
+        type: string
+      image_tag:
+        default: ''
+        required: false
+        type: string
+      git_ref:
+        description: Git ref to checkout the cloud-api-adaptor repository.
+        required: true
+        type: string
+      arch:
+        description: Which arch we are building the mkosi image for
+        default: 'amd64'
+        required: false
+        type: string
 
   workflow_call:
     inputs:
-      binaries-image:
-        description: "Prebuild fedora binaries image, as produced by this workflow under ghcr.io/confidential-containers/cloud-api-adaptor/podvm/binaries-fedora"
+      registry:
+        default: 'quay.io/confidential-containers'
+        required: false
+        type: string
+      image_tag:
+        default: ''
+        required: false
+        type: string
+      git_ref:
+        description: Git ref to checkout the cloud-api-adaptor repository.
+        required: true
+        type: string
+      arch:
+        description: Which arch we are building the mkosi image for
+        default: 'amd64'
         required: false
         type: string
 
@@ -19,117 +46,159 @@ defaults:
     working-directory: src/cloud-api-adaptor
 
 jobs:
-  build-binaries:
-    name: Build binaries
-    if : ${{ inputs.binaries-image == '' }}
-    runs-on: ubuntu-24.04
+  build-image:
+    name: Build mkosi image
+    runs-on: ${{ inputs.arch == 's390x' && 's390x' || 'ubuntu-24.04' }}
     permissions:
       contents: read
       packages: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: "${{ inputs.git_ref }}"
+
+      - name: Rebase the code
+        if: github.event_name == 'pull_request_target'
+        working-directory: ./
+        run: |
+          ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Log in to GitHub Container Registry
+      - name: Login to quay Container Registry
+        if: ${{ startsWith(inputs.registry, 'quay.io') }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Login to the ghcr Container registry
+        if: ${{ startsWith(inputs.registry, 'ghcr.io') }}
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build builder
-        uses: docker/build-push-action@v6
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y bubblewrap alien dnf qemu-utils uidmap
+          sudo snap install yq
+
+      - name: Read properties from versions.yaml
+        if: ${{ inputs.arch == 's390x' }}
+        run: |
+          go_version="$(yq '.tools.golang' versions.yaml)"
+          echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV"
+
+      - name: Setup Golang version ${{ env.GO_VERSION }}
+        if: ${{ inputs.arch == 's390x' }}
+        uses: actions/setup-go@v5
         with:
-          tags: ghcr.io/${{ github.repository }}/podvm/builder-fedora:${{ github.sha }}
-          context: src
-          file: src/cloud-api-adaptor/podvm/Dockerfile.podvm_builder.fedora
-          push: true
+          go-version: ${{ env.GO_VERSION }}
 
-      # Build binaries need git commit id as part of image name
-      - name: Prepare .git folder
-        working-directory: ./
+      - name: Build builder
+        id: build_builder
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: |
-          cp -rf .git src/.git
+          # If the builder image matching our git sha exists, then skip this build
+          builder_image=${{ inputs.registry }}/podvm-builder-fedora-${{ inputs.arch }}
+          tag=$(git rev-parse --short HEAD)
+
+          if ! docker manifest inspect "${builder_image}:${tag}"; then
+            PODVM_BUILDER_IMAGE="${builder_image}:${tag}" make fedora-binaries-builder
+          fi
+          echo "image=${builder_image}:${tag}" | tee -a "$GITHUB_OUTPUT"
+
+          # If the input has a different image-tag then also push it with that tag
+          if [ -n "${{ inputs.image_tag }}" ] && [ "${{ inputs.image_tag }}" != "${tag}" ];then
+            docker pull "${builder_image}:${tag}"
+            docker tag "${builder_image}:${tag}" ${builder_image}:${{ inputs.image_tag }}
+            docker push "${builder_image}:${{ inputs.image_tag }}"
+          fi
+        env:
+          PUSH: true
+          ARCH: ${{ inputs.arch }}
 
       - name: Build binaries
-        uses: docker/build-push-action@v6
-        with:
-          tags: ghcr.io/${{ github.repository }}/podvm/binaries-fedora:${{ github.sha }}
-          context: src
-          file: src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora
-          push: true
-          build-args:
-            "BUILDER_IMG=ghcr.io/${{ github.repository }}/podvm/builder-fedora:${{ github.sha }}"
+        id: build_binaries
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: |
+          # If the binaries which matching git sha exists, then skip this build
+          binaries_image=${{ inputs.registry }}/podvm-binaries-fedora-${{ inputs.arch }}
+          tag="$(git rev-parse --short HEAD)"
+          echo "image_tag=${image_tag}" | tee -a "$GITHUB_OUTPUT"
 
-  build-image:
-    name: Build mkosi image
-    needs: [build-binaries]
-    if: |
-      always() && (
-          needs.build-binaries.result == 'success' || (
-              needs.build-binaries.result == 'skipped' &&
-              inputs.binaries-image != ''
-          )
-      )
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+          if ! docker manifest inspect "${binaries_image}:${tag}"; then
+            PODVM_BINARIES_IMAGE="${binaries_image}:${tag}" make binaries
+          fi
+          echo "image=${binaries_image}:${tag}" | tee -a "$GITHUB_OUTPUT"
+
+          # If the input has a different image-tag then also push it with that tag
+          if [ -n "${{ inputs.image_tag }}" ] && [ "${{ inputs.image_tag }}" != "${tag}" ];then
+            docker pull "${binaries_image}:${tag}"
+            docker tag "${binaries_image}:${tag}" ${binaries_image}:${{ inputs.image_tag }}
+            docker push "${binaries_image}:${{ inputs.image_tag }}"
+          fi
+        env:
+          PUSH: true
+          ARCH: ${{ inputs.arch }}
+          PODVM_BUILDER_IMAGE: ${{ steps.build_builder.outputs.image }}
+
+      - name: Install mkosi
+        if: ${{ inputs.arch == 's390x' }}
+        run: |
+          git clone -b v22 https://github.com/systemd/mkosi
+          sudo rm -f /usr/local/bin/mkosi
+          sudo ln -s "$PWD/mkosi/bin/mkosi" /usr/local/bin/mkosi
+          mkosi --version
 
       - name: Install Nix
+        if: ${{ inputs.arch == 'amd64' }}
         uses: cachix/install-nix-action@v30
 
       - name: Build nix shell to cache dependencies
+        if: ${{ inputs.arch == 'amd64' }}
         run: nix build .#devShells.x86_64-linux.podvm-mkosi
 
-      - name: Install crane
+      - name: Install crane with nix
+        if: ${{ inputs.arch == 'amd64' }}
         run: nix profile install nixpkgs#crane
 
-        # This removes the checkout and creates a btrfs volume with maximized
-        # build space.
-      - name: Maximize build space
-        uses: katexochen/maximize-build-space@btrfs
-        with:
-          swap-size-mb: 1024
-          remove-dotnet: "true"
-          remove-android: "true"
-          remove-haskell: "true"
-          remove-codeql: "true"
-          remove-docker-images: "true"
+      - name: Install crane with go
+        if: ${{ inputs.arch == 's390x' }}
+        run: |
+          go install github.com/google/go-containerregistry/cmd/crane@latest
+          echo "PATH=${PATH}:$(go env GOPATH)/bin" >> "$GITHUB_ENV"
 
       - name: Second checkout
+        if: ${{ inputs.arch == 'amd64' }}
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: "${{ inputs.git_ref }}"
 
-      - name: Decide on image to use
-        id: binaries-image
+
+      - name: Rebase the code
+        if: github.event_name == 'pull_request_target'
+        working-directory: ./
         run: |
-          if [ -z "${{ inputs.binaries-image }}" ]; then
-            echo "image=ghcr.io/${{ github.repository }}/podvm/binaries-fedora:${{ github.sha }}" | tee -a "$GITHUB_OUTPUT"
-          else
-            echo "image=${{ inputs.binaries-image }}" | tee -a "$GITHUB_OUTPUT"
-          fi
+          ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch
 
       - name: Download binaries and unpack into binaries-tree
         run: |
           crane export \
-            ${{ steps.binaries-image.outputs.image }} \
+            ${{ steps.build_binaries.outputs.image }} \
             podvm-binaries.tar
           mkdir -p podvm-mkosi/resources/binaries-tree
           tar xf podvm-binaries.tar -C podvm-mkosi/resources/binaries-tree
           rm podvm-binaries.tar
 
-      - name: Build mkosi image
+      - name: Build mkosi debug image
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: make image-debug
-
-      - name: Upload image
-        uses: actions/upload-artifact@v4
-        with:
-          name: podvm-mkosi-${{ github.sha }}
-          path: src/cloud-api-adaptor/podvm-mkosi/build/system.raw
diff --git a/.github/workflows/podvm_mkosi_image.yaml b/.github/workflows/podvm_mkosi_image.yaml
deleted file mode 100644
index e985c91da..000000000
--- a/.github/workflows/podvm_mkosi_image.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-name: mkosi build podvm builder, binaries and image
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build-podvm-image-mkosi:
-    name: Build podvm image via mkosi
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        runner:
-          - S390X
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Quay container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_USERNAME }}
-          password: ${{ secrets.QUAY_PASSWORD }}
-          logout: false
-
-      - name: Install build dependencies
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y bubblewrap alien dnf qemu-utils uidmap
-          sudo snap install yq
-
-      - name: Install mkosi
-        run: |
-          git clone -b v22 https://github.com/systemd/mkosi
-          sudo rm /usr/local/bin/mkosi | true
-          sudo ln -s "$PWD/mkosi/bin/mkosi" /usr/local/bin/mkosi
-          mkosi --version
-
-      - name: Build builder
-        working-directory: src/cloud-api-adaptor/podvm-mkosi
-        run: make fedora-binaries-builder
-
-      - name: Build binaries
-        working-directory: src/cloud-api-adaptor/podvm-mkosi
-        run: make binaries
-
-      - name: Build image
-        run: make image-debug
-        working-directory: src/cloud-api-adaptor/podvm-mkosi
-
-      - name: Push image
-        run: make push-image
-        working-directory: src/cloud-api-adaptor/podvm-mkosi

From 02fed99955db92baf3ecd2548d42f8e487eb423b Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 9 Sep 2024 14:27:03 +0100
Subject: [PATCH 03/22] podvm-mkosi: Update image targets

- Update image names to match the existing naming scheme
- Add debug suffix to the container image name for clarity
- Add push/load switch based on the PUSH env

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 src/cloud-api-adaptor/podvm-mkosi/Makefile | 46 ++++++++++++----------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/src/cloud-api-adaptor/podvm-mkosi/Makefile b/src/cloud-api-adaptor/podvm-mkosi/Makefile
index 63fc7189f..433b12a5e 100644
--- a/src/cloud-api-adaptor/podvm-mkosi/Makefile
+++ b/src/cloud-api-adaptor/podvm-mkosi/Makefile
@@ -1,17 +1,24 @@
 include ../Makefile.defaults
 
 ARCH      ?= $(subst x86_64,amd64,$(shell uname -m))
-BUILDER   = fedora-binaries-builder-$(ARCH)
 SE_BOOT   ?= false
 IS_DEBIAN := $(shell if grep -q 'ID_LIKE=debian' /etc/os-release; then echo "true"; else echo "false"; fi)
 
 REGISTRY ?= quay.io/confidential-containers
 PODVM_DISTRO ?= fedora
-PODVM_TAG ?= $(VERSIONS_HASH)
-PODVM_NAME ?= $(REGISTRY)/podvm-generic-$(PODVM_DISTRO)-$(ARCH)
-PODVM_CONTAINER_NAME ?= $(REGISTRY)/podvm-docker-image
+ifeq ($(PODVM_TAG),)
+	PODVM_TAG := $(VERSIONS_HASH)
+endif
+PODVM_BUILDER_IMAGE ?= $(REGISTRY)/podvm-builder-$(PODVM_DISTRO)-$(ARCH):$(PODVM_TAG)
+PODVM_BINARIES_IMAGE ?= $(REGISTRY)/podvm-binaries-$(PODVM_DISTRO)-$(ARCH):$(PODVM_TAG)
+PODVM_IMAGE ?= $(REGISTRY)/podvm-generic-$(PODVM_DISTRO)$(if $(filter $(SE_BOOT),true),-se,)-$(ARCH):$(PODVM_TAG)
+PODVM_CONTAINER_NAME ?= $(REGISTRY)/podvm-docker-image-$(ARCH)
 VERIFY_PROVENANCE ?= no
 
+PUSH ?= false
+# If not pushing `--load` into the local docker cache
+DOCKER_OPTS := $(if $(filter $(PUSH),true),--push,--load) $(EXTRA_DOCKER_OPTS)
+
 .DEFAULT_GOAL := all
 .PHONY: all
 all: fedora-binaries-builder binaries image
@@ -30,7 +37,7 @@ PHONY: fedora-binaries-builder
 fedora-binaries-builder:
 	@echo "Building $(BUILDER) image..."
 	docker buildx build \
-		-t $(BUILDER) \
+		-t $(PODVM_BUILDER_IMAGE) \
 		--build-arg GO_VERSION=$(GO_VERSION) \
 		--build-arg ARCH=$(ARCH) \
 		--build-arg PROTOC_VERSION=$(PROTOC_VERSION) \
@@ -39,7 +46,7 @@ fedora-binaries-builder:
 		--build-arg YQ_ARCH=$(ARCH) \
 		--build-arg PROTOC_ARCH=$(if $(filter amd64,$(ARCH)),x86_64,s390x) \
 		--build-arg ORAS_VERSION=$(ORAS_VERSION) \
-		--load \
+		$(DOCKER_OPTS) \
 		-f ../podvm/Dockerfile.podvm_builder.fedora ../.
 
 PHONY: binaries
@@ -50,8 +57,9 @@ ifeq ($(IS_DEBIAN),true)
 	docker buildx use default
 endif
 	docker buildx build \
+		-t $(PODVM_BINARIES_IMAGE) \
 		--progress=plain \
-		--build-arg BUILDER_IMG=$(BUILDER) \
+		--build-arg BUILDER_IMG=$(PODVM_BUILDER_IMAGE) \
 		--build-arg TEE_PLATFORM=$(TEE_PLATFORM) \
 		--build-arg PAUSE_REPO=$(PAUSE_REPO) \
 		--build-arg PAUSE_VERSION=$(PAUSE_VERSION) \
@@ -60,7 +68,8 @@ endif
 		--build-arg VERIFY_PROVENANCE=$(VERIFY_PROVENANCE) \
 		$(if $(AUTHFILE),--build-arg AUTHFILE=$(AUTHFILE),) \
 		$(if $(DEFAULT_AGENT_POLICY_FILE),--build-arg DEFAULT_AGENT_POLICY_FILE=$(DEFAULT_AGENT_POLICY_FILE),) \
-		-o type=local,dest="./resources/binaries-tree" \
+		$(if $(filter $(PUSH),true),,-o type=local,dest="./resources/binaries-tree") \
+		$(DOCKER_OPTS) \
 		-f ../podvm/Dockerfile.podvm_binaries.fedora ../../
 
 PHONY: image
@@ -75,9 +84,8 @@ ifeq ($(SE_BOOT),true)
 	sudo -E ../hack/build-s390x-se-image.sh
 	@echo "Building docker image..."
 	docker buildx build \
-		-t $(PODVM_NAME)-se:$(PODVM_TAG) \
-		-t $(PODVM_NAME)-se:latest \
-		--load \
+		-t $(PODVM_IMAGE) \
+		$(DOCKER_OPTS) \
 		-f ../podvm/Dockerfile.podvm.fedora .
 else ifeq ($(ARCH),s390x)
 	touch resources/buildS390xImage
@@ -85,9 +93,8 @@ else ifeq ($(ARCH),s390x)
 	sudo -E ../hack/build-s390x-image.sh
 	@echo "Building docker image..."
 	docker buildx build \
-		-t $(PODVM_NAME):$(PODVM_TAG) \
-		-t $(PODVM_NAME):latest \
-		--load \
+		-t $(PODVM_IMAGE) \
+		$(DOCKER_OPTS) \
 		-f ../podvm/Dockerfile.podvm.fedora .
 else
 	touch resources/buildBootableImage
@@ -107,9 +114,8 @@ ifeq ($(SE_BOOT),true)
 	sudo -E ../hack/build-s390x-se-image.sh
 	@echo "Building docker image..."
 	docker buildx build \
-		-t $(PODVM_NAME)-se:$(PODVM_TAG) \
-		-t $(PODVM_NAME)-se:latest \
-		--load \
+		-t $(addsuffix -debug,$(PODVM_IMAGE)) \
+		$(DOCKER_OPTS) \
 		-f ../podvm/Dockerfile.podvm.fedora .
 else ifeq ($(ARCH),s390x)
 	touch resources/buildS390xImage
@@ -117,9 +123,8 @@ else ifeq ($(ARCH),s390x)
 	sudo -E ../hack/build-s390x-image.sh
 	@echo "Building docker image..."
 	docker buildx build \
-		-t $(PODVM_NAME):$(PODVM_TAG) \
-		-t $(PODVM_NAME):latest \
-		--load \
+		-t $(addsuffix -debug,$(PODVM_IMAGE)) \
+		$(DOCKER_OPTS) \
 		-f ../podvm/Dockerfile.podvm.fedora .
 else
 	touch resources/buildBootableImage
@@ -132,6 +137,7 @@ image-container:
 	docker buildx build \
 		-t $(PODVM_CONTAINER_NAME):$(PODVM_TAG) \
 		-t $(PODVM_CONTAINER_NAME):latest \
+		$(DOCKER_OPTS) \
 		-f Dockerfile.podvm .
 
 PHONY: push-image

From 2e2b88b2164f9ade895b0467a6d9b200e69e51fb Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 9 Sep 2024 15:12:00 +0100
Subject: [PATCH 04/22] podvm: docker provider: Rename Dockerfile

- Rename the docker provider's podvm Dockerfile
to Dockerfile.podvm_docker_provider for more clarity and to enable us
to use Dockerfile.podvm for the "main" version of the podvm in future
- Add arch awareness, so we can build images for multiple architectures.

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .../{Dockerfile.podvm => Dockerfile.podvm_docker_provider}      | 0
 src/cloud-api-adaptor/podvm-mkosi/Makefile                      | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename src/cloud-api-adaptor/podvm-mkosi/{Dockerfile.podvm => Dockerfile.podvm_docker_provider} (100%)

diff --git a/src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm b/src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm_docker_provider
similarity index 100%
rename from src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm
rename to src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm_docker_provider
diff --git a/src/cloud-api-adaptor/podvm-mkosi/Makefile b/src/cloud-api-adaptor/podvm-mkosi/Makefile
index 433b12a5e..464ada478 100644
--- a/src/cloud-api-adaptor/podvm-mkosi/Makefile
+++ b/src/cloud-api-adaptor/podvm-mkosi/Makefile
@@ -138,7 +138,7 @@ image-container:
 		-t $(PODVM_CONTAINER_NAME):$(PODVM_TAG) \
 		-t $(PODVM_CONTAINER_NAME):latest \
 		$(DOCKER_OPTS) \
-		-f Dockerfile.podvm .
+		-f Dockerfile.podvm_docker_provider .
 
 PHONY: push-image
 push-image:

From 20ad65d2f1aa2f5e7d54e31cba77f57e4cad3294 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 9 Sep 2024 15:48:48 +0100
Subject: [PATCH 05/22] workflows: Add podvm-mkosi workflow triggers

- Add calls to build the podvm-mkosi image
in the same places we build the current podvm image

- Note: we have decoupled the s390x and amd64
podvm_mkosi image jobs, so that their total
e2e workflows can run independently and delays
in one step doesn't impact the other.

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_run_all.yaml   | 18 ++++++++++++++++++
 .github/workflows/podvm_publish.yaml | 12 ++++++++++++
 .github/workflows/release.yaml       | 12 ++++++++++++
 3 files changed, 42 insertions(+)

diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml
index d9dff137c..6ffe13faf 100644
--- a/.github/workflows/e2e_run_all.yaml
+++ b/.github/workflows/e2e_run_all.yaml
@@ -66,6 +66,24 @@ jobs:
       git_ref: ${{ inputs.git_ref }}
     secrets: inherit
 
+  podvm_mkosi_amd64:
+    uses: ./.github/workflows/podvm_mkosi.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      image_tag: ${{ inputs.podvm_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+      arch: amd64
+    secrets: inherit
+
+  podvm_mkosi_s390x:
+    uses: ./.github/workflows/podvm_mkosi.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      image_tag: ${{ inputs.podvm_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+      arch: s390x
+    secrets: inherit
+
   # Build and push the cloud-api-adaptor image
   #
   # By using a reusable `workflow_call` workflow we are hitting two
diff --git a/.github/workflows/podvm_publish.yaml b/.github/workflows/podvm_publish.yaml
index 4122eeda9..333de2e29 100644
--- a/.github/workflows/podvm_publish.yaml
+++ b/.github/workflows/podvm_publish.yaml
@@ -30,3 +30,15 @@ jobs:
       git_ref: ${{ github.sha }}
       image_tag: ${{ github.sha }}
     secrets: inherit
+
+  podvm-mkosi:
+    uses: ./.github/workflows/podvm_mkosi.yaml
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [amd64, s390x]
+    with:
+      git_ref: ${{ github.sha }}
+      image_tag: ${{ github.sha }}
+      arch: ${{ matrix.arch}}
+    secrets: inherit
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index fa18fc9cd..cd32b7124 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -54,6 +54,18 @@ jobs:
       git_ref: ${{ github.ref }}
     secrets: inherit
 
+  podvm-mkosi:
+    uses: ./.github/workflows/podvm_mkosi.yaml
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [amd64, s390x]
+    with:
+      image_tag: ${{ github.event.release.tag_name }}
+      git_ref: ${{ github.ref }}
+      arch: ${{ matrix.arch}}
+    secrets: inherit
+
   webhook:
     uses: ./.github/workflows/webhook_image.yaml
     with:

From e2f052625e4bf55f205051358eaf0c58bd80e3a4 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 9 Sep 2024 16:44:02 +0100
Subject: [PATCH 06/22] workflows: podvm_mkosi: Add debug input

- Add input option to select between the debug and non-debug mkosi image build
- Initially use the debug build for e2e tests and non-debug for
release and image publish

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_run_all.yaml   |  2 ++
 .github/workflows/podvm_mkosi.yaml   | 16 ++++++++++++++++
 .github/workflows/podvm_publish.yaml |  1 +
 .github/workflows/release.yaml       |  1 +
 4 files changed, 20 insertions(+)

diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml
index 6ffe13faf..cfae80b6b 100644
--- a/.github/workflows/e2e_run_all.yaml
+++ b/.github/workflows/e2e_run_all.yaml
@@ -73,6 +73,7 @@ jobs:
       image_tag: ${{ inputs.podvm_image_tag }}
       git_ref: ${{ inputs.git_ref }}
       arch: amd64
+      debug: true
     secrets: inherit
 
   podvm_mkosi_s390x:
@@ -82,6 +83,7 @@ jobs:
       image_tag: ${{ inputs.podvm_image_tag }}
       git_ref: ${{ inputs.git_ref }}
       arch: s390x
+      debug: true
     secrets: inherit
 
   # Build and push the cloud-api-adaptor image
diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml
index a975fb020..16842bbe2 100644
--- a/.github/workflows/podvm_mkosi.yaml
+++ b/.github/workflows/podvm_mkosi.yaml
@@ -20,6 +20,11 @@ on:
         default: 'amd64'
         required: false
         type: string
+      debug:
+        description: Whether to build the image in debug mode
+        default: false
+        required: false
+        type: boolean
 
   workflow_call:
     inputs:
@@ -40,6 +45,11 @@ on:
         default: 'amd64'
         required: false
         type: string
+      debug:
+        description: Whether to build the image in debug mode
+        default: false
+        required: false
+        type: boolean
 
 defaults:
   run:
@@ -200,5 +210,11 @@ jobs:
           rm podvm-binaries.tar
 
       - name: Build mkosi debug image
+        if: ${{ inputs.debug == 'true' }}
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: make image-debug
+
+      - name: Build mkosi image
+        if: ${{ inputs.debug != 'true' }}
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: make image
diff --git a/.github/workflows/podvm_publish.yaml b/.github/workflows/podvm_publish.yaml
index 333de2e29..6a5edaeea 100644
--- a/.github/workflows/podvm_publish.yaml
+++ b/.github/workflows/podvm_publish.yaml
@@ -41,4 +41,5 @@ jobs:
       git_ref: ${{ github.sha }}
       image_tag: ${{ github.sha }}
       arch: ${{ matrix.arch}}
+      debug: false
     secrets: inherit
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index cd32b7124..90fca28c5 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -64,6 +64,7 @@ jobs:
       image_tag: ${{ github.event.release.tag_name }}
       git_ref: ${{ github.ref }}
       arch: ${{ matrix.arch}}
+      debug: false
     secrets: inherit
 
   webhook:

From 2e20c71dac987b27ebaf93fd0f8c63f0d471944b Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Mon, 29 Jul 2024 18:43:03 -0300
Subject: [PATCH 07/22] workflows: build podvm-docker-fedora-image in
 podvm_mkosi

- Re-use the resources/binaries-tree binaries already fetched in
the podvm_mkosi.yaml to build the podvm image for the docker provider.
- We also need to clean up the image build artifacts to stop the
runner going out of space

Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/podvm_mkosi.yaml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml
index 16842bbe2..a90f9ce16 100644
--- a/.github/workflows/podvm_mkosi.yaml
+++ b/.github/workflows/podvm_mkosi.yaml
@@ -218,3 +218,20 @@ jobs:
         if: ${{ inputs.debug != 'true' }}
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: make image
+
+      - name: Clean up some space for the docker provider build
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: |
+          sudo du --max-depth=2 /home/runner || true
+          sudo du --max-depth=2 /var/lib || true
+          sudo rm -rf /nix
+          sudo rm -rf ./build
+          sudo rm -rf ./mkosi.cache
+
+      - name: Build image for docker provider
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: |
+          PODVM_TAG=$(git rev-parse --short HEAD) make image-container
+        env:
+          PUSH: true
+          REGISTRY: ghcr.io/${{ github.repository }}

From 7e5578ccc720c6df516368bdd9b2cf335810c348 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Fri, 6 Sep 2024 11:29:19 +0100
Subject: [PATCH 08/22] podvm_binaries: Add fedora cross-build support

Add fedora-like OS support for cross-build-extras

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .../podvm/Dockerfile.podvm_binaries.fedora          |  2 ++
 .../podvm/hack/cross-build-extras.sh                | 13 +++++++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora b/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora
index 74d9c294d..75459daca 100644
--- a/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora
+++ b/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora
@@ -48,6 +48,8 @@ ENV IMAGE_CHECKSUM="none"
 COPY . /src
 
 WORKDIR /src/cloud-api-adaptor/podvm
+# Installs add-ons for foreign target, if required
+RUN ./hack/cross-build-extras.sh
 
 RUN LIBC=gnu make binaries
 
diff --git a/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh b/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh
index f0093ff36..b36e6632d 100755
--- a/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh
+++ b/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh
@@ -17,5 +17,14 @@ libc=$([[ $ARCH =~ s390x ]] && echo "gnu" || echo "musl")
 rustTarget="$ARCH-unknown-linux-$libc"
 
 rustup target add "$rustTarget"
-apt install -y "qemu-system-$ARCH"
-apt install -y "gcc-$ARCH-linux-$libc"
+
+source /etc/os-release || source /usr/lib/os-release
+if [[ ${ID_LIKE:-} == *"debian"* ]]; then
+    apt install -y "qemu-system-$ARCH"
+    apt install -y "gcc-$ARCH-linux-$libc"
+elif [[ "${ID_LIKE:-}" =~ "fedora" ]] || [[ "${ID:-}" =~ "fedora" ]]; then
+    dnf install -y "qemu-system-$ARCH"
+    dnf install -y "gcc-$ARCH-linux-$libc"
+else
+    echo "Unsupported distro $ID"; exit 1
+fi

From 54296b1d55b4d5aafb7dcc794325999d3ce12481 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 16 Sep 2024 17:21:18 +0100
Subject: [PATCH 09/22] podvm_mkosi: Create qcow2 file for testing

- Convert the mkosi image to a qcow2 file
- Remove the Dockerfile.podvm.fedora file as
we want to upload it via oras rather than our
own wrapping container image
- Run nix develop sudo to avoid permissions error

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .../hack/build-s390x-image.sh                 |  2 +-
 .../hack/build-s390x-se-image.sh              |  2 +-
 src/cloud-api-adaptor/podvm-mkosi/Makefile    | 27 ++++---------------
 .../podvm/Dockerfile.podvm.fedora             | 13 ---------
 4 files changed, 7 insertions(+), 37 deletions(-)
 delete mode 100644 src/cloud-api-adaptor/podvm/Dockerfile.podvm.fedora

diff --git a/src/cloud-api-adaptor/hack/build-s390x-image.sh b/src/cloud-api-adaptor/hack/build-s390x-image.sh
index 2cf11a75a..eb96706f0 100755
--- a/src/cloud-api-adaptor/hack/build-s390x-image.sh
+++ b/src/cloud-api-adaptor/hack/build-s390x-image.sh
@@ -76,7 +76,7 @@ umount "${dst_mnt}"
 
 qemu-nbd --disconnect "${tmp_nbd}"
 
-output_img_name="podvm-s390x.qcow2"
+output_img_name="podvm-fedora-s390x.qcow2"
 qemu-img convert -O qcow2 -c "${tmp_img_path}" "${output_img_name}"
 chmod 644 "${output_img_name}"
 
diff --git a/src/cloud-api-adaptor/hack/build-s390x-se-image.sh b/src/cloud-api-adaptor/hack/build-s390x-se-image.sh
index d9dabff8b..7c9e11eba 100755
--- a/src/cloud-api-adaptor/hack/build-s390x-se-image.sh
+++ b/src/cloud-api-adaptor/hack/build-s390x-se-image.sh
@@ -172,7 +172,7 @@ cryptsetup close "$LUKS_NAME"
 
 qemu-nbd --disconnect "${tmp_nbd}"
 
-output_img_name="podvm-s390x-se.qcow2"
+output_img_name="podvm-fedora-s390x-se.qcow2"
 qemu-img convert -O qcow2 -c "${tmp_img_path}" "${output_img_name}"
 output_img_path=$(realpath "${output_img_name}")
 echo "podvm se-image is generated: ${output_img_path}"
diff --git a/src/cloud-api-adaptor/podvm-mkosi/Makefile b/src/cloud-api-adaptor/podvm-mkosi/Makefile
index 464ada478..2d7267c1a 100644
--- a/src/cloud-api-adaptor/podvm-mkosi/Makefile
+++ b/src/cloud-api-adaptor/podvm-mkosi/Makefile
@@ -82,23 +82,14 @@ ifeq ($(SE_BOOT),true)
 	touch resources/buildS390xImage
 	sudo mkosi --profile production.conf --image system
 	sudo -E ../hack/build-s390x-se-image.sh
-	@echo "Building docker image..."
-	docker buildx build \
-		-t $(PODVM_IMAGE) \
-		$(DOCKER_OPTS) \
-		-f ../podvm/Dockerfile.podvm.fedora .
 else ifeq ($(ARCH),s390x)
 	touch resources/buildS390xImage
 	sudo mkosi --profile production.conf --image system
 	sudo -E ../hack/build-s390x-image.sh
-	@echo "Building docker image..."
-	docker buildx build \
-		-t $(PODVM_IMAGE) \
-		$(DOCKER_OPTS) \
-		-f ../podvm/Dockerfile.podvm.fedora .
 else
 	touch resources/buildBootableImage
-	nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=production
+	sudo -E env PATH=$(PATH) nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=production
+	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
 endif
 
 PHONY: image-debug
@@ -112,24 +103,16 @@ ifeq ($(SE_BOOT),true)
 	touch resources/buildS390xImage
 	sudo mkosi --profile debug.conf
 	sudo -E ../hack/build-s390x-se-image.sh
-	@echo "Building docker image..."
-	docker buildx build \
-		-t $(addsuffix -debug,$(PODVM_IMAGE)) \
-		$(DOCKER_OPTS) \
-		-f ../podvm/Dockerfile.podvm.fedora .
 else ifeq ($(ARCH),s390x)
 	touch resources/buildS390xImage
 	sudo mkosi --profile debug.conf
 	sudo -E ../hack/build-s390x-image.sh
-	@echo "Building docker image..."
-	docker buildx build \
-		-t $(addsuffix -debug,$(PODVM_IMAGE)) \
-		$(DOCKER_OPTS) \
-		-f ../podvm/Dockerfile.podvm.fedora .
 else
 	touch resources/buildBootableImage
-	nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=debug
+	sudo -E env PATH=$(PATH) nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=debug
+	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
 endif
+	sudo chown -R $(USER): build
 
 PHONY: image-container
 image-container:
diff --git a/src/cloud-api-adaptor/podvm/Dockerfile.podvm.fedora b/src/cloud-api-adaptor/podvm/Dockerfile.podvm.fedora
deleted file mode 100644
index d8496f8a5..000000000
--- a/src/cloud-api-adaptor/podvm/Dockerfile.podvm.fedora
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright Confidential Containers Contributors
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# Place pod vm image built via mkosi
-#
-FROM scratch
-
-ARG ARCH=s390x
-
-ENV ARCH=${ARCH}
-
-COPY build/podvm-*.qcow2 /

From 8bf929bbdef5e6dafc09e198f267af30381cc518 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Fri, 20 Sep 2024 16:05:29 +0100
Subject: [PATCH 10/22] podvm: Update platform logic in download-image

- It used to be that the fedora se image was the only
image that was built on s390x, but now with the other
mkosi builds, it feels better to make the s390x ubuntu
podvm image which is arguably set-up incorrectly the
special case. When we drop packer support we can
remove this code.

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .../podvm/hack/download-image.sh               | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/cloud-api-adaptor/podvm/hack/download-image.sh b/src/cloud-api-adaptor/podvm/hack/download-image.sh
index 4087ad063..4ebe3efb8 100755
--- a/src/cloud-api-adaptor/podvm/hack/download-image.sh
+++ b/src/cloud-api-adaptor/podvm/hack/download-image.sh
@@ -43,18 +43,18 @@ fi
 
 [ -z "$container_binary" ] && error "please install docker or podman"
 
-if [[ -z "$platform" ]]; then
-    # Check if the image name includes "podvm-generic-fedora-s390x-se"
-    # The "podvm-generic-fedora-s390x-se" docker image is built on s390x host, so here must use s390x platform
-    if [[ "$image" == *"podvm-generic-fedora-s390x-se"* ]]; then
-        platform="s390x"
-    else
-        platform="amd64"
-    fi
+platform_flag=""
+if [[ -n "${platform}" ]]; then
+    platform_flag="--platform=${platform}"
+elif [[ "${image}" == *"podvm-generic-ubuntu-s390x"* ]]; then
+    # The podvm-generic-ubuntu-s390x image (which will be deprecated soon) is built on amd64
+    # and therefore has the incorrect platform type, so we need to override it to create the
+    # container to extract the image from. Others can use their default platform
+    platform_flag="--platform=amd64"
 fi
 
 # Create a non-running container to extract image
-$container_binary create --pull=always --platform="$platform" --name "$container_name" "$image" /bin/sh >/dev/null 2>&1;
+$container_binary create --pull=always ${platform_flag} --name "$container_name" "$image" /bin/sh >/dev/null 2>&1;
 # Destory container after use
 rm-container(){
     $container_binary rm -f "$container_name" >/dev/null 2>&1;

From 1fcaf1be5810f3a3ebc93a2e574ca75a16cdef38 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Thu, 24 Oct 2024 16:13:54 +0100
Subject: [PATCH 11/22] workflows: e2e_libvirt: Add runner as input

Add runner as an input parameter to e2e_libvirt
tests, so we can control it from calling workflow

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_libvirt.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml
index 7de7111b6..fce3549ff 100644
--- a/.github/workflows/e2e_libvirt.yaml
+++ b/.github/workflows/e2e_libvirt.yaml
@@ -7,6 +7,11 @@ name: (Callable) libvirt e2e tests
 on:
   workflow_call:
     inputs:
+      runner:
+        default: 'ubuntu-24.04'
+        description: The runner to execute the workflow on. Defaults to 'ubuntu-24.04'.
+        required: false
+        type: string
       podvm_image:
         required: true
         type: string
@@ -38,7 +43,7 @@ defaults:
 
 jobs:
   test:
-    runs-on: ubuntu-24.04
+    runs-on: ${{ inputs.runner }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4

From 7052ece8c942c5b24275db2fbd2ed8a7bbfd30a9 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Fri, 25 Oct 2024 16:09:47 +0100
Subject: [PATCH 12/22] workflows: mkosi: Publish qcow2 with oras

Publish the podvm qcow2 files with oras

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/podvm_mkosi.yaml | 41 +++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml
index a90f9ce16..998f63668 100644
--- a/.github/workflows/podvm_mkosi.yaml
+++ b/.github/workflows/podvm_mkosi.yaml
@@ -62,6 +62,8 @@ jobs:
     permissions:
       contents: read
       packages: write
+      id-token: write
+      attestations: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -101,10 +103,10 @@ jobs:
           sudo snap install yq
 
       - name: Read properties from versions.yaml
-        if: ${{ inputs.arch == 's390x' }}
         run: |
           go_version="$(yq '.tools.golang' versions.yaml)"
           echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV"
+          echo "ORAS_VERSION=$(yq -e '.tools.oras' versions.yaml)" >> "$GITHUB_ENV"
 
       - name: Setup Golang version ${{ env.GO_VERSION }}
         if: ${{ inputs.arch == 's390x' }}
@@ -112,6 +114,10 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
 
+      - uses: oras-project/setup-oras@v1
+        with:
+          version: ${{ env.ORAS_VERSION }}
+
       - name: Build builder
         id: build_builder
         working-directory: src/cloud-api-adaptor/podvm-mkosi
@@ -219,6 +225,39 @@ jobs:
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: make image
 
+      - name: Upload the qcow2 with oras
+        id: publish
+        working-directory: src/cloud-api-adaptor/podvm-mkosi
+        run: |
+          mkdir oras
+          cd oras
+          cp ../build/podvm-*.qcow2 .
+          tar cJf podvm.tar.xz podvm-*.qcow2
+          image=${{ inputs.registry }}/podvm-generic-fedora
+          if [ "${{inputs.debug}}" = "true" ]; then
+            image=${image}-debug
+          fi
+          image=${image}-${{ inputs.arch }}
+          tag=$(git rev-parse --short HEAD)
+          oras push "${image}:${tag}" podvm.tar.xz
+
+          # If the input has a different image-tag then also push it with that tag
+          if [ -n "${{ inputs.image_tag }}" ] && [ "${{ inputs.image_tag }}" != "${tag}" ];then
+            oras push "${image}:${{ inputs.image_tag }}" podvm.tar.xz
+          fi
+
+          # add image and digest to output for attestation
+          echo "image=${image}" >> "$GITHUB_OUTPUT"
+          digest="$(oras manifest fetch "${image}:${tag}" --descriptor | jq -r .digest)"
+          echo "digest=${digest}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ steps.publish.outputs.image }}
+          subject-digest: ${{ steps.publish.outputs.digest }}
+          push-to-registry: true
+
+
       - name: Clean up some space for the docker provider build
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: |

From e3d76e6728fd9edd182ff47625a40f8edc0cb3a5 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 28 Oct 2024 12:22:59 +0000
Subject: [PATCH 13/22] workflows: podvm_mkosi: Output qcow2 oras image

In the workflow output the qcow2 oras image location,
so that we can use this in the e2e testing step

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/podvm_mkosi.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml
index 998f63668..b3c835150 100644
--- a/.github/workflows/podvm_mkosi.yaml
+++ b/.github/workflows/podvm_mkosi.yaml
@@ -50,6 +50,10 @@ on:
         default: false
         required: false
         type: boolean
+    outputs:
+      qcow2_oras_image:
+        description: The location of the qcow2 oras container this workflow pushed
+        value: ${{ jobs.build-image.outputs.qcow2_oras_image }}
 
 defaults:
   run:
@@ -64,6 +68,8 @@ jobs:
       packages: write
       id-token: write
       attestations: write
+    outputs:
+      qcow2_oras_image: ${{ steps.publish_oras_qcow2.outputs.image }}:${{ steps.publish_oras_qcow2.outputs.tag }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -250,6 +256,7 @@ jobs:
           echo "image=${image}" >> "$GITHUB_OUTPUT"
           digest="$(oras manifest fetch "${image}:${tag}" --descriptor | jq -r .digest)"
           echo "digest=${digest}" >> "$GITHUB_OUTPUT"
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
 
       - uses: actions/attest-build-provenance@v1
         with:

From 7c7a022e10533186fcede3f5795e0b8f18cb511b Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 28 Oct 2024 12:37:29 +0000
Subject: [PATCH 14/22] workflows: podvm_mkosi: Output docker provider oci
 image

In the workflow output the docker provider oci image location,
so that we can use this in the e2e testing step

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/podvm_mkosi.yaml | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml
index b3c835150..ac7f59e14 100644
--- a/.github/workflows/podvm_mkosi.yaml
+++ b/.github/workflows/podvm_mkosi.yaml
@@ -54,6 +54,9 @@ on:
       qcow2_oras_image:
         description: The location of the qcow2 oras container this workflow pushed
         value: ${{ jobs.build-image.outputs.qcow2_oras_image }}
+      docker_oci_image:
+        description: The location of the docker oci container image this workflow pushed
+        value: ${{ jobs.build-image.outputs.docker_oci_image }}
 
 defaults:
   run:
@@ -70,6 +73,7 @@ jobs:
       attestations: write
     outputs:
       qcow2_oras_image: ${{ steps.publish_oras_qcow2.outputs.image }}:${{ steps.publish_oras_qcow2.outputs.tag }}
+      docker_oci_image: ${{ steps.build_docker_oci.outputs.image }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -232,7 +236,7 @@ jobs:
         run: make image
 
       - name: Upload the qcow2 with oras
-        id: publish
+        id: publish_oras_qcow2
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: |
           mkdir oras
@@ -260,8 +264,8 @@ jobs:
 
       - uses: actions/attest-build-provenance@v1
         with:
-          subject-name: ${{ steps.publish.outputs.image }}
-          subject-digest: ${{ steps.publish.outputs.digest }}
+          subject-name: ${{ steps.publish_oras_qcow2.outputs.image }}
+          subject-digest: ${{ steps.publish_oras_qcow2.outputs.digest }}
           push-to-registry: true
 
 
@@ -275,9 +279,14 @@ jobs:
           sudo rm -rf ./mkosi.cache
 
       - name: Build image for docker provider
+        id: build_docker_oci
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: |
-          PODVM_TAG=$(git rev-parse --short HEAD) make image-container
+          tag=$(git rev-parse --short HEAD)
+          PODVM_TAG=${tag} make image-container
+          arch=$(uname -m)
+          arch=${arch/x86_64/amd64}
+          echo "image=ghcr.io/${{ github.repository }}/podvm-docker-image-${arch}:${tag}" >> "$GITHUB_OUTPUT"
         env:
           PUSH: true
           REGISTRY: ghcr.io/${{ github.repository }}

From 84ccbe75128e0d62f420bbff97c398b635528c7b Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 28 Oct 2024 12:55:59 +0000
Subject: [PATCH 15/22] workflows: libvirt_e2e: Add support to pull podvm with
 oras

In the mkosi flow we publish the podvm qcow2 image
with oras, so add support to pull this down in e2e testing

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_libvirt.yaml | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml
index fce3549ff..859096413 100644
--- a/.github/workflows/e2e_libvirt.yaml
+++ b/.github/workflows/e2e_libvirt.yaml
@@ -33,6 +33,12 @@ on:
         description: SecureComms configuration. Defaults to none.
         required: false
         type: string
+      oras:
+        description: Whether the podvm_image is oras published
+        default: false
+        required: false
+        type: boolean
+
 env:
   CLOUD_PROVIDER: libvirt
   DEBIAN_FRONTEND: noninteractive
@@ -77,7 +83,12 @@ jobs:
           sudo apt-get install -y docker.io
           sudo usermod -aG docker "$USER"
 
+      - uses: oras-project/setup-oras@v1
+        with:
+          version: ${{ env.ORAS_VERSION }}
+
       - name: Extract qcow2 from ${{ inputs.podvm_image }}
+        if: ${{ !inputs.oras }}
         run: |
            qcow2=$(echo ${{ inputs.podvm_image }} | sed -e "s#.*/\(.*\):.*#\1.qcow2#")
            ./hack/download-image.sh ${{ inputs.podvm_image }} . -o "${qcow2}" --clean-up
@@ -86,6 +97,15 @@ jobs:
            docker system prune -a -f
         working-directory: src/cloud-api-adaptor/podvm
 
+      - name: Use oras to get qcow2 from ${{ inputs.podvm_image }}
+        if: ${{ inputs.oras }}
+        run: |
+          oras pull ${{ inputs.podvm_image }}
+          tar xvJpf podvm.tar.xz
+          qcow2=$(find ./*.qcow2)
+          echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV"
+        working-directory: src/cloud-api-adaptor/podvm
+
       - name: Get the install directory
         if: ${{ inputs.install_directory_artifact != '' }}
         uses: actions/download-artifact@v4
@@ -101,10 +121,6 @@ jobs:
           # For debugging
           cat libvirt.properties
 
-      - uses: oras-project/setup-oras@v1
-        with:
-          version: ${{ env.ORAS_VERSION }}
-
       - name: Install gh cli
         run: |
           sudo apt install -y gh

From 9d5cf015fa6566a97ee5bc6a6d5203a932e7b790 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 28 Oct 2024 13:25:19 +0000
Subject: [PATCH 16/22] workflows: e2e_run_all: Add new decoupled jobs

- Added two new decoupled jobs that depends on the
amd64 and s390x mkosi podvm builds to test each
of those images with libvirt
- Added the option of arch specific label triggers in
case of specific testing restrictions

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_run_all.yaml | 38 ++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml
index cfae80b6b..dd67ef533 100644
--- a/.github/workflows/e2e_run_all.yaml
+++ b/.github/workflows/e2e_run_all.yaml
@@ -194,3 +194,41 @@ jobs:
       git_ref: ${{ inputs.git_ref }}
       secure_comms: ${{ matrix.secure_comms }}
     secrets: inherit
+
+  # Run libvirt amd64 e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt'
+  libvirt_amd64:
+    name: E2E tests on libvirt for the amd64 architecture
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64')
+    needs: [podvm_mkosi_amd64, image, prep_install]
+    uses: ./.github/workflows/e2e_libvirt.yaml
+    with:
+      runner: ubuntu-24.04
+      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev
+      podvm_image: ${{ needs.podvm_mkosi_amd64.outputs.qcow2_oras_image }}
+      install_directory_artifact: install_directory
+      git_ref: ${{ inputs.git_ref }}
+      oras: true
+    secrets: inherit
+
+  # Run libvirt s390x e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt'
+  libvirt_s390x:
+    name: E2E tests on libvirt for the s390x architecture
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x')
+    needs: [podvm_mkosi_s390x, image, prep_install]
+    uses: ./.github/workflows/e2e_libvirt.yaml
+    with:
+      runner: S390X
+      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev
+      podvm_image: ${{ needs.podvm_mkosi_s390x.outputs.qcow2_oras_image }}
+      install_directory_artifact: install_directory
+      git_ref: ${{ inputs.git_ref }}
+      oras: true
+    secrets: inherit

From d39e1acb7ca423b4db144135d2bdc8797fed7c9a Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Wed, 30 Oct 2024 13:27:52 +0000
Subject: [PATCH 17/22] workflows: e2e_libvirt: Enforce clean-up

We want to run clean up even if previous steps fail,
so try adding `always` to the condition

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_libvirt.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml
index 859096413..e1d519a63 100644
--- a/.github/workflows/e2e_libvirt.yaml
+++ b/.github/workflows/e2e_libvirt.yaml
@@ -211,5 +211,5 @@ jobs:
         shell: bash {0}
 
       - name: Clean-up cluster
-        if: ${{ runner.environment == 'self-hosted' }}
+        if: ${{ always() && runner.environment == 'self-hosted' }}
         run: ./libvirt/kcli_cluster.sh delete

From 744c56187bc8bbb944fff9a217f73076c7527683 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Wed, 30 Oct 2024 13:45:59 +0000
Subject: [PATCH 18/22] workflows: e2e_run_all: Decouple multi-arch CAA builds

Create separate s390x and amd64 CAA builds in the e2e
test, so we can reduce the coupling of different architecture's
jobs.

Also run the s390x dev build natively on the s390x runner,
so it doesn't run emulated and take a long time.

Signed-off-by: stevenhorsman <steven@uk.ibm.com>

fixup multi-arch caa
---
 .github/workflows/caa_build_and_push.yaml     |  8 +++++-
 .../caa_build_and_push_per_arch.yaml          |  2 +-
 .github/workflows/e2e_run_all.yaml            | 28 +++++++++++++++++--
 3 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/caa_build_and_push.yaml b/.github/workflows/caa_build_and_push.yaml
index 58d44e251..d9630438e 100644
--- a/.github/workflows/caa_build_and_push.yaml
+++ b/.github/workflows/caa_build_and_push.yaml
@@ -33,6 +33,11 @@ on:
         description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main.
         required: false
         type: string
+      runner:
+        default: 'ubuntu-24.04'
+        description: The runner to execute the workflow on. Defaults to 'ubuntu-24.04'.
+        required: false
+        type: string
 
 defaults:
   run:
@@ -41,7 +46,7 @@ defaults:
 jobs:
   build_push_job:
     name: build and push
-    runs-on: ubuntu-24.04
+    runs-on: ${{ inputs.runner }}
     strategy:
       fail-fast: false
       matrix:
@@ -65,6 +70,7 @@ jobs:
 
       - name: Read properties from versions.yaml
         run: |
+          sudo snap install yq
           go_version="$(yq '.tools.golang' versions.yaml)"
           [ -n "$go_version" ]
           echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV"
diff --git a/.github/workflows/caa_build_and_push_per_arch.yaml b/.github/workflows/caa_build_and_push_per_arch.yaml
index e58f5d178..86fa7efa6 100644
--- a/.github/workflows/caa_build_and_push_per_arch.yaml
+++ b/.github/workflows/caa_build_and_push_per_arch.yaml
@@ -58,7 +58,7 @@ jobs:
   build_push_job:
     name: build and push
     needs: [upload_tags]
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.type == 'dev-s390x' && 's390x' || 'ubuntu-24.04' }}
     strategy:
       fail-fast: false
       matrix:
diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml
index dd67ef533..757b01e77 100644
--- a/.github/workflows/e2e_run_all.yaml
+++ b/.github/workflows/e2e_run_all.yaml
@@ -117,7 +117,6 @@ jobs:
   # then please update the PROVIDERS list (space-separated names, e.g.,
   # "aws libvirt").
   prep_install:
-    needs: [image]
     runs-on: ubuntu-24.04
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
@@ -195,6 +194,29 @@ jobs:
       secure_comms: ${{ matrix.secure_comms }}
     secrets: inherit
 
+  caa_image_amd64:
+    uses: ./.github/workflows/caa_build_and_push.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      dev_arches: 'linux/amd64'
+      release_arches: 'linux/amd64'
+      dev_tags: ${{ inputs.caa_image_tag }}-dev
+      release_tags: ${{ inputs.caa_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+    secrets: inherit
+
+  caa_image_s390x:
+    uses: ./.github/workflows/caa_build_and_push.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      dev_arches: 'linux/s390x'
+      release_arches: 'linux/s390x'
+      dev_tags: ${{ inputs.caa_image_tag }}-dev
+      release_tags: ${{ inputs.caa_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+      runner: 's390x'
+    secrets: inherit
+
   # Run libvirt amd64 e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt'
   libvirt_amd64:
     name: E2E tests on libvirt for the amd64 architecture
@@ -203,7 +225,7 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64')
-    needs: [podvm_mkosi_amd64, image, prep_install]
+    needs: [podvm_mkosi_amd64, caa_image_amd64, prep_install]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: ubuntu-24.04
@@ -222,7 +244,7 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x')
-    needs: [podvm_mkosi_s390x, image, prep_install]
+    needs: [podvm_mkosi_s390x, caa_image_s390x, prep_install]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: S390X

From 26a1f1e4c1190f881c1f9f244784ace4035ba888 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 11 Nov 2024 17:01:34 +0000
Subject: [PATCH 19/22] workflows: e2e_run_all: Decouple prep_install

Given that we want to decouple the libvirt testing and
run each architecture independently, having a cached
kustomization.yaml doesn't seem to be very helpful,
so remove that dependency and instead update the
kustomization.yaml in the libvirt e2e step.

It might even be worth considering moving this into
the e2e framework provisioning itself via an ENVVAR

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_libvirt.yaml | 17 +++++++++--------
 .github/workflows/e2e_run_all.yaml | 16 ++++++++--------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml
index e1d519a63..a017338d1 100644
--- a/.github/workflows/e2e_libvirt.yaml
+++ b/.github/workflows/e2e_libvirt.yaml
@@ -106,18 +106,10 @@ jobs:
           echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV"
         working-directory: src/cloud-api-adaptor/podvm
 
-      - name: Get the install directory
-        if: ${{ inputs.install_directory_artifact != '' }}
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ inputs.install_directory_artifact }}
-          path: src/cloud-api-adaptor/install
-
       - name: Config Libvirt
         run: |
           export TEST_E2E_SECURE_COMMS="${{ inputs.secure_comms }}"
           ./libvirt/config_libvirt.sh
-          echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties
           # For debugging
           cat libvirt.properties
 
@@ -131,6 +123,15 @@ jobs:
           curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \
             sudo bash -s /usr/local/bin
 
+      - name: Update kustomization configuration
+        run: |
+          cd "install/overlays/libvirt"
+          kustomize edit set image "cloud-api-adaptor=${{ inputs.caa_image }}"
+          # Print for debugging
+          echo "::group::libvirt kustomization"
+          cat kustomization.yaml
+          echo "::endgroup::"
+
       - name: Checkout KBS Repository
         run: |
           test/utils/checkout_kbs.sh
diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml
index 757b01e77..2156a03d8 100644
--- a/.github/workflows/e2e_run_all.yaml
+++ b/.github/workflows/e2e_run_all.yaml
@@ -200,8 +200,8 @@ jobs:
       registry: ${{ inputs.registry }}
       dev_arches: 'linux/amd64'
       release_arches: 'linux/amd64'
-      dev_tags: ${{ inputs.caa_image_tag }}-dev
-      release_tags: ${{ inputs.caa_image_tag }}
+      dev_tags: ${{ inputs.caa_image_tag }}-amd64-dev
+      release_tags: ${{ inputs.caa_image_tag }}-amd64
       git_ref: ${{ inputs.git_ref }}
     secrets: inherit
 
@@ -211,8 +211,8 @@ jobs:
       registry: ${{ inputs.registry }}
       dev_arches: 'linux/s390x'
       release_arches: 'linux/s390x'
-      dev_tags: ${{ inputs.caa_image_tag }}-dev
-      release_tags: ${{ inputs.caa_image_tag }}
+      dev_tags: ${{ inputs.caa_image_tag }}-s390x-dev
+      release_tags: ${{ inputs.caa_image_tag }}-s390x
       git_ref: ${{ inputs.git_ref }}
       runner: 's390x'
     secrets: inherit
@@ -225,11 +225,11 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64')
-    needs: [podvm_mkosi_amd64, caa_image_amd64, prep_install]
+    needs: [podvm_mkosi_amd64, caa_image_amd64]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: ubuntu-24.04
-      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev
+      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-amd64-dev
       podvm_image: ${{ needs.podvm_mkosi_amd64.outputs.qcow2_oras_image }}
       install_directory_artifact: install_directory
       git_ref: ${{ inputs.git_ref }}
@@ -244,11 +244,11 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x')
-    needs: [podvm_mkosi_s390x, caa_image_s390x, prep_install]
+    needs: [podvm_mkosi_s390x, caa_image_s390x]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: S390X
-      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev
+      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-s390x-dev
       podvm_image: ${{ needs.podvm_mkosi_s390x.outputs.qcow2_oras_image }}
       install_directory_artifact: install_directory
       git_ref: ${{ inputs.git_ref }}

From aa13c2a2fa7a4ae11642257350cad8ad2749b1fe Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Wed, 13 Nov 2024 13:42:56 +0000
Subject: [PATCH 20/22] workflows: e2e_run_all: Add secure comms test support

Re-work the secure_comms matrix input, so it works for the
de-coupled per-arch libvirt test jobs.

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_run_all.yaml            | 32 +++++++++++++++++--
 .../libvirt/libvirt_e2e_arch_matrix.json      |  5 +++
 2 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 src/cloud-api-adaptor/libvirt/libvirt_e2e_arch_matrix.json

diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml
index 2156a03d8..bba0124bf 100644
--- a/.github/workflows/e2e_run_all.yaml
+++ b/.github/workflows/e2e_run_all.yaml
@@ -217,6 +217,28 @@ jobs:
       runner: 's390x'
     secrets: inherit
 
+  libvirt_e2e_arch_prep:
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
+
+      - name: Rebase the code
+        if: github.event_name == 'pull_request_target'
+        working-directory: ./
+        run: |
+          ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch
+
+      - name: Define Test Matrix
+        id: matrix
+        run: |
+          echo "matrix=$(jq -c . < ./libvirt/libvirt_e2e_arch_matrix.json)" >> "$GITHUB_OUTPUT"
+
   # Run libvirt amd64 e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt'
   libvirt_amd64:
     name: E2E tests on libvirt for the amd64 architecture
@@ -225,7 +247,10 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64')
-    needs: [podvm_mkosi_amd64, caa_image_amd64]
+    needs: [podvm_mkosi_amd64, libvirt_e2e_arch_prep, caa_image_amd64]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.libvirt_e2e_arch_prep.outputs.matrix) }}
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: ubuntu-24.04
@@ -244,7 +269,10 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x')
-    needs: [podvm_mkosi_s390x, caa_image_s390x]
+    needs: [podvm_mkosi_s390x, libvirt_e2e_arch_prep, caa_image_s390x]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.libvirt_e2e_arch_prep.outputs.matrix) }}
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: S390X
diff --git a/src/cloud-api-adaptor/libvirt/libvirt_e2e_arch_matrix.json b/src/cloud-api-adaptor/libvirt/libvirt_e2e_arch_matrix.json
new file mode 100644
index 000000000..9824c8966
--- /dev/null
+++ b/src/cloud-api-adaptor/libvirt/libvirt_e2e_arch_matrix.json
@@ -0,0 +1,5 @@
+{
+    "secure_comms": [
+        "none"
+    ]
+}

From 408da63552dc4f623331fc983b88db6e65dc2165 Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Thu, 28 Nov 2024 13:07:37 +0000
Subject: [PATCH 21/22] libvirt: Add the option of EFI firmware support for
 mkosi

- The packer build doesn't want a custom firmware,
so we can't easily keep the default to be ovmf.
Instead reverse the logic and set it as blank unless provided.
- In order to preserve the existing behaviour set the
kustomization.yaml with the old default and update
the packer e2e testing to comment it out
- Also rename to LIBVIRT_EFI_FIRMWARE to clarify that it's
explicitly used for EFI firmware

- If the EFI firmware path is set, then set domain firmware to efi
and set the device disk to sata & sdb as
mkosi needs sata and packer needs IDE. In future once we
drop packer we can potentially simplify this.

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 .github/workflows/e2e_libvirt.yaml             | 18 ++++++++++++++++++
 .../overlays/libvirt/kustomization.yaml        |  2 +-
 src/cloud-providers/libvirt/libvirt.go         | 18 ++++++++++++++++++
 src/cloud-providers/libvirt/manager.go         |  4 ++--
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml
index a017338d1..d364a8ccf 100644
--- a/.github/workflows/e2e_libvirt.yaml
+++ b/.github/workflows/e2e_libvirt.yaml
@@ -80,6 +80,7 @@ jobs:
       - name: Setup docker
         if: ${{ runner.environment == 'self-hosted' }}
         run: |
+          sudo apt-get update -y
           sudo apt-get install -y docker.io
           sudo usermod -aG docker "$USER"
 
@@ -115,14 +116,31 @@ jobs:
 
       - name: Install gh cli
         run: |
+          sudo apt update -y
           sudo apt install -y gh
 
+      - name: Double check that OVMF is installed
+        run: |
+          sudo apt update -y
+          sudo apt install -y ovmf
+
       - name: Install kustomize
         run: |
           command -v kustomize >/dev/null || \
           curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \
             sudo bash -s /usr/local/bin
 
+# For the legacy packer approach we don't want to use the default firmware, so comment it out
+      - name: Set blank firmware for packer libvirt tests
+        if: ${{ !inputs.oras }}
+        run: |
+          cd "install/overlays/libvirt"
+          sed -i 's/\(- LIBVIRT_EFI_FIRMWARE=.*\)/#\1/g' kustomization.yaml
+          # Print for debugging
+          echo "::group::Kustomization.yaml"
+          cat kustomization.yaml
+          echo "::endgroup::"
+
       - name: Update kustomization configuration
         run: |
           cd "install/overlays/libvirt"
diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml
index 3b7280332..b2c56a898 100644
--- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml
+++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml
@@ -25,8 +25,8 @@ configMapGenerator:
   - DISABLECVM="true" # set as false to enable confidential VM
   - SECURE_COMMS="false" # set as true to enable Secure Comms
   - INITDATA="" # set default initdata for podvm
+  - LIBVIRT_EFI_FIRMWARE="/usr/share/OVMF/OVMF_CODE_4M.fd" # Edit to change the EFI firmware path, or comment to unset, if not using EFI.
   #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv
-  #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd
   #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
diff --git a/src/cloud-providers/libvirt/libvirt.go b/src/cloud-providers/libvirt/libvirt.go
index e7b223125..4cb0c2cb9 100644
--- a/src/cloud-providers/libvirt/libvirt.go
+++ b/src/cloud-providers/libvirt/libvirt.go
@@ -388,6 +388,24 @@ func createDomainXMLx86_64(client *libvirtClient, cfg *domainConfig, vm *vmConfi
 		},
 	}
 
+	if vm.firmware != "" {
+		domain.OS.Loader = &libvirtxml.DomainLoader{
+			Path:     vm.firmware,
+			Readonly: "yes",
+			Type:     "pflash",
+		}
+
+		domain.OS.Firmware = "efi"
+
+		// TODO - IDE seems to only work with packer builds and sata only with mkosi,
+		// so we temporarily use the firmware being non-blank to assume this is mkosi
+		cidataDiskIndex := 1
+		var cidataDiskAddr uint = 1
+		domain.Devices.Disks[cidataDiskIndex].Target.Bus = "sata"
+		domain.Devices.Disks[cidataDiskIndex].Target.Dev = "sdb"
+		domain.Devices.Disks[cidataDiskIndex].Address.Drive.Unit = &cidataDiskAddr
+	}
+
 	switch l := vm.launchSecurityType; l {
 	case NoLaunchSecurity:
 		return domain, nil
diff --git a/src/cloud-providers/libvirt/manager.go b/src/cloud-providers/libvirt/manager.go
index ca8964de7..6a73b63b9 100644
--- a/src/cloud-providers/libvirt/manager.go
+++ b/src/cloud-providers/libvirt/manager.go
@@ -22,7 +22,7 @@ const (
 	defaultDataDir        = "/var/lib/libvirt/images"
 	defaultVolName        = "podvm-base.qcow2"
 	defaultLaunchSecurity = ""
-	defaultFirmware       = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
+	defaultFirmware       = ""
 )
 
 func init() {
@@ -47,7 +47,7 @@ func (_ *Manager) LoadEnv() {
 	provider.DefaultToEnv(&libvirtcfg.NetworkName, "LIBVIRT_NET", defaultNetworkName)
 	provider.DefaultToEnv(&libvirtcfg.VolName, "LIBVIRT_VOL_NAME", defaultVolName)
 	provider.DefaultToEnv(&libvirtcfg.LaunchSecurity, "LIBVIRT_LAUNCH_SECURITY", defaultLaunchSecurity)
-	provider.DefaultToEnv(&libvirtcfg.Firmware, "LIBVIRT_FIRMWARE", defaultFirmware)
+	provider.DefaultToEnv(&libvirtcfg.Firmware, "LIBVIRT_EFI_FIRMWARE", defaultFirmware)
 }
 
 func (_ *Manager) NewProvider() (provider.Provider, error) {

From d9007e8c02565eec989756e52da977cbf546e0fd Mon Sep 17 00:00:00 2001
From: stevenhorsman <steven@uk.ibm.com>
Date: Mon, 2 Dec 2024 15:31:19 +0000
Subject: [PATCH 22/22] test/e2e: Skip s390x encrypted image test

The current encrypted image seems to only be amd64 and
the kata CI's multi-arch image has a decryption key that is
44 bytes and our code rejects it as it only works with 32 byte keys

Signed-off-by: stevenhorsman <steven@uk.ibm.com>
---
 src/cloud-api-adaptor/test/e2e/common_suite.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go
index 13a4dce61..cc9822618 100644
--- a/src/cloud-api-adaptor/test/e2e/common_suite.go
+++ b/src/cloud-api-adaptor/test/e2e/common_suite.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
 	"testing"
@@ -501,6 +502,11 @@ func DoTestPodsMTLSCommunication(t *testing.T, e env.Environment, assert CloudAs
 }
 
 func DoTestImageDecryption(t *testing.T, e env.Environment, assert CloudAssert, kbs *pv.KeyBrokerService) {
+	// TODO create a multi-arch encrypted image. Note the Kata CI version doesn't work as the key length is 44, not 32 which is wanted
+	if runtime.GOARCH == "s390x" {
+		t.Skip("Encrypted image test not currently support on s390x")
+	}
+
 	image := "ghcr.io/confidential-containers/cloud-api-adaptor/nginx-encrypted:20240123"
 	var kbsEndpoint string
 	if ep := os.Getenv("KBS_ENDPOINT"); ep != "" {