diff --git a/.github/workflows/caa_build_and_push.yaml b/.github/workflows/caa_build_and_push.yaml index 58d44e251..d9630438e 100644 --- a/.github/workflows/caa_build_and_push.yaml +++ b/.github/workflows/caa_build_and_push.yaml @@ -33,6 +33,11 @@ on: description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main. required: false type: string + runner: + default: 'ubuntu-24.04' + description: The runner to execute the workflow on. Defaults to 'ubuntu-24.04'. + required: false + type: string defaults: run: @@ -41,7 +46,7 @@ defaults: jobs: build_push_job: name: build and push - runs-on: ubuntu-24.04 + runs-on: ${{ inputs.runner }} strategy: fail-fast: false matrix: @@ -65,6 +70,7 @@ jobs: - name: Read properties from versions.yaml run: | + sudo snap install yq go_version="$(yq '.tools.golang' versions.yaml)" [ -n "$go_version" ] echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV" diff --git a/.github/workflows/caa_build_and_push_per_arch.yaml b/.github/workflows/caa_build_and_push_per_arch.yaml index e58f5d178..86fa7efa6 100644 --- a/.github/workflows/caa_build_and_push_per_arch.yaml +++ b/.github/workflows/caa_build_and_push_per_arch.yaml @@ -58,7 +58,7 @@ jobs: build_push_job: name: build and push needs: [upload_tags] - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.type == 'dev-s390x' && 's390x' || 'ubuntu-24.04' }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml index 7de7111b6..d364a8ccf 100644 --- a/.github/workflows/e2e_libvirt.yaml +++ b/.github/workflows/e2e_libvirt.yaml @@ -7,6 +7,11 @@ name: (Callable) libvirt e2e tests on: workflow_call: inputs: + runner: + default: 'ubuntu-24.04' + description: The runner to execute the workflow on. Defaults to 'ubuntu-24.04'. + required: false + type: string podvm_image: required: true type: string @@ -28,6 +33,12 @@ on: description: SecureComms configuration. Defaults to none. required: false type: string + oras: + description: Whether the podvm_image is oras published + default: false + required: false + type: boolean + env: CLOUD_PROVIDER: libvirt DEBIAN_FRONTEND: noninteractive @@ -38,7 +49,7 @@ defaults: jobs: test: - runs-on: ubuntu-24.04 + runs-on: ${{ inputs.runner }} steps: - name: Checkout Code uses: actions/checkout@v4 @@ -69,10 +80,16 @@ jobs: - name: Setup docker if: ${{ runner.environment == 'self-hosted' }} run: | + sudo apt-get update -y sudo apt-get install -y docker.io sudo usermod -aG docker "$USER" + - uses: oras-project/setup-oras@v1 + with: + version: ${{ env.ORAS_VERSION }} + - name: Extract qcow2 from ${{ inputs.podvm_image }} + if: ${{ !inputs.oras }} run: | qcow2=$(echo ${{ inputs.podvm_image }} | sed -e "s#.*/\(.*\):.*#\1.qcow2#") ./hack/download-image.sh ${{ inputs.podvm_image }} . -o "${qcow2}" --clean-up @@ -81,35 +98,58 @@ jobs: docker system prune -a -f working-directory: src/cloud-api-adaptor/podvm - - name: Get the install directory - if: ${{ inputs.install_directory_artifact != '' }} - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.install_directory_artifact }} - path: src/cloud-api-adaptor/install + - name: Use oras to get qcow2 from ${{ inputs.podvm_image }} + if: ${{ inputs.oras }} + run: | + oras pull ${{ inputs.podvm_image }} + tar xvJpf podvm.tar.xz + qcow2=$(find ./*.qcow2) + echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV" + working-directory: src/cloud-api-adaptor/podvm - name: Config Libvirt run: | export TEST_E2E_SECURE_COMMS="${{ inputs.secure_comms }}" ./libvirt/config_libvirt.sh - echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties # For debugging cat libvirt.properties - - uses: oras-project/setup-oras@v1 - with: - version: ${{ env.ORAS_VERSION }} - - name: Install gh cli run: | + sudo apt update -y sudo apt install -y gh + - name: Double check that OVMF is installed + run: | + sudo apt update -y + sudo apt install -y ovmf + - name: Install kustomize run: | command -v kustomize >/dev/null || \ curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \ sudo bash -s /usr/local/bin +# For the legacy packer approach we don't want to use the default firmware, so comment it out + - name: Set blank firmware for packer libvirt tests + if: ${{ !inputs.oras }} + run: | + cd "install/overlays/libvirt" + sed -i 's/\(- LIBVIRT_EFI_FIRMWARE=.*\)/#\1/g' kustomization.yaml + # Print for debugging + echo "::group::Kustomization.yaml" + cat kustomization.yaml + echo "::endgroup::" + + - name: Update kustomization configuration + run: | + cd "install/overlays/libvirt" + kustomize edit set image "cloud-api-adaptor=${{ inputs.caa_image }}" + # Print for debugging + echo "::group::libvirt kustomization" + cat kustomization.yaml + echo "::endgroup::" + - name: Checkout KBS Repository run: | test/utils/checkout_kbs.sh @@ -190,5 +230,5 @@ jobs: shell: bash {0} - name: Clean-up cluster - if: ${{ runner.environment == 'self-hosted' }} + if: ${{ always() && runner.environment == 'self-hosted' }} run: ./libvirt/kcli_cluster.sh delete diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml index d9dff137c..bba0124bf 100644 --- a/.github/workflows/e2e_run_all.yaml +++ b/.github/workflows/e2e_run_all.yaml @@ -66,6 +66,26 @@ jobs: git_ref: ${{ inputs.git_ref }} secrets: inherit + podvm_mkosi_amd64: + uses: ./.github/workflows/podvm_mkosi.yaml + with: + registry: ${{ inputs.registry }} + image_tag: ${{ inputs.podvm_image_tag }} + git_ref: ${{ inputs.git_ref }} + arch: amd64 + debug: true + secrets: inherit + + podvm_mkosi_s390x: + uses: ./.github/workflows/podvm_mkosi.yaml + with: + registry: ${{ inputs.registry }} + image_tag: ${{ inputs.podvm_image_tag }} + git_ref: ${{ inputs.git_ref }} + arch: s390x + debug: true + secrets: inherit + # Build and push the cloud-api-adaptor image # # By using a reusable `workflow_call` workflow we are hitting two @@ -97,7 +117,6 @@ jobs: # then please update the PROVIDERS list (space-separated names, e.g., # "aws libvirt"). prep_install: - needs: [image] runs-on: ubuntu-24.04 outputs: matrix: ${{ steps.matrix.outputs.matrix }} @@ -174,3 +193,92 @@ jobs: git_ref: ${{ inputs.git_ref }} secure_comms: ${{ matrix.secure_comms }} secrets: inherit + + caa_image_amd64: + uses: ./.github/workflows/caa_build_and_push.yaml + with: + registry: ${{ inputs.registry }} + dev_arches: 'linux/amd64' + release_arches: 'linux/amd64' + dev_tags: ${{ inputs.caa_image_tag }}-amd64-dev + release_tags: ${{ inputs.caa_image_tag }}-amd64 + git_ref: ${{ inputs.git_ref }} + secrets: inherit + + caa_image_s390x: + uses: ./.github/workflows/caa_build_and_push.yaml + with: + registry: ${{ inputs.registry }} + dev_arches: 'linux/s390x' + release_arches: 'linux/s390x' + dev_tags: ${{ inputs.caa_image_tag }}-s390x-dev + release_tags: ${{ inputs.caa_image_tag }}-s390x + git_ref: ${{ inputs.git_ref }} + runner: 's390x' + secrets: inherit + + libvirt_e2e_arch_prep: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - name: Rebase the code + if: github.event_name == 'pull_request_target' + working-directory: ./ + run: | + ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch + + - name: Define Test Matrix + id: matrix + run: | + echo "matrix=$(jq -c . < ./libvirt/libvirt_e2e_arch_matrix.json)" >> "$GITHUB_OUTPUT" + + # Run libvirt amd64 e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt' + libvirt_amd64: + name: E2E tests on libvirt for the amd64 architecture + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') || + contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64') + needs: [podvm_mkosi_amd64, libvirt_e2e_arch_prep, caa_image_amd64] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.libvirt_e2e_arch_prep.outputs.matrix) }} + uses: ./.github/workflows/e2e_libvirt.yaml + with: + runner: ubuntu-24.04 + caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-amd64-dev + podvm_image: ${{ needs.podvm_mkosi_amd64.outputs.qcow2_oras_image }} + install_directory_artifact: install_directory + git_ref: ${{ inputs.git_ref }} + oras: true + secrets: inherit + + # Run libvirt s390x e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt' + libvirt_s390x: + name: E2E tests on libvirt for the s390x architecture + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') || + contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x') + needs: [podvm_mkosi_s390x, libvirt_e2e_arch_prep, caa_image_s390x] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.libvirt_e2e_arch_prep.outputs.matrix) }} + uses: ./.github/workflows/e2e_libvirt.yaml + with: + runner: S390X + caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-s390x-dev + podvm_image: ${{ needs.podvm_mkosi_s390x.outputs.qcow2_oras_image }} + install_directory_artifact: install_directory + git_ref: ${{ inputs.git_ref }} + oras: true + secrets: inherit diff --git a/.github/workflows/podvm_mkosi.yaml b/.github/workflows/podvm_mkosi.yaml index 91a073c25..ac7f59e14 100644 --- a/.github/workflows/podvm_mkosi.yaml +++ b/.github/workflows/podvm_mkosi.yaml @@ -3,126 +3,290 @@ name: Create a Pod VM image with mkosi on: workflow_dispatch: inputs: - binaries-image: - description: "Prebuild fedora binaries image, as produced by this workflow under ghcr.io/confidential-containers/cloud-api-adaptor/podvm/binaries-fedora" + registry: + default: 'quay.io/confidential-containers' required: false + type: string + image_tag: + default: '' + required: false + type: string + git_ref: + description: Git ref to checkout the cloud-api-adaptor repository. + required: true + type: string + arch: + description: Which arch we are building the mkosi image for + default: 'amd64' + required: false + type: string + debug: + description: Whether to build the image in debug mode + default: false + required: false + type: boolean + + workflow_call: + inputs: + registry: + default: 'quay.io/confidential-containers' + required: false + type: string + image_tag: + default: '' + required: false + type: string + git_ref: + description: Git ref to checkout the cloud-api-adaptor repository. + required: true + type: string + arch: + description: Which arch we are building the mkosi image for + default: 'amd64' + required: false + type: string + debug: + description: Whether to build the image in debug mode + default: false + required: false + type: boolean + outputs: + qcow2_oras_image: + description: The location of the qcow2 oras container this workflow pushed + value: ${{ jobs.build-image.outputs.qcow2_oras_image }} + docker_oci_image: + description: The location of the docker oci container image this workflow pushed + value: ${{ jobs.build-image.outputs.docker_oci_image }} defaults: run: working-directory: src/cloud-api-adaptor jobs: - build-binaries: - name: Build binaries - if : ${{ github.event.inputs.binaries-image == '' }} - runs-on: ubuntu-24.04 + build-image: + name: Build mkosi image + runs-on: ${{ inputs.arch == 's390x' && 's390x' || 'ubuntu-24.04' }} permissions: contents: read packages: write + id-token: write + attestations: write + outputs: + qcow2_oras_image: ${{ steps.publish_oras_qcow2.outputs.image }}:${{ steps.publish_oras_qcow2.outputs.tag }} + docker_oci_image: ${{ steps.build_docker_oci.outputs.image }} steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: "${{ inputs.git_ref }}" + + - name: Rebase the code + if: github.event_name == 'pull_request_target' + working-directory: ./ + run: | + ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Log in to GitHub Container Registry + - name: Login to quay Container Registry + if: ${{ startsWith(inputs.registry, 'quay.io') }} + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Login to the ghcr Container registry + if: ${{ startsWith(inputs.registry, 'ghcr.io') }} uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build builder - uses: docker/build-push-action@v6 + - name: Install build dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y bubblewrap alien dnf qemu-utils uidmap + sudo snap install yq + + - name: Read properties from versions.yaml + run: | + go_version="$(yq '.tools.golang' versions.yaml)" + echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV" + echo "ORAS_VERSION=$(yq -e '.tools.oras' versions.yaml)" >> "$GITHUB_ENV" + + - name: Setup Golang version ${{ env.GO_VERSION }} + if: ${{ inputs.arch == 's390x' }} + uses: actions/setup-go@v5 with: - tags: ghcr.io/${{ github.repository }}/podvm/builder-fedora:${{ github.sha }} - context: src - file: src/cloud-api-adaptor/podvm/Dockerfile.podvm_builder.fedora - push: true + go-version: ${{ env.GO_VERSION }} - # Build binaries need git commit id as part of image name - - name: Prepare .git folder - working-directory: ./ + - uses: oras-project/setup-oras@v1 + with: + version: ${{ env.ORAS_VERSION }} + + - name: Build builder + id: build_builder + working-directory: src/cloud-api-adaptor/podvm-mkosi run: | - cp -rf .git src/.git + # If the builder image matching our git sha exists, then skip this build + builder_image=${{ inputs.registry }}/podvm-builder-fedora-${{ inputs.arch }} + tag=$(git rev-parse --short HEAD) + + if ! docker manifest inspect "${builder_image}:${tag}"; then + PODVM_BUILDER_IMAGE="${builder_image}:${tag}" make fedora-binaries-builder + fi + echo "image=${builder_image}:${tag}" | tee -a "$GITHUB_OUTPUT" + + # If the input has a different image-tag then also push it with that tag + if [ -n "${{ inputs.image_tag }}" ] && [ "${{ inputs.image_tag }}" != "${tag}" ];then + docker pull "${builder_image}:${tag}" + docker tag "${builder_image}:${tag}" ${builder_image}:${{ inputs.image_tag }} + docker push "${builder_image}:${{ inputs.image_tag }}" + fi + env: + PUSH: true + ARCH: ${{ inputs.arch }} - name: Build binaries - uses: docker/build-push-action@v6 - with: - tags: ghcr.io/${{ github.repository }}/podvm/binaries-fedora:${{ github.sha }} - context: src - file: src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora - push: true - build-args: - "BUILDER_IMG=ghcr.io/${{ github.repository }}/podvm/builder-fedora:${{ github.sha }}" + id: build_binaries + working-directory: src/cloud-api-adaptor/podvm-mkosi + run: | + # If the binaries which matching git sha exists, then skip this build + binaries_image=${{ inputs.registry }}/podvm-binaries-fedora-${{ inputs.arch }} + tag="$(git rev-parse --short HEAD)" + echo "image_tag=${image_tag}" | tee -a "$GITHUB_OUTPUT" - build-image: - name: Build image - needs: [build-binaries] - if: | - always() && ( - needs.build-binaries.result == 'success' || ( - needs.build-binaries.result == 'skipped' && - github.event.inputs.binaries-image != '' - ) - ) - runs-on: ubuntu-24.04 - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v4 + if ! docker manifest inspect "${binaries_image}:${tag}"; then + PODVM_BINARIES_IMAGE="${binaries_image}:${tag}" make binaries + fi + echo "image=${binaries_image}:${tag}" | tee -a "$GITHUB_OUTPUT" + + # If the input has a different image-tag then also push it with that tag + if [ -n "${{ inputs.image_tag }}" ] && [ "${{ inputs.image_tag }}" != "${tag}" ];then + docker pull "${binaries_image}:${tag}" + docker tag "${binaries_image}:${tag}" ${binaries_image}:${{ inputs.image_tag }} + docker push "${binaries_image}:${{ inputs.image_tag }}" + fi + env: + PUSH: true + ARCH: ${{ inputs.arch }} + PODVM_BUILDER_IMAGE: ${{ steps.build_builder.outputs.image }} + + - name: Install mkosi + if: ${{ inputs.arch == 's390x' }} + run: | + git clone -b v22 https://github.com/systemd/mkosi + sudo rm -f /usr/local/bin/mkosi + sudo ln -s "$PWD/mkosi/bin/mkosi" /usr/local/bin/mkosi + mkosi --version - name: Install Nix + if: ${{ inputs.arch == 'amd64' }} uses: cachix/install-nix-action@v30 - name: Build nix shell to cache dependencies + if: ${{ inputs.arch == 'amd64' }} run: nix build .#devShells.x86_64-linux.podvm-mkosi - - name: Install crane + - name: Install crane with nix + if: ${{ inputs.arch == 'amd64' }} run: nix profile install nixpkgs#crane - # This removes the checkout and creates a btrfs volume with maximized - # build space. - - name: Maximize build space - uses: katexochen/maximize-build-space@btrfs - with: - swap-size-mb: 1024 - remove-dotnet: "true" - remove-android: "true" - remove-haskell: "true" - remove-codeql: "true" - remove-docker-images: "true" + - name: Install crane with go + if: ${{ inputs.arch == 's390x' }} + run: | + go install github.com/google/go-containerregistry/cmd/crane@latest + echo "PATH=${PATH}:$(go env GOPATH)/bin" >> "$GITHUB_ENV" - name: Second checkout + if: ${{ inputs.arch == 'amd64' }} uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: "${{ inputs.git_ref }}" - - name: Decide on image to use - id: binaries-image + + - name: Rebase the code + if: github.event_name == 'pull_request_target' + working-directory: ./ run: | - if [ -z "${{ github.event.inputs.binaries-image }}" ]; then - echo "image=ghcr.io/${{ github.repository }}/podvm/binaries-fedora:${{ github.sha }}" | tee -a "$GITHUB_OUTPUT" - else - echo "image=${{ github.event.inputs.binaries-image }}" | tee -a "$GITHUB_OUTPUT" - fi + ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch - name: Download binaries and unpack into binaries-tree run: | crane export \ - ${{ steps.binaries-image.outputs.image }} \ + ${{ steps.build_binaries.outputs.image }} \ podvm-binaries.tar mkdir -p podvm-mkosi/resources/binaries-tree tar xf podvm-binaries.tar -C podvm-mkosi/resources/binaries-tree rm podvm-binaries.tar - - name: Build image + - name: Build mkosi debug image + if: ${{ inputs.debug == 'true' }} working-directory: src/cloud-api-adaptor/podvm-mkosi run: make image-debug - - name: Upload image - uses: actions/upload-artifact@v4 + - name: Build mkosi image + if: ${{ inputs.debug != 'true' }} + working-directory: src/cloud-api-adaptor/podvm-mkosi + run: make image + + - name: Upload the qcow2 with oras + id: publish_oras_qcow2 + working-directory: src/cloud-api-adaptor/podvm-mkosi + run: | + mkdir oras + cd oras + cp ../build/podvm-*.qcow2 . + tar cJf podvm.tar.xz podvm-*.qcow2 + image=${{ inputs.registry }}/podvm-generic-fedora + if [ "${{inputs.debug}}" = "true" ]; then + image=${image}-debug + fi + image=${image}-${{ inputs.arch }} + tag=$(git rev-parse --short HEAD) + oras push "${image}:${tag}" podvm.tar.xz + + # If the input has a different image-tag then also push it with that tag + if [ -n "${{ inputs.image_tag }}" ] && [ "${{ inputs.image_tag }}" != "${tag}" ];then + oras push "${image}:${{ inputs.image_tag }}" podvm.tar.xz + fi + + # add image and digest to output for attestation + echo "image=${image}" >> "$GITHUB_OUTPUT" + digest="$(oras manifest fetch "${image}:${tag}" --descriptor | jq -r .digest)" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + + - uses: actions/attest-build-provenance@v1 with: - name: podvm-mkosi-${{ github.sha }} - path: src/cloud-api-adaptor/podvm-mkosi/build/system.raw + subject-name: ${{ steps.publish_oras_qcow2.outputs.image }} + subject-digest: ${{ steps.publish_oras_qcow2.outputs.digest }} + push-to-registry: true + + + - name: Clean up some space for the docker provider build + working-directory: src/cloud-api-adaptor/podvm-mkosi + run: | + sudo du --max-depth=2 /home/runner || true + sudo du --max-depth=2 /var/lib || true + sudo rm -rf /nix + sudo rm -rf ./build + sudo rm -rf ./mkosi.cache + + - name: Build image for docker provider + id: build_docker_oci + working-directory: src/cloud-api-adaptor/podvm-mkosi + run: | + tag=$(git rev-parse --short HEAD) + PODVM_TAG=${tag} make image-container + arch=$(uname -m) + arch=${arch/x86_64/amd64} + echo "image=ghcr.io/${{ github.repository }}/podvm-docker-image-${arch}:${tag}" >> "$GITHUB_OUTPUT" + env: + PUSH: true + REGISTRY: ghcr.io/${{ github.repository }} diff --git a/.github/workflows/podvm_mkosi_image.yaml b/.github/workflows/podvm_mkosi_image.yaml deleted file mode 100644 index e985c91da..000000000 --- a/.github/workflows/podvm_mkosi_image.yaml +++ /dev/null @@ -1,60 +0,0 @@ -name: mkosi build podvm builder, binaries and image - -on: - workflow_dispatch: - -jobs: - build-podvm-image-mkosi: - name: Build podvm image via mkosi - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - runner: - - S390X - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Quay container Registry - uses: docker/login-action@v3 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - logout: false - - - name: Install build dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y bubblewrap alien dnf qemu-utils uidmap - sudo snap install yq - - - name: Install mkosi - run: | - git clone -b v22 https://github.com/systemd/mkosi - sudo rm /usr/local/bin/mkosi | true - sudo ln -s "$PWD/mkosi/bin/mkosi" /usr/local/bin/mkosi - mkosi --version - - - name: Build builder - working-directory: src/cloud-api-adaptor/podvm-mkosi - run: make fedora-binaries-builder - - - name: Build binaries - working-directory: src/cloud-api-adaptor/podvm-mkosi - run: make binaries - - - name: Build image - run: make image-debug - working-directory: src/cloud-api-adaptor/podvm-mkosi - - - name: Push image - run: make push-image - working-directory: src/cloud-api-adaptor/podvm-mkosi diff --git a/.github/workflows/podvm_publish.yaml b/.github/workflows/podvm_publish.yaml index 4122eeda9..6a5edaeea 100644 --- a/.github/workflows/podvm_publish.yaml +++ b/.github/workflows/podvm_publish.yaml @@ -30,3 +30,16 @@ jobs: git_ref: ${{ github.sha }} image_tag: ${{ github.sha }} secrets: inherit + + podvm-mkosi: + uses: ./.github/workflows/podvm_mkosi.yaml + strategy: + fail-fast: false + matrix: + arch: [amd64, s390x] + with: + git_ref: ${{ github.sha }} + image_tag: ${{ github.sha }} + arch: ${{ matrix.arch}} + debug: false + secrets: inherit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fa18fc9cd..90fca28c5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -54,6 +54,19 @@ jobs: git_ref: ${{ github.ref }} secrets: inherit + podvm-mkosi: + uses: ./.github/workflows/podvm_mkosi.yaml + strategy: + fail-fast: false + matrix: + arch: [amd64, s390x] + with: + image_tag: ${{ github.event.release.tag_name }} + git_ref: ${{ github.ref }} + arch: ${{ matrix.arch}} + debug: false + secrets: inherit + webhook: uses: ./.github/workflows/webhook_image.yaml with: diff --git a/src/cloud-api-adaptor/hack/build-s390x-image.sh b/src/cloud-api-adaptor/hack/build-s390x-image.sh index 2cf11a75a..eb96706f0 100755 --- a/src/cloud-api-adaptor/hack/build-s390x-image.sh +++ b/src/cloud-api-adaptor/hack/build-s390x-image.sh @@ -76,7 +76,7 @@ umount "${dst_mnt}" qemu-nbd --disconnect "${tmp_nbd}" -output_img_name="podvm-s390x.qcow2" +output_img_name="podvm-fedora-s390x.qcow2" qemu-img convert -O qcow2 -c "${tmp_img_path}" "${output_img_name}" chmod 644 "${output_img_name}" diff --git a/src/cloud-api-adaptor/hack/build-s390x-se-image.sh b/src/cloud-api-adaptor/hack/build-s390x-se-image.sh index d9dabff8b..7c9e11eba 100755 --- a/src/cloud-api-adaptor/hack/build-s390x-se-image.sh +++ b/src/cloud-api-adaptor/hack/build-s390x-se-image.sh @@ -172,7 +172,7 @@ cryptsetup close "$LUKS_NAME" qemu-nbd --disconnect "${tmp_nbd}" -output_img_name="podvm-s390x-se.qcow2" +output_img_name="podvm-fedora-s390x-se.qcow2" qemu-img convert -O qcow2 -c "${tmp_img_path}" "${output_img_name}" output_img_path=$(realpath "${output_img_name}") echo "podvm se-image is generated: ${output_img_path}" diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 3b7280332..b2c56a898 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -25,8 +25,8 @@ configMapGenerator: - DISABLECVM="true" # set as false to enable confidential VM - SECURE_COMMS="false" # set as true to enable Secure Comms - INITDATA="" # set default initdata for podvm + - LIBVIRT_EFI_FIRMWARE="/usr/share/OVMF/OVMF_CODE_4M.fd" # Edit to change the EFI firmware path, or comment to unset, if not using EFI. #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv - #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 diff --git a/src/cloud-api-adaptor/libvirt/libvirt_e2e_arch_matrix.json b/src/cloud-api-adaptor/libvirt/libvirt_e2e_arch_matrix.json new file mode 100644 index 000000000..9824c8966 --- /dev/null +++ b/src/cloud-api-adaptor/libvirt/libvirt_e2e_arch_matrix.json @@ -0,0 +1,5 @@ +{ + "secure_comms": [ + "none" + ] +} diff --git a/src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm b/src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm_docker_provider similarity index 100% rename from src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm rename to src/cloud-api-adaptor/podvm-mkosi/Dockerfile.podvm_docker_provider diff --git a/src/cloud-api-adaptor/podvm-mkosi/Makefile b/src/cloud-api-adaptor/podvm-mkosi/Makefile index 63fc7189f..2d7267c1a 100644 --- a/src/cloud-api-adaptor/podvm-mkosi/Makefile +++ b/src/cloud-api-adaptor/podvm-mkosi/Makefile @@ -1,17 +1,24 @@ include ../Makefile.defaults ARCH ?= $(subst x86_64,amd64,$(shell uname -m)) -BUILDER = fedora-binaries-builder-$(ARCH) SE_BOOT ?= false IS_DEBIAN := $(shell if grep -q 'ID_LIKE=debian' /etc/os-release; then echo "true"; else echo "false"; fi) REGISTRY ?= quay.io/confidential-containers PODVM_DISTRO ?= fedora -PODVM_TAG ?= $(VERSIONS_HASH) -PODVM_NAME ?= $(REGISTRY)/podvm-generic-$(PODVM_DISTRO)-$(ARCH) -PODVM_CONTAINER_NAME ?= $(REGISTRY)/podvm-docker-image +ifeq ($(PODVM_TAG),) + PODVM_TAG := $(VERSIONS_HASH) +endif +PODVM_BUILDER_IMAGE ?= $(REGISTRY)/podvm-builder-$(PODVM_DISTRO)-$(ARCH):$(PODVM_TAG) +PODVM_BINARIES_IMAGE ?= $(REGISTRY)/podvm-binaries-$(PODVM_DISTRO)-$(ARCH):$(PODVM_TAG) +PODVM_IMAGE ?= $(REGISTRY)/podvm-generic-$(PODVM_DISTRO)$(if $(filter $(SE_BOOT),true),-se,)-$(ARCH):$(PODVM_TAG) +PODVM_CONTAINER_NAME ?= $(REGISTRY)/podvm-docker-image-$(ARCH) VERIFY_PROVENANCE ?= no +PUSH ?= false +# If not pushing `--load` into the local docker cache +DOCKER_OPTS := $(if $(filter $(PUSH),true),--push,--load) $(EXTRA_DOCKER_OPTS) + .DEFAULT_GOAL := all .PHONY: all all: fedora-binaries-builder binaries image @@ -30,7 +37,7 @@ PHONY: fedora-binaries-builder fedora-binaries-builder: @echo "Building $(BUILDER) image..." docker buildx build \ - -t $(BUILDER) \ + -t $(PODVM_BUILDER_IMAGE) \ --build-arg GO_VERSION=$(GO_VERSION) \ --build-arg ARCH=$(ARCH) \ --build-arg PROTOC_VERSION=$(PROTOC_VERSION) \ @@ -39,7 +46,7 @@ fedora-binaries-builder: --build-arg YQ_ARCH=$(ARCH) \ --build-arg PROTOC_ARCH=$(if $(filter amd64,$(ARCH)),x86_64,s390x) \ --build-arg ORAS_VERSION=$(ORAS_VERSION) \ - --load \ + $(DOCKER_OPTS) \ -f ../podvm/Dockerfile.podvm_builder.fedora ../. PHONY: binaries @@ -50,8 +57,9 @@ ifeq ($(IS_DEBIAN),true) docker buildx use default endif docker buildx build \ + -t $(PODVM_BINARIES_IMAGE) \ --progress=plain \ - --build-arg BUILDER_IMG=$(BUILDER) \ + --build-arg BUILDER_IMG=$(PODVM_BUILDER_IMAGE) \ --build-arg TEE_PLATFORM=$(TEE_PLATFORM) \ --build-arg PAUSE_REPO=$(PAUSE_REPO) \ --build-arg PAUSE_VERSION=$(PAUSE_VERSION) \ @@ -60,7 +68,8 @@ endif --build-arg VERIFY_PROVENANCE=$(VERIFY_PROVENANCE) \ $(if $(AUTHFILE),--build-arg AUTHFILE=$(AUTHFILE),) \ $(if $(DEFAULT_AGENT_POLICY_FILE),--build-arg DEFAULT_AGENT_POLICY_FILE=$(DEFAULT_AGENT_POLICY_FILE),) \ - -o type=local,dest="./resources/binaries-tree" \ + $(if $(filter $(PUSH),true),,-o type=local,dest="./resources/binaries-tree") \ + $(DOCKER_OPTS) \ -f ../podvm/Dockerfile.podvm_binaries.fedora ../../ PHONY: image @@ -73,25 +82,14 @@ ifeq ($(SE_BOOT),true) touch resources/buildS390xImage sudo mkosi --profile production.conf --image system sudo -E ../hack/build-s390x-se-image.sh - @echo "Building docker image..." - docker buildx build \ - -t $(PODVM_NAME)-se:$(PODVM_TAG) \ - -t $(PODVM_NAME)-se:latest \ - --load \ - -f ../podvm/Dockerfile.podvm.fedora . else ifeq ($(ARCH),s390x) touch resources/buildS390xImage sudo mkosi --profile production.conf --image system sudo -E ../hack/build-s390x-image.sh - @echo "Building docker image..." - docker buildx build \ - -t $(PODVM_NAME):$(PODVM_TAG) \ - -t $(PODVM_NAME):latest \ - --load \ - -f ../podvm/Dockerfile.podvm.fedora . else touch resources/buildBootableImage - nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=production + sudo -E env PATH=$(PATH) nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=production + qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 endif PHONY: image-debug @@ -105,26 +103,16 @@ ifeq ($(SE_BOOT),true) touch resources/buildS390xImage sudo mkosi --profile debug.conf sudo -E ../hack/build-s390x-se-image.sh - @echo "Building docker image..." - docker buildx build \ - -t $(PODVM_NAME)-se:$(PODVM_TAG) \ - -t $(PODVM_NAME)-se:latest \ - --load \ - -f ../podvm/Dockerfile.podvm.fedora . else ifeq ($(ARCH),s390x) touch resources/buildS390xImage sudo mkosi --profile debug.conf sudo -E ../hack/build-s390x-image.sh - @echo "Building docker image..." - docker buildx build \ - -t $(PODVM_NAME):$(PODVM_TAG) \ - -t $(PODVM_NAME):latest \ - --load \ - -f ../podvm/Dockerfile.podvm.fedora . else touch resources/buildBootableImage - nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=debug + sudo -E env PATH=$(PATH) nix develop ..#podvm-mkosi --command mkosi --environment=VARIANT_ID=debug + qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 endif + sudo chown -R $(USER): build PHONY: image-container image-container: @@ -132,7 +120,8 @@ image-container: docker buildx build \ -t $(PODVM_CONTAINER_NAME):$(PODVM_TAG) \ -t $(PODVM_CONTAINER_NAME):latest \ - -f Dockerfile.podvm . + $(DOCKER_OPTS) \ + -f Dockerfile.podvm_docker_provider . PHONY: push-image push-image: diff --git a/src/cloud-api-adaptor/podvm/Dockerfile.podvm.fedora b/src/cloud-api-adaptor/podvm/Dockerfile.podvm.fedora deleted file mode 100644 index d8496f8a5..000000000 --- a/src/cloud-api-adaptor/podvm/Dockerfile.podvm.fedora +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Confidential Containers Contributors -# -# SPDX-License-Identifier: Apache-2.0 -# -# Place pod vm image built via mkosi -# -FROM scratch - -ARG ARCH=s390x - -ENV ARCH=${ARCH} - -COPY build/podvm-*.qcow2 / diff --git a/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora b/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora index 74d9c294d..75459daca 100644 --- a/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora +++ b/src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora @@ -48,6 +48,8 @@ ENV IMAGE_CHECKSUM="none" COPY . /src WORKDIR /src/cloud-api-adaptor/podvm +# Installs add-ons for foreign target, if required +RUN ./hack/cross-build-extras.sh RUN LIBC=gnu make binaries diff --git a/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh b/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh index f0093ff36..b36e6632d 100755 --- a/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh +++ b/src/cloud-api-adaptor/podvm/hack/cross-build-extras.sh @@ -17,5 +17,14 @@ libc=$([[ $ARCH =~ s390x ]] && echo "gnu" || echo "musl") rustTarget="$ARCH-unknown-linux-$libc" rustup target add "$rustTarget" -apt install -y "qemu-system-$ARCH" -apt install -y "gcc-$ARCH-linux-$libc" + +source /etc/os-release || source /usr/lib/os-release +if [[ ${ID_LIKE:-} == *"debian"* ]]; then + apt install -y "qemu-system-$ARCH" + apt install -y "gcc-$ARCH-linux-$libc" +elif [[ "${ID_LIKE:-}" =~ "fedora" ]] || [[ "${ID:-}" =~ "fedora" ]]; then + dnf install -y "qemu-system-$ARCH" + dnf install -y "gcc-$ARCH-linux-$libc" +else + echo "Unsupported distro $ID"; exit 1 +fi diff --git a/src/cloud-api-adaptor/podvm/hack/download-image.sh b/src/cloud-api-adaptor/podvm/hack/download-image.sh index 4087ad063..4ebe3efb8 100755 --- a/src/cloud-api-adaptor/podvm/hack/download-image.sh +++ b/src/cloud-api-adaptor/podvm/hack/download-image.sh @@ -43,18 +43,18 @@ fi [ -z "$container_binary" ] && error "please install docker or podman" -if [[ -z "$platform" ]]; then - # Check if the image name includes "podvm-generic-fedora-s390x-se" - # The "podvm-generic-fedora-s390x-se" docker image is built on s390x host, so here must use s390x platform - if [[ "$image" == *"podvm-generic-fedora-s390x-se"* ]]; then - platform="s390x" - else - platform="amd64" - fi +platform_flag="" +if [[ -n "${platform}" ]]; then + platform_flag="--platform=${platform}" +elif [[ "${image}" == *"podvm-generic-ubuntu-s390x"* ]]; then + # The podvm-generic-ubuntu-s390x image (which will be deprecated soon) is built on amd64 + # and therefore has the incorrect platform type, so we need to override it to create the + # container to extract the image from. Others can use their default platform + platform_flag="--platform=amd64" fi # Create a non-running container to extract image -$container_binary create --pull=always --platform="$platform" --name "$container_name" "$image" /bin/sh >/dev/null 2>&1; +$container_binary create --pull=always ${platform_flag} --name "$container_name" "$image" /bin/sh >/dev/null 2>&1; # Destory container after use rm-container(){ $container_binary rm -f "$container_name" >/dev/null 2>&1; diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 13a4dce61..cc9822618 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -8,6 +8,7 @@ import ( "fmt" "math/rand" "os" + "runtime" "strconv" "strings" "testing" @@ -501,6 +502,11 @@ func DoTestPodsMTLSCommunication(t *testing.T, e env.Environment, assert CloudAs } func DoTestImageDecryption(t *testing.T, e env.Environment, assert CloudAssert, kbs *pv.KeyBrokerService) { + // TODO create a multi-arch encrypted image. Note the Kata CI version doesn't work as the key length is 44, not 32 which is wanted + if runtime.GOARCH == "s390x" { + t.Skip("Encrypted image test not currently support on s390x") + } + image := "ghcr.io/confidential-containers/cloud-api-adaptor/nginx-encrypted:20240123" var kbsEndpoint string if ep := os.Getenv("KBS_ENDPOINT"); ep != "" { diff --git a/src/cloud-providers/libvirt/libvirt.go b/src/cloud-providers/libvirt/libvirt.go index e7b223125..4cb0c2cb9 100644 --- a/src/cloud-providers/libvirt/libvirt.go +++ b/src/cloud-providers/libvirt/libvirt.go @@ -388,6 +388,24 @@ func createDomainXMLx86_64(client *libvirtClient, cfg *domainConfig, vm *vmConfi }, } + if vm.firmware != "" { + domain.OS.Loader = &libvirtxml.DomainLoader{ + Path: vm.firmware, + Readonly: "yes", + Type: "pflash", + } + + domain.OS.Firmware = "efi" + + // TODO - IDE seems to only work with packer builds and sata only with mkosi, + // so we temporarily use the firmware being non-blank to assume this is mkosi + cidataDiskIndex := 1 + var cidataDiskAddr uint = 1 + domain.Devices.Disks[cidataDiskIndex].Target.Bus = "sata" + domain.Devices.Disks[cidataDiskIndex].Target.Dev = "sdb" + domain.Devices.Disks[cidataDiskIndex].Address.Drive.Unit = &cidataDiskAddr + } + switch l := vm.launchSecurityType; l { case NoLaunchSecurity: return domain, nil diff --git a/src/cloud-providers/libvirt/manager.go b/src/cloud-providers/libvirt/manager.go index ca8964de7..6a73b63b9 100644 --- a/src/cloud-providers/libvirt/manager.go +++ b/src/cloud-providers/libvirt/manager.go @@ -22,7 +22,7 @@ const ( defaultDataDir = "/var/lib/libvirt/images" defaultVolName = "podvm-base.qcow2" defaultLaunchSecurity = "" - defaultFirmware = "/usr/share/edk2/ovmf/OVMF_CODE.fd" + defaultFirmware = "" ) func init() { @@ -47,7 +47,7 @@ func (_ *Manager) LoadEnv() { provider.DefaultToEnv(&libvirtcfg.NetworkName, "LIBVIRT_NET", defaultNetworkName) provider.DefaultToEnv(&libvirtcfg.VolName, "LIBVIRT_VOL_NAME", defaultVolName) provider.DefaultToEnv(&libvirtcfg.LaunchSecurity, "LIBVIRT_LAUNCH_SECURITY", defaultLaunchSecurity) - provider.DefaultToEnv(&libvirtcfg.Firmware, "LIBVIRT_FIRMWARE", defaultFirmware) + provider.DefaultToEnv(&libvirtcfg.Firmware, "LIBVIRT_EFI_FIRMWARE", defaultFirmware) } func (_ *Manager) NewProvider() (provider.Provider, error) {