ROCKS 1452 - Refactor Build Rock workflow to be externally reusable

.github/workflows/Build-Rock.yaml

@@ -1,54 +1,62 @@
-name: Build rock
+name: Build Rock
 
 on:
   workflow_call:
     inputs:
+      # build configuration
       oci-archive-name:
-        description: "Final filename of the rock's OCI archive"
+        description: "Final filename of the Rock OCI archive."
         type: string
         required: true
-      oci-factory-path:
-        description: "Path, in the OCI Factory, to this rock"
+      arch-map:
+        description: "JSON string mapping target architecture to runner."
         type: string
-        required: true
-      rock-name:
-        description: "Name of the rock"
-        type: string
-        required: true
+        default: '{"amd64": "ubuntu-22.04"}'
+      lpci-fallback:
+        description: 'Enable fallback to Launchpad build when runners for target arch are not available.'
+        type: boolean
+        default: false
+
+      # source configuration
       rock-repo:
-        description: "Public Git repo where to build the rock from"
+        description: "Public Git repo where to build the rock from."
         type: string
         required: true
       rock-repo-commit:
-        description: "Git ref from where to build the rock from"
+        description: "Git ref from where to build the rock from."
         type: string
         required: true
       rockfile-directory:
-        description: "Directory, in 'rock-repo', where to find the rockcraft.yaml file"
+        description: "Directory in repository where to find the rockcraft.yaml file."
         type: string
         required: true
 
+
 env:
-  ROCKS_CI_FOLDER: ci-rocks
+  ROCK_REPO_DIR: rock-repo  # path where the image repo is cloned to
+  ROCK_CI_FOLDER: ci-rocks  # path of uploaded/downloaded artifacts
+
+  OCI_FACTORY_REPO: canonical/oci-factory
+  # TODO: update before final merge
+  OCI_FACTORY_BRANCH: ROCKS-1452/oci-factory-refactor-build-rock-to-be-externally-reusable  # required for external workflow calls.
 
 jobs:
-  prepare-multi-arch-matrix:
+
+  configure-build:
+    # configure-build reads the rockcraft.yaml, creating one or more *-build job runs
+    # depending on the target architecture.
     runs-on: ubuntu-22.04
     outputs:
-      build-for: ${{ steps.rock-platforms.outputs.build-for }}
-      build-with-lpci: ${{ steps.rock-platforms.outputs.build-with-lpci }}
+      runner-build-matrix: ${{ steps.configure.outputs.runner-build-matrix }} 
+      lpci-build-matrix: ${{ steps.configure.outputs.lpci-build-matrix }} 
     steps:
-      - name: Clone GitHub image repository
-        uses: actions/checkout@v4
-        id: clone-image-repo
-        continue-on-error: true
+
+      # Job Setup
+      - uses: actions/checkout@v4
         with:
-          repository: ${{ inputs.rock-repo }}
-          fetch-depth: 0
-      - name: Clone generic image repository
-        if: ${{ steps.clone-image-repo.outcome == 'failure' }}
-        run: |
-          git clone ${{ inputs.rock-repo }} .
+          repository: ${{ env.OCI_FACTORY_REPO }}
+          ref: ${{ env.OCI_FACTORY_BRANCH }}
+          fetch-depth: 1
 
       - name: Validate access to triggered image
         uses: ./.github/actions/validate-actor
@@ -58,159 +66,171 @@ jobs:
           image-path: ${{ inputs.oci-factory-path }}
           github-token: ${{ secrets.ROCKSBOT_TOKEN }}
 
-      - run: git checkout ${{ inputs.rock-repo-commit }}
-
-      - run: sudo snap install yq --channel=v4/stable
-      - name: Validate image naming and base
-        working-directory: ${{ inputs.rockfile-directory }}
-        run: |
-          rock_name=`cat rockcraft.y*ml | yq -r .name`
-          if [[ "${{ inputs.oci-factory-path }}" != *"${rock_name}"* ]]
-          then
-            echo "ERROR: the rock's name '${rock_name}' must match the OCI folder name!"
-            exit 1
-          fi
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
-      - run: pip install pyyaml
-      - name: Get rock archs
-        uses: jannekem/run-python-script-action@v1
-        id: rock-platforms
+          python-version: "3.x"
+
+      - run: pip install -r src/build_rock/configure/requirements.txt
+
+
+      - uses: actions/checkout@v4
         with:
-          script: |
-            import yaml
-            import os
-
-            BUILD_WITH_LPCI = 0
-
-            with open("${{ inputs.rockfile-directory }}/rockcraft.yaml") as rf:
-              rockcraft_yaml = yaml.safe_load(rf)
-
-            platforms = rockcraft_yaml["platforms"]
-
-            target_archs = []
-            for platf, values in platforms.items():
-                if isinstance(values, dict) and "build-for" in values:
-                    target_archs += list(values["build-for"])
-                    continue
-                target_archs.append(platf)
-
-            print(f"Target architectures: {set(target_archs)}")
-
-            matrix = {"include": []}
-            gh_supported_archs = {"amd64": "ubuntu-22.04", "arm64": "Ubuntu_ARM64_4C_16G_01"}
-            if set(target_archs) - set(gh_supported_archs.keys()):
-              # Then there are other target archs, so we need to build in LP
-              matrix["include"].append(
-                {"architecture": "-".join(set(target_archs)), "runner": gh_supported_archs["amd64"]}
-              )
-              BUILD_WITH_LPCI = 1
-            else:
-              for runner_arch, runner_name in gh_supported_archs.items():
-                if runner_arch in target_archs:
-                  matrix["include"].append(
-                    {"architecture": runner_arch, "runner": runner_name}
-                  )
-
-            with open(os.environ["GITHUB_OUTPUT"], "a") as gh_out:
-              print(f"build-for={matrix}", file=gh_out)
-              print(f"build-with-lpci={BUILD_WITH_LPCI}", file=gh_out)
-
-  build:
-    needs: [prepare-multi-arch-matrix]
+          repository: ${{ inputs.rock-repo }}
+          path: ${{ env.ROCK_REPO_DIR }}
+          ref: ${{ inputs.rock-repo-commit }}
+          submodules: 'recursive'
+
+
+      # Configure matrices for each *-build job
+      - name: Configure
+        id: configure
+        run: |
+          python3 -m src.build_rock.configure.generate_build_matrix \
+            --rockfile-directory "${{ env.ROCK_REPO_DIR }}/${{ inputs.rockfile-directory }}" \
+            --lpci-fallback "${{ toJSON(inputs.lpci-fallback) }}" \
+            --config ${{ toJSON(inputs.arch-map) }} # important: do not use quotes here
+
+
+  runner-build:
+    # runner-build builds rocks per target architecture using pre configured runner images.
+    needs: [configure-build]
+    if: fromJSON(needs.configure-build.outputs.runner-build-matrix).include[0] != ''
     strategy:
       fail-fast: true
-      matrix: ${{ fromJSON(needs.prepare-multi-arch-matrix.outputs.build-for) }}
-    runs-on: ${{ matrix.runner }}
-    name: 'Build ${{ inputs.rock-name }} | ${{ matrix.architecture }}'
+      matrix: ${{ fromJSON(needs.configure-build.outputs.runner-build-matrix) }}
+    runs-on: ${{ matrix.runner }} 
+    name: 'runner-build | ${{ matrix.architecture }} '
     steps:
-      - name: Clone GitHub image repository
+
+      - name: Clone Repo
         uses: actions/checkout@v4
-        id: clone-image-repo
-        continue-on-error: true
         with:
           repository: ${{ inputs.rock-repo }}
-          fetch-depth: 0
-      - name: Clone generic image repository
-        if: ${{ steps.clone-image-repo.outcome == 'failure' }}
-        run: |
-          git clone ${{ inputs.rock-repo }} .
-      - run: git checkout ${{ inputs.rock-repo-commit }}
-      - name: Build rock ${{ inputs.rock-name }}
+          path: ${{ env.ROCK_REPO_DIR }}
+          ref: ${{ inputs.rock-repo-commit }}
+          submodules: 'recursive'
+
+
+      - name: Build Target
         id: rockcraft
-        if: needs.prepare-multi-arch-matrix.outputs.build-with-lpci == 0
         uses: canonical/craft-actions/rockcraft-pack@main
         with:
-          path: "${{ inputs.rockfile-directory }}"
+          path: "${{ env.ROCK_REPO_DIR }}/${{ inputs.rockfile-directory }}"
           verbosity: debug
-      - uses: actions/setup-python@v5
-        if: needs.prepare-multi-arch-matrix.outputs.build-with-lpci == 1
+
+
+      - name: Collect Artifacts
+        id: collect-artifacts
+        run: |
+              mkdir -p ${{ env.ROCK_CI_FOLDER }} && cp ${{ steps.rockcraft.outputs.rock }} "$_"
+              echo "filename=$(basename ${{ steps.rockcraft.outputs.rock }})" >> $GITHUB_OUTPUT
+
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v4
         with:
-          python-version: '3.x'
-      - uses: nick-fields/[email protected]
-        name: Build multi-arch ${{ inputs.rock-name }} in Launchpad
-        if: needs.prepare-multi-arch-matrix.outputs.build-with-lpci == 1
+          name: ${{ inputs.oci-archive-name }}-${{ steps.collect-artifacts.outputs.filename }}
+          path: ${{ env.ROCK_CI_FOLDER }}
+          if-no-files-found: error
+
+
+  lpci-build:
+    # lpci-build is a fallback for building rocks if no suitable runners are 
+    # configured for the required architecture. Builds in this job will be
+    # outsourced to Launchpad for completion. 
+    # Note the Secret
+    needs: [configure-build]
+    if: fromJSON(needs.configure-build.outputs.lpci-build-matrix).include[0] != ''
+    strategy: 
+      fail-fast: true
+      matrix: ${{ fromJSON(needs.configure-build.outputs.lpci-build-matrix) }}
+    runs-on: ubuntu-22.04
+    name: 'lpci-build | ${{ matrix.architecture }} '
+    steps:
+
+      # Job Setup
+      - uses: actions/checkout@v4
+        with:
+          repository: ${{ env.OCI_FACTORY_REPO }}
+          ref: ${{ env.OCI_FACTORY_BRANCH }}
+          fetch-depth: 1
+
+
+      - name: Clone Repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.rock-repo }}
+          path: ${{ env.ROCK_REPO_DIR }}
+          ref: ${{ inputs.rock-repo-commit }}
+          submodules: 'recursive'
+
+
+      - name: Build Target
+        # TODO: Replace this retry action with bash equivalent for better testing
+        uses: nick-fields/[email protected]
         with:
           timeout_minutes: 180
           max_attempts: 4
           polling_interval_seconds: 5
           retry_wait_seconds: 30
           command: |
-            set -ex
-            cd ${{ inputs.rockfile-directory }}
-            rocks_toolbox="$(mktemp -d)"
-            git clone --depth 1 --branch v1.1.2 https://github.com/canonical/rocks-toolbox $rocks_toolbox
-            ${rocks_toolbox}/rockcraft_lpci_build/requirements.sh
-            pip3 install -r ${rocks_toolbox}/rockcraft_lpci_build/requirements.txt
-
-            python3 ${rocks_toolbox}/rockcraft_lpci_build/rockcraft_lpci_build.py \
-              --lp-credentials-b64 "${{ secrets.LP_CREDENTIALS_B64 }}" \
-              --launchpad-accept-public-upload
-      - name: Rename rock OCI archive
-        id: rock
+            src/build_rock/lpci_build/lpci_build.sh \
+              -c "${{ secrets.LP_CREDENTIALS_B64 }}" \
+              -d "${{ env.ROCK_REPO_DIR }}/${{ inputs.rockfile-directory }}"
+
+
+      - name: Collect Artifacts
+        id: collect-artifacts
         run: |
-          mkdir ${{ env.ROCKS_CI_FOLDER }}
-          if [ ${{ needs.prepare-multi-arch-matrix.outputs.build-with-lpci }} -eq 0 ]
-          then
-            cp ${{ steps.rockcraft.outputs.rock }} ${{ env.ROCKS_CI_FOLDER }}/$(basename ${{ steps.rockcraft.outputs.rock }})
-            echo "filename=$(basename ${{ steps.rockcraft.outputs.rock }})" >> $GITHUB_OUTPUT
-          else
-            cp ${{ inputs.rockfile-directory }}/*.rock ${{ env.ROCKS_CI_FOLDER }}
-            echo "filename=${{ inputs.rock-name }}_${{ matrix.architecture }}" >> $GITHUB_OUTPUT
-          fi
-      - name: Upload ${{ inputs.rock-name }} for ${{ matrix.architecture }}
+            mkdir -p ${{ env.ROCK_CI_FOLDER }} && cp ${{ env.ROCK_REPO_DIR }}/${{ inputs.rockfile-directory }}/*.rock "$_"
+            echo "filename=${{ matrix.rock-name }}_${{ matrix.architecture }}" >> $GITHUB_OUTPUT
+
+
+      - name: Upload Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.oci-archive-name }}-${{ steps.rock.outputs.filename }}
-          path: ${{ env.ROCKS_CI_FOLDER }}
+          name: ${{ inputs.oci-archive-name }}-${{ steps.collect-artifacts.outputs.filename }}
+          path: ${{ env.ROCK_CI_FOLDER }}
           if-no-files-found: error
 
+
   assemble-rock:
-    needs: [prepare-multi-arch-matrix, build]
+    # Assemble individual single-arch rocks into multi-arch rocks
+    needs: [runner-build, lpci-build]
     runs-on: ubuntu-22.04
+    # Always run even if one of the *-build jobs are skipped
+    # Nice example from benjamin-bergia/github-workflow-patterns...
+    if: ${{ always() && contains(needs.*.result, 'success') && !(contains(needs.*.result, 'failure')) }}
     steps:
-      - uses: actions/download-artifact@v4
+
+      # Job Setup
+      - uses: actions/checkout@v4
+        with:
+          repository: ${{ env.OCI_FACTORY_REPO }}
+          ref: ${{ env.OCI_FACTORY_BRANCH }}
+          fetch-depth: 1
+
+      - run: src/build_rock/assemble_rock/requirements.sh
+
+
+      - name: Download Single Arch Rocks
+        uses: actions/download-artifact@v4
         id: download
-      - run: sudo apt update && sudo apt install buildah -y
-      - name: Merge single-arch rocks into multi-arch OCI archive
+        with:
+          path: ${{ env.ROCK_CI_FOLDER }} 
+          pattern: ${{ inputs.oci-archive-name }}-*
+
+
+      - name: Assemble Multi Arch Rock
         run: |
-          set -xe
-          ls ./${{ inputs.oci-archive-name }}*
-          buildah manifest create multi-arch-rock
-          for rock in `find ${{ inputs.oci-archive-name }}*/*.rock`
-          do
-            test -f $rock
-            buildah manifest add multi-arch-rock oci-archive:$rock
-          done
-          buildah manifest push --all multi-arch-rock oci-archive:${{ inputs.oci-archive-name }}
-      - name: Upload multi-arch ${{ inputs.oci-archive-name }} OCI archive
+          src/build_rock/assemble_rock/assemble.sh \
+            -n "${{ inputs.oci-archive-name }}" \
+            -d "${{ env.ROCK_CI_FOLDER }}"
+
+
+      - name: Upload Multi Arch Rock
         uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.oci-archive-name }}
           path: ${{ inputs.oci-archive-name }}
           if-no-files-found: error
-      - uses: actions/cache/save@v4
-        with:
-          path: ${{ inputs.oci-archive-name }}
-          key: ${{ github.run_id }}-${{ inputs.oci-archive-name }}