feat: add new ansible build matrix

spacelift-io · Aug 1, 2024 · 211c4ef · 211c4ef
1 parent 98be0ab
commit 211c4ef
Show file tree

Hide file tree

Showing 6 changed files with 485 additions and 0 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,299 @@
+name: Build docker images
+on:
+  push:
+env:
+  REGISTRY_IMAGE: ghcr.io/${{ github.repository }}
+concurrency:
+  group: docker-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  matrix:
+    name: Compute build matrix from pypi API
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: './build-matrix/go.mod'
+          cache-dependency-path: './build-matrix/go.mod'
+
+      - name: Run matrix generator tests
+        working-directory: build-matrix
+        run: go test ./
+
+      - id: matrix
+        working-directory: build-matrix
+        run: |
+          MATRIX=$(go run ./)
+          echo ${MATRIX} | jq
+          echo 'matrix=[{"ansible":"10.2","additional_tags":["10"]},{"ansible":"10.1","additional_tags":[]}]' >> $GITHUB_OUTPUT
+          #echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
+  build:
+    needs: [ matrix ]
+    runs-on: ubuntu-latest
+    name: Build ansible ${{ matrix.versions.ansible }}-${{ matrix.target }}/${{ matrix.platform }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      matrix:
+        target:
+          - base
+          - aws
+          - gcp
+        platform:
+          - linux/amd64
+          - linux/arm64
+        versions: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Github Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Docker images
+        uses: docker/build-push-action@v6
+        id: build
+        with:
+          pull: true
+          target: ${{ matrix.target }}
+          build-args: |
+            ANSIBLE_VERSION=${{ matrix.versions.ansible }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=docker,dest=/tmp/${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}.tar
+          tags: ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}-${{ github.sha }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp/${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}.tar
+          retention-days: 1
+          if-no-files-found: error
+
+  test:
+    name: Test image ${{ matrix.versions.ansible }}-${{ matrix.target }}/${{ matrix.platform }}
+    needs: [ matrix, build ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - base
+          - aws
+          - gcp
+        platform:
+          - linux/amd64
+        versions: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp
+      - name: Load image
+        run: |
+          docker load --input /tmp/${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}.tar
+          docker image ls -a
+      - name: Test ansible version
+        run: docker run ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}-${{ github.sha }} ansible-community --version | grep 'Ansible community version ${{ matrix.versions.ansible }}'
+
+  security:
+    name: Security scan
+    needs: [ matrix, build ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - base
+          - aws
+          - gcp
+        platform:
+          - linux/amd64
+          - linux/arm64
+        versions: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp
+
+      - name: Load image
+        run: |
+          docker load --input /tmp/${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}.tar
+          docker image ls -a
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/[email protected]
+        with:
+          exit-code: '1'
+          image-ref: "ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}-${{ github.sha }}"
+          format: "template"
+          template: "@/contrib/sarif.tpl"
+          output: "trivy-results.sarif"
+          severity: "CRITICAL,HIGH"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: "trivy-results.sarif"
+
+  deploy:
+    name: Push image ${{ matrix.versions.ansible }}
+    needs: [ matrix, security, test ]
+    runs-on: ubuntu-latest
+    env:
+      AWS_REGION: "us-east-1"
+    strategy:
+      matrix:
+        versions: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/ansible_build_matrix' }} # TODO(eliecharra): Remove condition
+    permissions:
+      #id-token: write
+      packages: write
+      contents: read
+    steps:
+      - name: Download base/amd64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-base-linux-amd64
+          path: /tmp
+
+      - name: Download gcp/amd64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-gcp-linux-amd64
+          path: /tmp
+
+      - name: Download aws/amd64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-aws-linux-amd64
+          path: /tmp
+
+      - name: Download base/arm64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-base-linux-arm64
+          path: /tmp
+
+      - name: Download gcp/arm64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-gcp-linux-arm64
+          path: /tmp
+
+      - name: Download aws/arm64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-aws-linux-arm64
+          path: /tmp
+
+      - name: Load image
+        run: |
+          docker load --input /tmp/${{ matrix.versions.ansible }}-base-linux-amd64.tar
+          docker load --input /tmp/${{ matrix.versions.ansible }}-gcp-linux-amd64.tar
+          docker load --input /tmp/${{ matrix.versions.ansible }}-aws-linux-amd64.tar
+          docker load --input /tmp/${{ matrix.versions.ansible }}-base-linux-arm64.tar
+          docker load --input /tmp/${{ matrix.versions.ansible }}-gcp-linux-arm64.tar
+          docker load --input /tmp/${{ matrix.versions.ansible }}-aws-linux-arm64.tar
+          docker image ls -a
+
+#      - name: Configure AWS credentials
+#        uses: aws-actions/configure-aws-credentials@v4
+#        with:
+#          aws-region: ${{ env.AWS_REGION }}
+#          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+#          role-duration-seconds: 900
+#
+#      - name: Login to Amazon ECR
+#        run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${REPOSITORY_PATH}
+#        env:
+#          REPOSITORY_PATH: ${{ secrets.PUBLIC_RUNNER_ANSIBLE_ECR_REPOSITORY_URL }}
+
+      - name: Log in to Github Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # TODO Push to ECR
+      - name: Push images
+        run: |
+          echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64"
+          docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-base-linux-amd64-${{ github.sha }}\
+            ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64
+          docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64
+          
+          echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64"
+          docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64-${{ github.sha }}\
+            ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64
+          docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64
+
+          echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64"
+          docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64-${{ github.sha }}\
+            ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64
+          docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64
+
+          echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64"
+          docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-base-linux-arm64-${{ github.sha }}\
+            ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64
+          docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64
+
+          echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64"
+          docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64-${{ github.sha }}\
+            ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64
+          docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64
+          
+          echo "Pushing ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64"
+          docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64-${{ github.sha }}\
+            ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64
+          docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64
+
+          echo "Create manifest ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}"
+          docker manifest create ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }} \
+            --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-arm64 \
+            --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-linux-amd64
+          docker manifest push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}
+
+          echo "Create manifest ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp"
+          docker manifest create ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp \
+            --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-arm64 \
+            --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp-linux-amd64
+          docker manifest push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-gcp
+          
+          echo "Create manifest ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws"
+          docker manifest create ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws \
+            --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-arm64 \
+            --amend ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws-linux-amd64
+          docker manifest push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-aws
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3.12 as base
+ARG ANSIBLE_VERSION=10.0
+RUN apt update && DEBIAN_FRONTEND=noninteractive apt upgrade -y &&\
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* &&\
+    pip install --no-cache-dir ansible==${ANSIBLE_VERSION}.* ansible-runner~=2.4
+
+FROM base as gcp
+RUN pip install --no-cache-dir requests==2.32.3 google-auth==2.32.0
+
+FROM base as aws
+RUN pip install --no-cache-dir boto3==1.34.151
diff --git a/build-matrix/go.mod b/build-matrix/go.mod
@@ -0,0 +1,11 @@
+module github.com/spacelift-io/build-matrix
+
+go 1.22.1
+
+require (
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
diff --git a/build-matrix/go.sum b/build-matrix/go.sum
@@ -0,0 +1,11 @@
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=