new-build-core-template.yml

name: Build Core images
on:
  workflow_call:
    secrets:
      DOCKERHUB_USER:
        description: "DOCKERHUB_USER"
        required: true
      DOCKERHUB_TOKEN:
        description: "DOCKERHUB_TOKEN"
        required: true
    inputs:
      image_tag_suffix:
        description: "Optional suffix to override tag name generation"
        type: string
        required: false
      compilers:
        description: 'JSON of required compilers and their versions'
        type: string
        required: false
        default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18", "1.3.19", "1.3.21"] } , { "zkvyper": ["1.3.13"] }]'
      en_alpha_release:
        description: 'Flag that determins if EN release should be marked as alpha'
        type: boolean
        required: false
        default: false
      action:
        type: string
        required: false
        default: "do nothing"

jobs:
  prepare-contracts:
    name: Prepare contracts
    runs-on: matterlabs-ci-runner-high-performance
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          submodules: "recursive"

      - name: Prepare ENV
        shell: bash
        run: |
          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
          echo CI=1 >> $GITHUB_ENV
          echo $(pwd)/bin >> $GITHUB_PATH
          echo CI=1 >> .env
          echo IN_DOCKER=1 >> .env

      - name: Download contracts
        shell: bash
        run: |
          commit_sha=$(git submodule status contracts | awk '{print $1}' | tr -d '-')
          page=1
          filtered_tag=""
          while [ true ]; do
            echo "Page: $page"
            tags=$(run_retried curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github+json" \
              "https://api.github.com/repos/matter-labs/era-contracts/tags?per_page=100&page=${page}" | jq .)
            if [ $(jq length <<<"$tags") -eq 0 ]; then
              echo "No tag found on all pages."
              echo "BUILD_CONTRACTS=true" >> "$GITHUB_ENV"
              exit 0
            fi
            filtered_tag=$(jq -r --arg commit_sha "$commit_sha" 'map(select(.commit.sha == $commit_sha)) | .[].name' <<<"$tags")
            if [[ ! -z "$filtered_tag" ]]; then
              echo "BUILD_CONTRACTS=false" >> "$GITHUB_ENV"
              break
            fi
            ((page++))
          done
          echo "Contracts tag is: ${filtered_tag}"
          mkdir -p ./contracts
          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l1-contracts.tar.gz
          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l2-contracts.tar.gz
          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/system-contracts.tar.gz
          tar -C ./contracts -zxf l1-contracts.tar.gz
          tar -C ./contracts -zxf l2-contracts.tar.gz
          tar -C ./contracts -zxf system-contracts.tar.gz

      - name: Install Apt dependencies
        shell: bash
        run: |
          sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config

      - name: Install Node
        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
        with:
          node-version: 20
          cache: 'npm'

      - name: Install Yarn
        run: npm install -g yarn

      - name: Setup rust
        uses: actions-rust-lang/setup-rust-toolchain@1fbea72663f6d4c03efaab13560c8a24cfd2a7cc # v1.9.0
        with:
          toolchain: nightly-2024-08-01

      - name: Install cargo-nextest from crates.io
        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
        with:
          crate: cargo-nextest

      - name: Install sqlx-cli from crates.io
        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
        with:
          crate: sqlx-cli
          tag: 0.8.1

      - name: Install Foundry
        uses: foundry-rs/foundry-toolchain@8f1998e9878d786675189ef566a2e4bf24869773 # v1.2.0

      - name: Pre-download compilers
        shell: bash
        run: |
          # Download needed versions of vyper compiler
          # Not sanitized due to unconventional path and tags
          mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux
          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux
          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux
          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10
          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3

          COMPILERS_JSON='${{ inputs.compilers }}'
          echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do
            mkdir -p "./hardhat-nodejs/compilers-v2/$compiler"
            wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}"
            chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
          done

      - name: init
        shell: bash
        run: |
          mkdir -p ./volumes/postgres
          docker compose up -d postgres
          zkt || true

      - name: build contracts
        shell: bash
        run: |
          cp etc/tokens/{test,localhost}.json
          zk_supervisor contracts

      - name: Upload contracts
        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
        with:
          name: contacts
          path: |
            ./contracts

  build-images:
    name: Build and Push Docker Images
    needs: prepare-contracts
    env:
      IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}${{ (inputs.en_alpha_release && matrix.components == 'external-node') && '-alpha' || '' }}
    runs-on: ${{ fromJSON('["matterlabs-ci-runner-high-performance", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }}
    strategy:
      matrix:
        components:
          - server-v2
          - external-node
          - snapshots-creator
        platforms:
          - linux/amd64
        include:
          - components: external-node
            platforms: linux/arm64

    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          submodules: "recursive"

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1

      - name: Setup env
        shell: bash
        run: |
          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
          echo CI=1 >> $GITHUB_ENV
          echo $(pwd)/bin >> $GITHUB_PATH
          echo CI=1 >> .env
          echo IN_DOCKER=1 >> .env

      - name: Download setup key
        shell: bash
        run: |
          run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key

      - name: Set env vars
        shell: bash
        run: |
          echo PLATFORM=$(echo ${{ matrix.platforms }} | tr '/' '-') >> $GITHUB_ENV
          echo IMAGE_TAG_SHA=$(git rev-parse --short HEAD) >> $GITHUB_ENV
          # Support for custom tag suffix
          if [ -n "${{ inputs.image_tag_suffix }}" ]; then
            echo IMAGE_TAG_SHA_TS="${{ inputs.image_tag_suffix }}" >> $GITHUB_ENV
          else
            echo IMAGE_TAG_SHA_TS=$(git rev-parse --short HEAD)-$(date +%s) >> $GITHUB_ENV
          fi

      - name: Download contracts
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: contacts
          path: |
            ./contracts

      - name: login to Docker registries
        if: ${{ inputs.action == 'push' }}
        shell: bash
        run: |
          docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
          gcloud auth configure-docker us-docker.pkg.dev -q

      - name: Build and push
        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
        with:
          context: .
          push: ${{ inputs.action == 'push' }}
          file: docker/${{ matrix.components }}/Dockerfile
          build-args: |
            SCCACHE_GCS_BUCKET=matterlabs-infra-sccache-storage
            SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com
            SCCACHE_GCS_RW_MODE=READ_WRITE
            RUSTC_WRAPPER=sccache
          tags: |
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest
            matterlabs/${{ matrix.components }}:latest
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:latest2.0
            matterlabs/${{ matrix.components }}:latest2.0
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA }}
            matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA }}
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA }}
            matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA }}
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
            matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
            matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}
            matterlabs/${{ matrix.components }}:2.0-${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}
            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}
            matterlabs/${{ matrix.components }}:${{ env.IMAGE_TAG_SHA_TS }}-${{ env.PLATFORM }}

  create_manifest:
    name: Create release manifest
    runs-on: matterlabs-ci-runner
    needs: build-images
    if: ${{ inputs.action == 'push' }}
    strategy:
      matrix:
        component:
          - name: server-v2
            platform: linux/amd64
          - name: external-node
            platform: linux/amd64,linux/arm64
          - name: snapshots-creator
            platform: linux/amd64

    env:
      IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}${{ (inputs.en_alpha_release && matrix.component.name == 'external-node') && '-alpha' || '' }}
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4

      - name: login to Docker registries
        shell: bash
        run: |
          docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
          gcloud auth configure-docker us-docker.pkg.dev -q

      - name: Create Docker manifest
        shell: bash
        run: |
          docker_repositories=("matterlabs/${{ matrix.component.name }}" "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component.name }}")
          platforms=${{ matrix.component.platform }}
          for repo in "${docker_repositories[@]}"; do
            platform_tags=""
            for platform in ${platforms//,/ }; do
              platform=$(echo $platform | tr '/' '-')
              platform_tags+=" --amend ${repo}:${IMAGE_TAG_SUFFIX}-${platform}"
            done
            for manifest in "${repo}:${IMAGE_TAG_SUFFIX}" "${repo}:2.0-${IMAGE_TAG_SUFFIX}" "${repo}:latest" "${repo}:latest2.0"; do
              docker manifest create ${manifest} ${platform_tags}
              docker manifest push ${manifest}
            done
          done