Skip to content

Commit

Permalink
Merge pull request #290 from lambdaclass/da-eigen-merged-main
Browse files Browse the repository at this point in the history
chore: Eigen DA Merge Main
  • Loading branch information
juan518munoz authored Oct 8, 2024
2 parents aa6ea7d + bdb3a82 commit d06b433
Show file tree
Hide file tree
Showing 726 changed files with 22,371 additions and 26,812 deletions.
13 changes: 9 additions & 4 deletions .githooks/pre-push
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,13 @@ RED='\033[0;31m'
NC='\033[0m' # No Color

# Check that prettier formatting rules are not violated.
if ! zk fmt --check; then
echo -e "${RED}Push error!${NC}"
echo "Please format the code via 'zk fmt', cannot push unformatted code"
exit 1
if which zk_supervisor >/dev/null; then
if ! zk_supervisor fmt --check; then
echo -e "${RED}Push error!${NC}"
echo "Please format the code via 'zks fmt', cannot push unformatted code"
exit 1
fi
else
echo "Please install zk_toolbox using zkup from https://github.com/matter-labs/zksync-era/tree/main/zk_toolbox/zkup, and then run ./bin/zkt from the zksync-era repository."
exit 1
fi
2 changes: 1 addition & 1 deletion .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
- [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.
- [ ] Code has been formatted via `zk_supervisor fmt` and `zk_supervisor lint`.
2 changes: 1 addition & 1 deletion .github/release-please/manifest.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"core": "24.24.0",
"core": "24.28.0",
"prover": "16.5.0",
"zk_toolbox": "0.1.2"
}
43 changes: 43 additions & 0 deletions .github/scripts/rate_limit_check.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/bin/bash

set -o errexit
set -o pipefail


api_endpoint="https://api.github.com/users/zksync-era-bot"
wait_time=60
max_retries=60
retry_count=0

while [[ $retry_count -lt $max_retries ]]; do
response=$(run_retried curl -s -w "%{http_code}" -o temp.json "$api_endpoint")
http_code=$(echo "$response" | tail -n1)

if [[ "$http_code" == "200" ]]; then
echo "Request successful. Not rate-limited."
cat temp.json
rm temp.json
exit 0
elif [[ "$http_code" == "403" ]]; then
rate_limit_exceeded=$(jq -r '.message' temp.json | grep -i "API rate limit exceeded")
if [[ -n "$rate_limit_exceeded" ]]; then
retry_count=$((retry_count+1))
echo "API rate limit exceeded. Retry $retry_count of $max_retries. Retrying in $wait_time seconds..."
sleep $wait_time
else
echo "Request failed with HTTP status $http_code."
cat temp.json
rm temp.json
exit 1
fi
else
echo "Request failed with HTTP status $http_code."
cat temp.json
rm temp.json
exit 1
fi
done

echo "Reached the maximum number of retries ($max_retries). Exiting."
rm temp.json
exit 1
159 changes: 159 additions & 0 deletions .github/workflows/build-base.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
name: Build zksync-build-base Docker image
on:
workflow_dispatch:
inputs:
repo_ref:
description: "git reference of the zksync-era to build"
required: true
default: main
jobs:
build-images:
name: Build and Push Docker Images
runs-on: ${{ fromJSON('["matterlabs-ci-runner-high-performance", "matterlabs-ci-runner-arm"]')[contains(matrix.arch, 'arm')] }}
outputs:
image_tag_sha: ${{ steps.get-sha.outputs.image_tag_sha }}
# Needed to push to Gihub Package Registry
permissions:
packages: write
contents: read
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
REPO_REF: ${{ github.event.inputs.repo_ref }}
strategy:
matrix:
name: [ build-base ]
repository: [ zksync-build-base ]
arch: [ amd64, arm64 ]

steps:
- uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
with:
submodules: "recursive"

- name: Login to google container registry
run: |
gcloud auth configure-docker us-docker.pkg.dev -q
- name: Login to DockerHub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Login to GitHub Container Registry
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Get tag
id: get-sha
run: |
echo IMAGE_TAG_SHA=$(git rev-parse --short HEAD) >> $GITHUB_ENV
echo image_tag_sha=$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1

- name: Build and push
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
with:
push: true
context: .
file: docker/build-base/Dockerfile
labels: |
org.opencontainers.image.source=https://github.com/matter-labs/zksync-era
org.opencontainers.image.licenses="MIT OR Apache-2.0"
tags: |
matterlabs/zksync-build-base:${{ steps.get-sha.outputs.image_tag_sha }}-${{ matrix.arch }}
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:${{ steps.get-sha.outputs.image_tag_sha }}-${{ matrix.arch }}
ghcr.io/${{ github.repository_owner }}/zksync-build-base:${{ steps.get-sha.outputs.image_tag_sha }}-${{ matrix.arch }}
multiarch_manifest:
# Needed to push to Gihub Package Registry
permissions:
packages: write
contents: read
needs: [ build-images ]
env:
IMAGE_TAG_SUFFIX: ${{ needs.build-images.outputs.image_tag_sha }}
runs-on: [ matterlabs-ci-runner-high-performance ]
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1

- name: Login to google container registry
run: |
gcloud auth configure-docker us-docker.pkg.dev -q
- name: Login to DockerHub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Login to GitHub Container Registry
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Create and push multi-arch manifests for Dockerhub
shell: bash
run: |
images=("zksync-build-base")
archs=("amd64" "arm64")
for img in "${images[@]}"; do
multiarch_tag="matterlabs/zksync-build-base:latest"
individual_images=()
for arch in "${archs[@]}"; do
TAG="$IMAGE_TAG_SUFFIX"
docker pull matterlabs/zksync-build-base:${TAG}-${arch} --platform linux/${arch}
individual_images+=("matterlabs/zksync-build-base:${TAG}-${arch}")
done
docker buildx imagetools create --tag "${multiarch_tag}" "${individual_images[@]}"
done
- name: Create and push multi-arch manifests for GitHub Container Registry
shell: bash
run: |
images=("zksync-build-base")
archs=("amd64" "arm64")
for img in "${images[@]}"; do
multiarch_tag="ghcr.io/${{ github.repository_owner }}/zksync-build-base:latest"
individual_images=()
for arch in "${archs[@]}"; do
TAG="$IMAGE_TAG_SUFFIX"
docker pull ghcr.io/${{ github.repository_owner }}/zksync-build-base:${TAG}-${arch} --platform linux/${arch}
individual_images+=("ghcr.io/${{ github.repository_owner }}/zksync-build-base:${TAG}-${arch}")
done
docker buildx imagetools create --tag "${multiarch_tag}" "${individual_images[@]}"
done
- name: Create and push multi-arch manifests for Google Artifact Registry
shell: bash
run: |
images=("zksync-build-base")
archs=("amd64" "arm64")
for img in "${images[@]}"; do
multiarch_tag="us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:latest"
individual_images=()
for arch in "${archs[@]}"; do
TAG="$IMAGE_TAG_SUFFIX"
docker pull us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:${TAG}-${arch} --platform linux/${arch}
individual_images+=("us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-build-base:${TAG}-${arch}")
done
docker buildx imagetools create --tag "${multiarch_tag}" "${individual_images[@]}"
done
2 changes: 1 addition & 1 deletion .github/workflows/build-core-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ jobs:
echo "No tag found on all pages."
echo "BUILD_CONTRACTS=true" >> "$GITHUB_ENV"
# TODO Remove it when we migrate to foundry inside contracts repository
mkdir -p contracts/l1-contracts/artifacts/
mkdir -p contracts/l1-contracts/artifacts/
exit 0
fi
filtered_tag=$(jq -r --arg commit_sha "$commit_sha" 'map(select(.commit.sha == $commit_sha)) | .[].name' <<<"$tags")
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build-docker-from-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,10 @@ jobs:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

build-gar-prover-fri-gpu:
build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar:
name: Build GAR prover FRI GPU
needs: [ setup, build-push-prover-images ]
uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
if: contains(github.ref_name, 'prover')
with:
setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Build Prover FRI GPU with builtin setup data
name: Build Prover FRI GPU & Circuit Prover GPU with builtin setup data

on:
workflow_call:
Expand All @@ -17,7 +17,7 @@ on:
type: string

jobs:
build-gar-prover-fri-gpu:
build:
name: Build prover FRI GPU GAR
runs-on: [matterlabs-ci-runner-high-performance]
steps:
Expand All @@ -28,6 +28,7 @@ jobs:
- name: Download Setup data
run: |
gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/prover-gpu-fri-gar
cp -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
- name: Login to us-central1 GAR
run: |
Expand All @@ -39,7 +40,15 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1

- name: Build and push
- name: Login to Asia GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
- name: Login to Europe GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
- name: Build and push prover-gpu-fri-gar
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
with:
context: docker/prover-gpu-fri-gar
Expand All @@ -49,22 +58,36 @@ jobs:
tags: |
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
- name: Login to Asia GAR
- name: Build and push prover-gpu-fri-gar to Asia GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
docker buildx imagetools create \
--tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
- name: Build and push to Asia GAR
- name: Build and push prover-gpu-fri-gar to Europe GAR
run: |
docker buildx imagetools create \
--tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
--tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
- name: Login to Europe GAR
- name: Build and push circuit-prover-gpu-gar
uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
with:
context: docker/circuit-prover-gpu-gar
build-args: |
PROVER_IMAGE=${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
push: true
tags: |
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
- name: Build and push circuit-prover-gpu-gar to Asia GAR
run: |
gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
docker buildx imagetools create \
--tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
- name: Build and push to Europe GAR
- name: Build and push circuit-prover-gpu-gar to Europe GAR
run: |
docker buildx imagetools create \
--tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
--tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
1 change: 1 addition & 0 deletions .github/workflows/build-prover-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ jobs:
- witness-generator
- prover-gpu-fri
- witness-vector-generator
- circuit-prover-gpu
- prover-fri-gateway
- prover-job-monitor
- proof-fri-gpu-compressor
Expand Down
Loading

0 comments on commit d06b433

Please sign in to comment.