Hyperchain da removal

.github/workflows/build-docker-from-tag.yml

@@ -23,7 +23,7 @@ concurrency: docker-build
 jobs:
   setup:
     name: Setup
-    runs-on: [ubuntu-latest]
+    runs-on: [ ubuntu-latest ]
     outputs:
       image_tag_suffix: ${{ steps.set.outputs.image_tag_suffix }}
       prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }}
@@ -48,7 +48,7 @@ jobs:
 
   build-push-core-images:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-core-template.yml
     if: contains(github.ref_name, 'core')
     secrets:
@@ -60,7 +60,7 @@ jobs:
 
   build-push-tee-prover-images:
     name: Build and push images
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-tee-prover-template.yml
     if: contains(github.ref_name, 'core')
     secrets:
@@ -72,7 +72,7 @@ jobs:
 
   build-push-contract-verifier:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-contract-verifier-template.yml
     if: contains(github.ref_name, 'contract_verifier')
     secrets:
@@ -83,20 +83,20 @@ jobs:
 
   build-push-prover-images:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-prover-template.yml
     if: contains(github.ref_name, 'prover')
     with:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
       ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
-      CUDA_ARCH: "60;70;75;89"
+      CUDA_ARCH: "60;70;75;80;89"
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   build-push-witness-generator-image-avx512:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-witness-generator-template.yml
     if: contains(github.ref_name, 'prover')
     with:
@@ -110,7 +110,7 @@ jobs:
 
   build-gar-prover-fri-gpu:
     name: Build GAR prover FRI GPU
-    needs: [setup, build-push-prover-images]
+    needs: [ setup, build-push-prover-images ]
     uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
     if: contains(github.ref_name, 'prover')
     with:

.github/workflows/ci-core-reusable.yml

@@ -71,7 +71,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vm_mode: [ "old", "new" ]
+        vm_mode: [ "OLD", "NEW" ]
 
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
@@ -87,13 +87,12 @@ jobs:
 
       - name: Loadtest configuration
         run: |
-          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'new' && 21000 || 16000 }} >> .env
+          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'NEW' && 21000 || 16000 }} >> .env
           echo ACCOUNTS_AMOUNT="100" >> .env
           echo MAX_INFLIGHT_TXS="10" >> .env
           echo SYNC_API_REQUESTS_LIMIT="15" >> .env
           echo FAIL_FAST=true >> .env
           echo IN_DOCKER=1 >> .env
-          echo DATABASE_MERKLE_TREE_MODE=lightweight >> .env
 
       - name: Start services
         run: |
@@ -107,23 +106,34 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
 
-          ci_run zk
-          ci_run zk init --local-legacy-bridge-testing
+          ci_run ./bin/zkt
+          ci_run zk_inception chain create \
+            --chain-name legacy \
+            --chain-id sequential \
+            --prover-mode no-proofs \
+            --wallet-creation localhost \
+            --l1-batch-commit-data-generator-mode rollup \
+            --base-token-address 0x0000000000000000000000000000000000000001 \
+            --base-token-price-nominator 1 \
+            --base-token-price-denominator 1 \
+            --set-as-default false \
+            --ignore-prerequisites \
+            --legacy-bridge
+
+          ci_run zk_inception ecosystem init --dev --verbose
+          ci_run zk_supervisor contracts --test-contracts
 
       # `sleep 60` because we need to wait until server added all the tokens
       - name: Run server
         run: |
-          EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE=${{ matrix.vm_mode }} \
-          CHAIN_MEMPOOL_DELAY_INTERVAL=50 \
-          PASSED_ENV_VARS="EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE,CHAIN_MEMPOOL_DELAY_INTERVAL" \
-            ci_run zk server --uring --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
+          ci_run sed -i -e 's/mode: FULL/mode: LIGHTWEIGHT/' chains/legacy/configs/general.yaml
+          ci_run sed -i -e 's/state_keeper_fast_vm_mode:.*/state_keeper_fast_vm_mode: ${{ matrix.vm_mode }}/' chains/legacy/configs/general.yaml
+          ci_run sed -i -e 's/delay_interval:.*/delay_interval: 50/' chains/legacy/configs/general.yaml
+          ci_run zk_inception server --uring --chain=legacy --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
           ci_run sleep 60
 
-      - name: Deploy legacy era contracts
-        run: ci_run zk contract setup-legacy-bridge-era
-
       - name: Perform loadtest
-        run: ci_run zk run loadtest
+        run: ci_run zk_supervisor t loadtest -v --chain=legacy
 
       - name: Show server.log logs
         if: always()
@@ -293,7 +303,7 @@ jobs:
 
       - name: Show revert.log logs
         if: always()
-        run: ci_run cat core/tests/revert-test/revert.log || true
+        run: ci_run cat logs/revert/default/server.log || true
 
       - name: Show upgrade.log logs
         if: always()
@@ -382,7 +392,11 @@ jobs:
 
       - name: Run revert test
         run: |
-          ENABLE_CONSENSUS=${{ matrix.consensus }} DEPLOYMENT_MODE=${{ matrix.deployment_mode }} PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE" ci_run zk test i revert-en
+          ENABLE_CONSENSUS=${{ matrix.consensus }} \
+          DEPLOYMENT_MODE=${{ matrix.deployment_mode }} \
+          PASSED_ENV_VARS="ENABLE_CONSENSUS,DEPLOYMENT_MODE" \
+          ci_run zk test i revert-en
+
           # test terminates the nodes, so we restart them.
           if [[ "${{ matrix.deployment_mode }}" == "Rollup" ]]; then
             ZKSYNC_ENV=docker ci_run zk server --components=$SERVER_COMPONENTS &>>server.log &
@@ -414,13 +428,13 @@ jobs:
         if: always()
         run: ci_run cat ext-node.log || true
 
-      - name: Show revert_main.log logs
+      - name: Show revert logs (main node)
         if: always()
-        run: ci_run cat core/tests/revert-test/revert_main.log || true
+        run: ci_run cat logs/revert/en/default/server.log || true
 
-      - name: Show revert_ext.log logs
+      - name: Show revert logs (EN)
         if: always()
-        run: ci_run cat core/tests/revert-test/revert_ext.log || true
+        run: ci_run cat logs/revert/en/default/external_node.log || true
 
       - name: Show upgrade.log logs
         if: always()

.github/workflows/ci-zk-toolbox-reusable.yml

@@ -4,14 +4,19 @@ on:
 
 env:
   CLICOLOR: 1
+  # We run multiple binaries in parallel, and by default they will try to utilize all the
+  # available CPUs. In tests, there is not much CPU-intensive work (rayon), but a lot of
+  # async work (tokio), so we prioritize tokio.
+  TOKIO_WORKER_THREADS: 4
+  RAYON_NUM_THREADS: 2
 
 jobs:
   lint:
     name: lint
     uses: ./.github/workflows/ci-core-lint-reusable.yml
 
   tests:
-    runs-on: [ matterlabs-ci-runner ]
+    runs-on: [ matterlabs-ci-runner-ultra-performance ]
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
@@ -127,6 +132,30 @@ jobs:
           --prover-db-name=zksync_prover_localhost_custom_token \
           --port-offset 3000 \
           --chain custom_token
+
+      - name: Create and initialize Consensus chain
+        run: |
+          ci_run zk_inception chain create \
+          --chain-name consensus \
+          --chain-id sequential \
+          --prover-mode no-proofs \
+          --wallet-creation localhost \
+          --l1-batch-commit-data-generator-mode validium \
+          --base-token-address ${{ env.address }} \
+          --base-token-price-nominator 3 \
+          --base-token-price-denominator 2 \
+          --set-as-default false \
+          --ignore-prerequisites
+
+          ci_run zk_inception chain init \
+          --deploy-paymaster \
+          --l1-rpc-url=http://localhost:8545 \
+          --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
+          --server-db-name=zksync_server_localhost_consensus \
+          --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
+          --prover-db-name=zksync_prover_localhost_consensus \
+          --port-offset 4000 \
+          --chain consensus
 
       - name: Build test dependencies
         run: |
@@ -137,6 +166,10 @@ jobs:
           ci_run zk_inception server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log &
           ci_run zk_inception server --ignore-prerequisites --chain validium &> ${{ env.SERVER_LOGS_DIR }}/validium.log &
           ci_run zk_inception server --ignore-prerequisites --chain custom_token &> ${{ env.SERVER_LOGS_DIR }}/custom_token.log &
+          ci_run zk_inception server --ignore-prerequisites --chain consensus \
+          --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \
+          &> ${{ env.SERVER_LOGS_DIR }}/consensus.log &
+
           ci_run sleep 5
 
       - name: Run integration tests
@@ -150,9 +183,13 @@ jobs:
           ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --chain custom_token &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/custom_token.log &
           PID3=$!
 
+          ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --chain consensus &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log &
+          PID4=$!
+
           wait $PID1
           wait $PID2
           wait $PID3
+          wait $PID4
 
       - name: Init external nodes
         run: |
@@ -168,6 +205,10 @@ jobs:
           --db-name=zksync_en_localhost_era_custom_token --l1-rpc-url=http://localhost:8545 --chain custom_token
           ci_run zk_inception external-node init --ignore-prerequisites --chain custom_token
 
+          ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@localhost:5432 \
+          --db-name=zksync_en_localhost_era_consensus --l1-rpc-url=http://localhost:8545 --chain consensus
+          ci_run zk_inception external-node init --ignore-prerequisites --chain consensus
+
       - name: Run recovery tests (from snapshot)
         run: |
 
@@ -180,9 +221,13 @@ jobs:
           ci_run zk_supervisor test recovery --snapshot --no-deps --ignore-prerequisites --verbose --chain custom_token &> ${{ env.SNAPSHOT_RECOVERY_LOGS_DIR }}//custom_token.log &
           PID3=$!
 
+          ci_run zk_supervisor test recovery --snapshot --no-deps --ignore-prerequisites --verbose --chain consensus &> ${{ env.SNAPSHOT_RECOVERY_LOGS_DIR }}//consensus.log &
+          PID4=$!
+
           wait $PID1
           wait $PID2
           wait $PID3
+          wait $PID4
 
       - name: Run recovery tests (from genesis)
         run: |
@@ -195,15 +240,20 @@ jobs:
           ci_run zk_supervisor test recovery --no-deps --no-kill --ignore-prerequisites --verbose --chain custom_token &> ${{ env.GENESIS_RECOVERY_LOGS_DIR }}/custom_token.log &
           PID3=$!
 
+          ci_run zk_supervisor test recovery --no-deps --no-kill --ignore-prerequisites --verbose --chain consensus &> ${{ env.GENESIS_RECOVERY_LOGS_DIR }}/consensus.log &
+          PID4=$!
+
           wait $PID1
           wait $PID2
           wait $PID3
+          wait $PID4
 
       - name: Run external node server
         run: |
           ci_run zk_inception external-node run --ignore-prerequisites --chain era &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/rollup.log &
           ci_run zk_inception external-node run --ignore-prerequisites --chain validium &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/validium.log &
           ci_run zk_inception external-node run --ignore-prerequisites --chain custom_token &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/custom_token.log &
+          ci_run zk_inception external-node run --ignore-prerequisites --chain consensus --enable-consensus &> ${{ env.EXTERNAL_NODE_LOGS_DIR }}/consensus.log &
           ci_run sleep 5
 
       - name: Run integration tests en
@@ -217,9 +267,13 @@ jobs:
           ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --external-node --chain custom_token &> ${{ env.INTEGRATION_TESTS_EN_LOGS_DIR }}/custom_token.log &
           PID3=$!
 
+          ci_run zk_supervisor test integration --no-deps --ignore-prerequisites --external-node --chain consensus &> ${{ env.INTEGRATION_TESTS_EN_LOGS_DIR }}/consensus.log &
+          PID4=$!
+
           wait $PID1
           wait $PID2
           wait $PID3
+          wait $PID4
 
       - name: Run revert tests
         run: |
@@ -234,10 +288,14 @@ jobs:
 
           ci_run zk_supervisor test revert --no-deps --external-node --no-kill --ignore-prerequisites --chain custom_token &> ${{ env.REVERT_LOGS_DIR }}/custom_token.log &
           PID3=$!
+
+          ci_run zk_supervisor test revert --no-deps --external-node --no-kill --ignore-prerequisites --chain consensus &> ${{ env.REVERT_LOGS_DIR }}/consensus.log &
+          PID4=$!
 
           wait $PID1
           wait $PID2
           wait $PID3
+          wait $PID4
 
 
       # Upgrade tests should run last, because as soon as they

.github/workflows/ci.yml

@@ -76,6 +76,7 @@ jobs:
               - 'etc/**'
               - 'contracts/**'
               - 'infrastructure/zk/**'
+              - 'docker/zk-environment/**'
               - '!**/*.md'
               - '!**/*.MD'