Merge branch 'arith-dev' into 1727-feat-enable-popping-transaction-bu…

…ndles

.github/actions/setup-go-corset/action.yml

@@ -12,4 +12,4 @@ runs:
 
     - name: Install Go Corset
       shell: bash
-      run: go install github.com/consensys/go-corset/cmd/go-corset@e5e0938
+      run: go install github.com/consensys/go-corset/cmd/go-corset@75a88c5

.github/workflows/gradle-ethereum-tests.yml

@@ -8,54 +8,9 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-  
+
 jobs:
   ethereum-tests:
-    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Setup Test Environment
-        uses: ./.github/actions/setup-environment
-        with:
-          rust-corset: true
-
-      - name: Generate General State Reference Tests
-        run:  ./gradlew generateGeneralStateReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
-        env:
-          JAVA_OPTS: -Dorg.gradle.daemon=false
-          CORSET_FLAGS: fields,expand,expand,expand
-
-      - name: Run General State Reference Tests
-        run: ./gradlew -Dblockchain=Ethereum referenceGeneralStateTests
-        env:
-          REFERENCE_TESTS_PARALLELISM: 10
-          JAVA_OPTS: -Dorg.gradle.daemon=false
-          CORSET_FLAGS: fields,expand,expand,expand
-
-      - name: Upload test report
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: ethereum-tests-report
-          path: |
-            reference-tests/build/reports/tests/**/*
-            tmp/local/*
-
-      - name: Failure Notification
-        if: ${{ failure() || cancelled() }}
-        uses: slackapi/[email protected]
-        with:
-          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
-          webhook-type: webhook-trigger
-          payload: |
-             name: "Daily State Tests (Rust Corset)"
-             status: "${{ job.status }}"
-
-  ethereum-tests-go-corset:
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
     steps:
       - name: Checkout repository
@@ -72,21 +27,19 @@ jobs:
         run: ./gradlew generateGeneralStateReferenceTests -Dorg.gradle.caching=true
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
-          ZKEVM_BIN: "zkevm_for_reference_tests.go.bin"
 
       - name: Run General State Reference Tests
         run: GOMEMLIMIT=64GiB ./gradlew -Dblockchain=Ethereum referenceGeneralStateTests
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --air
-          ZKEVM_BIN: "zkevm_for_reference_tests.go.bin"
+          GOCORSET_FLAGS: -vw -b1024 --ansi-escapes=false --report --air
 
       - name: Upload test report
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: ethereum-tests-go-corset-report
+          name: ethereum-tests-report
           path: |
             reference-tests/build/reports/tests/**/*
             tmp/local/*

.github/workflows/gradle-nightly-tests.yml

@@ -28,8 +28,8 @@ jobs:
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --hir
-          ZKEVM_BIN: "zkevm.go.bin"
+          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --mir
+          ZKEVM_BIN: "zkevm.bin"
           NIGHTLY_TESTS_PARALLELISM: 2
 
       - name: Upload test report

.github/workflows/gradle-tests.yml

@@ -55,50 +55,7 @@ jobs:
   # ==================================================================
   # Unit Tests
   # ==================================================================
-  corset-unit-tests:
-    needs: [ build ]
-    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          submodules: false
-
-      - name: Setup Test Environment
-        uses: ./.github/actions/setup-environment
-        with:
-          enable-ssh: ${{ inputs.tests-with-ssh }}
-          rust-corset: true
-
-      - name: Run unit tests
-        run: ./gradlew :arithmetization:test --stacktrace
-        env:
-          JAVA_OPTS: -Dorg.gradle.daemon=false
-          CORSET_FLAGS: trace-span=3,fields,expand,expand,expand
-          UNIT_TESTS_PARALLELISM: 2
-
-      - name: Upload test report
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: corset-unit-test-report
-          path: arithmetization/build/reports/tests/**/*
-
-      - name: Upload jacoco test coverage report
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: jacoco-unit-tests-coverage-report
-          path: arithmetization/build/reports/jacoco/test/**/*
-
-      - name: Upload jacoco exec test
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: jacoco-unit-tests-exec-file
-          path: arithmetization/build/jacoco/test.exec
-
-  go-corset-unit-tests:
+  unit-tests:
     needs: [ build ]
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
     continue-on-error: true
@@ -121,48 +78,18 @@ jobs:
           CORSET_FLAGS: disable
           GOCORSET_FLAGS: -b1024 -vw --ansi-escapes=false --report --air
           UNIT_TESTS_PARALLELISM: 1
-          ZKEVM_BIN: "zkevm.go.bin"
 
       - name: Upload test report
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: go-corset-unit-tests-report
+          name: unit-tests-report
           path: arithmetization/build/reports/tests/**/*
 
   # ==================================================================
   # Fast Replay Tests
   # ==================================================================
-  corset-replay-tests:
-    needs: [ build ]
-    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          submodules: false
-
-      - name: Setup Environment
-        uses: ./.github/actions/setup-environment
-        with:
-          enable-ssh: ${{ inputs.tests-with-ssh }}
-          rust-corset: true
-
-      - name: Run Replay tests
-        run: ./gradlew :arithmetization:fastReplayTests --stacktrace
-        env:
-          JAVA_OPTS: -Dorg.gradle.daemon=false
-          CORSET_FLAGS: trace-span=3,fields,expand,expand,expand
-          REPLAY_TESTS_PARALLELISM: 2
-
-      - name: Upload test report
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: replay-test-report
-          path: arithmetization/build/reports/tests/**/*
-
-  go-corset-replay-tests:
+  replay-tests:
     needs: [ build ]
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
     continue-on-error: true
@@ -184,11 +111,10 @@ jobs:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
           GOCORSET_FLAGS: -b1024 -vw --ansi-escapes=false --report --air
-          ZKEVM_BIN: "zkevm.go.bin"
 
       - name: Upload test report
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: go-corset-replay-tests-report
+          name: replay-tests-report
           path: arithmetization/build/reports/tests/**/*

.github/workflows/gradle-weekly-tests.yml

@@ -9,7 +9,7 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-  
+
 jobs:
   tests:
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
@@ -48,7 +48,7 @@ jobs:
              name: "Weekly (Go-Corset)"
              status: "${{ job.status }}"
 
-  go-corset-tests:
+  weekly-tests:
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
     continue-on-error: true
     steps:
@@ -67,8 +67,7 @@ jobs:
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
-          GOCORSET_FLAGS: -vw --ansi-escapes=false --report --air
-          ZKEVM_BIN: "zkevm.go.bin"
+          GOCORSET_FLAGS: -vw -b1024 --ansi-escapes=false --report --air
 
       - name: Failure Notification
         if: ${{ failure() || cancelled() }}

.github/workflows/reference-blockchain-tests.yml

@@ -72,7 +72,6 @@ jobs:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           CORSET_FLAGS: disable
           GOCORSET_FLAGS: -b1024 -vw --ansi-escapes=false --report --air
-          ZKEVM_BIN: "zkevm_for_reference_tests.go.bin"
           FAILED_TEST_JSON_DIRECTORY: ${{ github.workspace }}/tmp/${{ steps.extract_branch.outputs.branch }}/
           FAILED_MODULE: ${{ inputs.failed_module || '' }}
           FAILED_CONSTRAINT: ${{ inputs.failed_constraint || '' }}

arithmetization/src/main/java/net/consensys/linea/corset/GoCorsetValidator.java

@@ -95,7 +95,8 @@ private void configGoCorset() {
   private List<String> buildCommandLine(Path traceFile, String zkEvmBin) {
     ArrayList<String> options = new ArrayList<>();
     // Determine options to use (either default or override)
-    String flags = System.getenv().getOrDefault("GOCORSET_FLAGS", "-w --air");
+    String flags =
+        System.getenv().getOrDefault("GOCORSET_FLAGS", "-w --report --report-context 2 --hir ");
     // Specify corset binary
     options.add("go-corset");
     // Specify corset "check" command.

arithmetization/src/main/java/net/consensys/linea/corset/RustCorsetValidator.java

@@ -80,7 +80,7 @@ public class RustCorsetValidator extends AbstractExecutable {
   @Getter private boolean active = false;
 
   public RustCorsetValidator() {
-    initCorset();
+    // initCorset();
     configCorset();
   }
 

arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/AccountSnapshot.java

@@ -156,17 +156,10 @@ public AccountSnapshot deepCopy() {
         address, nonce, balance, isWarm, code, deploymentNumber, deploymentStatus);
   }
 
-  public AccountSnapshot wipe(DeploymentInfo deploymentInfo) {
+  public void wipe(DeploymentInfo deploymentInfo) {
     final boolean deploymentStatus = deploymentInfo.getDeploymentStatus(address);
     checkArgument(!deploymentStatus);
-    return new AccountSnapshot(
-        address,
-        0,
-        Wei.of(0),
-        isWarm,
-        Bytecode.EMPTY,
-        deploymentInfo.deploymentNumber(address),
-        deploymentStatus);
+    this.nonce(0).balance(Wei.ZERO).code(Bytecode.EMPTY).setDeploymentInfo(deploymentInfo);
   }
 
   /**

arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/Hub.java

@@ -537,9 +537,9 @@ public void traceEndTransaction(
 
     // TODO: add the following resolution this.defers.resolvePostRollback(this, ...
 
-    txStack.current().completeLineaTransaction(this, isSuccessful, logs, selfDestructs);
-
+    txStack.current().completeLineaTransaction(this, world, isSuccessful, logs, selfDestructs);
     defers.resolveAtEndTransaction(this, world, tx, isSuccessful);
+    defers.resolveAfterTransactionFinalization(this, world);
 
     // Warn: we need to call MMIO after resolving the defers
     for (Module m : modules) {
@@ -736,8 +736,6 @@ public void tracePostExecution(MessageFrame frame, Operation.OperationResult ope
     if (isExceptional()) {
       this.currentTraceSection()
           .exceptionalContextFragment(ContextFragment.executionProvidesEmptyReturnData(this));
-      this.squashCurrentFrameOutputData();
-      this.squashParentFrameReturnData();
     }
 
     defers.resolvePostExecution(this, frame, operationResult);
@@ -748,7 +746,10 @@ public void tracePostExecution(MessageFrame frame, Operation.OperationResult ope
 
     if (frame.getDepth() == 0 && (isExceptional() || opCode().isHalt())) {
       state.processingPhase(TX_FINL);
-      coinbaseWarmthAtTransactionEnd = frame.isAddressWarm(coinbaseAddress);
+      coinbaseWarmthAtTransactionEnd =
+          isExceptional() || opCode() == REVERT
+              ? txStack.current().coinbaseWarmthAfterTxInit(this)
+              : frame.isAddressWarm(coinbaseAddress);
     }
 
     if (frame.getDepth() == 0 && (isExceptional() || opCode() == REVERT)) {
@@ -774,6 +775,8 @@ private void compareLineaAndBesuGasCosts(
     long lineaGasCostExcludingDeploymentCost =
         currentSection.commonValues.gasCostExcluduingDeploymentCost();
 
+    gasCostAccumulator += besuGasCost;
+
     if (operationResult.getHaltReason() != null) {
 
       return;
@@ -867,7 +870,7 @@ private void exitDeploymentFromDeploymentInfoPov(MessageFrame frame) {
     transients.conflation().deploymentInfo().markAsNotUnderDeployment(bytecodeAddress);
   }
 
-  public int getCfiByMetaData(
+  public int getCodeFragmentIndexByMetaData(
       final Address address, final int deploymentNumber, final boolean deploymentStatus) {
     return this.romLex()
         .getCodeFragmentIndexByMetadata(

arithmetization/src/main/java/net/consensys/linea/zktracer/module/hub/defer/AfterTransactionFinalizationDefer.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package net.consensys.linea.zktracer.module.hub.defer;
+
+import net.consensys.linea.zktracer.module.hub.Hub;
+import org.hyperledger.besu.evm.worldstate.WorldView;
+
+/**
+ * Actions which get deferred to {@link AfterTransactionFinalizationDefer} are those actions that
+ * should be performed after the TX_FINL phase of transaction processing. These include
+ *
+ * <p>- the account wiping of successful <b>SELFDESTRUCT</b>'s
+ */
+public interface AfterTransactionFinalizationDefer {
+
+  void resolveAfterTransactionFinalization(Hub hub, WorldView view);
+}