Implement t8ntool to use for execution-spec-tests / VM Block Builder Adjustments

packages/vm/src/buildBlock.ts

@@ -90,7 +90,7 @@ export class BlockBuilder {
 
     this.headerData = {
       ...opts.headerData,
-      parentHash: opts.parentBlock.hash(),
+      parentHash: opts.headerData?.parentHash ?? opts.parentBlock.hash(),
       number: opts.headerData?.number ?? opts.parentBlock.header.number + BIGINT_1,
       gasLimit: opts.headerData?.gasLimit ?? opts.parentBlock.header.gasLimit,
       timestamp: opts.headerData?.timestamp ?? Math.round(Date.now() / 1000),
@@ -213,7 +213,10 @@ export class BlockBuilder {
    */
   async addTransaction(
     tx: TypedTransaction,
-    { skipHardForkValidation }: { skipHardForkValidation?: boolean } = {},
+    {
+      skipHardForkValidation,
+      allowNoBlobs,
+    }: { skipHardForkValidation?: boolean; allowNoBlobs?: boolean } = {},
   ) {
     this.checkStatus()
 
@@ -242,7 +245,11 @@ export class BlockBuilder {
 
       // Guard against the case if a tx came into the pool without blobs i.e. network wrapper payload
       if (blobTx.blobs === undefined) {
-        throw new Error('blobs missing for 4844 transaction')
+        // TODO: verify if we want this, do we want to allow the block builder to accept blob txs without the actual blobs?
+        // (these must have at least one `blobVersionedHashes`, this is verified at tx-level)
+        if (allowNoBlobs !== true) {
+          throw new Error('blobs missing for 4844 transaction')
+        }
       }
 
       if (this.blobGasUsed + BigInt(blobTx.numBlobs()) * blobGasPerBlob > blobGasLimit) {

packages/vm/src/runBlock.ts

@@ -494,14 +494,12 @@ export async function accumulateParentBlockHash(
 
   // getAccount with historyAddress will throw error as witnesses are not bundled
   // but we need to put account so as to query later for slot
-  try {
-    if ((await vm.stateManager.getAccount(historyAddress)) === undefined) {
-      const emptyHistoryAcc = new Account(BigInt(1))
-      await vm.evm.journal.putAccount(historyAddress, emptyHistoryAcc)
-    }
-  } catch (_e) {
-    const emptyHistoryAcc = new Account(BigInt(1))
-    await vm.evm.journal.putAccount(historyAddress, emptyHistoryAcc)
+  const code = await vm.stateManager.getCode(historyAddress)
+
+  if (code.length === 0) {
+    // Exit early, system contract has no code so no storage is written
+    // TODO: verify with Gabriel that this is fine regarding verkle (should we put an empty account?)
+    return
   }
 
   async function putBlockHash(vm: VM, hash: Uint8Array, number: bigint) {
@@ -536,24 +534,17 @@ export async function accumulateParentBeaconBlockRoot(vm: VM, root: Uint8Array,
   const timestampIndex = timestamp % historicalRootsLength
   const timestampExtended = timestampIndex + historicalRootsLength
 
-  /**
-   * Note: (by Jochem)
-   * If we don't do vm (put account if undefined / non-existent), block runner crashes because the beacon root address does not exist
-   * vm is hence (for me) again a reason why it should /not/ throw if the address does not exist
-   * All ethereum accounts have empty storage by default
-   */
-
   /**
    * Note: (by Gabriel)
    * Get account will throw an error in stateless execution b/c witnesses are not bundled
    * But we do need an account so we are able to put the storage
    */
-  try {
-    if ((await vm.stateManager.getAccount(parentBeaconBlockRootAddress)) === undefined) {
-      await vm.evm.journal.putAccount(parentBeaconBlockRootAddress, new Account())
-    }
-  } catch (_) {
-    await vm.evm.journal.putAccount(parentBeaconBlockRootAddress, new Account())
+  const code = await vm.stateManager.getCode(parentBeaconBlockRootAddress)
+
+  if (code.length === 0) {
+    // Exit early, system contract has no code so no storage is written
+    // TODO: verify with Gabriel that this is fine regarding verkle (should we put an empty account?)
+    return
   }
 
   await vm.stateManager.putStorage(

packages/vm/test/t8n/README.md

@@ -0,0 +1 @@
+TODO write instructions

packages/vm/test/t8n/ethereumjs-t8ntool.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if [[ "$1" == "--version" ]]; then
+    echo "ethereumjs t8n v1"
+    exit 0
+fi
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+export NODE_OPTIONS="--max-old-space-size=4096" 
+tsx "$SCRIPT_DIR/launchT8N.ts" "$@"

packages/vm/test/t8n/helpers.ts

@@ -0,0 +1,122 @@
+import yargs from 'yargs'
+import { hideBin } from 'yargs/helpers'
+
+import type { T8NOptions } from './types.js'
+
+export function getArguments() {
+  const argsParsed = yargs(hideBin(process.argv))
+    .parserConfiguration({
+      'dot-notation': false,
+    })
+    .option('state.fork', {
+      describe: 'Fork to use',
+      type: 'string',
+      demandOption: true,
+    })
+    .option('state.chainid', {
+      describe: 'ChainID to use',
+      type: 'string',
+      default: '1',
+    })
+    .option('state.reward', {
+      describe:
+        'Coinbase reward after running txs. If 0: coinbase account is touched and rewarded 0 wei. If -1, the coinbase account is not touched (default)',
+      type: 'string',
+      default: '-1',
+    })
+    .option('input.alloc', {
+      describe: 'Initial state allocation',
+      type: 'string',
+      demandOption: true,
+    })
+    .option('input.txs', {
+      describe: 'JSON input of txs to run on top of the initial state allocation',
+      type: 'string',
+      demandOption: true,
+    })
+    .option('input.env', {
+      describe: 'Input environment (coinbase, difficulty, etc.)',
+      type: 'string',
+      demandOption: true,
+    })
+    .option('output.basedir', {
+      describe: 'Base directory to write output to',
+      type: 'string',
+      demandOption: true,
+    })
+    .option('output.result', {
+      describe: 'File to write output results to (relative to `output.basedir`)',
+      type: 'string',
+      demandOption: true,
+    })
+    .option('output.alloc', {
+      describe: 'File to write output allocation to (after running the transactions)',
+      type: 'string',
+      demandOption: true,
+    })
+    .option('output.body', {
+      describe: 'File to write the txs RLP to',
+      type: 'string',
+      default: 'txs.rlp',
+    })
+    .strict()
+    .help().argv
+
+  const args = argsParsed as any as T8NOptions
+
+  args.input = {
+    alloc: (<any>args)['input.alloc'],
+    txs: (<any>args)['input.txs'],
+    env: (<any>args)['input.env'],
+  }
+  args.output = {
+    basedir: (<any>args)['output.basedir'],
+    result: (<any>args)['output.result'],
+    alloc: (<any>args)['output.alloc'],
+    body: (<any>args)['output.body'],
+  }
+  args.state = {
+    fork: (<any>args)['state.fork'],
+    reward: BigInt((<any>args)['state.reward']),
+    chainid: BigInt((<any>args)['state.chainid']),
+  }
+
+  return args
+}
+
+/**
+ * This function accepts an `inputs.env` which converts non-hex-prefixed numbers
+ * to a BigInt value, to avoid errors when converting non-prefixed hex strings to
+ * numbers
+ * @param input
+ * @returns converted input
+ */
+export function normalizeNumbers(input: any) {
+  const keys = [
+    'currentGasLimit',
+    'currentNumber',
+    'currentTimestamp',
+    'currentRandom',
+    'currentDifficulty',
+    'currentBaseFee',
+    'currentBlobGasUsed',
+    'currentExcessBlobGas',
+    'parentDifficulty',
+    'parentTimestamp',
+    'parentBaseFee',
+    'parentGasUsed',
+    'parentGasLimit',
+    'parentBlobGasUsed',
+    'parentExcessBlobGas',
+  ]
+
+  for (const key of keys) {
+    const value = input[key]
+    if (value !== undefined) {
+      if (value.substring(0, 2) !== '0x') {
+        input[key] = BigInt(value)
+      }
+    }
+  }
+  return input
+}

packages/vm/test/t8n/launchT8N.ts

@@ -0,0 +1,6 @@
+import { getArguments } from './helpers.js'
+import { TransitionTool } from './t8ntool.js'
+
+const args = getArguments()
+
+await TransitionTool.run(args)

packages/vm/test/t8n/stateTracker.ts

@@ -0,0 +1,110 @@
+import {
+  bigIntToHex,
+  bytesToHex,
+  createAddressFromString,
+  hexToBytes,
+  setLengthLeft,
+  unpadBytes,
+} from '@ethereumjs/util'
+
+import type { VM } from '../../src/vm.js'
+import type { T8NAlloc } from './types.js'
+import type { Account, Address, PrefixedHexString } from '@ethereumjs/util'
+
+export class StateTracker {
+  private allocTracker: {
+    // TODO these are all PrefixexHexString
+    [address: string]: {
+      storage: string[]
+    }
+  } = {}
+
+  private alloc: T8NAlloc
+
+  private vm: VM
+
+  constructor(vm: VM, alloc: T8NAlloc) {
+    this.alloc = alloc
+    const originalPutAccount = vm.stateManager.putAccount
+    const originalPutCode = vm.stateManager.putCode
+    const originalPutStorage = vm.stateManager.putStorage
+
+    this.vm = vm
+
+    const self = this
+
+    vm.stateManager.putAccount = async function (...args: [Address, Account?]) {
+      const address = args[0]
+      self.addAddress(address.toString())
+      await originalPutAccount.apply(this, args)
+    }
+
+    vm.stateManager.putCode = async function (...args: [Address, Uint8Array]) {
+      const address = args[0]
+      self.addAddress(address.toString())
+      return originalPutCode.apply(this, args)
+    }
+
+    vm.stateManager.putStorage = async function (...args: [Address, Uint8Array, Uint8Array]) {
+      const address = args[0]
+      const key = args[1]
+      self.addStorage(address.toString(), <PrefixedHexString>bytesToHex(key))
+      return originalPutStorage.apply(this, args)
+    }
+  }
+
+  private addAddress(address: PrefixedHexString) {
+    if (this.allocTracker[address] === undefined) {
+      this.allocTracker[address] = { storage: [] }
+    }
+    return this.allocTracker[address]
+  }
+
+  private addStorage(address: PrefixedHexString, storage: PrefixedHexString) {
+    const storageList = this.addAddress(address).storage
+    if (!storageList.includes(storage)) {
+      storageList.push(storage)
+    }
+  }
+
+  public async dumpAlloc() {
+    // Build output alloc
+    const outputAlloc = this.alloc
+    for (const addressString in this.allocTracker) {
+      const address = createAddressFromString(addressString)
+      const account = await this.vm.stateManager.getAccount(address)
+      if (account === undefined) {
+        delete outputAlloc[addressString]
+        continue
+      }
+      if (outputAlloc[addressString] === undefined) {
+        outputAlloc[addressString] = {
+          balance: '0x0',
+        }
+      }
+      outputAlloc[addressString].nonce = bigIntToHex(account.nonce)
+      outputAlloc[addressString].balance = bigIntToHex(account.balance)
+      outputAlloc[addressString].code = bytesToHex(await this.vm.stateManager.getCode(address))
+
+      const storage = this.allocTracker[addressString].storage
+      outputAlloc[addressString].storage = outputAlloc[addressString].storage ?? {}
+
+      for (const key of storage) {
+        const keyBytes = hexToBytes(<PrefixedHexString>key)
+        let storageKeyTrimmed = bytesToHex(unpadBytes(keyBytes))
+        if (storageKeyTrimmed === '0x') {
+          storageKeyTrimmed = '0x00'
+        }
+        const value = await this.vm.stateManager.getStorage(address, setLengthLeft(keyBytes, 32))
+        if (value.length === 0) {
+          delete outputAlloc[addressString].storage[storageKeyTrimmed]
+          // To be sure, also delete any keys which are left-padded to 32 bytes
+          delete outputAlloc[addressString].storage[key]
+          continue
+        }
+        outputAlloc[addressString].storage[storageKeyTrimmed] = bytesToHex(value)
+      }
+    }
+    return outputAlloc
+  }
+}