Various Verkle Fixes (#3650)

* scaffolding * broken WIP * partial implementations * fix getAccount * add todo * make trie.get accept suffixes * clean up reference * Add reserved bytes to encodeBasicData function * FIx encoding again * spelling [no ci] * change param to account [no ci] * Add support for basic account delete * implement chunkify code * Add putCode * Move code to helpers * getCode and getCodeSize * Start work on tests * Update magic numbers to constants * Make get/putCode work * Fix various get/putCode bugs * add get/putstorage * export SFVKSM [no ci] * add commit/flush/revert * Tests for caching * make cspell happy * lint * add back missing method from interface * Update packages/util/test/verkle.spec.ts * Apply suggestions from code review * address some feedback * Update types and add test * FIx commitment format * Update verkle crypto and add proof test * add max chunks constant * delete account in put if no account * spelling * fix basic data encoding offsets * remove console log * Fix suffix logic * wasm update * update interface * update package lock * update verkle crypto wasm again * console logs * maybe another fix * various fixes * update tests * Remove obsolete proof helpers * add second test * add explanatory comment for leaf marker * Add safeguards to leafnode.create * use correct randomBytes --------- Co-authored-by: Gabriel Rocheleau <[email protected]>
ethereumjs · Sep 25, 2024 · 15f8ff2 · 15f8ff2
1 parent 3a7e07f
commit 15f8ff2
Show file tree

Hide file tree

Showing 6 changed files with 158 additions and 49 deletions.
diff --git a/packages/util/src/verkle.ts b/packages/util/src/verkle.ts
@@ -36,6 +36,7 @@ export interface VerkleCrypto {
   serializeCommitment: (commitment: Uint8Array) => Uint8Array
   createProof: (bytes: ProverInput[]) => Uint8Array
   verifyProof: (proof: Uint8Array, verifierInput: VerifierInput[]) => boolean
+  commitToScalars: (vector: Uint8Array[]) => Uint8Array
 }
 
 export interface ProverInput {
@@ -352,8 +353,8 @@ export function encodeVerkleLeafBasicData(account: Account): Uint8Array {
  */
 export const generateChunkSuffixes = (numChunks: number) => {
   if (numChunks === 0) return []
-  const chunkSuffixes = new Array<number>(numChunks)
-  const firstChunksSet = Math.min(numChunks, VERKLE_CODE_OFFSET)
+  const chunkSuffixes: number[] = new Array<number>(numChunks)
+  const firstChunksSet = numChunks > VERKLE_CODE_OFFSET ? VERKLE_CODE_OFFSET : numChunks
   for (let x = 0; x < firstChunksSet; x++) {
     // Fill up to first 128 suffixes
     chunkSuffixes[x] = x + VERKLE_CODE_OFFSET

diff --git a/packages/verkle/src/node/leafNode.ts b/packages/verkle/src/node/leafNode.ts
@@ -1,4 +1,4 @@
-import { equalsBytes, intToBytes, setLengthLeft, setLengthRight } from '@ethereumjs/util'
+import { equalsBytes, intToBytes, setLengthRight } from '@ethereumjs/util'
 
 import { BaseVerkleNode } from './baseVerkleNode.js'
 import { NODE_WIDTH, VerkleLeafNodeValue, VerkleNodeType } from './types.js'
@@ -41,7 +41,15 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
     values?: (Uint8Array | VerkleLeafNodeValue)[],
   ): Promise<LeafNode> {
     // Generate the value arrays for c1 and c2
-    values = values !== undefined ? values : createDefaultLeafValues()
+    if (values !== undefined) {
+      values = values.map((el) => {
+        // Checks for instances of zeros and replaces with the "deleted" leaf node value
+        if (el instanceof Uint8Array && equalsBytes(el, new Uint8Array(32)))
+          return VerkleLeafNodeValue.Deleted
+        return el
+      })
+    } else values = createDefaultLeafValues()
+
     const c1Values = createCValues(values.slice(0, 128))
     const c2Values = createCValues(values.slice(128))
     let c1 = verkleCrypto.zeroCommitment
@@ -66,7 +74,7 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
       verkleCrypto.zeroCommitment,
       0,
       new Uint8Array(32),
-      setLengthLeft(intToBytes(1), 32),
+      setLengthRight(intToBytes(1), 32),
     )
     commitment = verkleCrypto.updateCommitment(
       commitment,
@@ -148,33 +156,18 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
         value === VerkleLeafNodeValue.Untouched
           ? createUntouchedLeafValue()
           : createDeletedLeafValue()
+
+    // Set the new values in the values array
+    this.values[index] = val
+
     // First we update c1 or c2 (depending on whether the index is < 128 or not)
     // Generate the 16 byte values representing the 32 byte values in the half of the values array that
     // contain the old value for the leaf node
     const cValues =
       index < 128 ? createCValues(this.values.slice(0, 128)) : createCValues(this.values.slice(128))
-    // The commitment index is 2 * the suffix (i.e. the position of the value in the values array)
-    // here because each 32 byte value in the leaf node is represented as two 16 byte values in the
-    // cValues array.
-    const commitmentIndex = index < 128 ? index * 2 : (index - 128) * 2
-    let cCommitment = index < 128 ? this.c1 : this.c2
-    // Update the commitment for the first 16 bytes of the value
-    cCommitment = this.verkleCrypto.updateCommitment(
-      cCommitment!,
-      commitmentIndex,
-      cValues[commitmentIndex],
-      // Right pad the value with zeroes since commitments require 32 byte scalars
-      setLengthRight(val.slice(0, 16), 32),
-    )
-    // Update the commitment for the second 16 bytes of the value
-    cCommitment = this.verkleCrypto.updateCommitment(
-      cCommitment!,
-      commitmentIndex + 1,
-      cValues[commitmentIndex + 1],
-      // Right pad the value with zeroes since commitments require 32 byte scalars
-      setLengthRight(val.slice(16), 32),
-    )
-    // Update the cCommitment corresponding to the index
+    // Create a commitment to the cValues returned and then use this to replace the c1/c2 commitment value
+    const cCommitment = this.verkleCrypto.commitToScalars(cValues)
+
     let oldCCommitment: Uint8Array | undefined
     if (index < 128) {
       oldCCommitment = this.c1
@@ -184,8 +177,6 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
       this.c2 = cCommitment
     }
 
-    // Set the new values in the values array
-    this.values[index] = value
     // Update leaf node commitment -- c1 (2) if index is < 128 or c2 (3) otherwise
     const cIndex = index < 128 ? 2 : 3
     this.commitment = this.verkleCrypto.updateCommitment(

diff --git a/packages/verkle/src/node/util.ts b/packages/verkle/src/node/util.ts
@@ -42,18 +42,15 @@ export function isInternalNode(node: VerkleNode): node is InternalNode {
 export const createUntouchedLeafValue = () => new Uint8Array(32)
 
 /**
- * Generates a 32 byte array of zeroes and sets the 129th bit to 1, which the EIP
- * refers to as the leaf marker to indicate a leaf value that has been touched previously
+ * Generates a 32 byte array of zeroes and sets the 129th bit to 1 (if `setLeafMarker` is set),
+ * which the EIP refers to as the leaf marker to indicate a leaf value that has been touched previously
  * and contains only zeroes
- *
- * Note: this value should only used in the commitment update process
- *
- * @returns a 32 byte array of zeroes with the 129th bit set to 1
+ * @returns a 32 byte array of zeroes (optionally with 129th bit set to 1)
  */
-export const createDeletedLeafValue = () => {
+export const createDeletedLeafValue = (setLeafMarker = false) => {
   const bytes = new Uint8Array(32)
-  // Set the 129th bit to 1 directly by setting the 17th byte (index 16) to 0x80
-  bytes[16] = 0x80
+  // Set the 129th bit to 1 directly by setting the 17th byte (index 16) to 1 (since these bytes are little endian)
+  if (setLeafMarker) bytes[16] = 1
 
   return bytes
 }
@@ -81,16 +78,22 @@ export const createCValues = (values: (Uint8Array | VerkleLeafNodeValue)[]) => {
         val = createUntouchedLeafValue()
         break
       case VerkleLeafNodeValue.Deleted: // Leaf value that has been written with zeros (either zeroes or a deleted value)
-        val = createDeletedLeafValue()
+        val = createDeletedLeafValue(true)
         break
       default:
         val = retrievedValue
         break
     }
-    // We add 16 trailing zeros to each value since all commitments are padded to an array of 32 byte values
+    // We add 16 trailing zeros to each value since all commitments are little endian and padded to 32 bytes
     expandedValues[x * 2] = setLengthRight(val.slice(0, 16), 32)
-    // Apply leaf marker to all touched values (i.e. flip 129th bit)
-    if (retrievedValue !== VerkleLeafNodeValue.Untouched) expandedValues[x * 2][16] = 0x80
+    // Apply leaf marker to all touched values (i.e. flip 129th bit) of the lower value (the 16 lower bytes
+    // of the original 32 byte value array)
+    // This is counterintuitive since the 129th bit is little endian byte encoding so 10000000 in bits but
+    // each byte in a Javascript Uint8Array is still "big endian" so the 16th byte (which contains the 129-137th bits)
+    // should be 1 and not 256.  In other words, the little endian value 10000000 is represented as an integer 1 in the byte
+    // at index 16 of the Uint8Array since each byte is big endian at the system level so we have to invert that
+    // value to get the correct representation
+    if (retrievedValue !== VerkleLeafNodeValue.Untouched) expandedValues[x * 2][16] = 1
     expandedValues[x * 2 + 1] = setLengthRight(val.slice(16), 32)
   }
   return expandedValues

diff --git a/packages/verkle/test/interop.spec.ts b/packages/verkle/test/interop.spec.ts
@@ -0,0 +1,61 @@
+import { MapDB, bytesToHex } from '@ethereumjs/util'
+import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
+import { assert, beforeAll, describe, it } from 'vitest'
+
+import { createVerkleTree } from '../src/constructors.js'
+
+describe('rust-verkle test vectors', () => {
+  let verkleCrypto: Awaited<ReturnType<typeof loadVerkleCrypto>>
+  beforeAll(async () => {
+    verkleCrypto = await loadVerkleCrypto()
+  })
+  it('should produce the correct commitment', async () => {
+    // Test from python implementation
+    //https://github.com/crate-crypto/verkle-trie-ref/blob/483f40c737f27bc8f059870f862cf6c244159cd4/verkle/verkle_test.py#L63
+    // It inserts a single value and then verifies that the hash of the root node matches (not the `trie.root` which is a serialized commitment and not the hash)
+    const key = Uint8Array.from([
+      1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
+      27, 28, 29, 30, 31, 32,
+    ])
+    const trie = await createVerkleTree({ verkleCrypto, db: new MapDB() })
+    await trie.put(key.slice(0, 31), [key[31]], [key])
+
+    const path = await trie.findPath(key.slice(0, 31))
+
+    assert.equal(
+      bytesToHex(path.stack[0][0].hash()),
+      '0x029b6c4c8af9001f0ac76472766c6579f41eec84a73898da06eb97ebdab80a09',
+    )
+    assert.equal(
+      bytesToHex(trie.root()),
+      '0x6f5e7cfc3a158a64e5718b0d2f18f564171342380f5808f3d2a82f7e7f3c2778',
+    )
+  })
+  it('should produce correct commitments after value updates', async () => {
+    // Variant of previous test that puts 0s at a specific key and then updates that value
+    // https://github.com/crate-crypto/verkle-trie-ref/blob/483f40c737f27bc8f059870f862cf6c244159cd4/verkle/verkle_test.py#L96
+    const key = Uint8Array.from([
+      1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
+      27, 28, 29, 30, 31, 32,
+    ])
+    const stem = key.slice(0, 31)
+    const trie = await createVerkleTree({ verkleCrypto, db: new MapDB() })
+    await trie.put(stem, [key[31]], [new Uint8Array(32)])
+    let path = await trie.findPath(stem)
+    assert.equal(
+      bytesToHex(path.stack[0][0].hash()),
+      '0x77a0747bd526d9d9af60bd5665d24d6cb421f5c8e726b1de62f914f3ff9a361c',
+    )
+    await trie.put(stem, [key[31]], [key])
+    path = await trie.findPath(key.slice(0, 31))
+
+    assert.equal(
+      bytesToHex(path.stack[0][0].hash()),
+      '0x029b6c4c8af9001f0ac76472766c6579f41eec84a73898da06eb97ebdab80a09',
+    )
+    assert.equal(
+      bytesToHex(trie.root()),
+      '0x6f5e7cfc3a158a64e5718b0d2f18f564171342380f5808f3d2a82f7e7f3c2778',
+    )
+  })
+})
diff --git a/packages/verkle/test/leafNode.spec.ts b/packages/verkle/test/leafNode.spec.ts
@@ -1,4 +1,4 @@
-import { type VerkleCrypto, equalsBytes, randomBytes, setLengthLeft } from '@ethereumjs/util'
+import { type VerkleCrypto, equalsBytes, randomBytes, setLengthRight } from '@ethereumjs/util'
 import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
 import { assert, beforeAll, describe, it } from 'vitest'
 
@@ -61,8 +61,8 @@ describe('verkle node - leaf', () => {
     const node = await LeafNode.create(key.slice(0, 31), verkleCrypto)
     assert.ok(node instanceof LeafNode)
     assert.equal(node.getValue(0), undefined)
-    node.setValue(0, setLengthLeft(Uint8Array.from([5]), 32))
-    assert.deepEqual(node.getValue(0), setLengthLeft(Uint8Array.from([5]), 32))
+    node.setValue(0, setLengthRight(Uint8Array.from([5]), 32))
+    assert.deepEqual(node.getValue(0), setLengthRight(Uint8Array.from([5]), 32))
     node.setValue(0, VerkleLeafNodeValue.Deleted)
     assert.deepEqual(node.getValue(0), new Uint8Array(32))
   })
@@ -72,7 +72,7 @@ describe('verkle node - leaf', () => {
     const node = await LeafNode.create(key.slice(0, 31), verkleCrypto)
     node.setValue(0, VerkleLeafNodeValue.Deleted)
     const c1Values = createCValues(node.values.slice(0, 128))
-    assert.equal(c1Values[0][16], 0x80)
+    assert.equal(c1Values[0][16], 1)
   })
 
   it('should update a commitment when setting a value', async () => {
@@ -91,9 +91,11 @@ describe('verkle node - leaf', () => {
     const node = await LeafNode.create(stem, verkleCrypto, values)
     const serialized = node.serialize()
     const decodedNode = decodeNode(serialized, verkleCrypto)
+
     assert.deepEqual(node, decodedNode)
 
     const defaultNode = await LeafNode.create(randomBytes(31), verkleCrypto)
+
     assert.deepEqual(defaultNode, decodeNode(defaultNode.serialize(), verkleCrypto))
   })
 })
diff --git a/packages/verkle/test/proof.spec.ts b/packages/verkle/test/proof.spec.ts
@@ -1,11 +1,12 @@
-import { MapDB, hexToBytes } from '@ethereumjs/util'
+import { MapDB, bigIntToBytes, hexToBytes, randomBytes, setLengthRight } from '@ethereumjs/util'
 import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
 import { assert, beforeAll, describe, it } from 'vitest'
 
 import { createVerkleTree } from '../src/constructors.js'
+import { LeafNode } from '../src/index.js'
 
-import type { LeafNode } from '../src/index.js'
-import type { PrefixedHexString, ProverInput, VerifierInput, VerkleCrypto } from '@ethereumjs/util'
+import type { PrefixedHexString, VerkleCrypto } from '@ethereumjs/util'
+import type { ProverInput, VerifierInput } from 'verkle-cryptography-wasm'
 
 describe('lets make proofs', () => {
   let verkleCrypto: VerkleCrypto
@@ -69,4 +70,54 @@ describe('lets make proofs', () => {
       assert.fail(`Failed to verify proof: ${err}`)
     }
   })
+  it('should pass for empty trie', async () => {
+    const trie = await createVerkleTree({ verkleCrypto, db: new MapDB() })
+
+    await trie['_createRootNode']()
+    const proof = verkleCrypto.createProof([
+      {
+        // Get commitment from root node
+        serializedCommitment: verkleCrypto.serializeCommitment(
+          (await trie.findPath(new Uint8Array(31))).stack![0][0].commitment,
+        ),
+        vector: new Array(256).fill(new Uint8Array(32).fill(0)),
+        indices: [0],
+      },
+    ])
+    const res = verkleCrypto.verifyProof(proof, [
+      {
+        serializedCommitment: verkleCrypto.serializeCommitment(
+          (await trie.findPath(new Uint8Array(31))).stack![0][0].commitment,
+        ),
+        indexValuePairs: [{ index: 0, value: new Uint8Array(32) }],
+      },
+    ])
+    assert.ok(res)
+  })
+  it.skip('should verify proof for single leaf node', async () => {
+    const node = await LeafNode.create(randomBytes(31), verkleCrypto)
+    node.setValue(0, setLengthRight(bigIntToBytes(1n), 32))
+    const valuesArray = new Array<Uint8Array>(256)
+    for (let x = 0; x < 256; x++) {
+      let value = node.getValue(x)
+      if (value === undefined) value = new Uint8Array(32)
+      valuesArray[x] = value
+    }
+
+    const proof = verkleCrypto.createProof([
+      {
+        serializedCommitment: verkleCrypto.serializeCommitment(node.commitment),
+        vector: valuesArray,
+        indices: [0],
+      },
+    ])
+
+    const res = verkleCrypto.verifyProof(proof, [
+      {
+        serializedCommitment: verkleCrypto.serializeCommitment(node.commitment),
+        indexValuePairs: [{ index: 0, value: node.getValue(0)! }],
+      },
+    ])
+    assert.ok(res)
+  })
 })