From 04a80277ec76f4140fb373cf79943c40e944269e Mon Sep 17 00:00:00 2001 From: thomas-quadratic <116874460+thomas-quadratic@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:07:46 +0200 Subject: [PATCH] Feature/stored nodes using rlp (#65) Signed-off-by: Thomas Zamojski Co-authored-by: Karim Taam --- build.gradle | 1 + gradle/versions.gradle | 7 +- .../besu/ethereum/trie/NodeLoader.java | 28 +++ .../trie/verkle/SimpleBatchedVerkleTrie.java | 2 +- .../trie/verkle/VerkleTrieBatchHasher.java | 24 ++- .../verkle/factory/StoredNodeFactory.java | 164 +++++++++++++----- .../ethereum/trie/verkle/node/BranchNode.java | 2 +- .../trie/verkle/node/InternalNode.java | 67 ++++++- .../ethereum/trie/verkle/node/LeafNode.java | 21 ++- .../besu/ethereum/trie/verkle/node/Node.java | 46 ++++- .../trie/verkle/node/NullLeafNode.java | 11 ++ .../ethereum/trie/verkle/node/NullNode.java | 11 ++ .../ethereum/trie/verkle/node/StemNode.java | 77 ++++---- .../ethereum/trie/verkle/node/StoredNode.java | 36 +++- .../trie/verkle/visitor/CommitVisitor.java | 11 +- .../trie/verkle/visitor/FlattenVisitor.java | 34 ++-- .../trie/verkle/visitor/HashVisitor.java | 19 +- .../trie/verkle/visitor/PutVisitor.java | 10 +- .../trie/verkle/visitor/RemoveVisitor.java | 2 + .../ethereum/trie/verkle/GenesisTest.java | 7 +- .../ethereum/trie/verkle/NodeLoaderMock.java | 21 ++- .../ethereum/trie/verkle/NodeUpdaterMock.java | 13 +- .../verkle/StoredBatchedVerkleTrieTest.java | 24 +-- .../trie/verkle/StoredVerkleTrieTest.java | 24 +-- src/test/resources/VerkleTrie.gv | 8 +- .../resources/expectedTreeOneValueNoNulls.txt | 10 +- .../expectedTreeOneValueShowNulls.txt | 10 +- .../expectedTreeThreeValuesNoNulls.txt | 26 +-- .../expectedTreeTwoValuesNoNulls.txt | 18 +- .../expectedTreeTwoValuesShowNulls.txt | 18 +- 30 files changed, 513 insertions(+), 239 deletions(-) create mode 100644 src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java diff --git a/build.gradle b/build.gradle index 627d2aa..86eb6f4 100644 --- a/build.gradle +++ b/build.gradle @@ -60,6 +60,7 @@ dependencies { implementation 'io.tmio:tuweni-rlp' implementation 'org.hyperledger.besu:ipa-multipoint' implementation 'org.hyperledger.besu.internal:trie' + implementation 'org.hyperledger.besu.internal:rlp' implementation 'org.apache.logging.log4j:log4j-api' implementation 'org.apache.logging.log4j:log4j-core' diff --git a/gradle/versions.gradle b/gradle/versions.gradle index e82a454..77c0576 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -19,12 +19,15 @@ dependencyManagement { dependency 'net.java.dev.jna:jna:5.14.0' - dependency 'org.hyperledger.besu.internal:trie:24.6.0' - dependency 'org.hyperledger.besu:ipa-multipoint:0.8.5' dependency 'org.assertj:assertj-core:3.25.1' + dependencySet(group: 'org.hyperledger.besu.internal', version: '24.7.0') { + entry 'rlp' + entry 'trie' + } + dependencySet(group: 'org.apache.logging.log4j', version: '2.22.1') { entry 'log4j-api' entry 'log4j-core' diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java b/src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java new file mode 100644 index 0000000..9ed0286 --- /dev/null +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java @@ -0,0 +1,28 @@ +/* + * Copyright Hyperledger Besu Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public interface NodeLoader { + + Optional getNode(Bytes location, Bytes32 hash); + + record NearestKeyValue(Bytes key, Optional value) {} +} diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/SimpleBatchedVerkleTrie.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/SimpleBatchedVerkleTrie.java index 9c1cd42..2724fad 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/SimpleBatchedVerkleTrie.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/SimpleBatchedVerkleTrie.java @@ -35,7 +35,7 @@ * @param The type of values in the Verkle Trie. */ public class SimpleBatchedVerkleTrie - extends SimpleVerkleTrie implements VerkleTrie { + extends SimpleVerkleTrie { private final VerkleTrieBatchHasher batchProcessor; diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/VerkleTrieBatchHasher.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/VerkleTrieBatchHasher.java index 16ae3d5..abe3628 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/VerkleTrieBatchHasher.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/VerkleTrieBatchHasher.java @@ -111,7 +111,8 @@ public void calculateStateRoot() { } if (location.isEmpty()) { // We will end up updating the root node. Once all the batching is finished, - // we will update the previous states of the nodes by setting them to the new ones. + // we will update the previous states of the nodes by setting them to the new + // ones. calculateRootInternalNodeHash((InternalNode) node); updatedNodes.forEach( (__, n) -> { @@ -185,8 +186,9 @@ private void processBatch(List> nodes) { } private void calculateRootInternalNodeHash(final InternalNode internalNode) { - final Bytes32 hash = Bytes32.wrap(getRootNodeCommitments(internalNode).get(0)); - internalNode.replaceHash(hash, hash); + final Bytes commitment = getRootNodeCommitments(internalNode).get(0); + final Bytes32 hash = hasher.compress(commitment); + internalNode.replaceHash(hash, commitment); } private void calculateStemNodeHashes( @@ -229,9 +231,11 @@ private List getStemNodeLeftRightCommitments(StemNode stemNode) { Node node = stemNode.child((byte) idx); Optional oldValue = node.getPrevious().map(Bytes.class::cast); - // We should not recalculate a node if it is persisted and has not undergone an update since + // We should not recalculate a node if it is persisted and has not undergone an + // update since // its last save. - // If a child does not have a previous value, it means that it is a new node and we must + // If a child does not have a previous value, it means that it is a new node and + // we must // therefore recalculate it. if (!(node instanceof StoredNode) && (oldValue.isEmpty() || node.isDirty())) { if (idx < halfSize) { @@ -300,9 +304,11 @@ private List getInternalNodeCommitments(InternalNode internalNode) { for (int i = 0; i < size; i++) { final Node node = internalNode.child((byte) i); Optional oldValue = node.getPrevious().map(Bytes.class::cast); - // We should not recalculate a node if it is persisted and has not undergone an update since + // We should not recalculate a node if it is persisted and has not undergone an + // update since // its last save. - // If a child does not have a previous value, it means that it is a new node and we must + // If a child does not have a previous value, it means that it is a new node and + // we must // therefore recalculate it. if (!(node instanceof StoredNode) && (oldValue.isEmpty() || node.isDirty())) { indices.add((byte) i); @@ -318,12 +324,12 @@ private List getInternalNodeCommitments(InternalNode internalNode) { private List getRootNodeCommitments(InternalNode internalNode) { int size = InternalNode.maxChild(); final List commitmentsHashes = new ArrayList<>(); - final List newValues = new ArrayList<>(); + final List newValues = new ArrayList<>(); for (int i = 0; i < size; i++) { final Node node = internalNode.child((byte) i); newValues.add(node.getHash().get()); } - commitmentsHashes.add(hasher.commitRoot(newValues.toArray(new Bytes[] {}))); + commitmentsHashes.add(hasher.commit(newValues.toArray(new Bytes32[] {}))); return commitmentsHashes; } } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/factory/StoredNodeFactory.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/factory/StoredNodeFactory.java index c520dc7..ba9b8fa 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/factory/StoredNodeFactory.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/factory/StoredNodeFactory.java @@ -15,10 +15,14 @@ */ package org.hyperledger.besu.ethereum.trie.verkle.factory; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.NodeLoader; import org.hyperledger.besu.ethereum.trie.verkle.node.InternalNode; import org.hyperledger.besu.ethereum.trie.verkle.node.LeafNode; import org.hyperledger.besu.ethereum.trie.verkle.node.Node; +import org.hyperledger.besu.ethereum.trie.verkle.node.NullLeafNode; +import org.hyperledger.besu.ethereum.trie.verkle.node.NullNode; import org.hyperledger.besu.ethereum.trie.verkle.node.StemNode; import org.hyperledger.besu.ethereum.trie.verkle.node.StoredNode; @@ -29,7 +33,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.rlp.RLP; +import org.apache.tuweni.bytes.MutableBytes; /** Node types that are saved to storage. */ enum NodeType { @@ -47,6 +51,7 @@ enum NodeType { public class StoredNodeFactory implements NodeFactory { private final NodeLoader nodeLoader; private final Function valueDeserializer; + private final Boolean areCommitmentsCompressed; /** * Creates a new StoredNodeFactory with the given node loader and value deserializer. @@ -57,6 +62,23 @@ public class StoredNodeFactory implements NodeFactory { public StoredNodeFactory(NodeLoader nodeLoader, Function valueDeserializer) { this.nodeLoader = nodeLoader; this.valueDeserializer = valueDeserializer; + this.areCommitmentsCompressed = false; + } + + /** + * Creates a new StoredNodeFactory with the given node loader and value deserializer. + * + * @param nodeLoader The loader for retrieving stored nodes. + * @param valueDeserializer The function to deserialize values from Bytes. + * @param areCommitmentsCompressed Are commitments stored compressed (32bytes). + */ + public StoredNodeFactory( + NodeLoader nodeLoader, + Function valueDeserializer, + Boolean areCommitmentsCompressed) { + this.nodeLoader = nodeLoader; + this.valueDeserializer = valueDeserializer; + this.areCommitmentsCompressed = areCommitmentsCompressed; } /** @@ -69,74 +91,134 @@ public StoredNodeFactory(NodeLoader nodeLoader, Function valueDeserial */ @Override public Optional> retrieve(final Bytes location, final Bytes32 hash) { - /* Currently, Root and Leaf are distinguishable by location. + /* + * Currently, Root and Leaf are distinguishable by location. * To distinguish internal from stem, we further need values. * Currently, they are distinguished by values length. */ - Optional optionalEncodedValues = nodeLoader.getNode(location, hash); - if (optionalEncodedValues.isEmpty()) { + Optional> result; + Optional optionalKeyValue = nodeLoader.getNode(location, hash); + if (optionalKeyValue.isEmpty()) { return Optional.empty(); } - Bytes encodedValues = optionalEncodedValues.get(); - List values = RLP.decodeToList(encodedValues, reader -> reader.readValue().copy()); - final int locLength = location.size(); - final int nValues = values.size(); - NodeType type = - (locLength == 32 ? NodeType.LEAF : (nValues == 2 ? NodeType.INTERNAL : NodeType.STEM)); - return switch (type) { - case LEAF -> Optional.of(createLeafNode(location, values)); - case INTERNAL -> Optional.of(createInternalNode(location, values)); - case STEM -> Optional.of(createStemNode(location, values)); - default -> Optional.empty(); - }; + Bytes key = optionalKeyValue.get().key(); + Optional maybeEncodedValues = optionalKeyValue.get().value(); + Bytes encodedValues = + maybeEncodedValues.isPresent() ? Bytes.of(maybeEncodedValues.get()) : Bytes.EMPTY; + + if (key.size() == 0) { + result = Optional.of(decodeRootNode(encodedValues)); + } else if (key.size() > 0 && key.size() < 31) { + result = Optional.of(decodeInternalNode(key, encodedValues, hash)); + } else if (key.size() == 31) { + result = Optional.of(decodeStemNode(key, encodedValues, hash)); + } else { + result = Optional.empty(); + } + return result; + } + + private Bytes decodeCommitment(Bytes commitment) { + if (areCommitmentsCompressed && !commitment.isEmpty()) { + // TODO: uncompress commitment + } + if (commitment.isEmpty()) { + commitment = Node.EMPTY_COMMITMENT; + } + MutableBytes comm = MutableBytes.create(64); + comm.set(0, commitment); + return (Bytes) comm; + } + + /** + * Creates a rootNode using the provided location, hash, and path. + * + * @param encodedValues List of Bytes values retrieved from storage. + * @return A internalNode instance. + */ + InternalNode decodeRootNode(Bytes encodedValues) { + RLPInput input = new BytesValueRLPInput(encodedValues, false); + input.enterList(); + Bytes32 hash = Bytes32.rightPad(input.readBytes()); + Bytes commitment = decodeCommitment(input.readBytes()); + List scalars = input.readList(in -> Bytes32.rightPad(in.readBytes())); + input.leaveList(); + return createInternalNode(Bytes.EMPTY, hash, commitment, scalars); } /** * Creates a internalNode using the provided location, hash, and path. * * @param location The location of the internalNode. - * @param values List of Bytes values retrieved from storage. + * @param encodedValues List of Bytes values retrieved from storage. + * @param hash Node's hash value. * @return A internalNode instance. */ - InternalNode createInternalNode(Bytes location, List values) { - final int nChild = InternalNode.maxChild(); - ArrayList> children = new ArrayList>(nChild); + InternalNode decodeInternalNode(Bytes location, Bytes encodedValues, Bytes32 hash) { + RLPInput input = new BytesValueRLPInput(encodedValues, false); + input.enterList(); + Bytes commitment = decodeCommitment(input.readBytes()); + List scalars = input.readList(in -> Bytes32.rightPad(in.readBytes())); + input.leaveList(); + return createInternalNode(location, hash, commitment, scalars); + } + + private InternalNode createInternalNode( + Bytes location, Bytes32 hash, Bytes commitment, List scalars) { + int nChild = InternalNode.maxChild(); + List> children = new ArrayList<>(nChild); for (int i = 0; i < nChild; i++) { - children.add(new StoredNode<>(this, Bytes.concatenate(location, Bytes.of(i)))); + if (scalars.get(i) == Bytes32.ZERO) { + children.add(new NullNode()); + } else { + children.add( + new StoredNode(this, Bytes.concatenate(location, Bytes.of(i)), scalars.get(i))); + } } - final Bytes32 hash = (Bytes32) values.get(0); - final Bytes commitment = values.get(1); return new InternalNode(location, hash, commitment, children); } /** - * Creates a BranchNode using the provided location, hash, and path. + * Creates a StemNode using the provided stem, hash and encodedValues * - * @param location The location of the BranchNode. - * @param values List of Bytes values retrieved from storage. + * @param stem The stem of the BranchNode. + * @param encodedValues List of Bytes values retrieved from storage. + * @param hash Node's hash value. * @return A BranchNode instance. */ - StemNode createStemNode(Bytes location, List values) { + StemNode decodeStemNode(Bytes stem, Bytes encodedValues, Bytes32 hash) { + RLPInput input = new BytesValueRLPInput(encodedValues, false); + input.enterList(); + + int depth = input.readByte(); + Bytes commitment = decodeCommitment(input.readBytes()); + Bytes leftCommitment = decodeCommitment(input.readBytes()); + Bytes rightCommitment = decodeCommitment(input.readBytes()); + Bytes32 leftScalar = Bytes32.rightPad(input.readBytes()); + Bytes32 rightScalar = Bytes32.rightPad(input.readBytes()); + List values = input.readList(in -> in.readBytes()); + + // create StemNode + final Bytes location = stem.slice(0, depth); final int nChild = StemNode.maxChild(); - final Bytes stem = values.get(0); - final Bytes32 hash = (Bytes32) values.get(1); - final Bytes commitment = values.get(2); - final Bytes32 leftHash = (Bytes32) values.get(3); - final Bytes leftCommitment = values.get(4); - final Bytes32 rightHash = (Bytes32) values.get(5); - final Bytes rightCommitment = values.get(6); - ArrayList> children = new ArrayList>(nChild); + List> children = new ArrayList<>(nChild); for (int i = 0; i < nChild; i++) { - children.add(new StoredNode<>(this, Bytes.concatenate(stem, Bytes.of(i)))); + if (values.get(i) == Bytes.EMPTY) { + children.add(new NullLeafNode()); + } else { + children.add( + createLeafNode( + Bytes.concatenate(location, Bytes.of(i)), Bytes32.rightPad(values.get(i)))); + } } return new StemNode( location, stem, hash, commitment, - leftHash, + leftScalar, leftCommitment, - rightHash, + rightScalar, rightCommitment, children); } @@ -145,11 +227,11 @@ StemNode createStemNode(Bytes location, List values) { * Creates a LeafNode using the provided location, path, and value. * * @param key The key of the LeafNode. - * @param values List of Bytes values retrieved from storage. + * @param encodedValue Leaf value retrieved from storage. * @return A LeafNode instance. */ - LeafNode createLeafNode(Bytes key, List values) { - V value = valueDeserializer.apply(values.get(0)); + LeafNode createLeafNode(Bytes key, Bytes encodedValue) { + V value = valueDeserializer.apply(encodedValue); return new LeafNode(Optional.of(key), value); } } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/BranchNode.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/BranchNode.java index c74c1d0..1b8d98f 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/BranchNode.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/BranchNode.java @@ -31,7 +31,7 @@ * @param The type of the node's value. */ public abstract class BranchNode extends Node { - private final Optional location; // Location in the tree + protected Optional location; // Location in the tree protected Optional hash; // Vector commitment's hash protected Optional commitment; // Vector commitment serialized private final List> children; // List of children nodes diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/InternalNode.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/InternalNode.java index 9d481b0..881a1d8 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/InternalNode.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/InternalNode.java @@ -15,17 +15,16 @@ */ package org.hyperledger.besu.ethereum.trie.verkle.node; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.trie.verkle.visitor.NodeVisitor; import org.hyperledger.besu.ethereum.trie.verkle.visitor.PathNodeVisitor; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.rlp.RLP; -import org.apache.tuweni.rlp.RLPWriter; /** * Represents an internal node in the Verkle Trie. @@ -52,6 +51,33 @@ public InternalNode( this.previous = Optional.of(hash); } + /** + * Constructs a new InternalNode with location, hash, path, and children. + * + * @param location The location in the tree. + * @param hash Node's vector commitment's hash. + * @param commitment Node's vector commitment. + * @param children The list of children nodes. + */ + public InternalNode( + final Optional location, + final Optional hash, + final Optional commitment, + final List> children) { + super(location, hash, commitment, children); + this.previous = hash; + } + + public InternalNode( + final Optional location, + final Optional hash, + final Optional commitment, + final Optional previous, + final List> children) { + super(location, hash, commitment, children); + this.previous = previous; + } + /** * Constructs a new InternalNode with optional location and path, initializing children to * NullNodes. @@ -85,6 +111,25 @@ public Node accept(final NodeVisitor visitor) { return visitor.visit(this); } + /** + * Replace node's Location + * + * @param newLocation The new location for the Node + * @return The updated Node + */ + @SuppressWarnings("unchecked") + @Override + public InternalNode replaceLocation(Bytes newLocation) { + List> newChildren = new ArrayList<>(maxChild()); + for (int i = 0; i < maxChild(); i++) { + Bytes index = Bytes.of(i); + Bytes childLocation = Bytes.concatenate(newLocation, index); + newChildren.add(child((byte) i).replaceLocation(childLocation)); + } + return new InternalNode( + Optional.of(newLocation), hash, commitment, (Optional) previous, newChildren); + } + /** * Replace the vector commitment with a new one. * @@ -108,8 +153,16 @@ public Bytes getEncodedValue() { if (encodedValue.isPresent()) { return encodedValue.get(); } - List values = Arrays.asList((Bytes) getHash().get(), getCommitment().get()); - Bytes result = RLP.encodeList(values, RLPWriter::writeValue); + BytesValueRLPOutput out = new BytesValueRLPOutput(); + out.startList(); + if (getLocation().get().isEmpty()) { + out.writeBytes((Bytes) getHash().get()); + } + out.writeBytes(encodeCommitment(getCommitment().get())); + out.writeList( + getChildren(), (child, writer) -> writer.writeBytes(encodeScalar(child.getHash().get()))); + out.endList(); + Bytes result = out.encoded(); this.encodedValue = Optional.of(result); return result; } @@ -128,8 +181,8 @@ public String print() { if (!(child instanceof NullNode)) { if (!(child instanceof StoredNode) || !child.getEncodedValue().isEmpty()) { final String label = String.format("[%02x] ", i); - final String childRep = child.print().replaceAll("\n\t", "\n\t\t"); - builder.append("\n\t").append(label).append(childRep); + final String childRep = child.print().replaceAll("\n ", "\n "); + builder.append("\n ").append(label).append(childRep); } } } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/LeafNode.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/LeafNode.java index af61193..ad6d293 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/LeafNode.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/LeafNode.java @@ -18,14 +18,11 @@ import org.hyperledger.besu.ethereum.trie.verkle.visitor.NodeVisitor; import org.hyperledger.besu.ethereum.trie.verkle.visitor.PathNodeVisitor; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.rlp.RLP; -import org.apache.tuweni.rlp.RLPWriter; /** * Represents a leaf node in the Verkle Trie. @@ -113,6 +110,18 @@ public Optional getLocation() { return location; } + /** + * Replace node's Location + * + * @param newLocation The new location for the Node + * @return The updated Node + */ + @SuppressWarnings("unchecked") + @Override + public LeafNode replaceLocation(Bytes newLocation) { + return new LeafNode(Optional.of(newLocation), value, (Optional) previous); + } + /** * Get the children of the node. A leaf node does not have children, so this method throws an * UnsupportedOperationException. @@ -148,10 +157,8 @@ public Bytes getEncodedValue() { } Bytes encodedVal = getValue().isPresent() ? valueSerializer.apply(getValue().get()) : Bytes.EMPTY; - List values = Arrays.asList(encodedVal); - Bytes result = RLP.encodeList(values, RLPWriter::writeValue); - this.encodedValue = Optional.of(result); - return result; + this.encodedValue = Optional.of(encodedVal.trimTrailingZeros()); + return encodedVal; } /** diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/Node.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/Node.java index d6a1136..3cfcc90 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/Node.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/Node.java @@ -36,12 +36,13 @@ public abstract class Node { public static Bytes32 EMPTY_HASH = Bytes32.ZERO; /** A constant representing a commitment to NullNodes */ - public static Bytes EMPTY_COMMITMENT = Bytes.EMPTY; + public static Bytes EMPTY_COMMITMENT = + Bytes.fromHexString( + "0x0000000000000000000000000000000000000000000000000000000000000000" + + "0100000000000000000000000000000000000000000000000000000000000000"); Optional previous; - boolean dirty; - boolean persisted; public Node(final boolean dirty, final boolean persisted) { @@ -73,6 +74,15 @@ public Node(final Optional previous, final boolean dirty, final boolean p */ public abstract Node accept(NodeVisitor visitor); + /** + * Does the node have an extension path ? + * + * @return hasExtension + */ + public Boolean hasExtension() { + return false; + } + /** * Get the location of the node. * @@ -82,6 +92,14 @@ public Optional getLocation() { return Optional.empty(); } + /** + * Replace node's Location + * + * @param newLocation The new location for the Node + * @return The updated Node + */ + public abstract Node replaceLocation(Bytes newLocation); + /** * Get the value associated with the node. * @@ -190,7 +208,7 @@ public boolean isPersisted() { * * @return A string representation of the node. */ - abstract String print(); + public abstract String print(); /** * Generates DOT representation for the Node. @@ -239,4 +257,24 @@ public static Bytes32 getHighValue(Optional value) { .map((v) -> Bytes32.rightPad(Bytes32.rightPad((Bytes) v).slice(16, 16))) .orElse(Bytes32.ZERO); } + + /** + * Encode a commitment + * + * @param value A commitment value + * @return The encoded commitment + */ + public static Bytes encodeCommitment(Bytes value) { + return value != EMPTY_COMMITMENT ? value.trimTrailingZeros() : Bytes.EMPTY; + } + + /** + * Encode a scalar + * + * @param scalar A scalar value + * @return The encoded scalar + */ + public static Bytes encodeScalar(Bytes32 scalar) { + return scalar != EMPTY_HASH ? scalar.trimTrailingZeros() : Bytes.EMPTY; + } } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullLeafNode.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullLeafNode.java index b0876cd..6fd3397 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullLeafNode.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullLeafNode.java @@ -64,6 +64,17 @@ public Node accept(final NodeVisitor visitor) { return visitor.visit(this); } + /** + * Replace node's Location + * + * @param newLocation The new location for the Node + * @return The updated Node + */ + @Override + public NullLeafNode replaceLocation(Bytes newLocation) { + return this; + } + /** * Get the hash associated with the `NullNode`. * diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullNode.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullNode.java index 0b793f6..ae4c443 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullNode.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/NullNode.java @@ -59,6 +59,17 @@ public Node accept(final NodeVisitor visitor) { return visitor.visit(this); } + /** + * Replace node's Location + * + * @param newLocation The new location for the Node + * @return The updated Node + */ + @Override + public NullNode replaceLocation(Bytes newLocation) { + return this; + } + /** * Get the hash associated with the `NullNode`. * diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StemNode.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StemNode.java index f856f09..8c94b85 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StemNode.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StemNode.java @@ -15,17 +15,16 @@ */ package org.hyperledger.besu.ethereum.trie.verkle.node; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.trie.verkle.visitor.NodeVisitor; import org.hyperledger.besu.ethereum.trie.verkle.visitor.PathNodeVisitor; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.rlp.RLP; -import org.apache.tuweni.rlp.RLPWriter; /** * Represents a stem node in the Verkle Trie. @@ -161,6 +160,11 @@ public Bytes getStem() { return stem; } + @Override + public Boolean hasExtension() { + return true; + } + /** * Get Node's extension. * @@ -207,25 +211,31 @@ public Optional getRightCommitment() { } /** - * Creates a new node by replacing its location + * Replace node's Location * - * @param location The location in the tree. - * @return StemNode with new location. + * @param newLocation The new location for the Node + * @return The updated Node */ - public StemNode replaceLocation(final Bytes location) { - StemNode vStemNode = - new StemNode<>( - Optional.of(location), - stem, - getHash(), - Optional.empty(), - getCommitment(), - leftHash, - leftCommitment, - rightHash, - rightCommitment, - getChildren()); - return vStemNode; + @Override + @SuppressWarnings("unchecked") + public StemNode replaceLocation(Bytes newLocation) { + List> newChildren = new ArrayList<>(maxChild()); + for (int i = 0; i < maxChild(); i++) { + Bytes index = Bytes.of(i); + Bytes childLocation = Bytes.concatenate(newLocation, index); + newChildren.add(child((byte) i).replaceLocation(childLocation)); + } + return new StemNode( + Optional.of(newLocation), + stem, + hash, + (Optional) previous, + commitment, + leftHash, + leftCommitment, + rightHash, + rightCommitment, + newChildren); } /** @@ -262,16 +272,17 @@ public Bytes getEncodedValue() { if (encodedValue.isPresent()) { return encodedValue.get(); } - List values = - Arrays.asList( - getStem(), - (Bytes) getHash().get(), - getCommitment().get(), - (Bytes) getLeftHash().get(), - getLeftCommitment().get(), - (Bytes) getRightHash().get(), - getRightCommitment().get()); - Bytes result = RLP.encodeList(values, RLPWriter::writeValue); + BytesValueRLPOutput values = new BytesValueRLPOutput(); + values.startList(); + values.writeByte((byte) (getLocation().get().size() & 0xff)); + values.writeBytes(encodeCommitment(getCommitment().get())); + values.writeBytes(encodeCommitment(getLeftCommitment().get())); + values.writeBytes(encodeCommitment(getRightCommitment().get())); + values.writeBytes(encodeScalar(getLeftHash().get())); + values.writeBytes(encodeScalar(getRightHash().get())); + values.writeList(getChildren(), (child, writer) -> writer.writeBytes(child.getEncodedValue())); + values.endList(); + Bytes result = values.encoded(); this.encodedValue = Optional.of(result); return result; } @@ -287,10 +298,10 @@ public String print() { builder.append(String.format("Stem: %s", stem)); for (int i = 0; i < maxChild(); i++) { final Node child = child((byte) i); - if (!(child instanceof NullNode)) { + if (!(child instanceof NullNode) && !(child instanceof NullLeafNode)) { final String label = String.format("[%02x] ", i); - final String childRep = child.print().replaceAll("\n\t", "\n\t\t"); - builder.append("\n\t").append(label).append(childRep); + final String childRep = child.print().replaceAll("\n ", "\n "); + builder.append("\n ").append(label).append(childRep); } } return builder.toString(); diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StoredNode.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StoredNode.java index dba20be..339027a 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StoredNode.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/node/StoredNode.java @@ -33,7 +33,8 @@ * @param The type of the node's value. */ public class StoredNode extends Node { - private final Bytes location; + private Bytes location; + private final Optional hash; private final NodeFactory nodeFactory; private Optional> loadedNode; @@ -46,6 +47,22 @@ public class StoredNode extends Node { public StoredNode(final NodeFactory nodeFactory, final Bytes location) { super(false, true); this.location = location; + this.hash = Optional.empty(); + this.nodeFactory = nodeFactory; + loadedNode = Optional.empty(); + } + + /** + * Constructs a new StoredNode at location. + * + * @param nodeFactory The node factory for creating nodes from storage. + * @param location The location in the tree. + * @param hash The hash value of the node. + */ + public StoredNode(final NodeFactory nodeFactory, final Bytes location, final Bytes32 hash) { + super(false, true); + this.location = location; + this.hash = Optional.of(hash); this.nodeFactory = nodeFactory; loadedNode = Optional.empty(); } @@ -85,6 +102,18 @@ public Optional getLocation() { return Optional.of(location); } + /** + * Replace node's Location + * + * @param newLocation The new location for the Node + * @return The updated Node + */ + @Override + public StoredNode replaceLocation(Bytes newLocation) { + location = newLocation; + return this; + } + /** * Get the value associated with the node. * @@ -103,6 +132,9 @@ public Optional getValue() { */ @Override public Optional getHash() { + if (hash.isPresent()) { + return hash; + } final Node node = load(); return node.getHash(); } @@ -174,7 +206,7 @@ public String toDot(Boolean showNullNodes) { private Node load() { if (loadedNode.isEmpty()) { - loadedNode = nodeFactory.retrieve(location, null); + loadedNode = nodeFactory.retrieve(location, hash.orElse(null)); } if (loadedNode.isPresent()) { return loadedNode.get(); diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/CommitVisitor.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/CommitVisitor.java index dcc7a58..545a237 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/CommitVisitor.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/CommitVisitor.java @@ -93,10 +93,10 @@ public Node visit(final StemNode stemNode, final Bytes location) { Bytes index = Bytes.of(i); final Node child = stemNode.child((byte) i); if (!child.isPersisted()) { - child.accept(this, Bytes.concatenate(stem, index)); + child.accept(this, Bytes.concatenate(location, index)); } } - nodeUpdater.store(location, null, stemNode.getEncodedValue()); + nodeUpdater.store(stem, null, stemNode.getEncodedValue()); stemNode.markPersisted(); return stemNode; } @@ -110,13 +110,6 @@ public Node visit(final StemNode stemNode, final Bytes location) { */ @Override public Node visit(final LeafNode leafNode, final Bytes location) { - if (leafNode.isPersisted()) { - return leafNode; - } - if (leafNode.isDirty()) { - throw new RuntimeException("cannot persist dirty node"); - } - nodeUpdater.store(location, null, leafNode.getEncodedValue()); leafNode.markPersisted(); return leafNode; } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/FlattenVisitor.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/FlattenVisitor.java index 258a901..853c133 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/FlattenVisitor.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/FlattenVisitor.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.trie.verkle.VerkleTrieBatchHasher; import org.hyperledger.besu.ethereum.trie.verkle.node.InternalNode; import org.hyperledger.besu.ethereum.trie.verkle.node.Node; +import org.hyperledger.besu.ethereum.trie.verkle.node.NullLeafNode; import org.hyperledger.besu.ethereum.trie.verkle.node.NullNode; import org.hyperledger.besu.ethereum.trie.verkle.node.StemNode; @@ -52,19 +53,28 @@ public Node visit(StemNode stemNode) { final Bytes location = stemNode.getLocation().get(); final Bytes newLocation = location.slice(0, location.size() - 1); // Should not flatten root node - if (!newLocation.isEmpty()) { - final StemNode updateStemNode = stemNode.replaceLocation(newLocation); - batchProcessor.ifPresent( - processor -> { - final NullNode nullNode = new NullNode<>(); - nullNode.markDirty(); - processor.addNodeToBatch(stemNode.getLocation(), nullNode); - updateStemNode.markDirty(); - processor.addNodeToBatch(updateStemNode.getLocation(), updateStemNode); - }); - return updateStemNode; - } else { + if (newLocation.isEmpty()) { return new NullNode<>(); } + final StemNode updateStemNode = stemNode.replaceLocation(newLocation); + batchProcessor.ifPresent( + processor -> { + final NullNode nullNode = new NullNode<>(); + nullNode.markDirty(); + processor.addNodeToBatch(stemNode.getLocation(), nullNode); + updateStemNode.markDirty(); + processor.addNodeToBatch(updateStemNode.getLocation(), updateStemNode); + for (int i = 0; i < StemNode.maxChild(); i++) { + Byte index = Bytes.of(i).get(0); + if (!(stemNode.child(index) instanceof NullLeafNode)) { + final NullLeafNode childNullNode = new NullLeafNode<>(); + childNullNode.markDirty(); + processor.addNodeToBatch(stemNode.child(index).getLocation(), childNullNode); + processor.addNodeToBatch( + updateStemNode.child(index).getLocation(), updateStemNode.child(index)); + } + } + }); + return updateStemNode; } } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/HashVisitor.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/HashVisitor.java index aa773b3..5a8b4f0 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/HashVisitor.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/HashVisitor.java @@ -67,12 +67,13 @@ public Node visit(InternalNode internalNode, Bytes location) { hashes[i] = updatedChild.getHash().get(); } final Bytes32 hash; + final Bytes commitment = hasher.commit(hashes); if (location.isEmpty()) { - hash = hasher.commitRoot(hashes); + hash = hasher.compress(commitment); } else { - hash = hasher.hash(hasher.commit(hashes)); + hash = hasher.hash(commitment); } - final Node vNode = internalNode.replaceHash(hash, hash); // commitment should be different + final Node vNode = internalNode.replaceHash(hash, commitment); vNode.markClean(); return vNode; } @@ -109,15 +110,17 @@ public Node visit(StemNode stemNode, Bytes location) { rightValues[2 * i - size] = getLowValue(child.getValue()); rightValues[2 * i + 1 - size] = getHighValue(child.getValue()); } + Bytes leftCommitment = hasher.commit(leftValues); + Bytes rightCommitment = hasher.commit(rightValues); hashes[0] = Bytes32.rightPad(Bytes.of(1)); // extension marker hashes[1] = Bytes32.rightPad(stemNode.getStem()); - hashes[2] = hasher.hash(hasher.commit(leftValues)); - hashes[3] = hasher.hash(hasher.commit(rightValues)); - final Bytes32 hash = hasher.hash(hasher.commit(hashes)); + hashes[2] = hasher.hash(leftCommitment); + hashes[3] = hasher.hash(rightCommitment); + Bytes commitment = hasher.commit(hashes); + final Bytes32 hash = hasher.hash(commitment); StemNode vStemNode = stemNode.replaceHash( - hash, hash, hashes[2], hashes[2], hashes[3], - hashes[3]); // commitment should be different + hash, commitment, hashes[2], leftCommitment, hashes[3], rightCommitment); vStemNode.markClean(); return vStemNode; } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/PutVisitor.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/PutVisitor.java index 21eed6e..a5ed1db 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/PutVisitor.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/PutVisitor.java @@ -97,11 +97,12 @@ public Node visit(final StemNode stemNode, final Bytes path) { final Bytes newStem = fullPath.slice(0, stem.size()); if (stem.compareTo(newStem) == 0) { // Same stem => skip to leaf in StemNode final byte index = fullPath.get(newStem.size()); - visited = Bytes.concatenate(newStem, Bytes.of(index)); + visited = Bytes.concatenate(visited, Bytes.of(index)); final Node child = stemNode.child(index); - final Node updatedChild = stemNode.child(index).accept(this, fullPath); + final Node updatedChild = stemNode.child(index).accept(this, path.slice(1)); if (child instanceof NullNode || child instanceof NullLeafNode) { - // This call may lead to the removal of the node from the batch if a null node is inserted. + // This call may lead to the removal of the node from the batch if a null node + // is inserted. batchProcessor.ifPresent( processor -> processor.addNodeToBatch(updatedChild.getLocation(), updatedChild)); } @@ -149,7 +150,6 @@ public Node visit(final LeafNode leafNode, final Bytes path) { } else { newNode = leafNode; } - visited = Bytes.EMPTY; return newNode; } @@ -184,10 +184,10 @@ public Node visit(final NullNode nullNode, final Bytes path) { public Node visit(final NullLeafNode nullLeafNode, final Bytes path) { assert path.size() < 33; oldValue = Optional.empty(); + visited = Bytes.concatenate(visited, path.slice(path.size())); LeafNode newNode = new LeafNode<>(visited, value); batchProcessor.ifPresent(processor -> processor.addNodeToBatch(newNode.getLocation(), newNode)); newNode.markDirty(); - visited = Bytes.EMPTY; return newNode; } diff --git a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/RemoveVisitor.java b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/RemoveVisitor.java index 45c2d5f..7632518 100644 --- a/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/RemoveVisitor.java +++ b/src/main/java/org/hyperledger/besu/ethereum/trie/verkle/visitor/RemoveVisitor.java @@ -80,6 +80,8 @@ public Node visit(InternalNode internalNode, Bytes path) { final Node newNode = internalNode.child(onlyChildIndex.get()).accept(flatten); if (!(newNode instanceof NullNode)) { // Flatten StemNode one-level up newNode.markDirty(); + batchProcessor.ifPresent( + processor -> processor.addNodeToBatch(newNode.getLocation(), newNode)); return newNode; } return internalNode; // Unique child was a internalNode, do nothing diff --git a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/GenesisTest.java b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/GenesisTest.java index 9fe0ace..2bebe60 100644 --- a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/GenesisTest.java +++ b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/GenesisTest.java @@ -23,7 +23,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.util.HashMap; import java.util.stream.Stream; import org.apache.commons.csv.CSVFormat; @@ -54,8 +53,8 @@ private static Stream provideGenesisAndStateRootExpected() { @ParameterizedTest @MethodSource("provideGenesisAndStateRootExpected") public void putGenesis(String genesisCSVFile, String expectedStateRootHash) throws IOException { - HashMap storage = new HashMap(); - NodeLoaderMock nodeLoader = new NodeLoaderMock(storage); + NodeUpdaterMock nodeUpdater = new NodeUpdaterMock(); + NodeLoaderMock nodeLoader = new NodeLoaderMock(nodeUpdater.storage); VerkleTrieBatchHasher batchProcessor = new VerkleTrieBatchHasher(); StoredNodeFactory nodeFactory = new StoredNodeFactory<>(nodeLoader, value -> value); StoredBatchedVerkleTrie trie = @@ -69,7 +68,7 @@ public void putGenesis(String genesisCSVFile, String expectedStateRootHash) thro trie.put(key, value); } } - trie.commit((location, hash, value) -> storage.put(location, value)); + trie.commit(nodeUpdater); assertThat(trie.getRootHash()) .isEqualByComparingTo(Bytes32.fromHexString(expectedStateRootHash)); diff --git a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeLoaderMock.java b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeLoaderMock.java index e501c5f..045ec02 100644 --- a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeLoaderMock.java +++ b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeLoaderMock.java @@ -17,22 +17,33 @@ import org.hyperledger.besu.ethereum.trie.NodeLoader; -import java.util.Map; import java.util.Optional; +import java.util.SortedMap; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; public class NodeLoaderMock implements NodeLoader { - public Map storage; + public SortedMap storage; - public NodeLoaderMock(Map storage) { + public NodeLoaderMock(SortedMap storage) { this.storage = storage; } @Override - public Optional getNode(Bytes location, Bytes32 hash) { - return Optional.ofNullable(storage.get(location)); + public Optional getNode(Bytes location, Bytes32 hash) { + Bytes key; + Optional value; + try { + key = storage.tailMap(location).firstKey(); + } catch (Exception noSuchElementException) { + return Optional.empty(); + } + if (key.commonPrefixLength(location) < location.size()) { + return Optional.empty(); + } + value = Optional.ofNullable(storage.get(key)); + return Optional.of(new NodeLoader.NearestKeyValue(key, value)); } } diff --git a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeUpdaterMock.java b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeUpdaterMock.java index abe6501..035c343 100644 --- a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeUpdaterMock.java +++ b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/NodeUpdaterMock.java @@ -17,26 +17,27 @@ import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import java.util.HashMap; -import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; public class NodeUpdaterMock implements NodeUpdater { - public Map storage; + public SortedMap storage; public NodeUpdaterMock() { - this.storage = new HashMap(); + this.storage = + new TreeMap((b1, b2) -> b1.toHexString().compareTo(b2.toHexString())); } - public NodeUpdaterMock(Map storage) { + public NodeUpdaterMock(SortedMap storage) { this.storage = storage; } @Override public void store(Bytes location, Bytes32 hash, Bytes value) { - storage.put(location, value); + storage.put(location, value.toArray()); } } diff --git a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredBatchedVerkleTrieTest.java b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredBatchedVerkleTrieTest.java index 979ca75..3aca4b8 100644 --- a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredBatchedVerkleTrieTest.java +++ b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredBatchedVerkleTrieTest.java @@ -17,13 +17,8 @@ import static org.assertj.core.api.Assertions.assertThat; -import org.hyperledger.besu.ethereum.trie.NodeUpdater; import org.hyperledger.besu.ethereum.trie.verkle.factory.StoredNodeFactory; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.Test; @@ -243,14 +238,13 @@ public void testDeleteThreeValuesWithFlattening() throws Exception { @Test public void testDeleteManyValuesWithDivergentStemsAtDepth2() throws Exception { - final Map map = new HashMap<>(); + NodeUpdaterMock nodeUpdater = new NodeUpdaterMock(); + NodeLoaderMock nodeLoader = new NodeLoaderMock(nodeUpdater.storage); VerkleTrieBatchHasher batchProcessor = new VerkleTrieBatchHasher(); StoredBatchedVerkleTrie trie = new StoredBatchedVerkleTrie<>( - batchProcessor, - new StoredNodeFactory<>( - (location, hash) -> Optional.ofNullable(map.get(location)), value -> value)); + batchProcessor, new StoredNodeFactory<>(nodeLoader, value -> value)); assertThat(trie.getRootHash()).isEqualTo(Bytes32.ZERO); Bytes32 key0 = @@ -286,18 +280,10 @@ public void testDeleteManyValuesWithDivergentStemsAtDepth2() throws Exception { trie.put(key5, value5); trie.put(key6, value6); - trie.commit( - new NodeUpdater() { - @Override - public void store(final Bytes location, final Bytes32 hash, final Bytes value) { - map.put(location, value); - } - }); + trie.commit(nodeUpdater); StoredBatchedVerkleTrie trie2 = new StoredBatchedVerkleTrie<>( - batchProcessor, - new StoredNodeFactory<>( - (location, hash) -> Optional.ofNullable(map.get(location)), value -> value)); + batchProcessor, new StoredNodeFactory<>(nodeLoader, value -> value)); assertThat(trie2.getRootHash()).isEqualTo(trie.getRootHash()); trie2.remove(key0); trie2.remove(key4); diff --git a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredVerkleTrieTest.java b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredVerkleTrieTest.java index 1ec7fa5..05723b9 100644 --- a/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredVerkleTrieTest.java +++ b/src/test/java/org/hyperledger/besu/ethereum/trie/verkle/StoredVerkleTrieTest.java @@ -17,13 +17,8 @@ import static org.assertj.core.api.Assertions.assertThat; -import org.hyperledger.besu.ethereum.trie.NodeUpdater; import org.hyperledger.besu.ethereum.trie.verkle.factory.StoredNodeFactory; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.Test; @@ -225,11 +220,10 @@ public void testDeleteThreeValuesWithFlattening() throws Exception { @Test public void testDeleteManyValuesWithDivergentStemsAtDepth2() throws Exception { - final Map map = new HashMap<>(); + NodeUpdaterMock nodeUpdater = new NodeUpdaterMock(); + NodeLoaderMock nodeLoader = new NodeLoaderMock(nodeUpdater.storage); StoredVerkleTrie trie = - new StoredVerkleTrie<>( - new StoredNodeFactory<>( - (location, hash) -> Optional.ofNullable(map.get(location)), value -> value)); + new StoredVerkleTrie<>(new StoredNodeFactory<>(nodeLoader, value -> value)); assertThat(trie.getRootHash()).isEqualTo(Bytes32.ZERO); Bytes32 key0 = @@ -265,17 +259,9 @@ public void testDeleteManyValuesWithDivergentStemsAtDepth2() throws Exception { trie.put(key5, value5); trie.put(key6, value6); - trie.commit( - new NodeUpdater() { - @Override - public void store(final Bytes location, final Bytes32 hash, final Bytes value) { - map.put(location, value); - } - }); + trie.commit(nodeUpdater); StoredVerkleTrie trie2 = - new StoredVerkleTrie<>( - new StoredNodeFactory<>( - (location, hash) -> Optional.ofNullable(map.get(location)), value -> value)); + new StoredVerkleTrie<>(new StoredNodeFactory<>(nodeLoader, value -> value)); assertThat(trie2.getRootHash()).isEqualTo(trie.getRootHash()); trie2.remove(key0); trie2.remove(key4); diff --git a/src/test/resources/VerkleTrie.gv b/src/test/resources/VerkleTrie.gv index d3eea29..faf2304 100644 --- a/src/test/resources/VerkleTrie.gv +++ b/src/test/resources/VerkleTrie.gv @@ -7,10 +7,10 @@ Stem: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee LeftCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000 RightCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000"] StemNode0x00 -> NullLeafNode0x -StemNode0x00 -> LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="L: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff +StemNode0x00 -> LeafNode0x00ff +LeafNode0x00ff [label="L: 0x00ff Suffix: 0xff"] -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -> Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x00ff -> Value0x00ff +Value0x00ff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] InternalNode0x -> NullNode0x } \ No newline at end of file diff --git a/src/test/resources/expectedTreeOneValueNoNulls.txt b/src/test/resources/expectedTreeOneValueNoNulls.txt index d3eea29..d99cefb 100644 --- a/src/test/resources/expectedTreeOneValueNoNulls.txt +++ b/src/test/resources/expectedTreeOneValueNoNulls.txt @@ -7,10 +7,10 @@ Stem: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee LeftCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000 RightCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000"] StemNode0x00 -> NullLeafNode0x -StemNode0x00 -> LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="L: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff +StemNode0x00 -> LeafNode0x00ff +LeafNode0x00ff [label="L: 0x00ff Suffix: 0xff"] -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -> Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x00ff -> Value0x00ff +Value0x00ff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] InternalNode0x -> NullNode0x -} \ No newline at end of file +} diff --git a/src/test/resources/expectedTreeOneValueShowNulls.txt b/src/test/resources/expectedTreeOneValueShowNulls.txt index c5f8352..93c16f7 100644 --- a/src/test/resources/expectedTreeOneValueShowNulls.txt +++ b/src/test/resources/expectedTreeOneValueShowNulls.txt @@ -516,11 +516,11 @@ StemNode0x00 -> NullLeafNode0x NullLeafNode0x [label="NL: 0x"] StemNode0x00 -> NullLeafNode0x NullLeafNode0x [label="NL: 0x"] -StemNode0x00 -> LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="L: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff +StemNode0x00 -> LeafNode0x00ff +LeafNode0x00ff [label="L: 0x00ff Suffix: 0xff"] -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -> Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x00ff -> Value0x00ff +Value0x00ff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] InternalNode0x -> NullNode0x NullNode0x [label="NL: 0x"] InternalNode0x -> NullNode0x @@ -1031,4 +1031,4 @@ InternalNode0x -> NullNode0x NullNode0x [label="NL: 0x"] InternalNode0x -> NullNode0x NullNode0x [label="NL: 0x"] -} \ No newline at end of file +} diff --git a/src/test/resources/expectedTreeThreeValuesNoNulls.txt b/src/test/resources/expectedTreeThreeValuesNoNulls.txt index 52afd53..b73f5b1 100644 --- a/src/test/resources/expectedTreeThreeValuesNoNulls.txt +++ b/src/test/resources/expectedTreeThreeValuesNoNulls.txt @@ -6,11 +6,11 @@ StemNode0x00 [label="S: 0x00 Stem: 0x00000000000000000000000000000000000000000000000000000000000000 LeftCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000 RightCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000"] -StemNode0x00 -> LeafNode0x0000000000000000000000000000000000000000000000000000000000000000 -LeafNode0x0000000000000000000000000000000000000000000000000000000000000000 [label="L: 0x0000000000000000000000000000000000000000000000000000000000000000 +StemNode0x00 -> LeafNode0x0000 +LeafNode0x0000 [label="L: 0x0000 Suffix: 0x00"] -LeafNode0x0000000000000000000000000000000000000000000000000000000000000000 -> Value0x0000000000000000000000000000000000000000000000000000000000000000 -Value0x0000000000000000000000000000000000000000000000000000000000000000 [label="Value: 0x0000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x0000 -> Value0x0000 +Value0x0000 [label="Value: 0x0000000000000000000000000000000000000000000000000000000000000000"] StemNode0x00 -> NullLeafNode0x InternalNode0x -> NullNode0x InternalNode0x -> InternalNode0x40 @@ -21,11 +21,11 @@ StemNode0x4000 [label="S: 0x4000 Stem: 0x40000000000000000000000000000000000000000000000000000000000000 LeftCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000 RightCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000"] -StemNode0x4000 -> LeafNode0x4000000000000000000000000000000000000000000000000000000000000000 -LeafNode0x4000000000000000000000000000000000000000000000000000000000000000 [label="L: 0x4000000000000000000000000000000000000000000000000000000000000000 +StemNode0x4000 -> LeafNode0x400000 +LeafNode0x400000 [label="L: 0x400000 Suffix: 0x00"] -LeafNode0x4000000000000000000000000000000000000000000000000000000000000000 -> Value0x4000000000000000000000000000000000000000000000000000000000000000 -Value0x4000000000000000000000000000000000000000000000000000000000000000 [label="Value: 0x0000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x400000 -> Value0x400000 +Value0x400000 [label="Value: 0x0000000000000000000000000000000000000000000000000000000000000000"] StemNode0x4000 -> NullLeafNode0x InternalNode0x40 -> NullNode0x InternalNode0x40 -> StemNode0x4020 @@ -33,10 +33,10 @@ StemNode0x4020 [label="S: 0x4020 Stem: 0x40200000000000000000000000000000000000000000000000000000000000 LeftCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000 RightCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000"] -StemNode0x4020 -> LeafNode0x4020000000000000000000000000000000000000000000000000000000000000 -LeafNode0x4020000000000000000000000000000000000000000000000000000000000000 [label="L: 0x4020000000000000000000000000000000000000000000000000000000000000 +StemNode0x4020 -> LeafNode0x402000 +LeafNode0x402000 [label="L: 0x402000 Suffix: 0x00"] -LeafNode0x4020000000000000000000000000000000000000000000000000000000000000 -> Value0x4020000000000000000000000000000000000000000000000000000000000000 -Value0x4020000000000000000000000000000000000000000000000000000000000000 [label="Value: 0x0000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x402000 -> Value0x402000 +Value0x402000 [label="Value: 0x0000000000000000000000000000000000000000000000000000000000000000"] StemNode0x4020 -> NullLeafNode0x -} \ No newline at end of file +} diff --git a/src/test/resources/expectedTreeTwoValuesNoNulls.txt b/src/test/resources/expectedTreeTwoValuesNoNulls.txt index e41ba22..f348844 100644 --- a/src/test/resources/expectedTreeTwoValuesNoNulls.txt +++ b/src/test/resources/expectedTreeTwoValuesNoNulls.txt @@ -6,16 +6,16 @@ StemNode0x00 [label="S: 0x00 Stem: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee LeftCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000 RightCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000"] -StemNode0x00 -> LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 [label="L: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 +StemNode0x00 -> LeafNode0x0000 +LeafNode0x0000 [label="L: 0x0000 Suffix: 0x00"] -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 -> Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 -Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 [label="Value: 0x0100000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x0000 -> Value0x0000 +Value0x0000 [label="Value: 0x0100000000000000000000000000000000000000000000000000000000000000"] StemNode0x00 -> NullLeafNode0x -StemNode0x00 -> LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="L: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff +StemNode0x00 -> LeafNode0x00ff +LeafNode0x00ff [label="L: 0x00ff Suffix: 0xff"] -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -> Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x00ff -> Value0x00ff +Value0x00ff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] InternalNode0x -> NullNode0x -} \ No newline at end of file +} diff --git a/src/test/resources/expectedTreeTwoValuesShowNulls.txt b/src/test/resources/expectedTreeTwoValuesShowNulls.txt index 7a40b84..4cb8e3a 100644 --- a/src/test/resources/expectedTreeTwoValuesShowNulls.txt +++ b/src/test/resources/expectedTreeTwoValuesShowNulls.txt @@ -6,11 +6,11 @@ StemNode0x00 [label="S: 0x00 Stem: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee LeftCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000 RightCommitment: 0x0000000000000000000000000000000000000000000000000000000000000000"] -StemNode0x00 -> LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 [label="L: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 +StemNode0x00 -> LeafNode0x0000 +LeafNode0x0000 [label="L: 0x0000 Suffix: 0x00"] -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 -> Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 -Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddee00 [label="Value: 0x0100000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x0000 -> Value0x0000 +Value0x0000 [label="Value: 0x0100000000000000000000000000000000000000000000000000000000000000"] StemNode0x00 -> NullLeafNode0x NullLeafNode0x [label="NL: 0x"] StemNode0x00 -> NullLeafNode0x @@ -519,11 +519,11 @@ StemNode0x00 -> NullLeafNode0x NullLeafNode0x [label="NL: 0x"] StemNode0x00 -> NullLeafNode0x NullLeafNode0x [label="NL: 0x"] -StemNode0x00 -> LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="L: 0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff +StemNode0x00 -> LeafNode0x00ff +LeafNode0x00ff [label="L: 0x00ff Suffix: 0xff"] -LeafNode0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -> Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff -Value0x00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] +LeafNode0x00ff -> Value0x00ff +Value0x00ff [label="Value: 0x1000000000000000000000000000000000000000000000000000000000000000"] InternalNode0x -> NullNode0x NullNode0x [label="NL: 0x"] InternalNode0x -> NullNode0x @@ -1034,4 +1034,4 @@ InternalNode0x -> NullNode0x NullNode0x [label="NL: 0x"] InternalNode0x -> NullNode0x NullNode0x [label="NL: 0x"] -} \ No newline at end of file +}