From 69de8e09205d09b78d5dd1a1b29a0fe12e0c3d35 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Tue, 11 Jun 2024 14:53:18 +0200 Subject: [PATCH 1/3] add trie key preloader implementation Signed-off-by: Karim Taam --- .../BonsaiWorldStateUpdateAccumulator.java | 9 +- .../DiffBasedWorldStateUpdateAccumulator.java | 11 +- .../accumulator/preload/CodeConsumingMap.java | 53 ++++ .../verkle/cache/TrieKeyPreloader.java | 262 ++++++++++++++++++ .../verkle/worldview/VerkleWorldState.java | 87 ++---- .../VerkleWorldStateUpdateAccumulator.java | 16 +- .../verkle/TrieKeyPreloaderTests.java | 131 +++++++++ .../ethereum/verkletrie/TrieKeyPreloader.java | 81 ------ gradle/versions.gradle | 4 +- 9 files changed, 500 insertions(+), 154 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/CodeConsumingMap.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java delete mode 100644 ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/TrieKeyPreloader.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 4dd89cd7ca1..578f92e0a42 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -40,7 +40,14 @@ public BonsaiWorldStateUpdateAccumulator( final Consumer> accountPreloader, final Consumer storagePreloader, final EvmConfiguration evmConfiguration) { - super(world, accountPreloader, storagePreloader, evmConfiguration); + super( + world, + accountPreloader, + storagePreloader, + (__, ___) -> { + /*nothing to preload for the code*/ + }, + evmConfiguration); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 4f0ea91b189..891b1813ef6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.AccountConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.CodeConsumingMap; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.account.Account; @@ -63,9 +64,10 @@ public abstract class DiffBasedWorldStateUpdateAccumulator> accountPreloader; protected final Consumer storagePreloader; + private final Consumer codePreloader; private final AccountConsumingMap> accountsToUpdate; - private final Map> codeToUpdate = new ConcurrentHashMap<>(); + private final CodeConsumingMap codeToUpdate; private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); protected final EvmConfiguration evmConfiguration; @@ -82,11 +84,14 @@ public DiffBasedWorldStateUpdateAccumulator( final DiffBasedWorldView world, final Consumer> accountPreloader, final Consumer storagePreloader, + final Consumer codePreloader, final EvmConfiguration evmConfiguration) { super(world, evmConfiguration); this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader); + this.codeToUpdate = new CodeConsumingMap(new ConcurrentHashMap<>(), codePreloader); this.accountPreloader = accountPreloader; this.storagePreloader = storagePreloader; + this.codePreloader = codePreloader; this.isAccumulatorStateChanged = false; this.evmConfiguration = evmConfiguration; } @@ -109,6 +114,10 @@ protected Consumer getStoragePreloader() { return storagePreloader; } + public Consumer getCodePreloader() { + return codePreloader; + } + protected EvmConfiguration getEvmConfiguration() { return evmConfiguration; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/CodeConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/CodeConsumingMap.java new file mode 100644 index 00000000000..68b191b3ec8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/CodeConsumingMap.java @@ -0,0 +1,53 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; +import org.apache.tuweni.bytes.Bytes; + +public class CodeConsumingMap extends ForwardingMap> { + + private final ConcurrentMap> codes; + private final Consumer consumer; + + public CodeConsumingMap( + final ConcurrentMap> codes, final Consumer consumer) { + this.codes = codes; + this.consumer = consumer; + } + + @Override + public DiffBasedValue put( + @Nonnull final Address address, @Nonnull final DiffBasedValue value) { + consumer.process(address, value.getUpdated() != null ? value.getUpdated() : value.getPrior()); + return codes.put(address, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map> delegate() { + return codes; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java new file mode 100644 index 00000000000..f0944b6260c --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java @@ -0,0 +1,262 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; + +import static org.hyperledger.besu.ethereum.trie.verkle.util.Parameters.VERKLE_NODE_WIDTH; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.IntStream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public class TrieKeyPreloader { + private static final int TRIE_KEY_CACHE_SIZE = 10_000; + private final Cache> trieKeysCache = + CacheBuilder.newBuilder().maximumSize(TRIE_KEY_CACHE_SIZE).build(); + + private final TrieKeyAdapter trieKeyAdapter; + + private final Hasher hasher; + + private long missedTrieKeys = 0; + + public TrieKeyPreloader() { + this.hasher = new PedersenHasher(); + this.trieKeyAdapter = new TrieKeyAdapter(hasher); + this.trieKeyAdapter.versionKey( + Address.ZERO); // TODO REMOVE is just to preload the native library for performance check + } + + public HasherContext createPreloadedHasher( + final Address address, + final Map> storageUpdate, + final DiffBasedValue codeUpdate) { + + // generate account triekeys + final List accountKeyIds = new ArrayList<>(generateAccountKeyIds()); + + // generate storage triekeys + final List storageKeyIds = new ArrayList<>(); + boolean isStorageUpdateNeeded; + if (storageUpdate != null) { + final Set storageSlotKeys = storageUpdate.keySet(); + isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); + if (isStorageUpdateNeeded) { + storageKeyIds.addAll(generateStorageKeyIds(storageSlotKeys)); + } + } + + // generate code triekeys + final List codeChunkIds = new ArrayList<>(); + boolean isCodeUpdateNeeded; + if (codeUpdate != null) { + final Bytes previousCode = codeUpdate.getPrior(); + final Bytes updatedCode = codeUpdate.getUpdated(); + isCodeUpdateNeeded = + !codeUpdate.isUnchanged() && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); + if (isCodeUpdateNeeded) { + accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); + codeChunkIds.addAll( + generateCodeChunkKeyIds(updatedCode == null ? previousCode : updatedCode)); + } + } + + return new HasherContext( + new CachedPedersenHasher( + generateManyTrieKeyHashes(address, accountKeyIds, storageKeyIds, codeChunkIds, true)), + !storageKeyIds.isEmpty(), + !codeChunkIds.isEmpty()); + } + + public void preLoadAccount(final Address account) { + CompletableFuture.runAsync(() -> cacheAccountTrieKeys(account)); + } + + public void preLoadStorageSlot(final Address account, final StorageSlotKey slotKey) { + CompletableFuture.runAsync(() -> cacheSlotTrieKeys(account, slotKey)); + } + + public void preLoadCode(final Address account, final Bytes code) { + CompletableFuture.runAsync(() -> cacheCodeTrieKeys(account, code)); + } + + public Map generateManyTrieKeyHashes( + final Address address, + final List accountKeyIds, + final List storageKeyIds, + final List codeChunkIds, + final boolean checkCacheBeforeGeneration) { + + final Set offsetsToGenerate = new HashSet<>(); + + if (!accountKeyIds.isEmpty()) { + offsetsToGenerate.add(UInt256.ZERO); + } + for (Bytes32 storageKey : storageKeyIds) { + offsetsToGenerate.add(trieKeyAdapter.locateStorageKeyOffset(storageKey)); + } + for (Bytes32 codeChunkId : codeChunkIds) { + final UInt256 codeChunkOffset = trieKeyAdapter.locateCodeChunkKeyOffset(codeChunkId); + offsetsToGenerate.add(codeChunkOffset.divide(VERKLE_NODE_WIDTH)); + } + + if (checkCacheBeforeGeneration) { + + // not regenerate data already preloaded + final Map cachedTrieKeys = getCachedTrieKeysForAccount(address); + cachedTrieKeys.forEach((key, __) -> offsetsToGenerate.remove(key)); + missedTrieKeys += offsetsToGenerate.size(); + + if (offsetsToGenerate.isEmpty()) { + return cachedTrieKeys; + } + + final Map trieKeyHashes = + hasher.manyTrieKeyHashes(address, new ArrayList<>(offsetsToGenerate)); + trieKeyHashes.putAll(cachedTrieKeys); + + trieKeyHashes.forEach( + (bytes32, bytes322) -> { + System.out.println(bytes32 + " " + bytes322); + }); + return trieKeyHashes; + + } else { + return hasher.manyTrieKeyHashes(address, new ArrayList<>(offsetsToGenerate)); + } + } + + @VisibleForTesting + public Cache> getTrieKeysCache() { + return trieKeysCache; + } + + public long getMissedTrieKeys() { + return missedTrieKeys; + } + + public void reset() { + missedTrieKeys = 0; + } + + @VisibleForTesting + public void cacheAccountTrieKeys(final Address account) { + try { + final Map cache = trieKeysCache.get(account, this::createAccountCache); + cache.putAll( + generateManyTrieKeyHashes( + account, + generateAccountKeyIds(), + Collections.emptyList(), + Collections.emptyList(), + false)); + } catch (ExecutionException e) { + // no op + } + } + + @VisibleForTesting + public void cacheSlotTrieKeys(final Address account, final StorageSlotKey slotKey) { + try { + final Map cache = trieKeysCache.get(account, this::createAccountCache); + if (slotKey.getSlotKey().isPresent()) { + cache.putAll( + generateManyTrieKeyHashes( + account, + Collections.emptyList(), + List.of(slotKey.getSlotKey().get()), + Collections.emptyList(), + false)); + } + } catch (ExecutionException e) { + // no op + } + } + + @VisibleForTesting + public void cacheCodeTrieKeys(final Address account, final Bytes code) { + try { + final Map cache = trieKeysCache.get(account, this::createAccountCache); + cache.putAll( + generateManyTrieKeyHashes( + account, + Collections.emptyList(), + Collections.emptyList(), + generateCodeChunkKeyIds(code), + false)); + } catch (ExecutionException e) { + // no op + } + } + + private List generateAccountKeyIds() { + final List keys = new ArrayList<>(); + keys.add(Parameters.VERSION_LEAF_KEY); + keys.add(Parameters.BALANCE_LEAF_KEY); + keys.add(Parameters.NONCE_LEAF_KEY); + keys.add(Parameters.CODE_KECCAK_LEAF_KEY); + return keys; + } + + private List generateCodeChunkKeyIds(final Bytes code) { + return new ArrayList<>( + IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); + } + + private List generateStorageKeyIds(final Set storageSlotKeys) { + return storageSlotKeys.stream() + .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) + .map(Bytes32::wrap) + .toList(); + } + + private Map createAccountCache() { + return new ConcurrentHashMap<>(); + } + + private Map getCachedTrieKeysForAccount(final Address address) { + return Optional.ofNullable(trieKeysCache.getIfPresent(address)) + .orElseGet(ConcurrentHashMap::new); + } + + private boolean codeIsEmpty(final Bytes value) { + return value == null || value.isEmpty(); + } + + public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index b095b86b9f1..f2142b43f46 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -30,11 +30,10 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader.HasherContext; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; -import org.hyperledger.besu.ethereum.verkletrie.TrieKeyPreloader; -import org.hyperledger.besu.ethereum.verkletrie.TrieKeyPreloader.HasherContext; import org.hyperledger.besu.ethereum.verkletrie.VerkleEntryFactory; import org.hyperledger.besu.ethereum.verkletrie.VerkleTrie; import org.hyperledger.besu.evm.account.Account; @@ -42,11 +41,8 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; @@ -84,7 +80,11 @@ public VerkleWorldState( this.trieKeyPreloader = new TrieKeyPreloader(); this.setAccumulator( new VerkleWorldStateUpdateAccumulator( - this, (addr, value) -> {}, (addr, value) -> {}, evmConfiguration)); + this, + (addr, value) -> trieKeyPreloader.preLoadAccount(addr), + trieKeyPreloader::preLoadStorageSlot, + trieKeyPreloader::preLoadCode, + evmConfiguration)); } @Override @@ -116,46 +116,15 @@ protected Hash internalCalculateRootHash( .forEach( accountUpdate -> { final Address accountKey = accountUpdate.getKey(); - // generate account triekeys - final List accountKeyIds = - new ArrayList<>(trieKeyPreloader.generateAccountKeyIds()); - - // generate storage triekeys - final List storageKeyIds = new ArrayList<>(); final StorageConsumingMap> storageAccountUpdate = worldStateUpdater.getStorageToUpdate().get(accountKey); - boolean isStorageUpdateNeeded; - if (storageAccountUpdate != null) { - final Set storageSlotKeys = storageAccountUpdate.keySet(); - isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); - if (isStorageUpdateNeeded) { - storageKeyIds.addAll(trieKeyPreloader.generateStorageKeyIds(storageSlotKeys)); - } - } - - // generate code triekeys - final List codeKeyIds = new ArrayList<>(); final DiffBasedValue codeUpdate = worldStateUpdater.getCodeToUpdate().get(accountKey); - boolean isCodeUpdateNeeded; - if (codeUpdate != null) { - final Bytes previousCode = codeUpdate.getPrior(); - final Bytes updatedCode = codeUpdate.getUpdated(); - isCodeUpdateNeeded = - !codeUpdate.isUnchanged() - && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); - if (isCodeUpdateNeeded) { - accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); - codeKeyIds.addAll( - trieKeyPreloader.generateCodeChunkKeyIds( - updatedCode == null ? previousCode : updatedCode)); - } - } preloadedHashers.put( accountKey, trieKeyPreloader.createPreloadedHasher( - accountKey, accountKeyIds, storageKeyIds, codeKeyIds)); + accountKey, storageAccountUpdate, codeUpdate)); }); for (final Map.Entry> accountUpdate : @@ -190,11 +159,10 @@ protected Hash internalCalculateRootHash( value); })); - LOG.info("end commit "); // LOG.info(stateTrie.toDotTree()); final Bytes32 rootHash = stateTrie.getRootHash(); LOG.info("end commit " + rootHash); - + trieKeyPreloader.reset(); return Hash.wrap(rootHash); } @@ -279,36 +247,19 @@ private void updateCode( maybeStateUpdater.ifPresent( bonsaiUpdater -> bonsaiUpdater.removeCode(accountHash, priorCodeHash)); } else { - if (updatedCode.isEmpty()) { - final Hash codeHash = Hash.hash(updatedCode); - verkleEntryFactory - .generateKeyValuesForCode(accountKey, updatedCode) - .forEach( - (bytes, bytes2) -> { - // System.out.println("add code " + bytes + " " + bytes2); - stateTrie.put(bytes, bytes2); - }); - maybeStateUpdater.ifPresent( - bonsaiUpdater -> bonsaiUpdater.removeCode(accountHash, codeHash)); - } else { - final Hash codeHash = Hash.hash(updatedCode); - verkleEntryFactory - .generateKeyValuesForCode(accountKey, updatedCode) - .forEach( - (bytes, bytes2) -> { - System.out.println("add code " + bytes + " " + bytes2); - stateTrie.put(bytes, bytes2); - }); - maybeStateUpdater.ifPresent( - bonsaiUpdater -> bonsaiUpdater.putCode(accountHash, codeHash, updatedCode)); - } + final Hash codeHash = Hash.hash(updatedCode); + verkleEntryFactory + .generateKeyValuesForCode(accountKey, updatedCode) + .forEach( + (bytes, bytes2) -> { + System.out.println("add code " + bytes + " " + bytes2); + stateTrie.put(bytes, bytes2); + }); + maybeStateUpdater.ifPresent( + bonsaiUpdater -> bonsaiUpdater.putCode(accountHash, codeHash, updatedCode)); } } - private boolean codeIsEmpty(final Bytes value) { - return value == null || value.isEmpty(); - } - private void updateAccountStorageState( final Address accountKey, final VerkleTrie stateTrie, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java index 1bb15200415..3bf60923931 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java @@ -25,11 +25,13 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; +import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; import java.util.Optional; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; public class VerkleWorldStateUpdateAccumulator @@ -39,8 +41,9 @@ public VerkleWorldStateUpdateAccumulator( final DiffBasedWorldView world, final Consumer> accountPreloader, final Consumer storagePreloader, + final Consumer codePreloader, final EvmConfiguration evmConfiguration) { - super(world, accountPreloader, storagePreloader, evmConfiguration); + super(world, accountPreloader, storagePreloader, codePreloader, evmConfiguration); } @Override @@ -50,6 +53,7 @@ public DiffBasedWorldStateUpdateAccumulator copy() { wrappedWorldView(), getAccountPreloader(), getStoragePreloader(), + getCodePreloader(), getEvmConfiguration()); copy.cloneFromUpdater(this); return copy; @@ -109,6 +113,16 @@ protected Optional getStorageValueByStorageSlotKey( return worldState.getStorageValueByStorageSlotKey(address, storageSlotKey); } + @Override + public MutableAccount getAccount(final Address address) { + return super.getAccount(address); + } + + @Override + public Optional getCode(final Address address, final Hash codeHash) { + return super.getCode(address, codeHash); + } + @Override protected boolean shouldIgnoreIdenticalValuesDuringAccountRollingUpdate() { return false; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java new file mode 100644 index 00000000000..4b9d5026f9f --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java @@ -0,0 +1,131 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TrieKeyPreloaderTests { + + private TrieKeyPreloader trieKeyPreLoader; + private final List
accounts = + List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); + + @BeforeEach + public void setup() { + trieKeyPreLoader = new TrieKeyPreloader(); + } + + // Test to verify that a single account's trie key is correctly added to the cache during preload + @Test + void shouldAddAccountsTrieKeyInCacheDuringPreload() { + trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); + Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); + final Map accountCache = + trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is exactly 1, indicating a single trie key has + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that trie keys for multiple accounts are correctly added to the cache during + // preload + @Test + void shouldAddMultipleAccountsTrieKeyInCacheDuringPreload() { + trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); + trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(1)); + Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(2); + final Map accountCache = + trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); + // Assert that each account's cache size is 1, indicating that trie keys for both accounts have + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + final Map account2Cache = + trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(1)); + Assertions.assertThat(Objects.requireNonNull(account2Cache).size()).isEqualTo(1); + } + + // Test to verify that header slots trie keys are correctly added to the cache for a single + // account during preload + @Test + void shouldAddHeaderSlotsTrieKeysInCacheDuringPreload() { + trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ZERO)); + trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); + final Map accountCache = + trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 1, indicating that two slots in the header + // storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that multiple slots trie keys are correctly added to the cache for a single + // account during preload + @Test + void shouldAddMultipleSlotsTrieKeysInCacheDuringPreload() { + trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.valueOf(64))); + Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); + final Map accountCache = + trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that slots in both the header and + // main storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that code trie keys are correctly added to the cache for a single account during + // preload + @Test + void shouldAddCodeTrieKeysInCacheDuringPreload() { + trieKeyPreLoader.cacheCodeTrieKeys(accounts.get(0), Bytes.repeat((byte) 0x01, 4096)); + Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); + final Map accountCache = + trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that the code is too big and + // multiple offsets have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that missing trie keys are correctly generated and accounted for + @Test + void shouldGenerateMissingTrieKeys() { + trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); + Map trieKeyHashes = + trieKeyPreLoader.generateManyTrieKeyHashes( + accounts.get(0), + Collections.emptyList(), + List.of(UInt256.valueOf(64)), + List.of(UInt256.valueOf(128)), + true); + // Assert that two trie keys (slot and code) are missing from the cache + Assertions.assertThat(trieKeyPreLoader.getMissedTrieKeys()).isEqualTo(2); + // Assert that three trie keys have been generated: one for the header, one for the slot in main + // storage, and one for the code chunk + Assertions.assertThat(trieKeyHashes.size()).isEqualTo(3); + } +} diff --git a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/TrieKeyPreloader.java b/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/TrieKeyPreloader.java deleted file mode 100644 index 2bdf4961ed9..00000000000 --- a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/TrieKeyPreloader.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.verkletrie; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyBatchAdapter; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.IntStream; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; - -public class TrieKeyPreloader { - - private final TrieKeyBatchAdapter trieKeyAdapter; - - private final Hasher hasher; - - public TrieKeyPreloader() { - this.hasher = new PedersenHasher(); - trieKeyAdapter = new TrieKeyBatchAdapter(hasher); - trieKeyAdapter.versionKey( - Address.ZERO); // TODO REMOVE is just to preload the native library for performance check - } - - public List generateAccountKeyIds() { - final List keys = new ArrayList<>(); - keys.add(Parameters.VERSION_LEAF_KEY); - keys.add(Parameters.BALANCE_LEAF_KEY); - keys.add(Parameters.NONCE_LEAF_KEY); - keys.add(Parameters.CODE_KECCAK_LEAF_KEY); - return keys; - } - - public List generateCodeChunkKeyIds(final Bytes code) { - return new ArrayList<>( - IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); - } - - public List generateStorageKeyIds(final Set storageSlotKeys) { - return storageSlotKeys.stream() - .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) - .map(Bytes32::wrap) - .toList(); - } - - public HasherContext createPreloadedHasher( - final Address address, - final List accountKeyIds, - final List storageKeyIds, - final List codeChunkIds) { - return new HasherContext( - new CachedPedersenHasher( - trieKeyAdapter.manyTrieKeyHashes(address, accountKeyIds, storageKeyIds, codeChunkIds)), - !storageKeyIds.isEmpty(), - !codeChunkIds.isEmpty()); - } - - public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} -} diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 865af38019e..c92e1075302 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -156,7 +156,7 @@ dependencyManagement { dependency 'org.openjdk.jol:jol-core:0.17' dependency 'tech.pegasys:jc-kzg-4844:1.0.0' - dependencySet(group: 'org.hyperledger.besu', version: '0.8.4') { + dependencySet(group: 'org.hyperledger.besu', version: '0.8.5') { entry 'arithmetic' entry 'ipa-multipoint' entry 'bls12-381' @@ -165,7 +165,7 @@ dependencyManagement { entry 'blake2bf' } - dependencySet(group: 'org.hyperledger.besu', version: '0.8.4-SNAPSHOT') { + dependencySet(group: 'org.hyperledger.besu', version: '0.8.5') { entry 'ipa-multipoint' } From 4d0aee6c2dfba54cc1a383ef2d36b686ab6078b4 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Tue, 11 Jun 2024 14:59:26 +0200 Subject: [PATCH 2/3] clean code Signed-off-by: Karim Taam --- .../trie/diffbased/verkle/cache/TrieKeyPreloader.java | 4 ---- .../worldview/VerkleWorldStateUpdateAccumulator.java | 11 ----------- 2 files changed, 15 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java index f0944b6260c..75e946dc27c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java @@ -150,10 +150,6 @@ public Map generateManyTrieKeyHashes( hasher.manyTrieKeyHashes(address, new ArrayList<>(offsetsToGenerate)); trieKeyHashes.putAll(cachedTrieKeys); - trieKeyHashes.forEach( - (bytes32, bytes322) -> { - System.out.println(bytes32 + " " + bytes322); - }); return trieKeyHashes; } else { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java index 3bf60923931..1a685336f6a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java @@ -25,7 +25,6 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; -import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; @@ -113,16 +112,6 @@ protected Optional getStorageValueByStorageSlotKey( return worldState.getStorageValueByStorageSlotKey(address, storageSlotKey); } - @Override - public MutableAccount getAccount(final Address address) { - return super.getAccount(address); - } - - @Override - public Optional getCode(final Address address, final Hash codeHash) { - return super.getCode(address, codeHash); - } - @Override protected boolean shouldIgnoreIdenticalValuesDuringAccountRollingUpdate() { return false; From 56259cd55bb09b92cd409dee3b881735a200cd01 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Thu, 20 Jun 2024 11:11:36 +0200 Subject: [PATCH 3/3] update cache in order to only save stem Signed-off-by: Karim Taam --- .../diffbased/verkle/cache/StemPreloader.java | 380 ++++++++++++++++++ .../verkle/cache/TrieKeyPreloader.java | 258 ------------ .../verkle/worldview/VerkleWorldState.java | 26 +- .../diffbased/verkle/LogRollingTests.java | 1 - .../diffbased/verkle/StemPreloaderTests.java | 131 ++++++ .../verkle/TrieKeyPreloaderTests.java | 131 ------ .../stateless/Eip4762AccessWitness.java | 4 +- 7 files changed, 529 insertions(+), 402 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java new file mode 100644 index 00000000000..f9aff616071 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java @@ -0,0 +1,380 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.IntStream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * Preloads stems for accounts, storage slots, and code. This class is designed to optimize block + * processing by caching trie stems associated with specific accounts, storage slots, and code data. + * By preloading these stems, the class aims to reduce the computational overhead and access times + * typically required during the state root computation. + * + *

The preloading process involves generating and caching stems for: + * + *

    + *
  • Account-related keys (e.g., version, balance, nonce, and code hash keys) + *
  • Storage slot keys for a given account + *
  • Code chunk keys for smart contract code associated with an account + *
+ * + *

This class utilizes a {@link Cache} to store preloaded stems, with the cache size configurable + * through the {@code STEM_CACHE_SIZE} constant. The {@link Hasher} used for stem generation is + * configurable, allowing for different hashing strategies (e.g., Pedersen hashing) to be employed. + * + * @see org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter + * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher + * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher + */ +public class StemPreloader { + private static final int STEM_CACHE_SIZE = 10_000; + private final Cache> stemByAddressCache = + CacheBuilder.newBuilder().maximumSize(STEM_CACHE_SIZE).build(); + + private final TrieKeyAdapter trieKeyAdapter; + + private final Hasher hasher; + + private long missedStemCounter = 0; + + private long cachedStemCounter = 0; + + public StemPreloader() { + this.hasher = new PedersenHasher(); + this.trieKeyAdapter = new TrieKeyAdapter(hasher); + this.trieKeyAdapter.versionKey( + Address.ZERO); // TODO REMOVE is just to preload the native library for performance check + } + + /** + * Creates a preloaded hasher context for a given updated address, storage, and code. This method + * preloads stems for account keys, storage slot keys, and code chunk keys based on the provided + * updates. It then returns a {@link HasherContext} containing the preloaded stems and flags + * indicating whether storage or code trie keys are present. + * + *

This method is particularly useful for optimizing the hashing process during state root + * computation by ensuring that stems are preloaded and readily available, thereby reducing the + * need for on-the-fly stem generation. + * + * @param address the address for which to preload trie stems + * @param storageUpdate a map of storage slot keys to their updated values + * @param codeUpdate the updated code value, encapsulated in a {@link DiffBasedValue} + * @return a {@link HasherContext} containing the preloaded stems and flags for storage and code + * trie key presence + */ + public HasherContext createPreloadedHasher( + final Address address, + final Map> storageUpdate, + final DiffBasedValue codeUpdate) { + + // generate account triekeys + final List accountKeyIds = new ArrayList<>(generateAccountKeyIds()); + + // generate storage triekeys + final List storageKeyIds = new ArrayList<>(); + boolean isStorageUpdateNeeded; + if (storageUpdate != null) { + final Set storageSlotKeys = storageUpdate.keySet(); + isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); + if (isStorageUpdateNeeded) { + storageKeyIds.addAll(generateStorageKeyIds(storageSlotKeys)); + } + } + + // generate code triekeys + final List codeChunkIds = new ArrayList<>(); + boolean isCodeUpdateNeeded; + if (codeUpdate != null) { + final Bytes previousCode = codeUpdate.getPrior(); + final Bytes updatedCode = codeUpdate.getUpdated(); + isCodeUpdateNeeded = + !codeUpdate.isUnchanged() && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); + if (isCodeUpdateNeeded) { + accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); + codeChunkIds.addAll( + generateCodeChunkKeyIds(updatedCode == null ? previousCode : updatedCode)); + } + } + + return new HasherContext( + new CachedPedersenHasher( + STEM_CACHE_SIZE, + generateManyStems(address, accountKeyIds, storageKeyIds, codeChunkIds, true)), + !storageKeyIds.isEmpty(), + !codeChunkIds.isEmpty()); + } + + /** + * Asynchronously preloads stems for a given account address. This method is designed to optimize + * the access to account-related stems by caching them ahead of time, thus reducing the + * computational overhead during state root computation. + * + * @param account the address of the account for which stems are to be preloaded + */ + public void preLoadAccount(final Address account) { + CompletableFuture.runAsync(() -> cacheAccountStems(account)); + } + + /** + * Asynchronously preloads stems for a specific storage slot associated with an account. This + * method enhances the efficiency of accessing storage-related stems by ensuring they are cached + * in advance, thereby facilitating faster state root computation. + * + * @param account the address of the account associated with the storage slot + * @param slotKey the key of the storage slot for which stems are to be preloaded + */ + public void preLoadStorageSlot(final Address account, final StorageSlotKey slotKey) { + CompletableFuture.runAsync(() -> cacheSlotStems(account, slotKey)); + } + + /** + * Asynchronously preloads stems for the code associated with an account. + * + * @param account the address of the account associated with the code + * @param code the smart contract code for which stems are to be preloaded + */ + public void preLoadCode(final Address account, final Bytes code) { + CompletableFuture.runAsync(() -> cacheCodeStems(account, code)); + } + + /** + * Generates a comprehensive set of stems for an account, including stems for account keys, + * storage slot keys, and code chunk keys. + * + * @param address the address for which stems are to be generated + * @param accountKeyIds a list of trie keys associated with the account + * @param storageKeyIds a list of trie keys associated with the account's storage slots + * @param codeChunkIds a list of trie keys associated with the account's code chunks + * @param checkCacheBeforeGeneration flag indicating whether to check the cache before generating + * new stems + * @return a map of trie key index to their corresponding stems + */ + public Map generateManyStems( + final Address address, + final List accountKeyIds, + final List storageKeyIds, + final List codeChunkIds, + final boolean checkCacheBeforeGeneration) { + + final Set trieIndexes = new HashSet<>(); + + if (!accountKeyIds.isEmpty()) { + trieIndexes.add(UInt256.ZERO); + } + for (Bytes32 storageKey : storageKeyIds) { + trieIndexes.add(trieKeyAdapter.getStorageKeyTrieIndex(storageKey)); + } + for (Bytes32 codeChunkId : codeChunkIds) { + trieIndexes.add(trieKeyAdapter.getCodeChunkKeyTrieIndex(codeChunkId)); + } + + if (checkCacheBeforeGeneration) { + + // not regenerate stem already preloaded + final Map stemCached = getCachedStemForAccount(address); + stemCached.forEach( + (key, __) -> { + trieIndexes.remove(key); + cachedStemCounter++; + }); + missedStemCounter += trieIndexes.size(); + if (trieIndexes.isEmpty()) { + return stemCached; + } + + final Map stems = hasher.manyStems(address, new ArrayList<>(trieIndexes)); + stems.putAll(stemCached); + + return stems; + + } else { + return hasher.manyStems(address, new ArrayList<>(trieIndexes)); + } + } + + /** + * Retrieves the cache that maps account addresses to their corresponding cached stems. + * + * @return the cache mapping account addresses to trie stems + */ + @VisibleForTesting + public Cache> getStemByAddressCache() { + return stemByAddressCache; + } + + /** + * Retrieves the current number of stems that were not found in the cache and had to be generated. + * This counter can be used to monitor the effectiveness of the stem preloading mechanism. + * + * @return the number of stems that were missed and subsequently generated + */ + public long getNbMissedStems() { + return missedStemCounter; + } + + /** + * Retrieves the current number of stems that were found in the cache and had not to be generated. + * This counter can be used to monitor the effectiveness of the stem preloading mechanism. + * + * @return the number of stems that were present in the cache + */ + public long getNbCachedStems() { + return cachedStemCounter; + } + + /** + * Resets the counter of missed trie stems. This method can be used to clear the count of trie + * stems that were not found in the cache and had to be generated, typically used in conjunction + * with performance monitoring or testing. + */ + public void reset() { + cachedStemCounter = 0; + missedStemCounter = 0; + } + + /** + * Caches stems for all account-related keys for a given account address. + * + * @param account the address of the account for which stems are to be cached + */ + @VisibleForTesting + public void cacheAccountStems(final Address account) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + cache.putAll( + generateManyStems( + account, + generateAccountKeyIds(), + Collections.emptyList(), + Collections.emptyList(), + false)); + } catch (ExecutionException e) { + // no op + } + } + + /** + * Caches stems for a specific storage slot key associated with an address. + * + * @param account the address of the account associated with the storage slot + * @param slotKey the storage slot key for which trie stems are to be cached + */ + @VisibleForTesting + public void cacheSlotStems(final Address account, final StorageSlotKey slotKey) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + if (slotKey.getSlotKey().isPresent()) { + cache.putAll( + generateManyStems( + account, + Collections.emptyList(), + List.of(slotKey.getSlotKey().get()), + Collections.emptyList(), + false)); + } + } catch (ExecutionException e) { + // no op + } + } + + /** + * Caches stems for the code associated with an account address. + * + * @param account the address of the account associated with the code + * @param code the smart contract code for which trie stems are to be cached + */ + @VisibleForTesting + public void cacheCodeStems(final Address account, final Bytes code) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + cache.putAll( + generateManyStems( + account, + Collections.emptyList(), + Collections.emptyList(), + generateCodeChunkKeyIds(code), + false)); + } catch (ExecutionException e) { + // no op + } + } + + private List generateAccountKeyIds() { + final List keys = new ArrayList<>(); + keys.add(Parameters.VERSION_LEAF_KEY); + keys.add(Parameters.BALANCE_LEAF_KEY); + keys.add(Parameters.NONCE_LEAF_KEY); + keys.add(Parameters.CODE_KECCAK_LEAF_KEY); + return keys; + } + + private List generateCodeChunkKeyIds(final Bytes code) { + return new ArrayList<>( + IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); + } + + private List generateStorageKeyIds(final Set storageSlotKeys) { + return storageSlotKeys.stream() + .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) + .map(Bytes32::wrap) + .toList(); + } + + private Map getCachedStemForAccount(final Address address) { + return Optional.ofNullable(stemByAddressCache.getIfPresent(address)) + .orElseGet(ConcurrentHashMap::new); + } + + private boolean codeIsEmpty(final Bytes value) { + return value == null || value.isEmpty(); + } + + /** + * Represents the context for a hasher, including the hasher instance and flags indicating the + * presence of storage trie keys and code trie keys. This record is used to encapsulate the state + * and configuration of the hasher in relation to preloaded trie stems. + * + * @param hasher the hasher instance used for generating stems + * @param hasStorageTrieKeys flag indicating whether storage trie keys are present in the cache + * @param hasCodeTrieKeys flag indicating whether code trie keys are present in the cache + */ + public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java deleted file mode 100644 index 75e946dc27c..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; - -import static org.hyperledger.besu.ethereum.trie.verkle.util.Parameters.VERKLE_NODE_WIDTH; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.stream.IntStream; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; - -public class TrieKeyPreloader { - private static final int TRIE_KEY_CACHE_SIZE = 10_000; - private final Cache> trieKeysCache = - CacheBuilder.newBuilder().maximumSize(TRIE_KEY_CACHE_SIZE).build(); - - private final TrieKeyAdapter trieKeyAdapter; - - private final Hasher hasher; - - private long missedTrieKeys = 0; - - public TrieKeyPreloader() { - this.hasher = new PedersenHasher(); - this.trieKeyAdapter = new TrieKeyAdapter(hasher); - this.trieKeyAdapter.versionKey( - Address.ZERO); // TODO REMOVE is just to preload the native library for performance check - } - - public HasherContext createPreloadedHasher( - final Address address, - final Map> storageUpdate, - final DiffBasedValue codeUpdate) { - - // generate account triekeys - final List accountKeyIds = new ArrayList<>(generateAccountKeyIds()); - - // generate storage triekeys - final List storageKeyIds = new ArrayList<>(); - boolean isStorageUpdateNeeded; - if (storageUpdate != null) { - final Set storageSlotKeys = storageUpdate.keySet(); - isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); - if (isStorageUpdateNeeded) { - storageKeyIds.addAll(generateStorageKeyIds(storageSlotKeys)); - } - } - - // generate code triekeys - final List codeChunkIds = new ArrayList<>(); - boolean isCodeUpdateNeeded; - if (codeUpdate != null) { - final Bytes previousCode = codeUpdate.getPrior(); - final Bytes updatedCode = codeUpdate.getUpdated(); - isCodeUpdateNeeded = - !codeUpdate.isUnchanged() && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); - if (isCodeUpdateNeeded) { - accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); - codeChunkIds.addAll( - generateCodeChunkKeyIds(updatedCode == null ? previousCode : updatedCode)); - } - } - - return new HasherContext( - new CachedPedersenHasher( - generateManyTrieKeyHashes(address, accountKeyIds, storageKeyIds, codeChunkIds, true)), - !storageKeyIds.isEmpty(), - !codeChunkIds.isEmpty()); - } - - public void preLoadAccount(final Address account) { - CompletableFuture.runAsync(() -> cacheAccountTrieKeys(account)); - } - - public void preLoadStorageSlot(final Address account, final StorageSlotKey slotKey) { - CompletableFuture.runAsync(() -> cacheSlotTrieKeys(account, slotKey)); - } - - public void preLoadCode(final Address account, final Bytes code) { - CompletableFuture.runAsync(() -> cacheCodeTrieKeys(account, code)); - } - - public Map generateManyTrieKeyHashes( - final Address address, - final List accountKeyIds, - final List storageKeyIds, - final List codeChunkIds, - final boolean checkCacheBeforeGeneration) { - - final Set offsetsToGenerate = new HashSet<>(); - - if (!accountKeyIds.isEmpty()) { - offsetsToGenerate.add(UInt256.ZERO); - } - for (Bytes32 storageKey : storageKeyIds) { - offsetsToGenerate.add(trieKeyAdapter.locateStorageKeyOffset(storageKey)); - } - for (Bytes32 codeChunkId : codeChunkIds) { - final UInt256 codeChunkOffset = trieKeyAdapter.locateCodeChunkKeyOffset(codeChunkId); - offsetsToGenerate.add(codeChunkOffset.divide(VERKLE_NODE_WIDTH)); - } - - if (checkCacheBeforeGeneration) { - - // not regenerate data already preloaded - final Map cachedTrieKeys = getCachedTrieKeysForAccount(address); - cachedTrieKeys.forEach((key, __) -> offsetsToGenerate.remove(key)); - missedTrieKeys += offsetsToGenerate.size(); - - if (offsetsToGenerate.isEmpty()) { - return cachedTrieKeys; - } - - final Map trieKeyHashes = - hasher.manyTrieKeyHashes(address, new ArrayList<>(offsetsToGenerate)); - trieKeyHashes.putAll(cachedTrieKeys); - - return trieKeyHashes; - - } else { - return hasher.manyTrieKeyHashes(address, new ArrayList<>(offsetsToGenerate)); - } - } - - @VisibleForTesting - public Cache> getTrieKeysCache() { - return trieKeysCache; - } - - public long getMissedTrieKeys() { - return missedTrieKeys; - } - - public void reset() { - missedTrieKeys = 0; - } - - @VisibleForTesting - public void cacheAccountTrieKeys(final Address account) { - try { - final Map cache = trieKeysCache.get(account, this::createAccountCache); - cache.putAll( - generateManyTrieKeyHashes( - account, - generateAccountKeyIds(), - Collections.emptyList(), - Collections.emptyList(), - false)); - } catch (ExecutionException e) { - // no op - } - } - - @VisibleForTesting - public void cacheSlotTrieKeys(final Address account, final StorageSlotKey slotKey) { - try { - final Map cache = trieKeysCache.get(account, this::createAccountCache); - if (slotKey.getSlotKey().isPresent()) { - cache.putAll( - generateManyTrieKeyHashes( - account, - Collections.emptyList(), - List.of(slotKey.getSlotKey().get()), - Collections.emptyList(), - false)); - } - } catch (ExecutionException e) { - // no op - } - } - - @VisibleForTesting - public void cacheCodeTrieKeys(final Address account, final Bytes code) { - try { - final Map cache = trieKeysCache.get(account, this::createAccountCache); - cache.putAll( - generateManyTrieKeyHashes( - account, - Collections.emptyList(), - Collections.emptyList(), - generateCodeChunkKeyIds(code), - false)); - } catch (ExecutionException e) { - // no op - } - } - - private List generateAccountKeyIds() { - final List keys = new ArrayList<>(); - keys.add(Parameters.VERSION_LEAF_KEY); - keys.add(Parameters.BALANCE_LEAF_KEY); - keys.add(Parameters.NONCE_LEAF_KEY); - keys.add(Parameters.CODE_KECCAK_LEAF_KEY); - return keys; - } - - private List generateCodeChunkKeyIds(final Bytes code) { - return new ArrayList<>( - IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); - } - - private List generateStorageKeyIds(final Set storageSlotKeys) { - return storageSlotKeys.stream() - .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) - .map(Bytes32::wrap) - .toList(); - } - - private Map createAccountCache() { - return new ConcurrentHashMap<>(); - } - - private Map getCachedTrieKeysForAccount(final Address address) { - return Optional.ofNullable(trieKeysCache.getIfPresent(address)) - .orElseGet(ConcurrentHashMap::new); - } - - private boolean codeIsEmpty(final Bytes value) { - return value == null || value.isEmpty(); - } - - public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index f2142b43f46..a2c766cbc8d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -30,8 +30,8 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader; -import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader.HasherContext; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader.HasherContext; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.verkletrie.VerkleEntryFactory; @@ -58,7 +58,7 @@ public class VerkleWorldState extends DiffBasedWorldState { private static final Logger LOG = LoggerFactory.getLogger(VerkleWorldState.class); - private final TrieKeyPreloader trieKeyPreloader; + private final StemPreloader stemPreloader; public VerkleWorldState( final VerkleWorldStateProvider archive, @@ -77,13 +77,13 @@ public VerkleWorldState( final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); - this.trieKeyPreloader = new TrieKeyPreloader(); + this.stemPreloader = new StemPreloader(); this.setAccumulator( new VerkleWorldStateUpdateAccumulator( this, - (addr, value) -> trieKeyPreloader.preLoadAccount(addr), - trieKeyPreloader::preLoadStorageSlot, - trieKeyPreloader::preLoadCode, + (addr, value) -> stemPreloader.preLoadAccount(addr), + stemPreloader::preLoadStorageSlot, + stemPreloader::preLoadCode, evmConfiguration)); } @@ -123,7 +123,7 @@ protected Hash internalCalculateRootHash( preloadedHashers.put( accountKey, - trieKeyPreloader.createPreloadedHasher( + stemPreloader.createPreloadedHasher( accountKey, storageAccountUpdate, codeUpdate)); }); @@ -161,8 +161,14 @@ protected Hash internalCalculateRootHash( // LOG.info(stateTrie.toDotTree()); final Bytes32 rootHash = stateTrie.getRootHash(); - LOG.info("end commit " + rootHash); - trieKeyPreloader.reset(); + LOG.info( + "end commit " + + rootHash + + " " + + stemPreloader.getNbMissedStems() + + "/" + + stemPreloader.getNbCachedStems()); + stemPreloader.reset(); return Hash.wrap(rootHash); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java index 9db0811f70c..c0bcf968131 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java @@ -464,7 +464,6 @@ void rollBackFourTimes() { worldState.persist(headerThree); final TrieLogLayer layerThree = getTrieLogLayer(trieLogStorage, headerThree.getHash()); - System.out.println(layerThree.dump()); final WorldUpdater updater4 = worldState.updater(); final MutableAccount mutableAccount4 = updater4.getAccount(addressOne); mutableAccount4.setStorageValue(UInt256.ONE, UInt256.valueOf(1)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java new file mode 100644 index 00000000000..1fcdb5bb612 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java @@ -0,0 +1,131 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class StemPreloaderTests { + + private StemPreloader stemPreLoader; + private final List

accounts = + List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); + + @BeforeEach + public void setup() { + stemPreLoader = new StemPreloader(); + } + + // Test to verify that a single account's trie key is correctly added to the cache during preload + @Test + void shouldAddAccountsStemsInCacheDuringPreload() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is exactly 1, indicating a single trie key has + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that stems for multiple accounts are correctly added to the cache during + // preload + @Test + void shouldAddMultipleAccountsStemsInCacheDuringPreload() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + stemPreLoader.cacheAccountStems(accounts.get(1)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(2); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that each account's cache size is 1, indicating that stems for both accounts have + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + final Map account2Cache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(1)); + Assertions.assertThat(Objects.requireNonNull(account2Cache).size()).isEqualTo(1); + } + + // Test to verify that header slots stems are correctly added to the cache for a single + // account during preload + @Test + void shouldAddHeaderSlotsStemsInCacheDuringPreload() { + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ZERO)); + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 1, indicating that two slots in the header + // storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that multiple slots stems are correctly added to the cache for a single + // account during preload + @Test + void shouldAddMultipleSlotsStemsInCacheDuringPreload() { + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.valueOf(64))); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that slots in both the header and + // main storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that code stems are correctly added to the cache for a single account during + // preload + @Test + void shouldAddCodeStemsInCacheDuringPreload() { + stemPreLoader.cacheCodeStems(accounts.get(0), Bytes.repeat((byte) 0x01, 4096)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that the code is too big and + // multiple offsets have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that missing stems are correctly generated and accounted for + @Test + void shouldGenerateMissingStems() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + Map stems = + stemPreLoader.generateManyStems( + accounts.get(0), + Collections.emptyList(), + List.of(UInt256.valueOf(64)), + List.of(UInt256.valueOf(128)), + true); + // Assert that two stems (slot and code) are missing from the cache + Assertions.assertThat(stemPreLoader.getNbMissedStems()).isEqualTo(2); + // Assert that three stems have been generated: one for the header, one for the slot in main + // storage, and one for the code chunk + Assertions.assertThat(stems.size()).isEqualTo(3); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java deleted file mode 100644 index 4b9d5026f9f..00000000000 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.diffbased.verkle; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class TrieKeyPreloaderTests { - - private TrieKeyPreloader trieKeyPreLoader; - private final List
accounts = - List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); - - @BeforeEach - public void setup() { - trieKeyPreLoader = new TrieKeyPreloader(); - } - - // Test to verify that a single account's trie key is correctly added to the cache during preload - @Test - void shouldAddAccountsTrieKeyInCacheDuringPreload() { - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is exactly 1, indicating a single trie key has - // been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); - } - - // Test to verify that trie keys for multiple accounts are correctly added to the cache during - // preload - @Test - void shouldAddMultipleAccountsTrieKeyInCacheDuringPreload() { - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(1)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(2); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that each account's cache size is 1, indicating that trie keys for both accounts have - // been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); - final Map account2Cache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(1)); - Assertions.assertThat(Objects.requireNonNull(account2Cache).size()).isEqualTo(1); - } - - // Test to verify that header slots trie keys are correctly added to the cache for a single - // account during preload - @Test - void shouldAddHeaderSlotsTrieKeysInCacheDuringPreload() { - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ZERO)); - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ONE)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is 1, indicating that two slots in the header - // storage have been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); - } - - // Test to verify that multiple slots trie keys are correctly added to the cache for a single - // account during preload - @Test - void shouldAddMultipleSlotsTrieKeysInCacheDuringPreload() { - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ONE)); - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.valueOf(64))); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is 2, indicating that slots in both the header and - // main storage have been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); - } - - // Test to verify that code trie keys are correctly added to the cache for a single account during - // preload - @Test - void shouldAddCodeTrieKeysInCacheDuringPreload() { - trieKeyPreLoader.cacheCodeTrieKeys(accounts.get(0), Bytes.repeat((byte) 0x01, 4096)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is 2, indicating that the code is too big and - // multiple offsets have been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); - } - - // Test to verify that missing trie keys are correctly generated and accounted for - @Test - void shouldGenerateMissingTrieKeys() { - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); - Map trieKeyHashes = - trieKeyPreLoader.generateManyTrieKeyHashes( - accounts.get(0), - Collections.emptyList(), - List.of(UInt256.valueOf(64)), - List.of(UInt256.valueOf(128)), - true); - // Assert that two trie keys (slot and code) are missing from the cache - Assertions.assertThat(trieKeyPreLoader.getMissedTrieKeys()).isEqualTo(2); - // Assert that three trie keys have been generated: one for the header, one for the slot in main - // storage, and one for the code chunk - Assertions.assertThat(trieKeyHashes.size()).isEqualTo(3); - } -} diff --git a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java index 0001654862e..a73058f8b0c 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java @@ -422,7 +422,7 @@ public ChunkAccessKey( @Override public List getStorageSlotTreeIndexes(final UInt256 storageKey) { return List.of( - TRIE_KEY_ADAPTER.locateStorageKeyOffset(storageKey), - TRIE_KEY_ADAPTER.locateStorageKeySuffix(storageKey)); + TRIE_KEY_ADAPTER.getStorageKeyTrieIndex(storageKey), + TRIE_KEY_ADAPTER.getStorageKeySuffix(storageKey)); } }