diff --git a/packages/blockchain/examples/4444.ts b/packages/blockchain/examples/4444.ts new file mode 100644 index 0000000000..ca4e2d0ede --- /dev/null +++ b/packages/blockchain/examples/4444.ts @@ -0,0 +1,85 @@ +import { createBlock } from '@ethereumjs/block' +import { createBlockchain } from '@ethereumjs/blockchain' +import { Common, Hardfork, Mainnet } from '@ethereumjs/common' +import { bytesToHex, hexToBytes } from '@ethereumjs/util' + +const main = async () => { + const common = new Common({ chain: Mainnet, hardfork: Hardfork.Shanghai }) + // Use the safe static constructor which awaits the init method + const blockchain = await createBlockchain({ + common, + }) + + // We are using this to create minimal post merge blocks between shanghai and cancun in line with the + // block hardfork configuration of mainnet + const chainTTD = BigInt('58750000000000000000000') + const shanghaiTimestamp = 1681338455 + + // We use minimal data to provide a sequence of blocks (increasing number, difficulty, and then setting parent hash to previous block) + const block1 = createBlock( + { + header: { + // 15537393n is terminal block in mainnet config + number: 15537393n + 500n, + // Could be any parenthash other than 0x00..00 as we will set this block as a TRUSTED 4444 anchor + // instead of genesis to build blockchain on top of. One could use any criteria to set a block + // as trusted 4444 anchor + parentHash: hexToBytes(`0x${'20'.repeat(32)}`), + timestamp: shanghaiTimestamp + 12 * 500, + }, + }, + { common, setHardfork: true }, + ) + const block2 = createBlock( + { + header: { + number: block1.header.number + 1n, + parentHash: block1.header.hash(), + timestamp: shanghaiTimestamp + 12 * 501, + }, + }, + { common, setHardfork: true }, + ) + const block3 = createBlock( + { + header: { + number: block2.header.number + 1n, + parentHash: block2.header.hash(), + timestamp: shanghaiTimestamp + 12 * 502, + }, + }, + { common, setHardfork: true }, + ) + + let headBlock, blockByHash, blockByNumber + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 0 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3 + + // 1. We can put any post merge block as 4444 anchor by using TTD as parentTD + // 2. For pre-merge blocks its prudent to supply correct parentTD so as to respect the + // hardfork configuration as well as to determine the canonicality of the chain on future putBlocks + await blockchain.putBlock(block1, { parentTd: chainTTD }) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537893 0x26cb3bfda75027016c17d737fdabe56f412925311b42178a675da88a41bbb7e7 + + await blockchain.putBlock(block2) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537894 0x6c33728cd8aa21db683d94418fec1f7ee1cfdaa9b77781762ec832da40ec3a7c + + await blockchain.putBlock(block3) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537895 0x4263ec367ce44e4092b79ea240f132250d0d341639afbaf8c0833fbdd6160d0f +} +void main() diff --git a/packages/blockchain/examples/optimistic.ts b/packages/blockchain/examples/optimistic.ts new file mode 100644 index 0000000000..568047cf2a --- /dev/null +++ b/packages/blockchain/examples/optimistic.ts @@ -0,0 +1,141 @@ +import { createBlock } from '@ethereumjs/block' +import { createBlockchain } from '@ethereumjs/blockchain' +import { Common, Hardfork, Mainnet } from '@ethereumjs/common' +import { bytesToHex, hexToBytes } from '@ethereumjs/util' + +const main = async () => { + const common = new Common({ chain: Mainnet, hardfork: Hardfork.Shanghai }) + // Use the safe static constructor which awaits the init method + const blockchain = await createBlockchain({ + common, + }) + + // We are using this to create minimal post merge blocks between shanghai and cancun in line with the + // block hardfork configuration of mainnet + const chainTTD = BigInt('58750000000000000000000') + const shanghaiTimestamp = 1681338455 + + // We use minimal data to construct random block sequence post merge/paris to worry not much about + // td's pre-merge while constructing chain + const block1 = createBlock( + { + header: { + // 15537393n is terminal block in mainnet config + number: 15537393n + 500n, + // Could be any parenthash other than 0x00..00 as we will set this block as a TRUSTED 4444 anchor + // instead of genesis to build blockchain on top of. One could use any criteria to set a block + // as trusted 4444 anchor + parentHash: hexToBytes(`0x${'20'.repeat(32)}`), + timestamp: shanghaiTimestamp + 12 * 500, + }, + }, + { common, setHardfork: true }, + ) + const block2 = createBlock( + { + header: { + number: block1.header.number + 1n, + parentHash: block1.header.hash(), + timestamp: shanghaiTimestamp + 12 * 501, + }, + }, + { common, setHardfork: true }, + ) + const block3 = createBlock( + { + header: { + number: block2.header.number + 1n, + parentHash: block2.header.hash(), + timestamp: shanghaiTimestamp + 12 * 502, + }, + }, + { common, setHardfork: true }, + ) + + let headBlock, blockByHash, blockByNumber + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 0 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3 + + // allows any block > head + 1 to be put as non canonical block + await blockchain.putBlock(block3, { canonical: true }) + headBlock = await blockchain.getCanonicalHeadBlock() + blockByHash = await blockchain.getBlock(block3.hash()).catch((e) => null) + blockByNumber = await blockchain.getBlock(block3.header.number).catch((e) => null) + console.log( + `putBlock ${block3.header.number} ${bytesToHex(block3.hash())} byHash: ${blockByHash ? true : false} byNumber: ${blockByNumber ? true : false} headBlock=${headBlock.header.number}`, + ) + // putBlock 15537895 0x4263ec367ce44e4092b79ea240f132250d0d341639afbaf8c0833fbdd6160d0f byHash: true byNumber: true headBlock=0 + + let hasBlock1, hasBlock2, hasBlock3 + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=0, 0 ... 15537893=false 15537894=false 15537895=true + + await blockchain.putBlock(block2, { canonical: true }) + headBlock = await blockchain.getCanonicalHeadBlock() + blockByHash = await blockchain.getBlock(block2.hash()).catch((e) => null) + blockByNumber = await blockchain.getBlock(block2.header.number).catch((e) => null) + console.log( + `putBlock ${block2.header.number} ${bytesToHex(block2.hash())} byHash: ${blockByHash ? true : false} byNumber: ${blockByNumber ? true : false} headBlock=${headBlock.header.number}`, + ) + // putBlock 15537894 0x6c33728cd8aa21db683d94418fec1f7ee1cfdaa9b77781762ec832da40ec3a7c byHash: true byNumber: true headBlock=0 + + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=0, 0 ... 15537893=false 15537894=true 15537895=true + + // 1. We can put any post merge block as 4444 anchor by using TTD as parentTD + // 2. For pre-merge blocks its prudent to supply correct parentTD so as to respect the + // hardfork configuration as well as to determine the canonicality of the chain on future putBlocks + await blockchain.putBlock(block1, { parentTd: chainTTD }) + headBlock = await blockchain.getCanonicalHeadBlock() + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=15537893, 0 ... 15537893=true 15537894=true 15537895=true + + await blockchain.putBlock(block2) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537894 0x6c33728cd8aa21db683d94418fec1f7ee1cfdaa9b77781762ec832da40ec3a7c + + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=15537894, 0 ... 15537893=true 15537894=true 15537895=true + + await blockchain.putBlock(block3) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537895 0x4263ec367ce44e4092b79ea240f132250d0d341639afbaf8c0833fbdd6160d0f + + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=15537895, 0 ... 15537893=true 15537894=true 15537895=true +} +void main() diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 89db94edda..bac5292857 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -4,6 +4,7 @@ import { AsyncEventEmitter, BIGINT_0, BIGINT_1, + BIGINT_2, BIGINT_8, KECCAK256_RLP, Lock, @@ -25,6 +26,7 @@ import { import { DBManager } from './db/manager.js' import { DBTarget } from './db/operation.js' +import type { PutOpts } from './db/operation.js' import type { BlockchainEvents, BlockchainInterface, @@ -272,8 +274,8 @@ export class Blockchain implements BlockchainInterface { * heads/hashes are overwritten. * @param block - The block to be added to the blockchain */ - async putBlock(block: Block) { - await this._putBlockOrHeader(block) + async putBlock(block: Block, opts?: PutOpts) { + await this._putBlockOrHeader(block, opts) } /** @@ -344,7 +346,7 @@ export class Blockchain implements BlockchainInterface { * header using the iterator method. * @hidden */ - private async _putBlockOrHeader(item: Block | BlockHeader) { + private async _putBlockOrHeader(item: Block | BlockHeader, opts?: PutOpts) { await this.runWithLock(async () => { // Save the current sane state incase _putBlockOrHeader midway with some // dirty changes in head trackers @@ -369,89 +371,177 @@ export class Blockchain implements BlockchainInterface { ) } - const { header } = block - const blockHash = header.hash() - const blockNumber = header.number - let td = header.difficulty - const currentTd = { header: BIGINT_0, block: BIGINT_0 } - let dbOps: DBOp[] = [] - if (block.common.chainId() !== this.common.chainId()) { throw new Error( `Chain mismatch while trying to put block or header. Chain ID of block: ${block.common.chainId}, chain ID of blockchain : ${this.common.chainId}`, ) } - if (this._validateBlocks && !isGenesis && item instanceof Block) { - // this calls into `getBlock`, which is why we cannot lock yet - await this.validateBlock(block) - } + const { header } = block + const blockHash = header.hash() + const blockNumber = header.number - if (this._validateConsensus) { - await this.consensus!.validateConsensus(block) - } + // check if head is still canonical i.e. if this is a block insertion on tail or reinsertion + // on the same canonical chain + // let isHeadChainStillCanonical + // if (opts?.canonical !== false) { + // const childHeaderHash = await this.dbManager.numberToHash(blockNumber + BIGINT_1) + // if (childHeaderHash !== undefined) { + // const childHeader = await this.dbManager + // .getHeaderSafe(childHeaderHash, blockNumber + BIGINT_1) + // .catch((_e) => undefined) + // isHeadChainStillCanonical = + // childHeader !== undefined && equalsBytes(childHeader.parentHash, blockHash) + // } else { + // isHeadChainStillCanonical = false + // } + // } else { + // isHeadChainStillCanonical = true + // } - // set total difficulty in the current context scope - if (this._headHeaderHash) { - currentTd.header = await this.getTotalDifficulty(this._headHeaderHash) - } - if (this._headBlockHash) { - currentTd.block = await this.getTotalDifficulty(this._headBlockHash) + let dbOps: DBOp[] = [] + dbOps = dbOps.concat(DBSetBlockOrHeader(item)) + dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) + + let parentTd = await this.dbManager + .getTotalDifficultySafe(header.parentHash, BigInt(blockNumber) - BIGINT_1) + .catch((_e) => undefined) + const isOptimistic = parentTd === undefined + parentTd = parentTd ?? opts?.parentTd + const isComplete = parentTd !== undefined + + if (!isOptimistic) { + if (this._validateBlocks && !isGenesis && item instanceof Block) { + // this calls into `getBlock`, which is why we cannot lock yet + await this.validateBlock(block) + } } - // calculate the total difficulty of the new block - const parentTd = await this.getParentTD(header) - if (!block.isGenesis()) { - td += parentTd - } + // 1. if canonical is explicit false then just dump the block + // 2. if canonical is explicit true then apply that even for the pow/poa blocks + // if they are optimistic, i.e. can't apply the normal rule + // 3. if canonical is not defined, then apply normal rules + if (opts?.canonical === false || (opts?.canonical === undefined && !isComplete)) { + if (parentTd !== undefined) { + const td = header.difficulty + parentTd + dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) + } + await this.dbManager.batch(dbOps) + } else { + let updatesHead = block.isGenesis() + let updatesHeadBlock + if (parentTd === undefined) { + // if the block is pow and optimistic, and has not been explicity marked canonical, then + // throw error since pow blocks can't be optimistically added without expicit instruction about + // their canonicality + if ( + !block.isGenesis() && + block.common.consensusType() !== ConsensusType.ProofOfStake && + opts?.canonical === undefined + ) { + throw Error( + `Invalid parentTd=${parentTd} for consensus=${block.common.consensusType()} putBlockOrHeader`, + ) + } + updatesHead = true + updatesHeadBlock = updatesHead && isComplete + } else { + const currentTd = { header: BIGINT_0, block: BIGINT_0 } + // set total difficulty in the current context scope + if (this._headHeaderHash) { + currentTd.header = await this.getTotalDifficulty(this._headHeaderHash) + } + if (this._headBlockHash) { + currentTd.block = await this.getTotalDifficulty(this._headBlockHash) + } + + const td = parentTd + header.difficulty + dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) + updatesHead = + block.isGenesis() || + td > currentTd.header || + block.common.consensusType() === ConsensusType.ProofOfStake + updatesHeadBlock = + isComplete && + (updatesHead || + td > currentTd.block || + block.common.consensusType() === ConsensusType.ProofOfStake) + } - // save total difficulty to the database - dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) + let commonAncestor: undefined | BlockHeader + let ancestorHeaders: undefined | BlockHeader[] + // if total difficulty is higher than current, add it to canonical chain + if (updatesHead) { + if (this._hardforkByHeadBlockNumber) { + await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp) + } - // save header/block to the database, but save the input not our wrapper block - dbOps = dbOps.concat(DBSetBlockOrHeader(item)) + if (this._validateConsensus) { + await this.consensus!.validateConsensus(block) + } - let commonAncestor: undefined | BlockHeader - let ancestorHeaders: undefined | BlockHeader[] - // if total difficulty is higher than current, add it to canonical chain - if ( - block.isGenesis() || - td > currentTd.header || - block.common.consensusType() === ConsensusType.ProofOfStake - ) { - const foundCommon = await this.findCommonAncestor(header) - commonAncestor = foundCommon.commonAncestor - ancestorHeaders = foundCommon.ancestorHeaders + if (!isOptimistic) { + const foundCommon = await this.findCommonAncestor(header) + commonAncestor = foundCommon.commonAncestor + ancestorHeaders = foundCommon.ancestorHeaders + } - this._headHeaderHash = blockHash - if (item instanceof Block) { - this._headBlockHash = blockHash - } - if (this._hardforkByHeadBlockNumber) { - await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp) - } + // delete higher number assignments and overwrite stale canonical chain + if (isComplete) { + this._headHeaderHash = blockHash + if (item instanceof Block) { + this._headBlockHash = blockHash + } - // delete higher number assignments and overwrite stale canonical chain - await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps) - // from the current header block, check the blockchain in reverse (i.e. - // traverse `parentHash`) until `numberToHash` matches the current - // number/hash in the canonical chain also: overwrite any heads if these - // heads are stale in `_heads` and `_headBlockHash` - await this._rebuildCanonical(header, dbOps) - } else { - // the TD is lower than the current highest TD so we will add the block - // to the DB, but will not mark it as the canonical chain. - if (td > currentTd.block && item instanceof Block) { - this._headBlockHash = blockHash + await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps) + // from the current header block, check the blockchain in reverse (i.e. + // traverse `parentHash`) until `numberToHash` matches the current + // number/hash in the canonical chain also: overwrite any heads if these + // heads are stale in `_heads` and `_headBlockHash` + await this._rebuildCanonical(header, dbOps) + } else { + // reset to blockNumber - 2, blockNumber - 1 is parent and that is not present else + // we would have isComplete true + const headHeader = await this._getHeader( + this._headHeaderHash ?? this.genesisBlock.hash(), + ) + let resetToNumber = blockNumber - BIGINT_2 + if (resetToNumber < BIGINT_0) { + resetToNumber = BIGINT_0 + } + if (headHeader.number >= resetToNumber) { + let resetToHash = await this.dbManager.numberToHash(resetToNumber) + if (resetToHash === undefined) { + resetToHash = this.genesisBlock.hash() + resetToNumber = BIGINT_0 + } + + this._headHeaderHash = resetToHash + if (item instanceof Block) { + this._headBlockHash = resetToHash + } + await this._deleteCanonicalChainReferences( + resetToNumber + BIGINT_1, + resetToHash, + dbOps, + ) + // save this number to hash + } + } + dbOps.push(DBOp.set(DBTarget.NumberToHash, blockHash, { blockNumber })) + } else { + // the TD is lower than the current highest TD so we will add the block + // to the DB, but will not mark it as the canonical chain. + if (updatesHeadBlock && item instanceof Block) { + this._headBlockHash = blockHash + } } - // save hash to number lookup info even if rebuild not needed - dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) - } - const ops = dbOps.concat(this._saveHeadOps()) - await this.dbManager.batch(ops) + const ops = dbOps.concat(this._saveHeadOps()) + await this.dbManager.batch(ops) - await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders) + await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders) + } } catch (e) { // restore head to the previously sane state this._heads = oldHeads @@ -1047,6 +1137,10 @@ export class Blockchain implements BlockchainInterface { hash = await this.safeNumberToHash(blockNumber) while (hash !== false) { + const blockTd = await this.dbManager.getTotalDifficultySafe(hash, blockNumber) + if (blockTd === undefined) { + return + } ops.push(DBOp.del(DBTarget.NumberToHash, { blockNumber })) if (this.events.listenerCount('deletedCanonicalBlocks') > 0) { @@ -1147,10 +1241,14 @@ export class Blockchain implements BlockchainInterface { staleHeadBlock = true } - header = await this._getHeader(header.parentHash, --currentNumber) - if (header === undefined) { + const parentHeader = await this.dbManager + .getHeader(header.parentHash, --currentNumber) + .catch((_e) => undefined) + if (parentHeader === undefined) { staleHeads = [] break + } else { + header = parentHeader } } // When the stale hash is equal to the blockHash of the provided header, diff --git a/packages/blockchain/src/db/constants.ts b/packages/blockchain/src/db/constants.ts index 5b365937ac..a15968b8db 100644 --- a/packages/blockchain/src/db/constants.ts +++ b/packages/blockchain/src/db/constants.ts @@ -28,7 +28,6 @@ const TD_SUFFIX = utf8ToBytes('t') * headerPrefix + number + numSuffix -> hash */ const NUM_SUFFIX = utf8ToBytes('n') - /** * blockHashPrefix + hash -> number */ diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 42f826c464..015e06a97e 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -95,6 +95,9 @@ export class DBManager { if (blockId instanceof Uint8Array) { hash = blockId number = await this.hashToNumber(blockId) + if (number === undefined) { + return undefined + } } else if (typeof blockId === 'bigint') { number = blockId hash = await this.numberToHash(blockId) @@ -163,14 +166,34 @@ export class DBManager { return createBlockHeaderFromBytesArray(headerValues as Uint8Array[], opts) } + async getHeaderSafe(blockHash: Uint8Array, blockNumber: bigint) { + const encodedHeader = await this.get(DBTarget.Header, { blockHash, blockNumber }) + + const opts: BlockOptions = { common: this.common, setHardfork: true } + return encodedHeader !== undefined + ? createBlockHeaderFromBytesArray(RLP.decode(encodedHeader) as Uint8Array[], opts) + : undefined + } + /** * Fetches total difficulty for a block given its hash and number. */ async getTotalDifficulty(blockHash: Uint8Array, blockNumber: bigint): Promise { const td = await this.get(DBTarget.TotalDifficulty, { blockHash, blockNumber }) + if (td === undefined) { + throw Error(`totalDifficulty not found`) + } return bytesToBigInt(RLP.decode(td) as Uint8Array) } + async getTotalDifficultySafe( + blockHash: Uint8Array, + blockNumber: bigint, + ): Promise { + const td = await this.get(DBTarget.TotalDifficulty, { blockHash, blockNumber }) + return td !== undefined ? bytesToBigInt(RLP.decode(td) as Uint8Array) : undefined + } + /** * Performs a block hash to block number lookup. */ diff --git a/packages/blockchain/src/db/operation.ts b/packages/blockchain/src/db/operation.ts index a861b9605f..47ac98597e 100644 --- a/packages/blockchain/src/db/operation.ts +++ b/packages/blockchain/src/db/operation.ts @@ -13,6 +13,8 @@ import { import type { CacheMap } from './manager.js' +export type PutOpts = { canonical?: boolean; parentTd?: bigint } + export enum DBTarget { Heads, HeadHeader, diff --git a/packages/blockchain/src/types.ts b/packages/blockchain/src/types.ts index 0eee200bf8..7d2f6f3804 100644 --- a/packages/blockchain/src/types.ts +++ b/packages/blockchain/src/types.ts @@ -1,3 +1,4 @@ +import type { PutOpts } from './db/operation.js' import type { Blockchain } from './index.js' import type { Block, BlockHeader } from '@ethereumjs/block' import type { Common, ConsensusAlgorithm } from '@ethereumjs/common' @@ -16,7 +17,7 @@ export interface BlockchainInterface { * * @param block - The block to be added to the blockchain. */ - putBlock(block: Block): Promise + putBlock(block: Block, opts?: PutOpts): Promise /** * Deletes a block from the blockchain. All child blocks in the chain are diff --git a/packages/blockchain/test/4444Optimistic.spec.ts b/packages/blockchain/test/4444Optimistic.spec.ts new file mode 100644 index 0000000000..ef9015d325 --- /dev/null +++ b/packages/blockchain/test/4444Optimistic.spec.ts @@ -0,0 +1,308 @@ +import { createBlock } from '@ethereumjs/block' +import { Mainnet, createCustomCommon } from '@ethereumjs/common' +import { equalsBytes } from '@ethereumjs/util' +import { assert, describe, it } from 'vitest' + +import mergeGenesisParams from '../../client/test/testdata/common/mergeTestnet.json' +import { createBlockchain } from '../src/index.js' + +describe('[Blockchain]: 4444/optimistic spec', () => { + const common = createCustomCommon(mergeGenesisParams, Mainnet, { name: 'post-merge' }) + const genesisBlock = createBlock({ header: { extraData: new Uint8Array(97) } }, { common }) + + const block1 = createBlock( + { header: { number: 1, parentHash: genesisBlock.hash(), difficulty: 100 } }, + { common, setHardfork: true }, + ) + const block2 = createBlock( + { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, + { common, setHardfork: true }, + ) + const block3PoS = createBlock( + { header: { number: 3, parentHash: block2.hash() } }, + { common, setHardfork: true }, + ) + const block4 = createBlock( + { header: { number: 4, parentHash: block3PoS.hash() } }, + { common, setHardfork: true }, + ) + const block5 = createBlock( + { header: { number: 5, parentHash: block4.hash() } }, + { common, setHardfork: true }, + ) + const block6 = createBlock( + { header: { number: 6, parentHash: block5.hash() } }, + { common, setHardfork: true }, + ) + const block7 = createBlock( + { header: { number: 7, parentHash: block6.hash() } }, + { common, setHardfork: true }, + ) + const block8 = createBlock( + { header: { number: 8, parentHash: block7.hash() } }, + { common, setHardfork: true }, + ) + + const block51 = createBlock( + { header: { number: 5, parentHash: block4.hash(), gasLimit: 999 } }, + { common, setHardfork: true }, + ) + const block61 = createBlock( + { header: { number: 6, parentHash: block51.hash() } }, + { common, setHardfork: true }, + ) + const block71 = createBlock( + { header: { number: 7, parentHash: block61.hash() } }, + { common, setHardfork: true }, + ) + const block81 = createBlock( + { header: { number: 8, parentHash: block71.hash() } }, + { common, setHardfork: true }, + ) + + it('should allow optimistic non canonical insertion unless specified explictly', async () => { + let dbBlock + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + + // randomly insert the blocks with gaps will be inserted as non canonical + for (const block of [block4, block3PoS, block7, block2]) { + // without explicit flag their insertion will fail as parent not present and canonicality + // can't be established + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + dbBlock = await blockchain.getBlock(block.hash()).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block should be inserted') + + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock === undefined, true, 'block number index should not exit') + } + const headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal( + headBlock.header.number, + genesisBlock.header.number, + 'head should still be at genesis', + ) + }) + + it('should allow explicit non canonical insertion', async () => { + let dbBlock, headBlock + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + await blockchain.putBlock(block, { canonical: false }).catch((_e) => { + undefined + }) + } + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + dbBlock = await blockchain.getBlock(block.hash()).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block should be inserted') + + const td = await blockchain.getTotalDifficulty(block.hash()).catch((_e) => undefined) + assert.equal(td !== undefined, true, 'block should be marked complete') + + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock === undefined, true, 'block number index should not exit') + } + + // head should not move + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal( + headBlock.header.number, + genesisBlock.header.number, + 'head should still be at genesis', + ) + + // however normal insertion should respect canonicality updates + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + } + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at genesis') + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block number index should not exit') + } + + // since block61 doesn't has the parent block51 inserted yet, they will still be inserted as non + // canonical + for (const block of [block61, block71]) { + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + } + // verify + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at genesis') + + for (const block of [block61, block71]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal( + equalsBytes(dbBlock.hash(), block.hash()), + false, + 'block number index should not exit', + ) + + const td = await blockchain.getTotalDifficulty(block.hash()).catch((_e) => undefined) + assert.equal(td, undefined, 'block should be not be marked complete') + } + + // insert the side chain blocks as non canonical and they should not impact the current canonical chain + for (const block of [block51, block61, block71]) { + await blockchain.putBlock(block, { canonical: false }).catch((_e) => { + undefined + }) + } + + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at block8') + assert.equal( + equalsBytes(headBlock.hash(), block8.hash()), + true, + 'head should still be at block 8', + ) + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block number index should not exit') + + const td = await blockchain.getTotalDifficulty(block.hash()).catch((_e) => undefined) + assert.equal(td !== undefined, true, 'block should be marked complete') + } + + // lets make the side chain canonical + await blockchain.putBlock(block81).catch((_e) => undefined) + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at block8') + assert.equal( + equalsBytes(headBlock.hash(), block81.hash()), + true, + 'head should still be at block 8', + ) + + // alternate chain should now be canonical + for (const block of [block1, block2, block3PoS, block4, block51, block61, block71, block81]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block number index should not exit') + assert.equal( + equalsBytes(dbBlock.hash(), block.hash()), + true, + 'head should still be at block 8', + ) + } + }) + + it('should be able to insert block optimistically', async () => { + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + await blockchain.putBlock(block5, { canonical: true }) + await blockchain.putBlock(block4, { canonical: true }) + await blockchain.putBlock(block3PoS, { canonical: true }) + + let dbBlock = await blockchain.getBlock(block5.hash()).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'optimistic block by hash should be found') + dbBlock = await blockchain.getBlock(block5.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'optimistic block by number should be found') + + // pow block should be allowed to inserted only in backfill mode + await blockchain.putBlock(block2).catch((_e) => undefined) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal( + dbBlock === undefined, + true, + 'pow block should not be inserted as a non canonical block', + ) + + await blockchain.putBlock(block2, { canonical: true }) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal( + dbBlock !== undefined, + true, + 'pow block should be inserted in optimistic block if marked canonical', + ) + + let headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, genesisBlock.header.number, 'head still at genesis block') + + await blockchain.putBlock(block1) + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block1.header.number, 'head should move') + + for (const block of [block2, block3PoS, block4, block5]) { + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'the forward index should still be untouched') + } + + for (const block of [block2, block3PoS, block4, block51]) { + await blockchain.putBlock(block).catch((_e) => undefined) + } + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal( + headBlock.header.number, + block5.header.number, + 'head should move along the new canonical chain', + ) + assert.equal( + equalsBytes(headBlock.hash(), block51.hash()), + true, + 'head should move along the new canonical chain', + ) + + for (const blockNumber of Array.from({ length: 5 }, (_v, i) => i + 1)) { + dbBlock = await blockchain.getBlock(blockNumber).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'the blocks should be available from index') + } + }) + + it('should be able to specify a 4444 checkpoint and forward fill', async () => { + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + + let dbBlock, headBlock + await blockchain.putBlock(block2).catch((_e) => undefined) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal(dbBlock === undefined, true, 'pow block2 should not be inserted without parents') + + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, genesisBlock.header.number, 'head should be at genesis') + + // insert block2 as checkpoint without needing block1 by specifying parent td which could be anything + // > difficulty of the checkpoint's parent + await blockchain.putBlock(block2, { parentTd: 102n }) + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal( + headBlock.header.number, + block2.header.number, + 'head should be at the new checkpoint', + ) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'pow block2 should be found with number') + + for (const block of [block3PoS, block4, block5, block6, block7, block8]) { + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + } + + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should be at block5') + + // block1 would not be available + dbBlock = await blockchain.getBlock(1n).catch((_e) => {}) + assert.equal(dbBlock === undefined, true, 'block 1 should be unavailable') + + // check from 2...8 by number + for (const blockNumber of Array.from({ length: 6 }, (_v, i) => i + 2)) { + dbBlock = await blockchain.getBlock(blockNumber).catch((_e) => {}) + assert.equal(dbBlock !== undefined, true, 'the blocks should be available from index') + } + }) +}) diff --git a/packages/client/src/service/skeleton.ts b/packages/client/src/service/skeleton.ts index 859a6454b3..98fbd09b8c 100644 --- a/packages/client/src/service/skeleton.ts +++ b/packages/client/src/service/skeleton.ts @@ -208,8 +208,12 @@ export class Skeleton extends MetaDBManager { // if its genesis we are linked if (tail === BIGINT_0) return true if (tail <= this.chain.blocks.height + BIGINT_1) { - const nextBlock = await this.chain.getBlock(tail - BIGINT_1) - const linked = equalsBytes(next, nextBlock.hash()) + // we here want the non optimistic block from canonical chain + const nextBlock = await this.chain.blockchain.getBlock(tail - BIGINT_1) + const tailTD = await this.chain.blockchain + .getTotalDifficulty(nextBlock.hash(), tail - BIGINT_1) + .catch((_e) => undefined) + const linked = equalsBytes(next, nextBlock.hash()) && tailTD !== undefined if (linked && this.status.progress.subchains.length > 1) { // Remove all other subchains as no more relevant const junkedSubChains = this.status.progress.subchains.splice(1) @@ -233,34 +237,42 @@ export class Skeleton extends MetaDBManager { return this.started > 0 } - async isLastAnnouncement(): Promise { - const subchain0 = this.status.progress.subchains[0] - if (subchain0 !== undefined) { - return this.getBlock(subchain0.head + BIGINT_1) !== undefined - } else { - return true - } - } - /** * Try fast forwarding the chain head to the number */ - private async fastForwardHead(lastchain: SkeletonSubchain, target: bigint) { + private async fastForwardHead( + lastchain: SkeletonSubchain, + target: bigint, + targetHash: Uint8Array, + ) { const head = lastchain.head let headBlock = await this.getBlock(head, true) if (headBlock === undefined) { return } + const fastForwardBlocks = [] + for (let newHead = head + BIGINT_1; newHead <= target; newHead += BIGINT_1) { - const newBlock = await this.getBlock(newHead, true) + const newBlockHash = await this.get(DBKey.SkeletonForwardNumberToHash, bigIntToBytes(newHead)) + const newBlock = newBlockHash + ? await this.chain.blockchain.getBlock(newBlockHash).catch((_e) => undefined) + : undefined if (newBlock === undefined || !equalsBytes(newBlock.header.parentHash, headBlock.hash())) { // Head can't be updated forward break } headBlock = newBlock + fastForwardBlocks.push(newBlock) } - lastchain.head = headBlock.header.number + + if (equalsBytes(headBlock.hash(), targetHash)) { + for (const block of fastForwardBlocks) { + await this.putBlock(block) + } + lastchain.head = headBlock.header.number + } + this.config.logger.debug( `lastchain head fast forwarded from=${head} to=${lastchain.head} tail=${lastchain.tail}`, ) @@ -316,7 +328,7 @@ export class Skeleton extends MetaDBManager { } else if (lastchain.head >= number) { // Check if its duplicate announcement, if not trim the head and let the match run // post this if block - const mayBeDupBlock = await this.getBlock(number) + const mayBeDupBlock = await this.getBlock(number, true) if (mayBeDupBlock !== undefined && equalsBytes(mayBeDupBlock.header.hash(), head.hash())) { this.config.logger.debug( `Skeleton duplicate ${force ? 'setHead' : 'announcement'} tail=${lastchain.tail} head=${ @@ -344,7 +356,7 @@ export class Skeleton extends MetaDBManager { } } else if (lastchain.head + BIGINT_1 < number) { if (force) { - await this.fastForwardHead(lastchain, number - BIGINT_1) + await this.fastForwardHead(lastchain, number - BIGINT_1, head.header.parentHash) // If its still less than number then its gapped head if (lastchain.head + BIGINT_1 < number) { this.config.logger.debug( @@ -359,7 +371,7 @@ export class Skeleton extends MetaDBManager { return true } } - const parent = await this.getBlock(number - BIGINT_1) + const parent = await this.getBlock(number - BIGINT_1, true) if (parent === undefined || !equalsBytes(parent.hash(), head.header.parentHash)) { if (force) { this.config.logger.warn( @@ -556,7 +568,7 @@ export class Skeleton extends MetaDBManager { } // only add to unfinalized cache if this is announcement and before canonical head - await this.putBlock(head, !force && head.header.number <= subchain0Head) + await this.putBlock(head, !force) if (init) { await this.trySubChainsMerge() @@ -818,7 +830,7 @@ export class Skeleton extends MetaDBManager { const syncedBlock = await this.getBlock( syncedHeight, // need to debug why this flag causes to return undefined when chain gets synced - //, true + true, ) if ( syncedBlock !== undefined && @@ -859,7 +871,7 @@ export class Skeleton extends MetaDBManager { async headHash(): Promise { const subchain = this.bounds() if (subchain !== undefined) { - const headBlock = await this.getBlock(subchain.head) + const headBlock = await this.getBlock(subchain.head, true) if (headBlock) { return headBlock.hash() } @@ -998,7 +1010,7 @@ export class Skeleton extends MetaDBManager { } else { // Critical error, we expect new incoming blocks to extend the canonical // subchain which is the [0]'th - const tailBlock = await this.getBlock(this.status.progress.subchains[0].tail) + const tailBlock = await this.getBlock(this.status.progress.subchains[0].tail, true) this.config.logger.warn( `Blocks don't extend canonical subchain tail=${ this.status.progress.subchains[0].tail @@ -1112,7 +1124,7 @@ export class Skeleton extends MetaDBManager { for (let i = 0; i < maxItems; i++) { const tailBlock = await this.getBlockByHash(next) - if (tailBlock === undefined) { + if (tailBlock === undefined || tailBlock.header.number <= BIGINT_1) { break } else { blocks.push(tailBlock) @@ -1162,7 +1174,6 @@ export class Skeleton extends MetaDBManager { async fillCanonicalChain() { if (this.filling) return this.filling = true - let canonicalHead = this.chain.blocks.height const subchain = this.status.progress.subchains[0]! if (this.status.canonicalHeadReset) { @@ -1181,7 +1192,7 @@ export class Skeleton extends MetaDBManager { `Resetting canonicalHead for fillCanonicalChain from=${canonicalHead} to=${newHead}`, ) canonicalHead = newHead - await this.chain.resetCanonicalHead(canonicalHead) + // await this.chain.resetCanonicalHead(canonicalHead) } // update in lock so as to not conflict/overwrite setHead/putBlock updates await this.runWithLock(async () => { @@ -1202,7 +1213,7 @@ export class Skeleton extends MetaDBManager { while (!this.status.canonicalHeadReset && canonicalHead < subchain.head) { // Get next block const number = canonicalHead + BIGINT_1 - const block = await this.getBlock(number) + const block = await this.getBlock(number, true) if (block === undefined) { // This can happen @@ -1362,21 +1373,24 @@ export class Skeleton extends MetaDBManager { */ private async putBlock(block: Block, onlyUnfinalized: boolean = false): Promise { // Serialize the block with its hardfork so that its easy to load the block latter - const rlp = this.serialize({ hardfork: block.common.hardfork(), blockRLP: block.serialize() }) - await this.put(DBKey.SkeletonUnfinalizedBlockByHash, block.hash(), rlp) - - if (!onlyUnfinalized) { - await this.put(DBKey.SkeletonBlock, bigIntToBytes(block.header.number), rlp) - // this is duplication of the unfinalized blocks but for now an easy reference - // will be pruned on finalization changes. this could be simplified and deduped - // but will anyway will move into blockchain class and db on upcoming skeleton refactor - await this.put( - DBKey.SkeletonBlockHashToNumber, - block.hash(), - bigIntToBytes(block.header.number), - ) + this.config.logger.debug( + `blockchain putBlock number=${block.header.number} hash=${short(block.hash())} onlyUnfinalized=${onlyUnfinalized}`, + ) + await this.chain.blockchain.putBlock(block, { canonical: !onlyUnfinalized }) + + if (onlyUnfinalized) { + // save the forward annoucement for fast forwarding if lucky + const subchain0 = this.status.progress.subchains[0] + if (subchain0 === undefined || block.header.number > subchain0.head) { + await this.put( + DBKey.SkeletonForwardNumberToHash, + bigIntToBytes(block.header.number), + block.hash(), + ) + } + } else { + await this.chain.update() } - return true } @@ -1394,23 +1408,33 @@ export class Skeleton extends MetaDBManager { /** * Gets a block from the skeleton or canonical db by number. */ - async getBlock(number: bigint, onlyCanonical = false): Promise { - try { - const skeletonBlockRlp = await this.get(DBKey.SkeletonBlock, bigIntToBytes(number)) - if (skeletonBlockRlp === null) { - throw Error(`SkeletonBlock rlp lookup failed for ${number} onlyCanonical=${onlyCanonical}`) - } - return this.skeletonBlockRlpToBlock(skeletonBlockRlp) - } catch (error: any) { - // If skeleton is linked, it probably has deleted the block and put it into the chain - if (onlyCanonical && !this.status.linked) return undefined - // As a fallback, try to get the block from the canonical chain in case it is available there - try { - return await this.chain.getBlock(number) - } catch (error) { - return undefined - } + async getBlock(number: bigint, fcUed: boolean = false): Promise { + const subchain0 = this.status.progress.subchains[0] + this.config.logger.debug( + `getBlock subchain0: head=${subchain0.head} tail=${subchain0.tail} next=${short(subchain0.next ?? 'na')} number=${number}`, + ) + + let block + if ( + !this.status.linked && + (subchain0 === undefined || number < subchain0.tail || number > subchain0.head) + ) { + block = undefined + } else { + block = await this.chain.blockchain.dbManager.getBlock(number).catch((_e) => undefined) } + + if (block === undefined && fcUed === false) { + const blockHash = await this.get(DBKey.SkeletonForwardNumberToHash, bigIntToBytes(number)) + block = blockHash + ? await this.chain.blockchain.getBlock(blockHash).catch((_e) => undefined) + : undefined + } + + this.config.logger.debug( + `found block number=${number} with hash=${block ? short(block.hash()) : undefined}`, + ) + return block } /** @@ -1420,63 +1444,24 @@ export class Skeleton extends MetaDBManager { hash: Uint8Array, onlyCanonical: boolean = false, ): Promise { - const number = await this.get(DBKey.SkeletonBlockHashToNumber, hash) - if (number) { - const block = await this.getBlock(bytesToBigInt(number), onlyCanonical) - if (block !== undefined && equalsBytes(block.hash(), hash)) { - return block + const block = await this.chain.blockchain.dbManager.getBlock(hash).catch((_e) => undefined) + if (onlyCanonical && block !== undefined) { + const canonicalBlock = await this.getBlock(block.header.number) + if (canonicalBlock === undefined || !equalsBytes(block.hash(), canonicalBlock.hash())) { + return undefined + } else { + return canonicalBlock } - } - - if (onlyCanonical === true && !this.status.linked) { - return undefined - } - - let block = onlyCanonical === false ? await this.getUnfinalizedBlock(hash) : undefined - if (block === undefined && (onlyCanonical === false || this.status.linked)) { - block = await this.chain.getBlock(hash).catch((_e) => undefined) - } - - if (onlyCanonical === false) { - return block } else { - if (this.status.linked && block !== undefined) { - const canBlock = await this.chain.getBlock(block.header.number).catch((_e) => undefined) - if (canBlock !== undefined && equalsBytes(canBlock.hash(), block.hash())) { - // block is canonical - return block - } - } - - // no canonical block found or the block was not canonical - return undefined - } - } - - async getUnfinalizedBlock(hash: Uint8Array): Promise { - try { - const skeletonBlockRlp = await this.get(DBKey.SkeletonUnfinalizedBlockByHash, hash) - if (skeletonBlockRlp === null) { - throw Error(`SkeletonUnfinalizedBlockByHash rlp lookup failed for hash=${short(hash)}`) - } - return this.skeletonBlockRlpToBlock(skeletonBlockRlp) - } catch (_e) { - return undefined + return block } } /** * Deletes a skeleton block from the db by number */ - async deleteBlock(block: Block): Promise { - try { - await this.delete(DBKey.SkeletonBlock, bigIntToBytes(block.header.number)) - await this.delete(DBKey.SkeletonBlockHashToNumber, block.hash()) - await this.delete(DBKey.SkeletonUnfinalizedBlockByHash, block.hash()) - return true - } catch (error: any) { - return false - } + async deleteBlock(_block: Block): Promise { + return true } /** diff --git a/packages/client/src/util/metaDBManager.ts b/packages/client/src/util/metaDBManager.ts index a553cc8c44..9b04a38d13 100644 --- a/packages/client/src/util/metaDBManager.ts +++ b/packages/client/src/util/metaDBManager.ts @@ -21,6 +21,7 @@ export enum DBKey { SkeletonStatus, SkeletonUnfinalizedBlockByHash, Preimage, + SkeletonForwardNumberToHash, } export interface MetaDBManagerOptions { diff --git a/packages/client/test/sync/skeleton.spec.ts b/packages/client/test/sync/skeleton.spec.ts index a26853cfa8..42cff8c1a0 100644 --- a/packages/client/test/sync/skeleton.spec.ts +++ b/packages/client/test/sync/skeleton.spec.ts @@ -1,11 +1,6 @@ import { createBlock } from '@ethereumjs/block' -import { - Common, - Mainnet, - createCommonFromGethGenesis, - createCustomCommon, -} from '@ethereumjs/common' -import { equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { Mainnet, createCommonFromGethGenesis, createCustomCommon } from '@ethereumjs/common' +import { equalsBytes, hexToBytes, utf8ToBytes } from '@ethereumjs/util' import { MemoryLevel } from 'memory-level' import { assert, describe, it } from 'vitest' @@ -24,15 +19,28 @@ type Subchain = { tail: bigint } -const common = new Common({ chain: Mainnet }) -const block49 = createBlock({ header: { number: 49 } }, { common }) -const block49B = createBlock({ header: { number: 49, extraData: utf8ToBytes('B') } }, { common }) -const block50 = createBlock({ header: { number: 50, parentHash: block49.hash() } }, { common }) +const common = createCustomCommon(mergeGenesisParams, Mainnet, { name: 'post-merge' }) +const block48 = createBlock({ header: { number: 48 } }, { common, setHardfork: true }) +const block49 = createBlock( + { header: { number: 49, parentHash: block48.hash() } }, + { common, setHardfork: true }, +) +const block49B = createBlock( + { header: { number: 49, parentHash: block48.hash(), extraData: utf8ToBytes('B') } }, + { common, setHardfork: true }, +) +const block50 = createBlock( + { header: { number: 50, parentHash: block49.hash() } }, + { common, setHardfork: true }, +) const block50B = createBlock( { header: { number: 50, parentHash: block49.hash(), gasLimit: 999 } }, - { common }, + { common, setHardfork: true }, +) +const block51 = createBlock( + { header: { number: 51, parentHash: block50.hash() } }, + { common, setHardfork: true }, ) -const block51 = createBlock({ header: { number: 51, parentHash: block50.hash() } }, { common }) describe('[Skeleton]/ startup scenarios ', () => { it('starts the chain when starting the skeleton', async () => { @@ -233,6 +241,7 @@ describe('[Skeleton] / initSync', async () => { storageCache: 1000, }) const chain = await Chain.create({ config }) + ;(chain.blockchain as any)._validateBlocks = false const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) await skeleton.open() @@ -350,6 +359,7 @@ describe('[Skeleton] / setHead', async () => { storageCache: 1000, }) const chain = await Chain.create({ config }) + ;(chain.blockchain as any)._validateBlocks = false const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) await skeleton.open() for (const block of testCase.blocks ?? []) { @@ -397,6 +407,7 @@ describe('[Skeleton] / setHead', async () => { assert.ok(true, `test ${testCaseIndex}: subchain[${i}] matched`) } } + skeleton.logSyncStatus('specLog') }) } @@ -425,7 +436,7 @@ describe('[Skeleton] / setHead', async () => { }) it('should init/setHead properly from genesis', async () => { - const config = new Config({ common }) + const config = new Config({ common, logger: getLogger({ logLevel: 'debug' }) }) const chain = await Chain.create({ config }) ;(chain.blockchain as any)._validateBlocks = false const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) @@ -440,8 +451,18 @@ describe('[Skeleton] / setHead', async () => { { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, { common, setHardfork: true }, ) + + // block3 from an alternate chain const block3 = createBlock( - { header: { number: 3, difficulty: 100 } }, + { + header: { + number: 3, + difficulty: 0, + parentHash: hexToBytes( + '0xa321d27cd2743617c1c1b0d7ecb607dd14febcdfca8f01b79c3f0249505ea069', + ), + }, + }, { common, setHardfork: true }, ) @@ -523,11 +544,17 @@ describe('[Skeleton] / setHead', async () => { 'head should be set to second block', ) assert.equal(skeleton.isLinked(), true, 'subchain status should stay linked') + skeleton.logSyncStatus('specLog') reorg = await skeleton.setHead(block3, true) + skeleton.logSyncStatus('specLog') assert.equal(reorg, true, 'should not extend on invalid third block') // since its not a forced update so shouldn't affect subchain status - assert.equal(skeleton['status'].progress.subchains.length, 2, 'new subchain should be created') + assert.equal( + skeleton['status'].progress.subchains.length, + 1, + 'new subchain should be created and previous truncated', + ) assert.equal( skeleton['status'].progress.subchains[0].head, BigInt(3), @@ -552,16 +579,18 @@ describe('[Skeleton] / setHead', async () => { { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, { common, setHardfork: true }, ) + + // block3 onwards is POS const block3 = createBlock( - { header: { number: 3, parentHash: block2.hash(), difficulty: 100 } }, + { header: { number: 3, parentHash: block2.hash() } }, { common, setHardfork: true }, ) const block4 = createBlock( - { header: { number: 4, parentHash: block3.hash(), difficulty: 100 } }, + { header: { number: 4, parentHash: block3.hash() } }, { common, setHardfork: true }, ) const block5 = createBlock( - { header: { number: 5, parentHash: block4.hash(), difficulty: 100 } }, + { header: { number: 5, parentHash: block4.hash() } }, { common, setHardfork: true }, ) @@ -577,7 +606,9 @@ describe('[Skeleton] / setHead', async () => { BigInt(4), 'canonical height should update after being linked', ) + skeleton.logSyncStatus('specLog') await skeleton.setHead(block5, false) + skeleton.logSyncStatus('specLog') await wait(200) assert.equal( chain.blocks.height, @@ -594,26 +625,32 @@ describe('[Skeleton] / setHead', async () => { 'canonical height should change when setHead is set with force=true', ) - // unlink the skeleton for the below check to check all blocks cleared + // unlink the skeleton for the below check and still find the blocks in blokchain skeleton['status'].linked = false for (const block of [block1, block2, block3, block4, block5]) { assert.equal( - (await skeleton.getBlock(block.header.number, true))?.hash(), - undefined, - `skeleton block number=${block.header.number} should be cleaned up after filling canonical chain`, + (await skeleton.getBlock(block.header.number, true))?.hash() !== undefined, + true, + `skeleton block number=${block.header.number} should be available even afer filling canonical chain`, ) assert.equal( - (await skeleton.getBlockByHash(block.hash(), true))?.hash(), - undefined, + (await skeleton.getBlockByHash(block.hash(), true))?.hash() !== undefined, + true, `skeleton block hash=${short( block.hash(), - )} should be cleaned up after filling canonical chain`, + )} should be available even after filling canonical chain`, ) } + + skeleton.logSyncStatus('specLog') }) it('should fill the canonical chain after being linked to a canonical block past genesis', async () => { - const config = new Config({ common, engineNewpayloadMaxExecute: 10 }) + const config = new Config({ + common, + engineNewpayloadMaxExecute: 10, + logger: getLogger({ logLevel: 'debug' }), + }) const chain = await Chain.create({ config }) ;(chain.blockchain as any)._validateBlocks = false @@ -632,23 +669,26 @@ describe('[Skeleton] / setHead', async () => { { common, setHardfork: true }, ) const block3 = createBlock( - { header: { number: 3, parentHash: block2.hash(), difficulty: 100 } }, + { header: { number: 3, parentHash: block2.hash() } }, { common, setHardfork: true }, ) const block4 = createBlock( - { header: { number: 4, parentHash: block3.hash(), difficulty: 100 } }, + { header: { number: 4, parentHash: block3.hash() } }, { common, setHardfork: true }, ) const block5 = createBlock( - { header: { number: 5, parentHash: block4.hash(), difficulty: 100 } }, + { header: { number: 5, parentHash: block4.hash() } }, { common, setHardfork: true }, ) await chain.putBlocks([block1, block2]) + skeleton.logSyncStatus('specLog') await skeleton.initSync(block4) + skeleton.logSyncStatus('specLog') assert.equal(chain.blocks.height, BigInt(2), 'canonical height should be at block 2') await skeleton.putBlocks([block3]) await wait(200) + skeleton.logSyncStatus('specLog') assert.equal( chain.blocks.height, BigInt(4), @@ -677,39 +717,39 @@ describe('[Skeleton] / setHead', async () => { skeleton['status'].linked = false for (const block of [block3, block4, block5]) { assert.equal( - (await skeleton.getBlock(block.header.number, true))?.hash(), - undefined, - `skeleton block number=${block.header.number} should be cleaned up after filling canonical chain`, + (await skeleton.getBlock(block.header.number, true))?.hash() !== undefined, + true, + `skeleton block number=${block.header.number} should still be available after filling canonical chain`, ) assert.equal( - (await skeleton.getBlockByHash(block.hash(), true))?.hash(), - undefined, + (await skeleton.getBlockByHash(block.hash(), true))?.hash() !== undefined, + true, `skeleton block hash=${short( block.hash(), - )} should be cleaned up after filling canonical chain`, + )} should still be available after filling canonical chain`, ) } // restore linkedStatus skeleton['status'].linked = prevLinked const block41 = createBlock( - { header: { number: 4, parentHash: block3.hash(), difficulty: 101 } }, + { header: { number: 4, parentHash: block3.hash() } }, { common, setHardfork: true }, ) const block51 = createBlock( - { header: { number: 5, parentHash: block41.hash(), difficulty: 100 } }, + { header: { number: 5, parentHash: block41.hash() } }, { common, setHardfork: true }, ) const block61 = createBlock( - { header: { number: 6, parentHash: block51.hash(), difficulty: 100 } }, + { header: { number: 6, parentHash: block51.hash() } }, { common, setHardfork: true }, ) await skeleton.setHead(block41, false) await skeleton.setHead(block51, false) - // should link the chains including the 41, 51 block backfilled from the unfinalized await skeleton.forkchoiceUpdate(block61) + assert.equal( skeleton['status'].progress.subchains[0]?.head, BigInt(6), @@ -717,22 +757,22 @@ describe('[Skeleton] / setHead', async () => { ) assert.equal( skeleton['status'].progress.subchains[0]?.tail, - BigInt(4), + BigInt(3), 'tail should be backfilled', ) assert.equal(skeleton['status'].linked, true, 'should be linked') assert.equal(chain.blocks.height, BigInt(6), 'all blocks should be in chain') const block71 = createBlock( - { header: { number: 7, parentHash: block61.hash(), difficulty: 100 } }, + { header: { number: 7, parentHash: block61.hash() } }, { common, setHardfork: true }, ) const block81 = createBlock( - { header: { number: 8, parentHash: block71.hash(), difficulty: 100 } }, + { header: { number: 8, parentHash: block71.hash() } }, { common, setHardfork: true }, ) const block91 = createBlock( - { header: { number: 9, parentHash: block81.hash(), difficulty: 100 } }, + { header: { number: 9, parentHash: block81.hash() } }, { common, setHardfork: true }, ) @@ -760,7 +800,7 @@ describe('[Skeleton] / setHead', async () => { ) assert.equal( skeleton['status'].progress.subchains[0]?.tail, - BigInt(7), + BigInt(8), 'tail should be backfilled', ) assert.equal(skeleton['status'].linked, true, 'should be linked') @@ -775,11 +815,11 @@ describe('[Skeleton] / setHead', async () => { // do a very common reorg that happens in a network: reorged head block const block92 = createBlock( - { header: { number: 9, parentHash: block81.hash(), difficulty: 101 } }, + { header: { number: 9, parentHash: block81.hash(), gasLimit: 999 } }, { common, setHardfork: true }, ) const block102 = createBlock( - { header: { number: 10, parentHash: block92.hash(), difficulty: 100 } }, + { header: { number: 10, parentHash: block92.hash(), gasLimit: 999 } }, { common, setHardfork: true }, ) @@ -817,6 +857,7 @@ describe('[Skeleton] / setHead', async () => { common, accountCache: 10000, storageCache: 1000, + logger: getLogger({ logLevel: 'debug' }), }) const chain = await Chain.create({ config }) ;(chain.blockchain as any)._validateBlocks = false @@ -856,9 +897,16 @@ describe('[Skeleton] / setHead', async () => { await skeleton.open() await skeleton.initSync(block4InvalidPoS) - await skeleton.putBlocks([block3PoW, block2]) + // the following putBlocks will fail ad block3 is pow block + try { + await skeleton.putBlocks([block3PoW]) + assert.fail('should have failed putting invalid pow') + // eslint-disable-next-line no-empty + } catch (_e) {} + assert.equal(chain.blocks.height, BigInt(0), 'canonical height should be at genesis') - await skeleton.putBlocks([block1]) + skeleton.logSyncStatus('specLog') + await skeleton.putBlocks([block2, block1]) await wait(200) assert.equal( chain.blocks.height, @@ -902,4 +950,54 @@ describe('[Skeleton] / setHead', async () => { await wait(200) assert.equal(skeleton.bounds().head, BigInt(5), 'should update to new height') }) + + it('should fast forward the chain after annoucement', async () => { + const config = new Config({ common, logger: getLogger({ logLevel: 'debug' }) }) + const chain = await Chain.create({ config }) + ;(chain.blockchain as any)._validateBlocks = false + const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) + await chain.open() + + const genesis = await chain.getBlock(BigInt(0)) + const block1 = createBlock( + { header: { number: 1, parentHash: genesis.hash(), difficulty: 100 } }, + { common, setHardfork: true }, + ) + const block2 = createBlock( + { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, + { common, setHardfork: true }, + ) + + // block3 onwards is POS + const block3 = createBlock( + { header: { number: 3, parentHash: block2.hash() } }, + { common, setHardfork: true }, + ) + const block4 = createBlock( + { header: { number: 4, parentHash: block3.hash() } }, + { common, setHardfork: true }, + ) + const block5 = createBlock( + { header: { number: 5, parentHash: block4.hash() } }, + { common, setHardfork: true }, + ) + + await skeleton.open() + + await skeleton.initSync(block2) + await skeleton.setHead(block3, false) + await skeleton.setHead(block4, false) + await skeleton.setHead(block5, true) + assert.equal( + skeleton['status'].progress.subchains[0]?.head, + BigInt(5), + 'subchain should be fastforwarded', + ) + assert.equal( + skeleton['status'].progress.subchains[0]?.tail, + BigInt(2), + 'subchain should be fastforwarded', + ) + skeleton.logSyncStatus('specLog') + }) })