From ea4bc2dbff949932050994d5213d9a41cf4cd6ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 24 Aug 2021 10:08:08 +0300 Subject: [PATCH 01/30] params: begin Geth v1.10.9 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 8d222ac047..dd629a100a 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 79bb9300c1b9fbc00701bd27d5f465df99752870 Mon Sep 17 00:00:00 2001 From: chuwt Date: Tue, 24 Aug 2021 16:51:28 +0800 Subject: [PATCH 02/30] trie, les: add missing calls to Ticker.Stop (#23415) --- les/pruner.go | 1 + trie/sync_bloom.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/les/pruner.go b/les/pruner.go index 622e648688..a1bd51d86c 100644 --- a/les/pruner.go +++ b/les/pruner.go @@ -62,6 +62,7 @@ func (p *pruner) loop() { // cleanTicker is the ticker used to trigger a history clean 2 times a day. var cleanTicker = time.NewTicker(12 * time.Hour) + defer cleanTicker.Stop() // pruning finds the sections that have been processed by all indexers // and deletes all historical chain data. diff --git a/trie/sync_bloom.go b/trie/sync_bloom.go index 49986fcf0a..91e5e6711d 100644 --- a/trie/sync_bloom.go +++ b/trie/sync_bloom.go @@ -129,6 +129,8 @@ func (b *SyncBloom) init(database ethdb.Iteratee) { func (b *SyncBloom) meter() { // check every second tick := time.NewTicker(1 * time.Second) + defer tick.Stop() + for { select { case <-tick.C: From 8dbf261fd9fc4cd6eca64ebeafee17826e608094 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 24 Aug 2021 12:22:56 +0200 Subject: [PATCH 03/30] p2p, p2p/enode: fix data races (#23434) In p2p/dial.go, conn.flags was accessed without using sync/atomic. This race is fixed by removing the access. In p2p/enode/iter_test.go, a similar race is resolved by writing the field atomically. Co-authored-by: Felix Lange --- p2p/dial.go | 6 +++--- p2p/enode/iter_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/dial.go b/p2p/dial.go index 83ced3cb32..0d70e6f4a3 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -107,7 +107,7 @@ type dialScheduler struct { // Everything below here belongs to loop and // should only be accessed by code on the loop goroutine. dialing map[enode.ID]*dialTask // active tasks - peers map[enode.ID]connFlag // all connected peers + peers map[enode.ID]struct{} // all connected peers dialPeers int // current number of dialed peers // The static map tracks all static dial tasks. The subset of usable static dial tasks @@ -166,7 +166,7 @@ func newDialScheduler(config dialConfig, it enode.Iterator, setupFunc dialSetupF setupFunc: setupFunc, dialing: make(map[enode.ID]*dialTask), static: make(map[enode.ID]*dialTask), - peers: make(map[enode.ID]connFlag), + peers: make(map[enode.ID]struct{}), doneCh: make(chan *dialTask), nodesIn: make(chan *enode.Node), addStaticCh: make(chan *enode.Node), @@ -259,7 +259,7 @@ loop: d.dialPeers++ } id := c.node.ID() - d.peers[id] = c.flags + d.peers[id] = struct{}{} // Remove from static pool because the node is now connected. task := d.static[id] if task != nil && task.staticPoolIndex >= 0 { diff --git a/p2p/enode/iter_test.go b/p2p/enode/iter_test.go index 6009661f3c..5014346af4 100644 --- a/p2p/enode/iter_test.go +++ b/p2p/enode/iter_test.go @@ -268,7 +268,7 @@ func (s *genIter) Node() *Node { } func (s *genIter) Close() { - s.index = ^uint32(0) + atomic.StoreUint32(&s.index, ^uint32(0)) } func testNode(id, seq uint64) *Node { From 8e0771c21825e9621f29d6c4018038b4505a59f4 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Tue, 24 Aug 2021 12:32:19 +0100 Subject: [PATCH 04/30] core/bloombits: avoid crash when storing errors of different type (#23437) This fixes a rare crash which could occur when two different errors happened in the same bloombits.MatcherSession. --- core/bloombits/matcher.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go index 927232be01..f2a8bda17c 100644 --- a/core/bloombits/matcher.go +++ b/core/bloombits/matcher.go @@ -510,8 +510,9 @@ type MatcherSession struct { closer sync.Once // Sync object to ensure we only ever close once quit chan struct{} // Quit channel to request pipeline termination - ctx context.Context // Context used by the light client to abort filtering - err atomic.Value // Global error to track retrieval failures deep in the chain + ctx context.Context // Context used by the light client to abort filtering + err error // Global error to track retrieval failures deep in the chain + errLock sync.Mutex pend sync.WaitGroup } @@ -529,10 +530,10 @@ func (s *MatcherSession) Close() { // Error returns any failure encountered during the matching session. func (s *MatcherSession) Error() error { - if err := s.err.Load(); err != nil { - return err.(error) - } - return nil + s.errLock.Lock() + defer s.errLock.Unlock() + + return s.err } // allocateRetrieval assigns a bloom bit index to a client process that can either @@ -630,7 +631,9 @@ func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan result := <-request if result.Error != nil { - s.err.Store(result.Error) + s.errLock.Lock() + s.err = result.Error + s.errLock.Unlock() s.Close() } s.deliverSections(result.Bit, result.Sections, result.Bitsets) From 5c66bab3b8e6368e66f6bc300ac6816ebd1f2c83 Mon Sep 17 00:00:00 2001 From: ligi Date: Tue, 24 Aug 2021 13:56:31 +0200 Subject: [PATCH 05/30] tests/testdata: make submodule shallow (#23412) Before: 316M .git/modules/tests/ After: 171M .git/modules/tests/ --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 32bdb3b6e5..90d1be0a3d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "tests"] path = tests/testdata url = https://github.com/ethereum/tests + shallow = true From 0b4097748016395b14ecdc316d2095eb7c968f2a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 24 Aug 2021 13:57:05 +0200 Subject: [PATCH 06/30] core/vm: fix typo in comment (#23450) --- core/vm/gas_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 944b6cf0a5..19d2198af6 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -60,7 +60,7 @@ func memoryGasCost(mem *Memory, newMemSize uint64) (uint64, error) { // as argument: // CALLDATACOPY (stack position 2) // CODECOPY (stack position 2) -// EXTCODECOPY (stack poition 3) +// EXTCODECOPY (stack position 3) // RETURNDATACOPY (stack position 2) func memoryCopierGas(stackpos int) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { From a0a4a153e98f21b014e7ad289816514751d86568 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Aug 2021 13:59:15 +0200 Subject: [PATCH 07/30] core/types: add benchmarks for rlp encoding/decoding (#23190) Co-authored-by: Felix Lange Co-authored-by: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> --- core/types/types_test.go | 148 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 core/types/types_test.go diff --git a/core/types/types_test.go b/core/types/types_test.go new file mode 100644 index 0000000000..1fb386d5de --- /dev/null +++ b/core/types/types_test.go @@ -0,0 +1,148 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" +) + +type devnull struct{ len int } + +func (d *devnull) Write(p []byte) (int, error) { + d.len += len(p) + return len(p), nil +} + +func BenchmarkEncodeRLP(b *testing.B) { + benchRLP(b, true) +} + +func BenchmarkDecodeRLP(b *testing.B) { + benchRLP(b, false) +} + +func benchRLP(b *testing.B, encode bool) { + key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer := NewLondonSigner(big.NewInt(1337)) + for _, tc := range []struct { + name string + obj interface{} + }{ + { + "legacy-header", + &Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1000), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + }, + }, + { + "london-header", + &Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1000), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + BaseFee: big.NewInt(10000000000), + }, + }, + { + "receipt-for-storage", + &ReceiptForStorage{ + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 0x888888888, + Logs: make([]*Log, 0), + }, + }, + { + "receipt-full", + &Receipt{ + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 0x888888888, + Logs: make([]*Log, 0), + }, + }, + { + "legacy-transaction", + MustSignNewTx(key, signer, + &LegacyTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }), + }, + { + "access-transaction", + MustSignNewTx(key, signer, + &AccessListTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }), + }, + { + "1559-transaction", + MustSignNewTx(key, signer, + &DynamicFeeTx{ + Nonce: 1, + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + GasTipCap: big.NewInt(500), + GasFeeCap: big.NewInt(500), + }), + }, + } { + if encode { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var null = &devnull{} + for i := 0; i < b.N; i++ { + rlp.Encode(null, tc.obj) + } + b.SetBytes(int64(null.len / b.N)) + }) + } else { + data, _ := rlp.EncodeToBytes(tc.obj) + // Test decoding + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := rlp.DecodeBytes(data, tc.obj); err != nil { + b.Fatal(err) + } + } + b.SetBytes(int64(len(data))) + }) + } + } +} From 85126c4eb99ab4b2ccd74c1ee66d2aba0b54a767 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 24 Aug 2021 08:10:52 -0400 Subject: [PATCH 08/30] node: add comment about --nousb being deprecated (#23439) Co-authored-by: Felix Lange --- node/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/config.go b/node/config.go index ef1da15d70..b7e99eb7bf 100644 --- a/node/config.go +++ b/node/config.go @@ -93,6 +93,7 @@ type Config struct { InsecureUnlockAllowed bool `toml:",omitempty"` // NoUSB disables hardware wallet monitoring and connectivity. + // Deprecated: USB monitoring is disabled by default and must be enabled explicitly. NoUSB bool `toml:",omitempty"` // USB enables hardware wallet monitoring and connectivity. From 5cee33eb7234756939b1ef8b23e990b672a03dff Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Tue, 24 Aug 2021 14:27:15 +0200 Subject: [PATCH 09/30] tests/fuzzers: fix go vet warning about ReadByte (#23380) --- tests/fuzzers/trie/trie-fuzzer.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 762ab5f347..d0b614c9ac 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -69,7 +69,7 @@ func newDataSource(input []byte) *dataSource { input, bytes.NewReader(input), } } -func (ds *dataSource) ReadByte() byte { +func (ds *dataSource) readByte() byte { if b, err := ds.reader.ReadByte(); err != nil { return 0 } else { @@ -89,22 +89,22 @@ func Generate(input []byte) randTest { r := newDataSource(input) genKey := func() []byte { - if len(allKeys) < 2 || r.ReadByte() < 0x0f { + if len(allKeys) < 2 || r.readByte() < 0x0f { // new key - key := make([]byte, r.ReadByte()%50) + key := make([]byte, r.readByte()%50) r.Read(key) allKeys = append(allKeys, key) return key } // use existing key - return allKeys[int(r.ReadByte())%len(allKeys)] + return allKeys[int(r.readByte())%len(allKeys)] } var steps randTest for i := 0; !r.Ended(); i++ { - step := randTestStep{op: int(r.ReadByte()) % opMax} + step := randTestStep{op: int(r.readByte()) % opMax} switch step.op { case opUpdate: step.key = genKey() From d705f5a5543402a1d505bdc411d264cd4f883402 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Aug 2021 20:48:36 +0200 Subject: [PATCH 10/30] core: make txpool reject too sudden changes (#23095) * core: make txpool reject too sudden changes * core: add some metrics to txpool --- core/tx_pool.go | 26 ++++++++++++++++++++++++++ core/tx_pool_test.go | 14 +++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index f6228df2fa..ee56dae888 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -111,6 +111,14 @@ var ( invalidTxMeter = metrics.NewRegisteredMeter("txpool/invalid", nil) underpricedTxMeter = metrics.NewRegisteredMeter("txpool/underpriced", nil) overflowedTxMeter = metrics.NewRegisteredMeter("txpool/overflowed", nil) + // throttleTxMeter counts how many transactions are rejected due to too-many-changes between + // txpool reorgs. + throttleTxMeter = metrics.NewRegisteredMeter("txpool/throttle", nil) + // reorgDurationTimer measures how long time a txpool reorg takes. + reorgDurationTimer = metrics.NewRegisteredTimer("txpool/reorgtime", nil) + // dropBetweenReorgHistogram counts how many drops we experience between two reorg runs. It is expected + // that this number is pretty low, since txpool reorgs happen very frequently. + dropBetweenReorgHistogram = metrics.NewRegisteredHistogram("txpool/dropbetweenreorg", nil, metrics.NewExpDecaySample(1028, 0.015)) pendingGauge = metrics.NewRegisteredGauge("txpool/pending", nil) queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) @@ -256,6 +264,8 @@ type TxPool struct { reorgDoneCh chan chan struct{} reorgShutdownCh chan struct{} // requests shutdown of scheduleReorgLoop wg sync.WaitGroup // tracks loop, scheduleReorgLoop + + changesSinceReorg int // A counter for how many drops we've performed in-between reorg. } type txpoolResetRequest struct { @@ -663,6 +673,15 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e underpricedTxMeter.Mark(1) return false, ErrUnderpriced } + // We're about to replace a transaction. The reorg does a more thorough + // analysis of what to remove and how, but it runs async. We don't want to + // do too many replacements between reorg-runs, so we cap the number of + // replacements to 25% of the slots + if pool.changesSinceReorg > int(pool.config.GlobalSlots/4) { + throttleTxMeter.Mark(1) + return false, ErrTxPoolOverflow + } + // New transaction is better than our worse ones, make room for it. // If it's a local transaction, forcibly discard all available transactions. // Otherwise if we can't make enough room for new one, abort the operation. @@ -674,6 +693,8 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e overflowedTxMeter.Mark(1) return false, ErrTxPoolOverflow } + // Bump the counter of rejections-since-reorg + pool.changesSinceReorg += len(drop) // Kick out the underpriced remote transactions. for _, tx := range drop { log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) @@ -1114,6 +1135,9 @@ func (pool *TxPool) scheduleReorgLoop() { // runReorg runs reset and promoteExecutables on behalf of scheduleReorgLoop. func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*txSortedMap) { + defer func(t0 time.Time) { + reorgDurationTimer.Update(time.Since(t0)) + }(time.Now()) defer close(done) var promoteAddrs []common.Address @@ -1163,6 +1187,8 @@ func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirt highestPending := list.LastElement() pool.pendingNonces.set(addr, highestPending.Nonce()+1) } + dropBetweenReorgHistogram.Update(int64(pool.changesSinceReorg)) + pool.changesSinceReorg = 0 // Reset change counter pool.mu.Unlock() // Notify subsystems for newly added transactions diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index e02096fe25..f86f64bf25 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -1946,20 +1946,20 @@ func TestDualHeapEviction(t *testing.T) { } add := func(urgent bool) { - txs := make([]*types.Transaction, 20) - for i := range txs { + for i := 0; i < 20; i++ { + var tx *types.Transaction // Create a test accounts and fund it key, _ := crypto.GenerateKey() testAddBalance(pool, crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000000)) if urgent { - txs[i] = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+1+i)), big.NewInt(int64(1+i)), key) - highTip = txs[i] + tx = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+1+i)), big.NewInt(int64(1+i)), key) + highTip = tx } else { - txs[i] = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+200+i)), big.NewInt(1), key) - highCap = txs[i] + tx = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+200+i)), big.NewInt(1), key) + highCap = tx } + pool.AddRemotesSync([]*types.Transaction{tx}) } - pool.AddRemotes(txs) pending, queued := pool.Stats() if pending+queued != 20 { t.Fatalf("transaction count mismatch: have %d, want %d", pending+queued, 10) From 0db0b277545383d952132f126d4a8edd05138e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 24 Aug 2021 21:52:58 +0300 Subject: [PATCH 11/30] Revert "Revert "eth: drop eth/65, the last non-reqid protocol version" (#23426)" (#23456) This reverts commit c368f728c19e7fd7a9613513edda68ffcb503af0. --- cmd/devp2p/internal/ethtest/suite_test.go | 2 +- eth/downloader/downloader.go | 4 +- eth/downloader/downloader_test.go | 95 +--------- eth/downloader/peer.go | 8 +- eth/handler.go | 5 +- eth/handler_eth_test.go | 38 ++-- eth/protocols/eth/handler.go | 47 ++--- eth/protocols/eth/handler_test.go | 134 +++++--------- eth/protocols/eth/handlers.go | 107 ----------- eth/protocols/eth/handshake_test.go | 1 - eth/protocols/eth/peer.go | 208 +++++++--------------- eth/protocols/eth/protocol.go | 34 ++-- eth/sync.go | 102 +---------- eth/sync_test.go | 1 - les/client_handler.go | 2 +- 15 files changed, 179 insertions(+), 609 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 6e3217151a..6d14404e66 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.AllEthTests() { + for _, test := range suite.Eth66Tests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 1767506a33..a6bf87acbd 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -448,8 +448,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < eth.ETH65 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH65) + if p.version < eth.ETH66 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) } mode := d.getMode() diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 794160993b..17cd3630c9 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -522,10 +522,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } -func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, eth.ETH65, FullSync) } -func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, eth.ETH65, FastSync) } -func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, eth.ETH65, LightSync) } - func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } @@ -549,9 +545,6 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling65Full(t *testing.T) { testThrottling(t, eth.ETH65, FullSync) } -func TestThrottling65Fast(t *testing.T) { testThrottling(t, eth.ETH65, FastSync) } - func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } @@ -634,10 +627,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync65Full(t *testing.T) { testForkedSync(t, eth.ETH65, FullSync) } -func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, eth.ETH65, FastSync) } -func TestForkedSync65Light(t *testing.T) { testForkedSync(t, eth.ETH65, LightSync) } - func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } @@ -667,10 +656,6 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FullSync) } -func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FastSync) } -func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, LightSync) } - func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } @@ -702,10 +687,6 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FullSync) } -func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FastSync) } -func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, LightSync) } - func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } @@ -736,16 +717,6 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync65Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FullSync) -} -func TestBoundedHeavyForkedSync65Fast(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FastSync) -} -func TestBoundedHeavyForkedSync65Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, LightSync) -} - func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) } @@ -800,10 +771,6 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel65Full(t *testing.T) { testCancel(t, eth.ETH65, FullSync) } -func TestCancel65Fast(t *testing.T) { testCancel(t, eth.ETH65, FastSync) } -func TestCancel65Light(t *testing.T) { testCancel(t, eth.ETH65, LightSync) } - func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } @@ -833,10 +800,6 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FullSync) } -func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FastSync) } -func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, LightSync) } - func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } @@ -863,10 +826,6 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FullSync) } -func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FastSync) } -func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, eth.ETH65, LightSync) } - func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } @@ -881,8 +840,8 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type - tester.newPeer("peer 65", eth.ETH65, chain) tester.newPeer("peer 66", eth.ETH66, chain) + //tester.newPeer("peer 65", eth.ETH67, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -891,7 +850,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{65, 66} { + for _, version := range []int{66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -901,10 +860,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FullSync) } -func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FastSync) } -func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, LightSync) } - func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } @@ -955,10 +910,6 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FullSync) } -func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FastSync) } -func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, LightSync) } - func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } @@ -987,10 +938,6 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. -func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FullSync) } -func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FastSync) } -func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, LightSync) } - func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } @@ -1024,7 +971,6 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. -func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH65, FastSync) } func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { @@ -1115,16 +1061,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack65Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FullSync) -} -func TestHighTDStarvationAttack65Fast(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FastSync) -} -func TestHighTDStarvationAttack65Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, LightSync) -} - func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, FullSync) } @@ -1149,7 +1085,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH65) } func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { @@ -1202,10 +1137,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, eth.ETH65, FullSync) } -func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, eth.ETH65, FastSync) } -func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, eth.ETH65, LightSync) } - func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } @@ -1286,10 +1217,6 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FullSync) } -func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FastSync) } -func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, LightSync) } - func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } @@ -1362,10 +1289,6 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FullSync) } -func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FastSync) } -func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, LightSync) } - func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } @@ -1435,10 +1358,6 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FullSync) } -func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FastSync) } -func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, LightSync) } - func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } @@ -1512,10 +1431,6 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FullSync) } -func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FastSync) } -func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, LightSync) } - func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } @@ -1673,12 +1588,6 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FullSync) } -func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FastSync) } -func TestCheckpointEnforcement65Light(t *testing.T) { - testCheckpointEnforcement(t, eth.ETH65, LightSync) -} - func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } func TestCheckpointEnforcement66Light(t *testing.T) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 066a366315..8632948329 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -413,7 +413,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +425,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +437,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +449,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/handler.go b/eth/handler.go index aff4871afa..06a8088bf0 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -117,7 +117,6 @@ type handler struct { whitelist map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop - txsyncCh chan *txsync quitSync chan struct{} chainSync *chainSyncer @@ -140,7 +139,6 @@ func newHandler(config *handlerConfig) (*handler, error) { chain: config.Chain, peers: newPeerSet(), whitelist: config.Whitelist, - txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { @@ -408,9 +406,8 @@ func (h *handler) Start(maxPeers int) { go h.minedBroadcastLoop() // start sync handlers - h.wg.Add(2) + h.wg.Add(1) go h.chainSync.loop() - go h.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (h *handler) Stop() { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 038de46990..039091244f 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -80,7 +80,6 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, eth.ETH65) } func TestForkIDSplit66(t *testing.T) { testForkIDSplit(t, eth.ETH66) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -236,7 +235,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, eth.ETH65) } func TestRecvTransactions66(t *testing.T) { testRecvTransactions(t, eth.ETH66) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -294,7 +292,6 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions65(t *testing.T) { testSendTransactions(t, eth.ETH65) } func TestSendTransactions66(t *testing.T) { testSendTransactions(t, eth.ETH66) } func testSendTransactions(t *testing.T, protocol uint) { @@ -306,7 +303,7 @@ func testSendTransactions(t *testing.T, protocol uint) { insert := make([]*types.Transaction, 100) for nonce := range insert { - tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, txsyncPackSize/10)) + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, 10240)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) insert[nonce] = tx @@ -380,7 +377,6 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, eth.ETH65) } func TestTransactionPropagation66(t *testing.T) { testTransactionPropagation(t, eth.ETH66) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -521,8 +517,8 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() @@ -543,30 +539,39 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { t.Fatalf("failed to run protocol handshake") } - // Connect a new peer and check that we receive the checkpoint challenge. if checkpoint { - if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { - t.Fatalf("challenge mismatch: %v", err) + msg, err := p2pRemote.ReadMsg() + if err != nil { + t.Fatalf("failed to read checkpoint challenge: %v", err) + } + request := new(eth.GetBlockHeadersPacket66) + if err := msg.Decode(request); err != nil { + t.Fatalf("failed to decode checkpoint challenge: %v", err) + } + query := request.GetBlockHeadersPacket + if query.Origin.Number != response.Number.Uint64() || query.Amount != 1 || query.Skip != 0 || query.Reverse { + t.Fatalf("challenge mismatch: have [%d, %d, %d, %v] want [%d, %d, %d, %v]", + query.Origin.Number, query.Amount, query.Skip, query.Reverse, + response.Number.Uint64(), 1, 0, false) } // Create a block to reply to the challenge if no timeout is simulated. if !timeout { if empty { - if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else if match { - if err := remote.SendBlockHeaders([]*types.Header{response}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{response}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else { - if err := remote.SendBlockHeaders([]*types.Header{{Number: response.Number}}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{{Number: response.Number}}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } } } - // Wait until the test timeout passes to ensure proper cleanup time.Sleep(syncChallengeTimeout + 300*time.Millisecond) @@ -619,8 +624,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -671,7 +676,6 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH65) } func TestBroadcastMalformedBlock66(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH66) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 6bbaa2f555..8289300141 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -171,39 +171,21 @@ type Decoder interface { Time() time.Time } -var eth65 = map[uint64]msgHandler{ - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetNodeDataMsg: handleGetNodeData, - NodeDataMsg: handleNodeData, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth66 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - // eth66 messages with request-id - GetBlockHeadersMsg: handleGetBlockHeaders66, - BlockHeadersMsg: handleBlockHeaders66, - GetBlockBodiesMsg: handleGetBlockBodies66, - BlockBodiesMsg: handleBlockBodies66, - GetNodeDataMsg: handleGetNodeData66, - NodeDataMsg: handleNodeData66, - GetReceiptsMsg: handleGetReceipts66, - ReceiptsMsg: handleReceipts66, - GetPooledTransactionsMsg: handleGetPooledTransactions66, - PooledTransactionsMsg: handlePooledTransactions66, + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetNodeDataMsg: handleGetNodeData66, + NodeDataMsg: handleNodeData66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, } // handleMessage is invoked whenever an inbound message is received from a remote @@ -219,10 +201,11 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth65 - if peer.Version() >= ETH66 { - handlers = eth66 - } + var handlers = eth66 + //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added + // handlers = eth67 + //} + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 473be3f9b7..809f17e36c 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -110,7 +110,6 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, ETH65) } func TestGetBlockHeaders66(t *testing.T) { testGetBlockHeaders(t, ETH66) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -254,44 +253,30 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 123, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 123, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 123, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 123, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } // If the test used number origins, repeat with hashes as the too if tt.query.Origin.Hash == (common.Hash{}) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 456, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 456, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 456, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 456, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } } } @@ -299,7 +284,6 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, ETH65) } func TestGetBlockBodies66(t *testing.T) { testGetBlockBodies(t, ETH66) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -369,28 +353,20 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockBodiesMsg, hashes) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ - RequestId: 123, - GetBlockBodiesPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ - RequestId: 123, - BlockBodiesPacket: bodies, - }); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + RequestId: 123, + GetBlockBodiesPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + RequestId: 123, + BlockBodiesPacket: bodies, + }); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) } } } // Tests that the state trie nodes can be retrieved based on hashes. -func TestGetNodeData65(t *testing.T) { testGetNodeData(t, ETH65) } func TestGetNodeData66(t *testing.T) { testGetNodeData(t, ETH66) } func testGetNodeData(t *testing.T, protocol uint) { @@ -449,14 +425,10 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - if protocol <= ETH65 { - p2p.Send(peer.app, GetNodeDataMsg, hashes) - } else { - p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ - RequestId: 123, - GetNodeDataPacket: hashes, - }) - } + p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + RequestId: 123, + GetNodeDataPacket: hashes, + }) msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) @@ -464,18 +436,14 @@ func testGetNodeData(t *testing.T, protocol uint) { if msg.Code != NodeDataMsg { t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } - var data [][]byte - if protocol <= ETH65 { - if err := msg.Decode(&data); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - } else { - var res NodeDataPacket66 - if err := msg.Decode(&res); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - data = res.NodeDataPacket + var ( + data [][]byte + res NodeDataPacket66 + ) + if err := msg.Decode(&res); err != nil { + t.Fatalf("failed to decode response node data: %v", err) } + data = res.NodeDataPacket // Verify that all hashes correspond to the requested data, and reconstruct a state tree for i, want := range hashes { if hash := crypto.Keccak256Hash(data[i]); hash != want { @@ -506,7 +474,6 @@ func testGetNodeData(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, ETH65) } func TestGetBlockReceipts66(t *testing.T) { testGetBlockReceipts(t, ETH66) } func testGetBlockReceipts(t *testing.T, protocol uint) { @@ -566,21 +533,14 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetReceiptsMsg, hashes) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { - t.Errorf("receipts mismatch: %v", err) - } - } else { - p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ - RequestId: 123, - GetReceiptsPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ - RequestId: 123, - ReceiptsPacket: receipts, - }); err != nil { - t.Errorf("receipts mismatch: %v", err) - } + p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + RequestId: 123, + GetReceiptsPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + RequestId: 123, + ReceiptsPacket: receipts, + }); err != nil { + t.Errorf("receipts mismatch: %v", err) } } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index d7d993a23d..e54838cbc5 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -27,17 +27,6 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// handleGetBlockHeaders handles Block header query, collect the requested headers and reply -func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // Decode the complex header query - var query GetBlockHeadersPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockHeadersQuery(backend, &query, peer) - return peer.SendBlockHeaders(response) -} - // handleGetBlockHeaders66 is the eth/66 version of handleGetBlockHeaders func handleGetBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query @@ -135,16 +124,6 @@ func answerGetBlockHeadersQuery(backend Backend, query *GetBlockHeadersPacket, p return headers } -func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block body retrieval message - var query GetBlockBodiesPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockBodiesQuery(backend, query, peer) - return peer.SendBlockBodiesRLP(response) -} - func handleGetBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block body retrieval message var query GetBlockBodiesPacket66 @@ -174,16 +153,6 @@ func answerGetBlockBodiesQuery(backend Backend, query GetBlockBodiesPacket, peer return bodies } -func handleGetNodeData(backend Backend, msg Decoder, peer *Peer) error { - // Decode the trie node data retrieval message - var query GetNodeDataPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetNodeDataQuery(backend, query, peer) - return peer.SendNodeData(response) -} - func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { // Decode the trie node data retrieval message var query GetNodeDataPacket66 @@ -223,16 +192,6 @@ func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer return nodes } -func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block receipts retrieval message - var query GetReceiptsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetReceiptsQuery(backend, query, peer) - return peer.SendReceiptsRLP(response) -} - func handleGetReceipts66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket66 @@ -312,15 +271,6 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, ann) } -func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // A batch of headers arrived to one of our previous requests - res := new(BlockHeadersPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket66) @@ -332,15 +282,6 @@ func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockHeadersPacket) } -func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // A batch of block bodies arrived to one of our previous requests - res := new(BlockBodiesPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // A batch of block bodies arrived to one of our previous requests res := new(BlockBodiesPacket66) @@ -352,15 +293,6 @@ func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockBodiesPacket) } -func handleNodeData(backend Backend, msg Decoder, peer *Peer) error { - // A batch of node state data arrived to one of our previous requests - res := new(NodeDataPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { // A batch of node state data arrived to one of our previous requests res := new(NodeDataPacket66) @@ -372,15 +304,6 @@ func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.NodeDataPacket) } -func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { - // A batch of receipts arrived to one of our previous requests - res := new(ReceiptsPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests res := new(ReceiptsPacket66) @@ -409,16 +332,6 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) return backend.Handle(peer, ann) } -func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Decode the pooled transactions retrieval message - var query GetPooledTransactionsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - hashes, txs := answerGetPooledTransactions(backend, query, peer) - return peer.SendPooledTransactionsRLP(hashes, txs) -} - func handleGetPooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Decode the pooled transactions retrieval message var query GetPooledTransactionsPacket66 @@ -477,26 +390,6 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &txs) } -func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Transactions arrived, make sure we have a valid and fresh chain to handle them - if !backend.AcceptTxs() { - return nil - } - // Transactions can be processed, parse all of them and deliver to the pool - var txs PooledTransactionsPacket - if err := msg.Decode(&txs); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("%w: transaction %d is nil", errDecode, i) - } - peer.markTransaction(tx.Hash()) - } - return backend.Handle(peer, &txs) -} - func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Transactions arrived, make sure we have a valid and fresh chain to handle them if !backend.AcceptTxs() { diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 3bebda2dcc..05d473e057 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,7 +27,6 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake65(t *testing.T) { testHandshake(t, ETH65) } func TestHandshake66(t *testing.T) { testHandshake(t, ETH66) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index e619c183ba..98273ccfc9 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -108,9 +108,8 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe // Start up all the broadcasters go peer.broadcastBlocks() go peer.broadcastTransactions() - if version >= ETH65 { - go peer.announceTransactions() - } + go peer.announceTransactions() + return peer } @@ -252,22 +251,6 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { } } -// SendPooledTransactionsRLP sends requested transactions to the peer and adds the -// hashes in its transaction hash set for future reference. -// -// Note, the method assumes the hashes are correct and correspond to the list of -// transactions being sent. -func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding -} - // ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { // Mark all the transactions as known, but ensure we don't overflow our limits @@ -346,11 +329,6 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { } } -// SendBlockHeaders sends a batch of block headers to the remote peer. -func (p *Peer) SendBlockHeaders(headers []*types.Header) error { - return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) -} - // ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket66{ @@ -359,12 +337,6 @@ func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { }) } -// SendBlockBodiesRLP sends a batch of block contents to the remote peer from -// an already RLP encoded format. -func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { - return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding -} - // ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // Not packed into BlockBodiesPacket to avoid RLP decoding @@ -374,12 +346,6 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { }) } -// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the -// hashes requested. -func (p *Peer) SendNodeData(data [][]byte) error { - return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) -} - // ReplyNodeData is the eth/66 response to GetNodeData. func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ @@ -388,12 +354,6 @@ func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { }) } -// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the -// ones requested from an already RLP encoded format. -func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding -} - // ReplyReceiptsRLP is the eth/66 response to GetReceipts. func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ @@ -406,138 +366,102 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { // single header. It is used solely by the fetcher. func (p *Peer) RequestOneHeader(hash common.Hash) error { p.Log().Debug("Fetching single header", "hash", hash) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: hash}, - Amount: uint64(1), - Skip: uint64(0), - Reverse: false, - } - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: hash}, + Amount: uint64(1), + Skip: uint64(0), + Reverse: false, + }, + }) } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) -} - -// ExpectRequestHeadersByNumber is a testing method to mirror the recipient side -// of the RequestHeadersByNumber operation. -func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - req := &GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - return p2p.ExpectMsg(p.rw, GetBlockHeadersMsg, req) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestBodies fetches a batch of blocks' bodies corresponding to the hashes // specified. func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) - return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ - RequestId: id, - GetBlockBodiesPacket: hashes, - }) - } - return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) + return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ + RequestId: id, + GetBlockBodiesPacket: hashes, + }) } // RequestNodeData fetches a batch of arbitrary data from a node's known state // data, corresponding to the specified hashes. func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) - return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ - RequestId: id, - GetNodeDataPacket: hashes, - }) - } - return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) + return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ + RequestId: id, + GetNodeDataPacket: hashes, + }) } // RequestReceipts fetches a batch of transaction receipts from a remote node. func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) - return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ - RequestId: id, - GetReceiptsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) + return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ + RequestId: id, + GetReceiptsPacket: hashes, + }) } // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) - return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ - RequestId: id, - GetPooledTransactionsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ + RequestId: id, + GetPooledTransactionsPacket: hashes, + }) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index de1b0ed1ee..3c3da30fa5 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH65 = 65 ETH66 = 66 ) @@ -40,31 +39,28 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66, ETH65} +var ProtocolVersions = []uint{ETH66} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} +var protocolLengths = map[uint]uint64{ETH66: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 const ( - // Protocol messages in eth/64 - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TransactionsMsg = 0x02 - GetBlockHeadersMsg = 0x03 - BlockHeadersMsg = 0x04 - GetBlockBodiesMsg = 0x05 - BlockBodiesMsg = 0x06 - NewBlockMsg = 0x07 - GetNodeDataMsg = 0x0d - NodeDataMsg = 0x0e - GetReceiptsMsg = 0x0f - ReceiptsMsg = 0x10 - - // Protocol messages overloaded in eth/65 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TransactionsMsg = 0x02 + GetBlockHeadersMsg = 0x03 + BlockHeadersMsg = 0x04 + GetBlockBodiesMsg = 0x05 + BlockBodiesMsg = 0x06 + NewBlockMsg = 0x07 + GetNodeDataMsg = 0x0d + NodeDataMsg = 0x0e + GetReceiptsMsg = 0x0f + ReceiptsMsg = 0x10 NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 PooledTransactionsMsg = 0x0a @@ -128,7 +124,7 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } -// GetBlockHeadersPacket represents a block header query over eth/66 +// GetBlockHeadersPacket66 represents a block header query over eth/66 type GetBlockHeadersPacket66 struct { RequestId uint64 *GetBlockHeadersPacket diff --git a/eth/sync.go b/eth/sync.go index ab114b59f3..27941158f3 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -18,7 +18,6 @@ package eth import ( "math/big" - "math/rand" "sync/atomic" "time" @@ -28,23 +27,13 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" ) const ( forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available defaultMinSyncPeers = 5 // Amount of peers desired to start syncing - - // This is the target size for the packs of transactions sent by txsyncLoop64. - // A pack can get larger than this if a single transactions exceeds this size. - txsyncPackSize = 100 * 1024 ) -type txsync struct { - p *eth.Peer - txs []*types.Transaction -} - // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { // Assemble the set of transaction to broadcast or announce to the remote @@ -64,94 +53,11 @@ func (h *handler) syncTransactions(p *eth.Peer) { // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). - if p.Version() >= eth.ETH65 { - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() - } - p.AsyncSendPooledTransactionHashes(hashes) - return - } - // Out of luck, peer is running legacy protocols, drop the txs over - select { - case h.txsyncCh <- &txsync{p: p, txs: txs}: - case <-h.quitSync: - } -} - -// txsyncLoop64 takes care of the initial transaction sync for each new -// connection. When a new peer appears, we relay all currently pending -// transactions. In order to minimise egress bandwidth usage, we send -// the transactions in small packs to one peer at a time. -func (h *handler) txsyncLoop64() { - defer h.wg.Done() - - var ( - pending = make(map[enode.ID]*txsync) - sending = false // whether a send is active - pack = new(txsync) // the pack that is being sent - done = make(chan error, 1) // result of the send - ) - - // send starts a sending a pack of transactions from the sync. - send := func(s *txsync) { - if s.p.Version() >= eth.ETH65 { - panic("initial transaction syncer running on eth/65+") - } - // Fill pack with transactions up to the target size. - size := common.StorageSize(0) - pack.p = s.p - pack.txs = pack.txs[:0] - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.Peer.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendTransactions(pack.txs) }() - } - // pick chooses the next pending sync. - pick := func() *txsync { - if len(pending) == 0 { - return nil - } - n := rand.Intn(len(pending)) + 1 - for _, s := range pending { - if n--; n == 0 { - return s - } - } - return nil - } - - for { - select { - case s := <-h.txsyncCh: - pending[s.p.Peer.ID()] = s - if !sending { - send(s) - } - case err := <-done: - sending = false - // Stop tracking peers that cause send failures. - if err != nil { - pack.p.Log().Debug("Transaction send failed", "err", err) - delete(pending, pack.p.Peer.ID()) - } - // Schedule the next send. - if s := pick(); s != nil { - send(s) - } - case <-h.quitSync: - return - } + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() } + p.AsyncSendPooledTransactionHashes(hashes) } // chainSyncer coordinates blockchain sync components. diff --git a/eth/sync_test.go b/eth/sync_test.go index a0c6f86023..e96b9ee81f 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,6 @@ import ( ) // Tests that fast sync is disabled after a successful sync cycle. -func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, eth.ETH65) } func TestFastSyncDisabling66(t *testing.T) { testFastSyncDisabling(t, eth.ETH66) } // Tests that fast sync gets disabled as soon as a real block is successfully diff --git a/les/client_handler.go b/les/client_handler.go index e95996c51f..b903b11065 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -472,7 +472,7 @@ func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { handler: h, peer: p, } - h.downloader.RegisterLightPeer(p.id, eth.ETH65, pc) + h.downloader.RegisterLightPeer(p.id, eth.ETH66, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { From b69f5ca7d47e47335b9d19075245f9ee9fa9bd31 Mon Sep 17 00:00:00 2001 From: Ivan Bogatyy Date: Tue, 24 Aug 2021 21:54:00 +0300 Subject: [PATCH 12/30] cmd/utils: fix typo in variable name (#23451) --- cmd/geth/main.go | 2 +- cmd/geth/snapshot.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index da62b8d4d4..cfc1a62b6d 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -124,7 +124,7 @@ var ( utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, - utils.MinerNoVerfiyFlag, + utils.MinerNoVerifyFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 35d027fb16..9976db49f6 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -232,7 +232,7 @@ func verifyState(ctx *cli.Context) error { } } if err := snaptree.Verify(root); err != nil { - log.Error("Failed to verfiy state", "root", root, "err", err) + log.Error("Failed to verify state", "root", root, "err", err) return err } log.Info("Verified the state", "root", root) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 708edcc793..f39e4f402f 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -185,7 +185,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, - utils.MinerNoVerfiyFlag, + utils.MinerNoVerifyFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8a53f525f6..2953a00329 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -460,7 +460,7 @@ var ( Usage: "Time interval to recreate the block being mined", Value: ethconfig.Defaults.Miner.Recommit, } - MinerNoVerfiyFlag = cli.BoolFlag{ + MinerNoVerifyFlag = cli.BoolFlag{ Name: "miner.noverify", Usage: "Disable remote sealing verification", } @@ -1398,8 +1398,8 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { cfg.Recommit = ctx.GlobalDuration(MinerRecommitIntervalFlag.Name) } - if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) { - cfg.Noverify = ctx.GlobalBool(MinerNoVerfiyFlag.Name) + if ctx.GlobalIsSet(MinerNoVerifyFlag.Name) { + cfg.Noverify = ctx.GlobalBool(MinerNoVerifyFlag.Name) } if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) { log.Warn("The generic --miner.gastarget flag is deprecated and will be removed in the future!") From a789dcc9780e3d2364a7c6879af26aa0ed256fca Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 24 Aug 2021 20:54:55 +0200 Subject: [PATCH 13/30] metrics: fix compilation for GOOS=js (#23449) --- metrics/cpu_disabled.go | 2 +- metrics/cpu_enabled.go | 2 +- metrics/{cpu_windows.go => cputime_nop.go} | 2 ++ metrics/{cpu_syscall.go => cputime_unix.go} | 2 +- metrics/runtime_cgo.go | 3 +-- metrics/runtime_no_cgo.go | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) rename metrics/{cpu_windows.go => cputime_nop.go} (97%) rename metrics/{cpu_syscall.go => cputime_unix.go} (98%) diff --git a/metrics/cpu_disabled.go b/metrics/cpu_disabled.go index 6c3428993f..b0b483ee4b 100644 --- a/metrics/cpu_disabled.go +++ b/metrics/cpu_disabled.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build ios +// +build ios js package metrics diff --git a/metrics/cpu_enabled.go b/metrics/cpu_enabled.go index 02192928b7..edb824d9aa 100644 --- a/metrics/cpu_enabled.go +++ b/metrics/cpu_enabled.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build !ios +// +build !ios,!js package metrics diff --git a/metrics/cpu_windows.go b/metrics/cputime_nop.go similarity index 97% rename from metrics/cpu_windows.go rename to metrics/cputime_nop.go index fb29a52a82..1bf5537277 100644 --- a/metrics/cpu_windows.go +++ b/metrics/cputime_nop.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// +build windows js + package metrics // getProcessCPUTime returns 0 on Windows as there is no system call to resolve diff --git a/metrics/cpu_syscall.go b/metrics/cputime_unix.go similarity index 98% rename from metrics/cpu_syscall.go rename to metrics/cputime_unix.go index 50e04ef1d3..55871e668c 100644 --- a/metrics/cpu_syscall.go +++ b/metrics/cputime_unix.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build !windows +// +build !windows,!js package metrics diff --git a/metrics/runtime_cgo.go b/metrics/runtime_cgo.go index e3391f4e89..11722f76cc 100644 --- a/metrics/runtime_cgo.go +++ b/metrics/runtime_cgo.go @@ -1,5 +1,4 @@ -// +build cgo -// +build !appengine +// +build cgo,!appengine,!js package metrics diff --git a/metrics/runtime_no_cgo.go b/metrics/runtime_no_cgo.go index 616a3b4751..e760af554f 100644 --- a/metrics/runtime_no_cgo.go +++ b/metrics/runtime_no_cgo.go @@ -1,4 +1,4 @@ -// +build !cgo appengine +// +build !cgo appengine js package metrics From a5a52371789f2e2a3144c8a842c4238ba92cf301 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 25 Aug 2021 03:00:42 +0800 Subject: [PATCH 14/30] core, light, tests, trie: add state metrics (#23433) --- core/state/database.go | 2 +- core/state/metrics.go | 28 ++++++++++++ core/state/snapshot/generate.go | 2 +- core/state/snapshot/generate_test.go | 16 +++---- core/state/state_object.go | 16 ++++--- core/state/statedb.go | 26 ++++++++++- light/postprocess.go | 4 +- light/trie.go | 4 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- tests/fuzzers/trie/trie-fuzzer.go | 4 +- trie/committer.go | 63 ++++++++++++++------------ trie/iterator_test.go | 2 +- trie/secure_trie.go | 2 +- trie/trie.go | 13 +++--- trie/trie_test.go | 22 ++++----- 15 files changed, 131 insertions(+), 75 deletions(-) create mode 100644 core/state/metrics.go diff --git a/core/state/database.go b/core/state/database.go index 1a06e33409..e25d059c05 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -86,7 +86,7 @@ type Trie interface { // Commit writes all nodes to the trie's memory database, tracking the internal // and external (for account tries) references. - Commit(onleaf trie.LeafCallback) (common.Hash, error) + Commit(onleaf trie.LeafCallback) (common.Hash, int, error) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. diff --git a/core/state/metrics.go b/core/state/metrics.go new file mode 100644 index 0000000000..7b40ff37af --- /dev/null +++ b/core/state/metrics.go @@ -0,0 +1,28 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import "github.com/ethereum/go-ethereum/metrics" + +var ( + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil) + storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil) +) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 7e29e51b21..3e11b4ac6b 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -436,7 +436,7 @@ func (dl *diskLayer) generateRange(root common.Hash, prefix []byte, kind string, for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, _ := snapTrie.Commit(nil) + root, _, _ := snapTrie.Commit(nil) snapTrieDb.Commit(root, false, nil) } tr := result.tr diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index a92517b315..582da6a2e7 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -60,7 +60,7 @@ func TestGeneration(t *testing.T) { acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} val, _ = rlp.EncodeToBytes(acc) accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd triedb.Commit(root, false, nil) if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { @@ -128,7 +128,7 @@ func TestGenerateExistentState(t *testing.T) { rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2")) rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3")) - root, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd triedb.Commit(root, false, nil) snap := generateSnapshot(diskdb, triedb, 16, root) @@ -215,12 +215,12 @@ func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte { for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } - root, _ := stTrie.Commit(nil) + root, _, _ := stTrie.Commit(nil) return root.Bytes() } func (t *testHelper) Generate() (common.Hash, *diskLayer) { - root, _ := t.accTrie.Commit(nil) + root, _, _ := t.accTrie.Commit(nil) t.triedb.Commit(root, false, nil) snap := generateSnapshot(t.diskdb, t.triedb, 16, root) return root, snap @@ -575,7 +575,7 @@ func TestGenerateWithExtraAccounts(t *testing.T) { rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) // To verify the test: If we now inspect the snap db, there should exist extraneous storage items @@ -637,7 +637,7 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { rawdb.WriteAccountSnapshot(diskdb, key, val) } } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) @@ -690,7 +690,7 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val) } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) @@ -734,7 +734,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk) } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) diff --git a/core/state/state_object.go b/core/state/state_object.go index 38621ffb61..7b366d64c7 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -329,7 +329,7 @@ func (s *stateObject) finalise(prefetch bool) { // It will return nil if the trie has not been loaded and no changes have been made func (s *stateObject) updateTrie(db Database) Trie { // Make sure all dirty slots are finalized into the pending storage area - s.finalise(false) // Don't prefetch any more, pull directly if need be + s.finalise(false) // Don't prefetch anymore, pull directly if need be if len(s.pendingStorage) == 0 { return s.trie } @@ -354,10 +354,12 @@ func (s *stateObject) updateTrie(db Database) Trie { var v []byte if (value == common.Hash{}) { s.setError(tr.TryDelete(key[:])) + s.db.StorageDeleted += 1 } else { // Encoding []byte cannot fail, ok to ignore the error. v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) s.setError(tr.TryUpdate(key[:], v)) + s.db.StorageUpdated += 1 } // If state snapshotting is active, cache the data til commit if s.db.snap != nil { @@ -368,7 +370,7 @@ func (s *stateObject) updateTrie(db Database) Trie { s.db.snapStorage[s.addrHash] = storage } } - storage[crypto.HashData(hasher, key[:])] = v // v will be nil if value is 0x00 + storage[crypto.HashData(hasher, key[:])] = v // v will be nil if it's deleted } usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure } @@ -396,23 +398,23 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. -func (s *stateObject) CommitTrie(db Database) error { +func (s *stateObject) CommitTrie(db Database) (int, error) { // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { - return nil + return 0, nil } if s.dbErr != nil { - return s.dbErr + return 0, s.dbErr } // Track the amount of time wasted on committing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, err := s.trie.Commit(nil) + root, committed, err := s.trie.Commit(nil) if err == nil { s.data.Root = root } - return err + return committed, err } // AddBalance adds amount to s's balance. diff --git a/core/state/statedb.go b/core/state/statedb.go index 61f0f5b9f6..a6d47179ba 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -117,6 +117,11 @@ type StateDB struct { SnapshotAccountReads time.Duration SnapshotStorageReads time.Duration SnapshotCommits time.Duration + + AccountUpdated int + StorageUpdated int + AccountDeleted int + StorageDeleted int } // New creates a new state from a given trie. @@ -860,8 +865,10 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; obj.deleted { s.deleteStateObject(obj) + s.AccountDeleted += 1 } else { s.updateStateObject(obj) + s.AccountUpdated += 1 } usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure } @@ -903,6 +910,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time + var storageCommitted int codeWriter := s.db.TrieDB().DiskDB().NewBatch() for addr := range s.stateObjectsDirty { if obj := s.stateObjects[addr]; !obj.deleted { @@ -912,9 +920,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - if err := obj.CommitTrie(s.db); err != nil { + committed, err := obj.CommitTrie(s.db) + if err != nil { return common.Hash{}, err } + storageCommitted += committed } } if len(s.stateObjectsDirty) > 0 { @@ -933,7 +943,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. var account Account - root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { + root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { if err := rlp.DecodeBytes(leaf, &account); err != nil { return nil } @@ -942,8 +952,20 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } return nil }) + if err != nil { + return common.Hash{}, err + } if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) + + accountUpdatedMeter.Mark(int64(s.AccountUpdated)) + storageUpdatedMeter.Mark(int64(s.StorageUpdated)) + accountDeletedMeter.Mark(int64(s.AccountDeleted)) + storageDeletedMeter.Mark(int64(s.StorageDeleted)) + accountCommittedMeter.Mark(int64(accountCommitted)) + storageCommittedMeter.Mark(int64(storageCommitted)) + s.AccountUpdated, s.AccountDeleted = 0, 0 + s.StorageUpdated, s.StorageDeleted = 0, 0 } // If snapshotting is enabled, update the snapshot tree with this new version if s.snap != nil { diff --git a/light/postprocess.go b/light/postprocess.go index 891c8a5869..ce38d091e8 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -217,7 +217,7 @@ func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) e // Commit implements core.ChainIndexerBackend func (c *ChtIndexerBackend) Commit() error { - root, err := c.trie.Commit(nil) + root, _, err := c.trie.Commit(nil) if err != nil { return err } @@ -454,7 +454,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.trie.Delete(encKey[:]) } } - root, err := b.trie.Commit(nil) + root, _, err := b.trie.Commit(nil) if err != nil { return err } diff --git a/light/trie.go b/light/trie.go index 0516b94486..39e928bbe1 100644 --- a/light/trie.go +++ b/light/trie.go @@ -125,9 +125,9 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } -func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) { +func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, int, error) { if t.trie == nil { - return t.id.Root, nil + return t.id.Root, 0, nil } return t.trie.Commit(onleaf) } diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 5cea7769c2..e73ef4851a 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -173,7 +173,7 @@ func (f *fuzzer) fuzz() int { return 0 } // Flush trie -> database - rootA, err := trieA.Commit(nil) + rootA, _, err := trieA.Commit(nil) if err != nil { panic(err) } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index d0b614c9ac..e993af47cf 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -162,11 +162,11 @@ func runRandTest(rt randTest) error { rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) } case opCommit: - _, rt[i].err = tr.Commit(nil) + _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() case opReset: - hash, err := tr.Commit(nil) + hash, _, err := tr.Commit(nil) if err != nil { return err } diff --git a/trie/committer.go b/trie/committer.go index ce4065f5fd..0721990a21 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -72,24 +72,24 @@ func returnCommitterToPool(h *committer) { committerPool.Put(h) } -// commit collapses a node down into a hash node and inserts it into the database -func (c *committer) Commit(n node, db *Database) (hashNode, error) { +// Commit collapses a node down into a hash node and inserts it into the database +func (c *committer) Commit(n node, db *Database) (hashNode, int, error) { if db == nil { - return nil, errors.New("no db provided") + return nil, 0, errors.New("no db provided") } - h, err := c.commit(n, db) + h, committed, err := c.commit(n, db) if err != nil { - return nil, err + return nil, 0, err } - return h.(hashNode), nil + return h.(hashNode), committed, nil } // commit collapses a node down into a hash node and inserts it into the database -func (c *committer) commit(n node, db *Database) (node, error) { +func (c *committer) commit(n node, db *Database) (node, int, error) { // if this path is clean, use available cached data hash, dirty := n.cache() if hash != nil && !dirty { - return hash, nil + return hash, 0, nil } // Commit children, then parent, and remove remove the dirty flag. switch cn := n.(type) { @@ -97,37 +97,38 @@ func (c *committer) commit(n node, db *Database) (node, error) { // Commit child collapsed := cn.copy() - // If the child is fullnode, recursively commit. - // Otherwise it can only be hashNode or valueNode. + // If the child is fullNode, recursively commit, + // otherwise it can only be hashNode or valueNode. + var childCommitted int if _, ok := cn.Val.(*fullNode); ok { - childV, err := c.commit(cn.Val, db) + childV, committed, err := c.commit(cn.Val, db) if err != nil { - return nil, err + return nil, 0, err } - collapsed.Val = childV + collapsed.Val, childCommitted = childV, committed } // The key needs to be copied, since we're delivering it to database collapsed.Key = hexToCompact(cn.Key) hashedNode := c.store(collapsed, db) if hn, ok := hashedNode.(hashNode); ok { - return hn, nil + return hn, childCommitted + 1, nil } - return collapsed, nil + return collapsed, childCommitted, nil case *fullNode: - hashedKids, err := c.commitChildren(cn, db) + hashedKids, childCommitted, err := c.commitChildren(cn, db) if err != nil { - return nil, err + return nil, 0, err } collapsed := cn.copy() collapsed.Children = hashedKids hashedNode := c.store(collapsed, db) if hn, ok := hashedNode.(hashNode); ok { - return hn, nil + return hn, childCommitted + 1, nil } - return collapsed, nil + return collapsed, childCommitted, nil case hashNode: - return cn, nil + return cn, 0, nil default: // nil, valuenode shouldn't be committed panic(fmt.Sprintf("%T: invalid node: %v", n, n)) @@ -135,8 +136,11 @@ func (c *committer) commit(n node, db *Database) (node, error) { } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, error) { - var children [17]node +func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, error) { + var ( + committed int + children [17]node + ) for i := 0; i < 16; i++ { child := n.Children[i] if child == nil { @@ -144,25 +148,26 @@ func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, error) } // If it's the hashed child, save the hash value directly. // Note: it's impossible that the child in range [0, 15] - // is a valuenode. + // is a valueNode. if hn, ok := child.(hashNode); ok { children[i] = hn continue } // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's - // possible the type is not hashnode. - hashed, err := c.commit(child, db) + // possible the type is not hashNode. + hashed, childCommitted, err := c.commit(child, db) if err != nil { - return children, err + return children, 0, err } children[i] = hashed + committed += childCommitted } // For the 17th child, it's possible the type is valuenode. if n.Children[16] != nil { children[16] = n.Children[16] } - return children, nil + return children, committed, nil } // store hashes the node n and if we have a storage layer specified, it writes @@ -176,7 +181,7 @@ func (c *committer) store(n node, db *Database) node { ) if hash == nil { // This was not generated - must be a small node stored in the parent. - // In theory we should apply the leafCall here if it's not nil(embedded + // In theory, we should apply the leafCall here if it's not nil(embedded // node usually contains value). But small value(less than 32bytes) is // not our target. return n @@ -224,7 +229,7 @@ func (c *committer) commitLoop(db *Database) { } case *fullNode: // For children in range [0, 15], it's impossible - // to contain valuenode. Only check the 17th child. + // to contain valueNode. Only check the 17th child. if n.Children[16] != nil { c.onleaf(nil, nil, n.Children[16].(valueNode), hash) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 2518f7bac8..95cafdd3bd 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -393,7 +393,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { for _, val := range testdata1 { ctr.Update([]byte(val.k), []byte(val.v)) } - root, _ := ctr.Commit(nil) + root, _, _ := ctr.Commit(nil) if !memonly { triedb.Commit(root, true, nil) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index e38471c1b7..4d0050bc59 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -144,7 +144,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { // // Committing flushes nodes from memory. Subsequent Get calls will load nodes // from the database. -func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { +func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock diff --git a/trie/trie.go b/trie/trie.go index e492a532cb..7ea7efa835 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -514,12 +514,12 @@ func (t *Trie) Hash() common.Hash { // Commit writes all nodes to the trie's memory database, tracking the internal // and external (for account tries) references. -func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { +func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) { if t.db == nil { panic("commit called on trie with nil database") } if t.root == nil { - return emptyRoot, nil + return emptyRoot, 0, nil } // Derive the hash for all dirty nodes first. We hold the assumption // in the following procedure that all nodes are hashed. @@ -531,7 +531,7 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { // up goroutines. This can happen e.g. if we load a trie for reading storage // values, but don't write to it. if _, dirty := t.root.cache(); !dirty { - return rootHash, nil + return rootHash, 0, nil } var wg sync.WaitGroup if onleaf != nil { @@ -543,8 +543,7 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { h.commitLoop(t.db) }() } - var newRoot hashNode - newRoot, err = h.Commit(t.root, t.db) + newRoot, committed, err := h.Commit(t.root, t.db) if onleaf != nil { // The leafch is created in newCommitter if there was an onleaf callback // provided. The commitLoop only _reads_ from it, and the commit @@ -554,10 +553,10 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { wg.Wait() } if err != nil { - return common.Hash{}, err + return common.Hash{}, 0, err } t.root = newRoot - return rootHash, nil + return rootHash, committed, nil } // hashRoot calculates the root hash of the given trie diff --git a/trie/trie_test.go b/trie/trie_test.go index 492b423c2f..ae1871e423 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -90,7 +90,7 @@ func testMissingNode(t *testing.T, memonly bool) { trie, _ := New(common.Hash{}, triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) if !memonly { triedb.Commit(root, true, nil) } @@ -172,7 +172,7 @@ func TestInsert(t *testing.T) { updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, err := trie.Commit(nil) + root, _, err := trie.Commit(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -270,7 +270,7 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - exp, err := trie.Commit(nil) + exp, _, err := trie.Commit(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -285,7 +285,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, err := trie2.Commit(nil) + hash, _, err := trie2.Commit(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -429,11 +429,11 @@ func runRandTest(rt randTest) bool { rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) } case opCommit: - _, rt[i].err = tr.Commit(nil) + _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() case opReset: - hash, err := tr.Commit(nil) + hash, _, err := tr.Commit(nil) if err != nil { rt[i].err = err return false @@ -633,7 +633,7 @@ func TestCommitAfterHash(t *testing.T) { if exp != root { t.Errorf("got %x, exp %x", root, exp) } - root, _ = trie.Commit(nil) + root, _, _ = trie.Commit(nil) if exp != root { t.Errorf("got %x, exp %x", root, exp) } @@ -740,7 +740,7 @@ func TestCommitSequence(t *testing.T) { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -792,7 +792,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { trie.Update(key, val) } // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -834,7 +834,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { stTrie.TryUpdate(key, val) } // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, nil) // And flush stacktrie -> disk @@ -879,7 +879,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie.TryUpdate(key, []byte{0x1}) stTrie.TryUpdate(key, []byte{0x1}) // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, nil) // And flush stacktrie -> disk From fe2f153b556a0f1225c571fd5df7967a33244bf1 Mon Sep 17 00:00:00 2001 From: ucwong Date: Wed, 25 Aug 2021 03:02:46 +0800 Subject: [PATCH 15/30] go.mod: update goleveldb (#23417) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ee9ffa0454..ce19afc850 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 - github.com/golang/snappy v0.0.3 + github.com/golang/snappy v0.0.4 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa github.com/google/uuid v1.1.5 github.com/gorilla/websocket v1.4.2 @@ -59,7 +59,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 - github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 diff --git a/go.sum b/go.sum index d6e9639d2e..3d8a04236f 100644 --- a/go.sum +++ b/go.sum @@ -185,9 +185,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -389,8 +389,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= From 83ad92c42191a47b3ded38a9fd22fd0bed2f05f6 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 25 Aug 2021 17:56:25 +0800 Subject: [PATCH 16/30] les: fix data races in tests (#23457) --- les/client_handler.go | 11 ++++++---- les/fetcher.go | 8 ------- les/fetcher_test.go | 49 +++++++++---------------------------------- les/odr_test.go | 6 +++--- les/server_handler.go | 3 +-- les/sync_test.go | 8 +++---- les/test_helper.go | 12 +++++------ les/ulc_test.go | 27 +++++++++++++++--------- 8 files changed, 48 insertions(+), 76 deletions(-) diff --git a/les/client_handler.go b/les/client_handler.go index b903b11065..4a550b2074 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -100,11 +100,11 @@ func (h *clientHandler) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) defer peer.close() h.wg.Add(1) defer h.wg.Done() - err := h.handle(peer) + err := h.handle(peer, false) return err } -func (h *clientHandler) handle(p *serverPeer) error { +func (h *clientHandler) handle(p *serverPeer, noInitAnnounce bool) error { if h.backend.peers.len() >= h.backend.config.LightPeers && !p.Peer.Info().Network.Trusted { return p2p.DiscTooManyPeers } @@ -143,8 +143,11 @@ func (h *clientHandler) handle(p *serverPeer) error { connectionTimer.Update(time.Duration(mclock.Now() - connectedAt)) serverConnectionGauge.Update(int64(h.backend.peers.len())) }() - h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) - + // It's mainly used in testing which requires discarding initial + // signal to prevent syncing. + if !noInitAnnounce { + h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) + } // Mark the peer starts to be served. atomic.StoreUint32(&p.serving, 1) defer atomic.StoreUint32(&p.serving, 0) diff --git a/les/fetcher.go b/les/fetcher.go index a6d1c93c4b..5eea996748 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -153,9 +153,7 @@ type lightFetcher struct { synchronise func(peer *serverPeer) // Test fields or hooks - noAnnounce bool newHeadHook func(*types.Header) - newAnnounce func(*serverPeer, *announceData) } // newLightFetcher creates a light fetcher instance. @@ -474,12 +472,6 @@ func (f *lightFetcher) mainloop() { // announce processes a new announcement message received from a peer. func (f *lightFetcher) announce(p *serverPeer, head *announceData) { - if f.newAnnounce != nil { - f.newAnnounce(p, head) - } - if f.noAnnounce { - return - } select { case f.announceCh <- &announce{peerid: p.ID(), trust: p.trusted, data: head}: case <-f.closeCh: diff --git a/les/fetcher_test.go b/les/fetcher_test.go index d3a74d25c2..ef700651e3 100644 --- a/les/fetcher_test.go +++ b/les/fetcher_test.go @@ -74,14 +74,12 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() - // Create connected peer pair. - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - p1, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler) + // Create connected peer pair, the initial signal from LES server + // is discarded to prevent syncing. + p1, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler, true) if err != nil { t.Fatalf("Failed to create peer pair %v", err) } - c.handler.fetcher.noAnnounce = false - importCh := make(chan interface{}) c.handler.fetcher.newHeadHook = func(header *types.Header) { importCh <- header @@ -114,14 +112,12 @@ func testGappedAnnouncements(t *testing.T, protocol int) { s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() - // Create connected peer pair. - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - peer, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler) + // Create connected peer pair, the initial signal from LES server + // is discarded to prevent syncing. + peer, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler, true) if err != nil { t.Fatalf("Failed to create peer pair %v", err) } - c.handler.fetcher.noAnnounce = false - done := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { done <- header } @@ -141,29 +137,11 @@ func testGappedAnnouncements(t *testing.T, protocol int) { verifyChainHeight(t, c.handler.fetcher, 4) // Send a reorged announcement - var newAnno = make(chan struct{}, 1) - c.handler.fetcher.noAnnounce = true - c.handler.fetcher.newAnnounce = func(*serverPeer, *announceData) { - newAnno <- struct{}{} - } blocks, _ := core.GenerateChain(rawdb.ReadChainConfig(s.db, s.backend.Blockchain().Genesis().Hash()), s.backend.Blockchain().GetBlockByNumber(3), ethash.NewFaker(), s.db, 2, func(i int, gen *core.BlockGen) { gen.OffsetTime(-9) // higher block difficulty }) s.backend.Blockchain().InsertChain(blocks) - <-newAnno - c.handler.fetcher.noAnnounce = false - c.handler.fetcher.newAnnounce = nil - - latest = blocks[len(blocks)-1].Header() - hash, number = latest.Hash(), latest.Number.Uint64() - td = rawdb.ReadTd(s.db, hash, number) - - announce = announceData{hash, number, td, 1, nil} - if peer.cpeer.announceType == announceTypeSigned { - announce.sign(s.handler.server.privateKey) - } - peer.cpeer.sendAnnounce(announce) <-done // Wait syncing verifyChainHeight(t, c.handler.fetcher, 5) @@ -206,20 +184,15 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { teardowns[i]() } }() - - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - // Connect all server instances. for i := 0; i < len(servers); i++ { - sp, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol) + sp, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) if err != nil { t.Fatalf("connect server and client failed, err %s", err) } cpeers = append(cpeers, cp) speers = append(speers, sp) } - c.handler.fetcher.noAnnounce = false - newHead := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { newHead <- header } @@ -262,14 +235,12 @@ func testInvalidAnnounces(t *testing.T, protocol int) { s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() - // Create connected peer pair. - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - peer, _, err := newTestPeerPair("peer", lpv3, s.handler, c.handler) + // Create connected peer pair, the initial signal from LES server + // is discarded to prevent syncing. + peer, _, err := newTestPeerPair("peer", lpv3, s.handler, c.handler, true) if err != nil { t.Fatalf("Failed to create peer pair %v", err) } - c.handler.fetcher.noAnnounce = false - done := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { done <- header } diff --git a/les/odr_test.go b/les/odr_test.go index ea88495d19..ad77abf5b9 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -401,9 +401,9 @@ func testGetTxStatusFromUnindexedPeers(t *testing.T, protocol int) { closeFns = append(closeFns, closePeer) // Create a one-time routine for serving message - go func(i int, peer *testPeer) { - serveMsg(peer, testspec.txLookups[i]) - }(i, peer) + go func(i int, peer *testPeer, lookup uint64) { + serveMsg(peer, lookup) + }(i, peer, testspec.txLookups[i]) } // Send out the GetTxStatus requests, compare the result with diff --git a/les/server_handler.go b/les/server_handler.go index 80fcf1c44e..fa20fd7b3b 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" @@ -408,7 +407,7 @@ func (h *serverHandler) broadcastLoop() { defer headSub.Unsubscribe() var ( - lastHead *types.Header + lastHead = h.blockchain.CurrentHeader() lastTd = common.Big0 ) for { diff --git a/les/sync_test.go b/les/sync_test.go index d3bb90df02..3fc2a9c154 100644 --- a/les/sync_test.go +++ b/les/sync_test.go @@ -116,7 +116,7 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { } // Create connected peer pair. - peer1, peer2, err := newTestPeerPair("peer", protocol, server.handler, client.handler) + peer1, peer2, err := newTestPeerPair("peer", protocol, server.handler, client.handler, false) if err != nil { t.Fatalf("Failed to connect testing peers %v", err) } @@ -218,7 +218,7 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool, protocol int) { } } // Create connected peer pair. - if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler, false); err != nil { t.Fatalf("Failed to connect testing peers %v", err) } select { @@ -291,7 +291,7 @@ func testSyncFromConfiguredCheckpoint(t *testing.T, protocol int) { } } // Create connected peer pair. - if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler, false); err != nil { t.Fatalf("Failed to connect testing peers %v", err) } @@ -364,7 +364,7 @@ func testSyncAll(t *testing.T, protocol int) { } } // Create connected peer pair. - if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler, false); err != nil { t.Fatalf("Failed to connect testing peers %v", err) } diff --git a/les/test_helper.go b/les/test_helper.go index 9ff2583b97..21d0f191c9 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -398,7 +398,7 @@ func (p *testPeer) close() { p.app.Close() } -func newTestPeerPair(name string, version int, server *serverHandler, client *clientHandler) (*testPeer, *testPeer, error) { +func newTestPeerPair(name string, version int, server *serverHandler, client *clientHandler, noInitAnnounce bool) (*testPeer, *testPeer, error) { // Create a message pipe to communicate through app, net := p2p.MsgPipe() @@ -423,16 +423,16 @@ func newTestPeerPair(name string, version int, server *serverHandler, client *cl select { case <-client.closeCh: errc2 <- p2p.DiscQuitting - case errc2 <- client.handle(peer2): + case errc2 <- client.handle(peer2, noInitAnnounce): } }() // Ensure the connection is established or exits when any error occurs for { select { case err := <-errc1: - return nil, nil, fmt.Errorf("Failed to establish protocol connection %v", err) + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) case err := <-errc2: - return nil, nil, fmt.Errorf("Failed to establish protocol connection %v", err) + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) default: } if atomic.LoadUint32(&peer1.serving) == 1 && atomic.LoadUint32(&peer2.serving) == 1 { @@ -473,7 +473,7 @@ func (client *testClient) newRawPeer(t *testing.T, name string, version int, rec select { case <-client.handler.closeCh: errCh <- p2p.DiscQuitting - case errCh <- client.handler.handle(peer): + case errCh <- client.handler.handle(peer, false): } }() tp := &testPeer{ @@ -623,7 +623,7 @@ func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testC if config.connect { done := make(chan struct{}) client.syncEnd = func(_ *types.Header) { close(done) } - cpeer, speer, err = newTestPeerPair("peer", config.protocol, server, client) + cpeer, speer, err = newTestPeerPair("peer", config.protocol, server, client, false) if err != nil { t.Fatalf("Failed to connect testing peers %v", err) } diff --git a/les/ulc_test.go b/les/ulc_test.go index d7308fa593..ecef58d979 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -20,6 +20,7 @@ import ( "crypto/rand" "fmt" "net" + "sync/atomic" "testing" "time" @@ -65,7 +66,7 @@ func testULCAnnounceThreshold(t *testing.T, protocol int) { // Connect all servers. for i := 0; i < len(servers); i++ { - connect(servers[i].handler, nodes[i].ID(), c.handler, protocol) + connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, false) } for i := 0; i < len(servers); i++ { for j := 0; j < testcase.height[i]; j++ { @@ -86,7 +87,7 @@ func testULCAnnounceThreshold(t *testing.T, protocol int) { } } -func connect(server *serverHandler, serverId enode.ID, client *clientHandler, protocol int) (*serverPeer, *clientPeer, error) { +func connect(server *serverHandler, serverId enode.ID, client *clientHandler, protocol int, noInitAnnounce bool) (*serverPeer, *clientPeer, error) { // Create a message pipe to communicate through app, net := p2p.MsgPipe() @@ -110,16 +111,22 @@ func connect(server *serverHandler, serverId enode.ID, client *clientHandler, pr select { case <-client.closeCh: errc1 <- p2p.DiscQuitting - case errc1 <- client.handle(peer1): + case errc1 <- client.handle(peer1, noInitAnnounce): } }() - - select { - case <-time.After(time.Millisecond * 100): - case err := <-errc1: - return nil, nil, fmt.Errorf("peerLight handshake error: %v", err) - case err := <-errc2: - return nil, nil, fmt.Errorf("peerFull handshake error: %v", err) + // Ensure the connection is established or exits when any error occurs + for { + select { + case err := <-errc1: + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) + case err := <-errc2: + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) + default: + } + if atomic.LoadUint32(&peer1.serving) == 1 && atomic.LoadUint32(&peer2.serving) == 1 { + break + } + time.Sleep(50 * time.Millisecond) } return peer1, peer2, nil } From 45f34430fd5fc5a93b666eba90f2c76bd3f770f3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 13:21:09 +0200 Subject: [PATCH 17/30] build: upgrade to golangci-lint v1.42.0 (#23461) The new linter version is built with go 1.17 and thus includes the go vet check for mismatched +build and go:build lines. Fortunately, no new warnings are reported with this update. --- build/checksums.txt | 37 ++++++++++++++++++++----------------- build/ci.go | 2 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index e667b30ce6..6df097b889 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -14,20 +14,23 @@ ccdd2b76de1941b60734408fda0d750aaa69330d8a07430eed4c56bdb3502f6f go1.16.4.freeb 80cfac566e344096a8df8f37bbd21f89e76a6fbe601406565d71a87a665fc125 go1.16.4.linux-ppc64le.tar.gz d6431881b3573dc29ecc24fbeab5e5ec25d8c9273aa543769c86a1a3bbac1ddf go1.16.4.linux-s390x.tar.gz -7e9a47ab540aa3e8472fbf8120d28bed3b9d9cf625b955818e8bc69628d7187c golangci-lint-1.39.0-darwin-amd64.tar.gz -574daa2c9c299b01672a6daeb1873b5f12e413cdb6dc0e30f2ff163956778064 golangci-lint-1.39.0-darwin-arm64.tar.gz -6225f7014987324ab78e9b511f294e3f25be013728283c33918c67c8576d543e golangci-lint-1.39.0-freebsd-386.tar.gz -6b3e76e1e5eaf0159411c8e2727f8d533989d3bb19f10e9caa6e0b9619ee267d golangci-lint-1.39.0-freebsd-amd64.tar.gz -a301cacfff87ed9b00313d95278533c25a4527a06b040a17d969b4b7e1b8a90d golangci-lint-1.39.0-freebsd-armv7.tar.gz -25bfd96a29c3112f508d5e4fc860dbad7afce657233c343acfa20715717d51e7 golangci-lint-1.39.0-freebsd-armv6.tar.gz -9687e4ff15545cfc722b0e46107a94195166a505023b48a316579af25ad09505 golangci-lint-1.39.0-linux-armv7.tar.gz -a7fa7ab2bfc99cbe5e5bcbf5684f5a997f920afbbe2f253d2feb1001d5e3c8b3 golangci-lint-1.39.0-linux-armv6.tar.gz -c8f9634115beddb4ed9129c1f7ecd4c97c99d07aeef33e3707234097eeb51b7b golangci-lint-1.39.0-linux-mips64le.tar.gz -d1234c213b74751f1af413302dde0e9a6d4d29aecef034af7abb07dc1b6e887f golangci-lint-1.39.0-linux-arm64.tar.gz -df25d9267168323b163147acb823ab0215a8a3bb6898a4a9320afdfedde66817 golangci-lint-1.39.0-linux-386.tar.gz -1767e75fba357b7651b1a796d38453558f371c60af805505ec99e166908c04b5 golangci-lint-1.39.0-linux-ppc64le.tar.gz -25fd75bf3186b3d930ecae10185689968fd18fd8fa6f9f555d6beb04348c20f6 golangci-lint-1.39.0-linux-s390x.tar.gz -3a73aa7468087caa62673c8adea99b4e4dff846dc72707222db85f8679b40cbf golangci-lint-1.39.0-linux-amd64.tar.gz -578caceccf81739bda67dbfec52816709d03608c6878888ecdc0e186a094a41b golangci-lint-1.39.0-linux-mips64.tar.gz -494b66ba0e32c8ddf6c4f6b1d05729b110900f6017eda943057e43598c17d7a8 golangci-lint-1.39.0-windows-386.zip -52ec2e13a3cbb47147244dff8cfc35103563deb76e0459133058086fc35fb2c7 golangci-lint-1.39.0-windows-amd64.zip +d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa golangci-lint-1.42.0-darwin-amd64.tar.gz +e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea golangci-lint-1.42.0-darwin-arm64.tar.gz +14d912a3fa856830339472fc4dc341933adf15f37bdb7130bbbfcf960ecf4809 golangci-lint-1.42.0-freebsd-386.tar.gz +337257fccc9baeb5ee1cd7e70c153e9d9f59d3afde46d631659500048afbdf80 golangci-lint-1.42.0-freebsd-amd64.tar.gz +6debcc266b629359fdd8eef4f4abb05a621604079d27016265afb5b4593b0eff golangci-lint-1.42.0-freebsd-armv6.tar.gz +878f0e190169db2ce9dde8cefbd99adc4fe28b90b68686bbfcfcc2085e6d693e golangci-lint-1.42.0-freebsd-armv7.tar.gz +42c78e31faf62b225363eff1b1d2aa74f9dbcb75686c8914aa3e90d6af65cece golangci-lint-1.42.0-linux-386.tar.gz +6937f62f8e2329e94822dc11c10b871ace5557ae1fcc4ee2f9980cd6aecbc159 golangci-lint-1.42.0-linux-amd64.tar.gz +2cf8d23d96cd854a537b355dab2962b960b88a06b615232599f066afd233f246 golangci-lint-1.42.0-linux-arm64.tar.gz +08b003d1ed61367473886defc957af5301066e62338e5d96a319c34dadc4c1d1 golangci-lint-1.42.0-linux-armv6.tar.gz +c7c00ec4845e806a1f32685f5b150219e180bd6d6a9d584be8d27f0c41d7a1bf golangci-lint-1.42.0-linux-armv7.tar.gz +3650fcf29eb3d8ee326d77791a896b15259eb2d5bf77437dc72e7efe5af6bd40 golangci-lint-1.42.0-linux-mips64.tar.gz +f51ae003fdbca4fef78ba73e2eb736a939c8eaa178cd452234213b489da5a420 golangci-lint-1.42.0-linux-mips64le.tar.gz +1b0bb7b8b22cc4ea7da44fd5ad5faaf6111d0677e01cc6f961b62a96537de2c6 golangci-lint-1.42.0-linux-ppc64le.tar.gz +8cb56927eb75e572450efbe0ff0f9cf3f56dc9faa81d9e8d30d6559fc1d06e6d golangci-lint-1.42.0-linux-riscv64.tar.gz +5ac41cd31825a176b21505a371a7b307cd9cdf17df0f35bbb3bf1466f9356ccc golangci-lint-1.42.0-linux-s390x.tar.gz +e1cebd2af621ac4b64c20937df92c3819264f2174c92f51e196db1e64ae097e0 golangci-lint-1.42.0-windows-386.zip +7e70fcde8e87a17cae0455df07d257ebc86669f3968d568e12727fa24bbe9883 golangci-lint-1.42.0-windows-amd64.zip +59da7ce1bda432616bfc28ae663e52c3675adee8d9bf5959fafd657c159576ab golangci-lint-1.42.0-windows-armv6.zip +65f62dda937bfcede0326ac77abe947ce1548931e6e13298ca036cb31f224db5 golangci-lint-1.42.0-windows-armv7.zip diff --git a/build/ci.go b/build/ci.go index d7d2ce72e0..540cddd531 100644 --- a/build/ci.go +++ b/build/ci.go @@ -330,7 +330,7 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.39.0" + const version = "1.42.0" csdb := build.MustLoadChecksums("build/checksums.txt") base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) From efee85378ef426ab1dffa2e9045a60ea411415e1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 14:22:33 +0200 Subject: [PATCH 18/30] build, .travis.yml: upgrade to Go 1.17 (#23464) --- .travis.yml | 24 ++++++++++++------------ build/checksums.txt | 27 ++++++++++++++------------- build/ci.go | 2 +- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7db711ccc..f1018cc11e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jobs: allow_failures: - stage: build os: osx - go: 1.15.x + go: 1.17.x env: - azure-osx - azure-ios @@ -16,7 +16,7 @@ jobs: - stage: lint os: linux dist: bionic - go: 1.16.x + go: 1.17.x env: - lint git: @@ -31,7 +31,7 @@ jobs: os: linux arch: amd64 dist: bionic - go: 1.16.x + go: 1.17.x env: - docker services: @@ -48,7 +48,7 @@ jobs: os: linux arch: arm64 dist: bionic - go: 1.16.x + go: 1.17.x env: - docker services: @@ -65,7 +65,7 @@ jobs: if: type = push os: linux dist: bionic - go: 1.16.x + go: 1.17.x env: - ubuntu-ppa - GO111MODULE=on @@ -90,7 +90,7 @@ jobs: os: linux dist: bionic sudo: required - go: 1.16.x + go: 1.17.x env: - azure-linux - GO111MODULE=on @@ -127,7 +127,7 @@ jobs: dist: bionic services: - docker - go: 1.16.x + go: 1.17.x env: - azure-linux-mips - GO111MODULE=on @@ -192,7 +192,7 @@ jobs: - stage: build if: type = push os: osx - go: 1.16.x + go: 1.17.x env: - azure-osx - azure-ios @@ -224,7 +224,7 @@ jobs: os: linux arch: amd64 dist: bionic - go: 1.16.x + go: 1.17.x env: - GO111MODULE=on script: @@ -235,7 +235,7 @@ jobs: os: linux arch: arm64 dist: bionic - go: 1.16.x + go: 1.17.x env: - GO111MODULE=on script: @@ -244,7 +244,7 @@ jobs: - stage: build os: linux dist: bionic - go: 1.15.x + go: 1.16.x env: - GO111MODULE=on script: @@ -255,7 +255,7 @@ jobs: if: type = cron os: linux dist: bionic - go: 1.16.x + go: 1.17.x env: - azure-purge - GO111MODULE=on diff --git a/build/checksums.txt b/build/checksums.txt index 6df097b889..686e1604b6 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,18 +1,19 @@ # This file contains sha256 checksums of optional build dependencies. -ae4f6b6e2a1677d31817984655a762074b5356da50fb58722b99104870d43503 go1.16.4.src.tar.gz -18fe94775763db3878717393b6d41371b0b45206055e49b3838328120c977d13 go1.16.4.darwin-amd64.tar.gz -cb6b972cc42e669f3585c648198cd5b6f6d7a0811d413ad64b50c02ba06ccc3a go1.16.4.darwin-arm64.tar.gz -cd1b146ef6e9006f27dd99e9687773e7fef30e8c985b7d41bff33e955a3bb53a go1.16.4.linux-386.tar.gz -7154e88f5a8047aad4b80ebace58a059e36e7e2e4eb3b383127a28c711b4ff59 go1.16.4.linux-amd64.tar.gz -8b18eb05ddda2652d69ab1b1dd1f40dd731799f43c6a58b512ad01ae5b5bba21 go1.16.4.linux-arm64.tar.gz -a53391a800ddec749ee90d38992babb27b95cfb864027350c737b9aa8e069494 go1.16.4.linux-armv6l.tar.gz -e75c0b114a09eb5499874162b208931dc260de0fedaeedac8621bf263c974605 go1.16.4.windows-386.zip -d40139b7ade8a3008e3240a6f86fe8f899a9c465c917e11dac8758af216f5eb0 go1.16.4.windows-amd64.zip -7cf2bc8a175d6d656861165bfc554f92dc78d2abf5afe5631db3579555d97409 go1.16.4.freebsd-386.tar.gz -ccdd2b76de1941b60734408fda0d750aaa69330d8a07430eed4c56bdb3502f6f go1.16.4.freebsd-amd64.tar.gz -80cfac566e344096a8df8f37bbd21f89e76a6fbe601406565d71a87a665fc125 go1.16.4.linux-ppc64le.tar.gz -d6431881b3573dc29ecc24fbeab5e5ec25d8c9273aa543769c86a1a3bbac1ddf go1.16.4.linux-s390x.tar.gz +3a70e5055509f347c0fb831ca07a2bf3b531068f349b14a3c652e9b5b67beb5d go1.17.src.tar.gz +355bd544ce08d7d484d9d7de05a71b5c6f5bc10aa4b316688c2192aeb3dacfd1 go1.17.darwin-amd64.tar.gz +da4e3e3c194bf9eed081de8842a157120ef44a7a8d7c820201adae7b0e28b20b go1.17.darwin-arm64.tar.gz +6819a7a11b8351d5d5768f2fff666abde97577602394f132cb7f85b3a7151f05 go1.17.freebsd-386.tar.gz +15c184c83d99441d719da201b26256455eee85a808747c404b4183e9aa6c64b4 go1.17.freebsd-amd64.tar.gz +c19e3227a6ac6329db91d1af77bbf239ccd760a259c16e6b9c932d527ff14848 go1.17.linux-386.tar.gz +6bf89fc4f5ad763871cf7eac80a2d594492de7a818303283f1366a7f6a30372d go1.17.linux-amd64.tar.gz +01a9af009ada22122d3fcb9816049c1d21842524b38ef5d5a0e2ee4b26d7c3e7 go1.17.linux-arm64.tar.gz +ae89d33f4e4acc222bdb04331933d5ece4ae71039812f6ccd7493cb3e8ddfb4e go1.17.linux-armv6l.tar.gz +ee84350114d532bf15f096198c675aafae9ff091dc4cc69eb49e1817ff94dbd7 go1.17.linux-ppc64le.tar.gz +a50aaecf054f393575f969a9105d5c6864dd91afc5287d772449033fbafcf7e3 go1.17.linux-s390x.tar.gz +c5afdd2ea4969f2b44637e913b04f7c15265d7beb60924a28063722670a52feb go1.17.windows-386.zip +2a18bd65583e221be8b9b7c2fbe3696c40f6e27c2df689bbdcc939d49651d151 go1.17.windows-amd64.zip +5256f92f643d9022394ddc84de5c74fe8660c2151daaa199b12e60e542d694ae go1.17.windows-arm64.zip d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa golangci-lint-1.42.0-darwin-amd64.tar.gz e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea golangci-lint-1.42.0-darwin-arm64.tar.gz diff --git a/build/ci.go b/build/ci.go index 540cddd531..3de1351205 100644 --- a/build/ci.go +++ b/build/ci.go @@ -153,7 +153,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.16.4" + dlgoVersion = "1.17" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) From 84c51bc5ec9038db6ab28195384a6bd720a28af9 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 25 Aug 2021 15:04:26 +0200 Subject: [PATCH 19/30] cmd/evm: add difficulty calculation to t8n tool (#23353) This PR adds functionality to the evm t8n to calculate ethash difficulty. If the caller does not provide a currentDifficulty, but instead provides the parentTimestamp (well, semi-optional, will default to 0 if not given), and parentDifficulty, we can calculate it for him. The caller can also provide a parentUncleHash. In most, but not all cases, the parent uncle hash also affects the formula. If no such hash is provided (or, if the empty all-zero hash is provided), it's assumed that there were no uncles. --- cmd/evm/internal/t8ntool/execution.go | 70 ++++++++++++++++++-------- cmd/evm/internal/t8ntool/gen_stenv.go | 55 +++++++++++++------- cmd/evm/internal/t8ntool/transition.go | 14 ++++++ cmd/evm/testdata/14/alloc.json | 12 +++++ cmd/evm/testdata/14/env.json | 9 ++++ cmd/evm/testdata/14/env.uncles.json | 10 ++++ cmd/evm/testdata/14/readme.md | 41 +++++++++++++++ cmd/evm/testdata/14/txs.json | 1 + 8 files changed, 172 insertions(+), 40 deletions(-) create mode 100644 cmd/evm/testdata/14/alloc.json create mode 100644 cmd/evm/testdata/14/env.json create mode 100644 cmd/evm/testdata/14/env.uncles.json create mode 100644 cmd/evm/testdata/14/readme.md create mode 100644 cmd/evm/testdata/14/txs.json diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 1ab2f001e2..fae65767be 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -46,13 +47,14 @@ type Prestate struct { // ExecutionResult contains the execution status after running a state test, any // error that might have occurred and a dump of the final state if requested. type ExecutionResult struct { - StateRoot common.Hash `json:"stateRoot"` - TxRoot common.Hash `json:"txRoot"` - ReceiptRoot common.Hash `json:"receiptRoot"` - LogsHash common.Hash `json:"logsHash"` - Bloom types.Bloom `json:"logsBloom" gencodec:"required"` - Receipts types.Receipts `json:"receipts"` - Rejected []*rejectedTx `json:"rejected,omitempty"` + StateRoot common.Hash `json:"stateRoot"` + TxRoot common.Hash `json:"txRoot"` + ReceiptRoot common.Hash `json:"receiptRoot"` + LogsHash common.Hash `json:"logsHash"` + Bloom types.Bloom `json:"logsBloom" gencodec:"required"` + Receipts types.Receipts `json:"receipts"` + Rejected []*rejectedTx `json:"rejected,omitempty"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` } type ommer struct { @@ -62,23 +64,28 @@ type ommer struct { //go:generate gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go type stEnv struct { - Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` - Difficulty *big.Int `json:"currentDifficulty" gencodec:"required"` - GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` - Number uint64 `json:"currentNumber" gencodec:"required"` - Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *big.Int `json:"currentBaseFee,omitempty"` + Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` + Difficulty *big.Int `json:"currentDifficulty"` + ParentDifficulty *big.Int `json:"parentDifficulty"` + GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` + Number uint64 `json:"currentNumber" gencodec:"required"` + Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp uint64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *big.Int `json:"currentBaseFee,omitempty"` + ParentUncleHash common.Hash `json:"parentUncleHash"` } type stEnvMarshaling struct { - Coinbase common.UnprefixedAddress - Difficulty *math.HexOrDecimal256 - GasLimit math.HexOrDecimal64 - Number math.HexOrDecimal64 - Timestamp math.HexOrDecimal64 - BaseFee *math.HexOrDecimal256 + Coinbase common.UnprefixedAddress + Difficulty *math.HexOrDecimal256 + ParentDifficulty *math.HexOrDecimal256 + GasLimit math.HexOrDecimal64 + Number math.HexOrDecimal64 + Timestamp math.HexOrDecimal64 + ParentTimestamp math.HexOrDecimal64 + BaseFee *math.HexOrDecimal256 } type rejectedTx struct { @@ -247,6 +254,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, LogsHash: rlpHash(statedb.Logs()), Receipts: receipts, Rejected: rejectedTxs, + Difficulty: (*math.HexOrDecimal256)(vmContext.Difficulty), } return statedb, execRs, nil } @@ -274,3 +282,23 @@ func rlpHash(x interface{}) (h common.Hash) { hw.Sum(h[:0]) return h } + +// calcDifficulty is based on ethash.CalcDifficulty. This method is used in case +// the caller does not provide an explicit difficulty, but instead provides only +// parent timestamp + difficulty. +// Note: this method only works for ethash engine. +func calcDifficulty(config *params.ChainConfig, number, currentTime, parentTime uint64, + parentDifficulty *big.Int, parentUncleHash common.Hash) *big.Int { + uncleHash := parentUncleHash + if uncleHash == (common.Hash{}) { + uncleHash = types.EmptyUncleHash + } + parent := &types.Header{ + ParentHash: common.Hash{}, + UncleHash: uncleHash, + Difficulty: parentDifficulty, + Number: new(big.Int).SetUint64(number - 1), + Time: parentTime, + } + return ethash.CalcDifficulty(config, currentTime, parent) +} diff --git a/cmd/evm/internal/t8ntool/gen_stenv.go b/cmd/evm/internal/t8ntool/gen_stenv.go index c7f079c02f..1bb3c6a46b 100644 --- a/cmd/evm/internal/t8ntool/gen_stenv.go +++ b/cmd/evm/internal/t8ntool/gen_stenv.go @@ -16,38 +16,47 @@ var _ = (*stEnvMarshaling)(nil) // MarshalJSON marshals as JSON. func (s stEnv) MarshalJSON() ([]byte, error) { type stEnv struct { - Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` - GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` + ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` + GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + ParentUncleHash common.Hash `json:"parentUncleHash"` } var enc stEnv enc.Coinbase = common.UnprefixedAddress(s.Coinbase) enc.Difficulty = (*math.HexOrDecimal256)(s.Difficulty) + enc.ParentDifficulty = (*math.HexOrDecimal256)(s.ParentDifficulty) enc.GasLimit = math.HexOrDecimal64(s.GasLimit) enc.Number = math.HexOrDecimal64(s.Number) enc.Timestamp = math.HexOrDecimal64(s.Timestamp) + enc.ParentTimestamp = math.HexOrDecimal64(s.ParentTimestamp) enc.BlockHashes = s.BlockHashes enc.Ommers = s.Ommers enc.BaseFee = (*math.HexOrDecimal256)(s.BaseFee) + enc.ParentUncleHash = s.ParentUncleHash return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (s *stEnv) UnmarshalJSON(input []byte) error { type stEnv struct { - Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` - GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` + ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` + GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp *math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + ParentUncleHash *common.Hash `json:"parentUncleHash"` } var dec stEnv if err := json.Unmarshal(input, &dec); err != nil { @@ -57,10 +66,12 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'currentCoinbase' for stEnv") } s.Coinbase = common.Address(*dec.Coinbase) - if dec.Difficulty == nil { - return errors.New("missing required field 'currentDifficulty' for stEnv") + if dec.Difficulty != nil { + s.Difficulty = (*big.Int)(dec.Difficulty) + } + if dec.ParentDifficulty != nil { + s.ParentDifficulty = (*big.Int)(dec.ParentDifficulty) } - s.Difficulty = (*big.Int)(dec.Difficulty) if dec.GasLimit == nil { return errors.New("missing required field 'currentGasLimit' for stEnv") } @@ -73,6 +84,9 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'currentTimestamp' for stEnv") } s.Timestamp = uint64(*dec.Timestamp) + if dec.ParentTimestamp != nil { + s.ParentTimestamp = uint64(*dec.ParentTimestamp) + } if dec.BlockHashes != nil { s.BlockHashes = dec.BlockHashes } @@ -82,5 +96,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { if dec.BaseFee != nil { s.BaseFee = (*big.Int)(dec.BaseFee) } + if dec.ParentUncleHash != nil { + s.ParentUncleHash = *dec.ParentUncleHash + } return nil } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 8334aa01d4..9fdbbe29cb 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -252,6 +252,20 @@ func Main(ctx *cli.Context) error { return NewError(ErrorVMConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section")) } } + if env := prestate.Env; env.Difficulty == nil { + // If difficulty was not provided by caller, we need to calculate it. + switch { + case env.ParentDifficulty == nil: + return NewError(ErrorVMConfig, errors.New("currentDifficulty was not provided, and cannot be calculated due to missing parentDifficulty")) + case env.Number == 0: + return NewError(ErrorVMConfig, errors.New("currentDifficulty needs to be provided for block number 0")) + case env.Timestamp <= env.ParentTimestamp: + return NewError(ErrorVMConfig, fmt.Errorf("currentDifficulty cannot be calculated -- currentTime (%d) needs to be after parent time (%d)", + env.Timestamp, env.ParentTimestamp)) + } + prestate.Env.Difficulty = calcDifficulty(chainConfig, env.Number, env.Timestamp, + env.ParentTimestamp, env.ParentDifficulty, env.ParentUncleHash) + } // Run the test and aggregate the result s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) if err != nil { diff --git a/cmd/evm/testdata/14/alloc.json b/cmd/evm/testdata/14/alloc.json new file mode 100644 index 0000000000..cef1a25ff0 --- /dev/null +++ b/cmd/evm/testdata/14/alloc.json @@ -0,0 +1,12 @@ +{ + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x5ffd4878be161d74", + "code": "0x", + "nonce": "0xac", + "storage": {} + }, + "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192":{ + "balance": "0xfeedbead", + "nonce" : "0x00" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/14/env.json b/cmd/evm/testdata/14/env.json new file mode 100644 index 0000000000..6b7457ffdf --- /dev/null +++ b/cmd/evm/testdata/14/env.json @@ -0,0 +1,9 @@ +{ + "currentCoinbase": "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "currentGasLimit": "0x750a163df65e8a", + "currentBaseFee": "0x500", + "currentNumber": "12800000", + "currentTimestamp": "10015", + "parentTimestamp" : "99999", + "parentDifficulty" : "0x2000000000000" +} diff --git a/cmd/evm/testdata/14/env.uncles.json b/cmd/evm/testdata/14/env.uncles.json new file mode 100644 index 0000000000..9c2149b39e --- /dev/null +++ b/cmd/evm/testdata/14/env.uncles.json @@ -0,0 +1,10 @@ +{ + "currentCoinbase": "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "currentGasLimit": "0x750a163df65e8a", + "currentBaseFee": "0x500", + "currentNumber": "12800000", + "currentTimestamp": "10035", + "parentTimestamp" : "99999", + "parentDifficulty" : "0x2000000000000", + "parentUncleHash" : "0x000000000000000000000000000000000000000000000000000000000000beef" +} diff --git a/cmd/evm/testdata/14/readme.md b/cmd/evm/testdata/14/readme.md new file mode 100644 index 0000000000..ce31abb802 --- /dev/null +++ b/cmd/evm/testdata/14/readme.md @@ -0,0 +1,41 @@ +## Difficulty calculation + +This test shows how the `evm t8n` can be used to calculate the (ethash) difficulty, if none is provided by the caller. + +Calculating it (with an empty set of txs) using `London` rules (and no provided unclehash for the parent block): +``` +[user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.json --output.result=stdout --state.fork=London +INFO [08-08|17:35:46.876] Trie dumping started root=6f0588..7f4bdc +INFO [08-08|17:35:46.876] Trie dumping complete accounts=2 elapsed="89.313µs" +INFO [08-08|17:35:46.877] Wrote file file=alloc.json +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": 3311729559732224 + } +} +``` +Same thing, but this time providing a non-empty (and non-`emptyKeccak`) unclehash, which leads to a slightly different result: +``` +[user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.uncles.json --output.result=stdout --state.fork=London +INFO [08-08|17:35:49.232] Trie dumping started root=6f0588..7f4bdc +INFO [08-08|17:35:49.232] Trie dumping complete accounts=2 elapsed="83.069µs" +INFO [08-08|17:35:49.233] Wrote file file=alloc.json +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": 3311179803918336 + } +} +``` + diff --git a/cmd/evm/testdata/14/txs.json b/cmd/evm/testdata/14/txs.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/cmd/evm/testdata/14/txs.json @@ -0,0 +1 @@ +[] From 3784e15106c8502b0a612d932be2e995fd182ea6 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 25 Aug 2021 15:07:58 +0200 Subject: [PATCH 20/30] tests: avoid performance tests during CI runs (#23304) --- tests/state_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/state_test.go b/tests/state_test.go index c2ca0e8d69..a7d9653a33 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -42,6 +42,7 @@ func TestState(t *testing.T) { // Very time consuming st.skipLoad(`^stTimeConsuming/`) + st.skipLoad(`.*vmPerformance/loop.*`) // Uses 1GB RAM per tested fork st.skipLoad(`^stStaticCall/static_Call1MB`) From d16214228f40079af8db6752ca5f51b2022878c2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 15:18:31 +0200 Subject: [PATCH 21/30] Dockerfile: upgrade to Go 1.17 (#23465) --- Dockerfile | 2 +- Dockerfile.alltools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e76c5765bc..7badbc1320 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.17-alpine as builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 71f63b7a47..3ae5377e4f 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.17-alpine as builder RUN apk add --no-cache gcc musl-dev linux-headers git From 887902ea4d7ee77118ce803e05085bd9055aa46d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 25 Aug 2021 17:33:09 +0200 Subject: [PATCH 22/30] crypto/cloudflare/bn256: fix in-place addition and unmarshalling (#23419) --- crypto/bn256/cloudflare/bn256_test.go | 13 +++++++++++++ crypto/bn256/cloudflare/curve.go | 6 +++--- crypto/bn256/cloudflare/gfp.go | 1 + crypto/bn256/cloudflare/twist.go | 6 +++--- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crypto/bn256/cloudflare/bn256_test.go b/crypto/bn256/cloudflare/bn256_test.go index 0c8016d86c..481e2f78c3 100644 --- a/crypto/bn256/cloudflare/bn256_test.go +++ b/crypto/bn256/cloudflare/bn256_test.go @@ -92,6 +92,19 @@ func TestTripartiteDiffieHellman(t *testing.T) { } } +func TestG2SelfAddition(t *testing.T) { + s, _ := rand.Int(rand.Reader, Order) + p := new(G2).ScalarBaseMult(s) + + if !p.p.IsOnCurve() { + t.Fatal("p isn't on curve") + } + m := p.Add(p, p).Marshal() + if _, err := p.Unmarshal(m); err != nil { + t.Fatalf("p.Add(p, p) ∉ G₂: %v", err) + } +} + func BenchmarkG1(b *testing.B) { x, _ := rand.Int(rand.Reader, Order) b.ResetTimer() diff --git a/crypto/bn256/cloudflare/curve.go b/crypto/bn256/cloudflare/curve.go index 18e9b38f3f..16f0489e33 100644 --- a/crypto/bn256/cloudflare/curve.go +++ b/crypto/bn256/cloudflare/curve.go @@ -171,15 +171,15 @@ func (c *curvePoint) Double(a *curvePoint) { gfpAdd(t, d, d) gfpSub(&c.x, f, t) + gfpMul(&c.z, &a.y, &a.z) + gfpAdd(&c.z, &c.z, &c.z) + gfpAdd(t, C, C) gfpAdd(t2, t, t) gfpAdd(t, t2, t2) gfpSub(&c.y, d, &c.x) gfpMul(t2, e, &c.y) gfpSub(&c.y, t2, t) - - gfpMul(t, &a.y, &a.z) - gfpAdd(&c.z, t, t) } func (c *curvePoint) Mul(a *curvePoint, scalar *big.Int) { diff --git a/crypto/bn256/cloudflare/gfp.go b/crypto/bn256/cloudflare/gfp.go index e8e84e7b3b..b15e1697e1 100644 --- a/crypto/bn256/cloudflare/gfp.go +++ b/crypto/bn256/cloudflare/gfp.go @@ -61,6 +61,7 @@ func (e *gfP) Marshal(out []byte) { func (e *gfP) Unmarshal(in []byte) error { // Unmarshal the bytes into little endian form for w := uint(0); w < 4; w++ { + e[3-w] = 0 for b := uint(0); b < 8; b++ { e[3-w] += uint64(in[8*w+b]) << (56 - 8*b) } diff --git a/crypto/bn256/cloudflare/twist.go b/crypto/bn256/cloudflare/twist.go index 0c2f80d4ed..2c7a69a4d7 100644 --- a/crypto/bn256/cloudflare/twist.go +++ b/crypto/bn256/cloudflare/twist.go @@ -150,15 +150,15 @@ func (c *twistPoint) Double(a *twistPoint) { t.Add(d, d) c.x.Sub(f, t) + c.z.Mul(&a.y, &a.z) + c.z.Add(&c.z, &c.z) + t.Add(C, C) t2.Add(t, t) t.Add(t2, t2) c.y.Sub(d, &c.x) t2.Mul(e, &c.y) c.y.Sub(t2, t) - - t.Mul(&a.y, &a.z) - c.z.Add(t, t) } func (c *twistPoint) Mul(a *twistPoint, scalar *big.Int) { From 8a134014b4b370b4a3632e32a2fc8e84ee2b6947 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 18:46:29 +0200 Subject: [PATCH 23/30] all: add go:build lines (#23468) Generated by go1.17 fmt ./... --- accounts/keystore/watch.go | 1 + accounts/keystore/watch_fallback.go | 1 + cmd/utils/diskusage.go | 1 + cmd/utils/diskusage_openbsd.go | 1 + common/fdlimit/fdlimit_bsd.go | 1 + common/fdlimit/fdlimit_unix.go | 1 + core/mkalloc.go | 1 + crypto/blake2b/blake2bAVX2_amd64.go | 1 + crypto/blake2b/blake2b_amd64.go | 1 + crypto/blake2b/blake2b_f_fuzz.go | 1 + crypto/blake2b/blake2b_ref.go | 1 + crypto/blake2b/register.go | 1 + crypto/bls12381/arithmetic_decl.go | 1 + crypto/bls12381/arithmetic_fallback.go | 1 + crypto/bls12381/arithmetic_x86_adx.go | 1 + crypto/bls12381/arithmetic_x86_noadx.go | 1 + crypto/bn256/bn256_fast.go | 1 + crypto/bn256/bn256_slow.go | 1 + crypto/bn256/cloudflare/gfp_decl.go | 1 + crypto/bn256/cloudflare/gfp_generic.go | 1 + crypto/secp256k1/dummy.go | 1 + crypto/secp256k1/panic_cb.go | 4 ++-- crypto/secp256k1/scalar_mult_cgo.go | 4 ++-- crypto/secp256k1/scalar_mult_nocgo.go | 1 + crypto/secp256k1/secp256.go | 4 ++-- crypto/signature_cgo.go | 1 + crypto/signature_nocgo.go | 1 + crypto/signify/signify_fuzz.go | 1 + ethdb/leveldb/leveldb.go | 1 + internal/debug/loudpanic.go | 1 + internal/debug/loudpanic_fallback.go | 1 + internal/debug/trace.go | 3 ++- internal/debug/trace_fallback.go | 3 ++- log/handler_go13.go | 1 + log/handler_go14.go | 1 + log/syslog.go | 1 + metrics/cpu_disabled.go | 1 + metrics/cpu_enabled.go | 1 + metrics/cputime_nop.go | 1 + metrics/cputime_unix.go | 1 + metrics/disk_nop.go | 1 + metrics/runtime_cgo.go | 1 + metrics/runtime_gccpufraction.go | 1 + metrics/runtime_no_cgo.go | 1 + metrics/runtime_no_gccpufraction.go | 1 + metrics/syslog.go | 1 + mobile/geth_android.go | 1 + mobile/geth_ios.go | 1 + mobile/geth_other.go | 1 + p2p/netutil/toobig_notwindows.go | 3 ++- p2p/netutil/toobig_windows.go | 3 ++- rlp/safe.go | 1 + rlp/unsafe.go | 1 + rpc/constants_unix.go | 1 + rpc/constants_unix_nocgo.go | 1 + rpc/ipc_js.go | 1 + rpc/ipc_unix.go | 1 + rpc/ipc_windows.go | 1 + tests/fuzzers/bls12381/bls12381_fuzz.go | 1 + 59 files changed, 66 insertions(+), 10 deletions(-) diff --git a/accounts/keystore/watch.go b/accounts/keystore/watch.go index d6ef53327d..ad176040d6 100644 --- a/accounts/keystore/watch.go +++ b/accounts/keystore/watch.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build (darwin && !ios && cgo) || freebsd || (linux && !arm64) || netbsd || solaris // +build darwin,!ios,cgo freebsd linux,!arm64 netbsd solaris package keystore diff --git a/accounts/keystore/watch_fallback.go b/accounts/keystore/watch_fallback.go index de0e87f8a5..e40eca42fe 100644 --- a/accounts/keystore/watch_fallback.go +++ b/accounts/keystore/watch_fallback.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build (darwin && !cgo) || ios || (linux && arm64) || windows || (!darwin && !freebsd && !linux && !netbsd && !solaris) // +build darwin,!cgo ios linux,arm64 windows !darwin,!freebsd,!linux,!netbsd,!solaris // This is the fallback implementation of directory watching. diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index c3d51765f8..09844652ef 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !windows && !openbsd // +build !windows,!openbsd package utils diff --git a/cmd/utils/diskusage_openbsd.go b/cmd/utils/diskusage_openbsd.go index 54f759d291..52502d0cfa 100644 --- a/cmd/utils/diskusage_openbsd.go +++ b/cmd/utils/diskusage_openbsd.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build openbsd // +build openbsd package utils diff --git a/common/fdlimit/fdlimit_bsd.go b/common/fdlimit/fdlimit_bsd.go index 86181337a2..a3a6902c09 100644 --- a/common/fdlimit/fdlimit_bsd.go +++ b/common/fdlimit/fdlimit_bsd.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build freebsd || dragonfly // +build freebsd dragonfly package fdlimit diff --git a/common/fdlimit/fdlimit_unix.go b/common/fdlimit/fdlimit_unix.go index e5a575f7a7..a1f388ebb7 100644 --- a/common/fdlimit/fdlimit_unix.go +++ b/common/fdlimit/fdlimit_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build linux || netbsd || openbsd || solaris // +build linux netbsd openbsd solaris package fdlimit diff --git a/core/mkalloc.go b/core/mkalloc.go index 5118a4fcb3..df167d7082 100644 --- a/core/mkalloc.go +++ b/core/mkalloc.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build none // +build none /* diff --git a/crypto/blake2b/blake2bAVX2_amd64.go b/crypto/blake2b/blake2bAVX2_amd64.go index 0d52b18699..3a85d0e73a 100644 --- a/crypto/blake2b/blake2bAVX2_amd64.go +++ b/crypto/blake2b/blake2bAVX2_amd64.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.7 && amd64 && !gccgo && !appengine // +build go1.7,amd64,!gccgo,!appengine package blake2b diff --git a/crypto/blake2b/blake2b_amd64.go b/crypto/blake2b/blake2b_amd64.go index 4dbe90da8f..a318b2b617 100644 --- a/crypto/blake2b/blake2b_amd64.go +++ b/crypto/blake2b/blake2b_amd64.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !go1.7 && amd64 && !gccgo && !appengine // +build !go1.7,amd64,!gccgo,!appengine package blake2b diff --git a/crypto/blake2b/blake2b_f_fuzz.go b/crypto/blake2b/blake2b_f_fuzz.go index ab73342803..b2f4057074 100644 --- a/crypto/blake2b/blake2b_f_fuzz.go +++ b/crypto/blake2b/blake2b_f_fuzz.go @@ -1,3 +1,4 @@ +//go:build gofuzz // +build gofuzz package blake2b diff --git a/crypto/blake2b/blake2b_ref.go b/crypto/blake2b/blake2b_ref.go index 9d0ade473a..095c71a648 100644 --- a/crypto/blake2b/blake2b_ref.go +++ b/crypto/blake2b/blake2b_ref.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !amd64 || appengine || gccgo // +build !amd64 appengine gccgo package blake2b diff --git a/crypto/blake2b/register.go b/crypto/blake2b/register.go index efd689af4b..9d8633963c 100644 --- a/crypto/blake2b/register.go +++ b/crypto/blake2b/register.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.9 // +build go1.9 package blake2b diff --git a/crypto/bls12381/arithmetic_decl.go b/crypto/bls12381/arithmetic_decl.go index ec0b21e805..f6d232d658 100644 --- a/crypto/bls12381/arithmetic_decl.go +++ b/crypto/bls12381/arithmetic_decl.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build (amd64 && blsasm) || (amd64 && blsadx) // +build amd64,blsasm amd64,blsadx package bls12381 diff --git a/crypto/bls12381/arithmetic_fallback.go b/crypto/bls12381/arithmetic_fallback.go index 91cabf4f3d..c09ae0d91c 100644 --- a/crypto/bls12381/arithmetic_fallback.go +++ b/crypto/bls12381/arithmetic_fallback.go @@ -31,6 +31,7 @@ // Package bls (generated by goff) contains field arithmetics operations +//go:build !amd64 || (!blsasm && !blsadx) // +build !amd64 !blsasm,!blsadx package bls12381 diff --git a/crypto/bls12381/arithmetic_x86_adx.go b/crypto/bls12381/arithmetic_x86_adx.go index 9c30741e6a..a40c7384eb 100644 --- a/crypto/bls12381/arithmetic_x86_adx.go +++ b/crypto/bls12381/arithmetic_x86_adx.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build amd64 && blsadx // +build amd64,blsadx package bls12381 diff --git a/crypto/bls12381/arithmetic_x86_noadx.go b/crypto/bls12381/arithmetic_x86_noadx.go index eaac4b45d7..679b30ec8c 100644 --- a/crypto/bls12381/arithmetic_x86_noadx.go +++ b/crypto/bls12381/arithmetic_x86_noadx.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build amd64 && blsasm // +build amd64,blsasm package bls12381 diff --git a/crypto/bn256/bn256_fast.go b/crypto/bn256/bn256_fast.go index 14b5965393..e3c9b60518 100644 --- a/crypto/bn256/bn256_fast.go +++ b/crypto/bn256/bn256_fast.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. +//go:build amd64 || arm64 // +build amd64 arm64 // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve. diff --git a/crypto/bn256/bn256_slow.go b/crypto/bn256/bn256_slow.go index 49021082f5..4c0c351e2d 100644 --- a/crypto/bn256/bn256_slow.go +++ b/crypto/bn256/bn256_slow.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. +//go:build !amd64 && !arm64 // +build !amd64,!arm64 // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve. diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index fdea5c11a5..ec4018e88a 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -1,3 +1,4 @@ +//go:build (amd64 && !generic) || (arm64 && !generic) // +build amd64,!generic arm64,!generic package bn256 diff --git a/crypto/bn256/cloudflare/gfp_generic.go b/crypto/bn256/cloudflare/gfp_generic.go index 8e6be95961..7742dda4c9 100644 --- a/crypto/bn256/cloudflare/gfp_generic.go +++ b/crypto/bn256/cloudflare/gfp_generic.go @@ -1,3 +1,4 @@ +//go:build (!amd64 && !arm64) || generic // +build !amd64,!arm64 generic package bn256 diff --git a/crypto/secp256k1/dummy.go b/crypto/secp256k1/dummy.go index c0f2ee52c0..65a75080f6 100644 --- a/crypto/secp256k1/dummy.go +++ b/crypto/secp256k1/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // This file is part of a workaround for `go mod vendor` which won't vendor diff --git a/crypto/secp256k1/panic_cb.go b/crypto/secp256k1/panic_cb.go index 5da2bea376..a30b04f51b 100644 --- a/crypto/secp256k1/panic_cb.go +++ b/crypto/secp256k1/panic_cb.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -// +build !gofuzz -// +build cgo +//go:build !gofuzz && cgo +// +build !gofuzz,cgo package secp256k1 diff --git a/crypto/secp256k1/scalar_mult_cgo.go b/crypto/secp256k1/scalar_mult_cgo.go index f28a1c7826..8afa9d023b 100644 --- a/crypto/secp256k1/scalar_mult_cgo.go +++ b/crypto/secp256k1/scalar_mult_cgo.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -// +build !gofuzz -// +build cgo +//go:build !gofuzz && cgo +// +build !gofuzz,cgo package secp256k1 diff --git a/crypto/secp256k1/scalar_mult_nocgo.go b/crypto/secp256k1/scalar_mult_nocgo.go index 55756b5be8..22f53ac6ae 100644 --- a/crypto/secp256k1/scalar_mult_nocgo.go +++ b/crypto/secp256k1/scalar_mult_nocgo.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. +//go:build gofuzz || !cgo // +build gofuzz !cgo package secp256k1 diff --git a/crypto/secp256k1/secp256.go b/crypto/secp256k1/secp256.go index a1bcf7796e..c9c01b3209 100644 --- a/crypto/secp256k1/secp256.go +++ b/crypto/secp256k1/secp256.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -// +build !gofuzz -// +build cgo +//go:build !gofuzz && cgo +// +build !gofuzz,cgo // Package secp256k1 wraps the bitcoin secp256k1 C library. package secp256k1 diff --git a/crypto/signature_cgo.go b/crypto/signature_cgo.go index 8433602980..bd72d97d3b 100644 --- a/crypto/signature_cgo.go +++ b/crypto/signature_cgo.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !nacl && !js && cgo && !gofuzz // +build !nacl,!js,cgo,!gofuzz package crypto diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index 77c8a1db0b..fd1e66c7e6 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build nacl || js || !cgo || gofuzz // +build nacl js !cgo gofuzz package crypto diff --git a/crypto/signify/signify_fuzz.go b/crypto/signify/signify_fuzz.go index f9167900ad..2dc9b2102f 100644 --- a/crypto/signify/signify_fuzz.go +++ b/crypto/signify/signify_fuzz.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build gofuzz // +build gofuzz package signify diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 5d19cc3577..9ff1a2ce1d 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !js // +build !js // Package leveldb implements the key-value database layer based on LevelDB. diff --git a/internal/debug/loudpanic.go b/internal/debug/loudpanic.go index 572ebcefa1..86e6bc88f8 100644 --- a/internal/debug/loudpanic.go +++ b/internal/debug/loudpanic.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build go1.6 // +build go1.6 package debug diff --git a/internal/debug/loudpanic_fallback.go b/internal/debug/loudpanic_fallback.go index 4ce4985da7..377490e5be 100644 --- a/internal/debug/loudpanic_fallback.go +++ b/internal/debug/loudpanic_fallback.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !go1.6 // +build !go1.6 package debug diff --git a/internal/debug/trace.go b/internal/debug/trace.go index cab5deaafd..a273e4a9db 100644 --- a/internal/debug/trace.go +++ b/internal/debug/trace.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build go1.5 +//go:build go1.5 +// +build go1.5 package debug diff --git a/internal/debug/trace_fallback.go b/internal/debug/trace_fallback.go index 4118ff4087..ec07d991ef 100644 --- a/internal/debug/trace_fallback.go +++ b/internal/debug/trace_fallback.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build !go1.5 +//go:build !go1.5 +// +build !go1.5 // no-op implementation of tracing methods for Go < 1.5. diff --git a/log/handler_go13.go b/log/handler_go13.go index 0843ed0e5f..4df694debe 100644 --- a/log/handler_go13.go +++ b/log/handler_go13.go @@ -1,3 +1,4 @@ +//go:build !go1.4 // +build !go1.4 package log diff --git a/log/handler_go14.go b/log/handler_go14.go index 05dedbf2a7..d0cb14aa06 100644 --- a/log/handler_go14.go +++ b/log/handler_go14.go @@ -1,3 +1,4 @@ +//go:build go1.4 // +build go1.4 package log diff --git a/log/syslog.go b/log/syslog.go index 71a17b30b3..451d831b6d 100644 --- a/log/syslog.go +++ b/log/syslog.go @@ -1,3 +1,4 @@ +//go:build !windows && !plan9 // +build !windows,!plan9 package log diff --git a/metrics/cpu_disabled.go b/metrics/cpu_disabled.go index b0b483ee4b..025d97aeb3 100644 --- a/metrics/cpu_disabled.go +++ b/metrics/cpu_disabled.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build ios || js // +build ios js package metrics diff --git a/metrics/cpu_enabled.go b/metrics/cpu_enabled.go index edb824d9aa..533d40b85a 100644 --- a/metrics/cpu_enabled.go +++ b/metrics/cpu_enabled.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !ios && !js // +build !ios,!js package metrics diff --git a/metrics/cputime_nop.go b/metrics/cputime_nop.go index 1bf5537277..0188735a78 100644 --- a/metrics/cputime_nop.go +++ b/metrics/cputime_nop.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build windows || js // +build windows js package metrics diff --git a/metrics/cputime_unix.go b/metrics/cputime_unix.go index 55871e668c..3c56a75d00 100644 --- a/metrics/cputime_unix.go +++ b/metrics/cputime_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !windows && !js // +build !windows,!js package metrics diff --git a/metrics/disk_nop.go b/metrics/disk_nop.go index 4319f8b277..58fa4e02f8 100644 --- a/metrics/disk_nop.go +++ b/metrics/disk_nop.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !linux // +build !linux package metrics diff --git a/metrics/runtime_cgo.go b/metrics/runtime_cgo.go index 11722f76cc..4307ebdba6 100644 --- a/metrics/runtime_cgo.go +++ b/metrics/runtime_cgo.go @@ -1,3 +1,4 @@ +//go:build cgo && !appengine && !js // +build cgo,!appengine,!js package metrics diff --git a/metrics/runtime_gccpufraction.go b/metrics/runtime_gccpufraction.go index ca12c05bac..28cd44752b 100644 --- a/metrics/runtime_gccpufraction.go +++ b/metrics/runtime_gccpufraction.go @@ -1,3 +1,4 @@ +//go:build go1.5 // +build go1.5 package metrics diff --git a/metrics/runtime_no_cgo.go b/metrics/runtime_no_cgo.go index e760af554f..1799bef63b 100644 --- a/metrics/runtime_no_cgo.go +++ b/metrics/runtime_no_cgo.go @@ -1,3 +1,4 @@ +//go:build !cgo || appengine || js // +build !cgo appengine js package metrics diff --git a/metrics/runtime_no_gccpufraction.go b/metrics/runtime_no_gccpufraction.go index be96aa6f1b..af1a4b63c8 100644 --- a/metrics/runtime_no_gccpufraction.go +++ b/metrics/runtime_no_gccpufraction.go @@ -1,3 +1,4 @@ +//go:build !go1.5 // +build !go1.5 package metrics diff --git a/metrics/syslog.go b/metrics/syslog.go index a0ed4b1b23..551a2bd0f0 100644 --- a/metrics/syslog.go +++ b/metrics/syslog.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package metrics diff --git a/mobile/geth_android.go b/mobile/geth_android.go index 8e4ebe638f..cfdf1c28c9 100644 --- a/mobile/geth_android.go +++ b/mobile/geth_android.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build android // +build android package geth diff --git a/mobile/geth_ios.go b/mobile/geth_ios.go index 307cd08580..aab839727f 100644 --- a/mobile/geth_ios.go +++ b/mobile/geth_ios.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build ios // +build ios package geth diff --git a/mobile/geth_other.go b/mobile/geth_other.go index 6f0c5dda68..c5cad4a7ba 100644 --- a/mobile/geth_other.go +++ b/mobile/geth_other.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !android && !ios // +build !android,!ios package geth diff --git a/p2p/netutil/toobig_notwindows.go b/p2p/netutil/toobig_notwindows.go index 47b6438572..f9f936ae56 100644 --- a/p2p/netutil/toobig_notwindows.go +++ b/p2p/netutil/toobig_notwindows.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build !windows +//go:build !windows +// +build !windows package netutil diff --git a/p2p/netutil/toobig_windows.go b/p2p/netutil/toobig_windows.go index dfbb6d44f0..652903e83c 100644 --- a/p2p/netutil/toobig_windows.go +++ b/p2p/netutil/toobig_windows.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build windows +//go:build windows +// +build windows package netutil diff --git a/rlp/safe.go b/rlp/safe.go index a80380aef4..6f3b151ffa 100644 --- a/rlp/safe.go +++ b/rlp/safe.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build nacl || js || !cgo // +build nacl js !cgo package rlp diff --git a/rlp/unsafe.go b/rlp/unsafe.go index 94ed5405a8..30bb9a6595 100644 --- a/rlp/unsafe.go +++ b/rlp/unsafe.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !nacl && !js && cgo // +build !nacl,!js,cgo package rlp diff --git a/rpc/constants_unix.go b/rpc/constants_unix.go index 2f98d6499b..1f04d15d7f 100644 --- a/rpc/constants_unix.go +++ b/rpc/constants_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build darwin || dragonfly || freebsd || linux || nacl || netbsd || openbsd || solaris // +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris package rpc diff --git a/rpc/constants_unix_nocgo.go b/rpc/constants_unix_nocgo.go index ecb231f927..a62e4ee529 100644 --- a/rpc/constants_unix_nocgo.go +++ b/rpc/constants_unix_nocgo.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !cgo && !windows // +build !cgo,!windows package rpc diff --git a/rpc/ipc_js.go b/rpc/ipc_js.go index 7e7554a768..453a20bc1a 100644 --- a/rpc/ipc_js.go +++ b/rpc/ipc_js.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build js // +build js package rpc diff --git a/rpc/ipc_unix.go b/rpc/ipc_unix.go index f4690cc0ab..249a9cf044 100644 --- a/rpc/ipc_unix.go +++ b/rpc/ipc_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build darwin || dragonfly || freebsd || linux || nacl || netbsd || openbsd || solaris // +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris package rpc diff --git a/rpc/ipc_windows.go b/rpc/ipc_windows.go index ca56a3ce43..adb1826f0c 100644 --- a/rpc/ipc_windows.go +++ b/rpc/ipc_windows.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build windows // +build windows package rpc diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index c0f452f3ed..b283ed11fe 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build gofuzz // +build gofuzz package bls From 32c576bd3c64bb19e0b6aa7b67342092a2022242 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 19:01:10 +0200 Subject: [PATCH 24/30] rlp: minor optimizations for slice/array encoding (#23467) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per benchmark results below, these changes speed up encoding/decoding of consensus objects a bit. name old time/op new time/op delta EncodeRLP/legacy-header-8 384ns ± 1% 331ns ± 3% -13.83% (p=0.000 n=7+8) EncodeRLP/london-header-8 411ns ± 1% 359ns ± 2% -12.53% (p=0.000 n=8+8) EncodeRLP/receipt-for-storage-8 251ns ± 0% 239ns ± 0% -4.97% (p=0.000 n=8+8) EncodeRLP/receipt-full-8 319ns ± 0% 300ns ± 0% -5.89% (p=0.000 n=8+7) EncodeRLP/legacy-transaction-8 389ns ± 1% 387ns ± 1% ~ (p=0.099 n=8+8) EncodeRLP/access-transaction-8 607ns ± 0% 581ns ± 0% -4.26% (p=0.000 n=8+8) EncodeRLP/1559-transaction-8 627ns ± 0% 606ns ± 1% -3.44% (p=0.000 n=8+8) DecodeRLP/legacy-header-8 831ns ± 1% 813ns ± 1% -2.20% (p=0.000 n=8+8) DecodeRLP/london-header-8 824ns ± 0% 804ns ± 1% -2.44% (p=0.000 n=8+7) * rlp: pass length to byteArrayBytes This makes it possible to inline byteArrayBytes. For arrays, the length is known at encoder construction time, so the call to v.Len() can be avoided. * rlp: avoid IsNil for pointer encoding It's actually cheaper to use Elem first, because it performs less checks on the value. If the pointer was nil, the result of Elem is 'invalid'. * rlp: minor optimizations for slice/array encoding For empty slices/arrays, we can avoid storing a list header entry in the encoder buffer. Also avoid doing the tail check at encoding time because it is already known at encoder construction time. --- rlp/decode.go | 2 +- rlp/encode.go | 105 ++++++++++++++++++++++++++++----------------- rlp/encode_test.go | 28 ++++++++++++ rlp/safe.go | 4 +- rlp/unsafe.go | 7 ++- 5 files changed, 100 insertions(+), 46 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index ac04d5d569..5f2e5ad5fe 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -379,7 +379,7 @@ func decodeByteArray(s *Stream, val reflect.Value) error { if err != nil { return err } - slice := byteArrayBytes(val) + slice := byteArrayBytes(val, val.Len()) switch kind { case Byte: if len(slice) == 0 { diff --git a/rlp/encode.go b/rlp/encode.go index 3348644342..1623e97a3e 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -432,7 +432,20 @@ func makeByteArrayWriter(typ reflect.Type) writer { case 1: return writeLengthOneByteArray default: - return writeByteArray + length := typ.Len() + return func(val reflect.Value, w *encbuf) error { + if !val.CanAddr() { + // Getting the byte slice of val requires it to be addressable. Make it + // addressable by copying. + copy := reflect.New(val.Type()).Elem() + copy.Set(val) + val = copy + } + slice := byteArrayBytes(val, length) + w.encodeStringHeader(len(slice)) + w.str = append(w.str, slice...) + return nil + } } } @@ -451,21 +464,6 @@ func writeLengthOneByteArray(val reflect.Value, w *encbuf) error { return nil } -func writeByteArray(val reflect.Value, w *encbuf) error { - if !val.CanAddr() { - // Getting the byte slice of val requires it to be addressable. Make it - // addressable by copying. - copy := reflect.New(val.Type()).Elem() - copy.Set(val) - val = copy - } - - slice := byteArrayBytes(val) - w.encodeStringHeader(len(slice)) - w.str = append(w.str, slice...) - return nil -} - func writeString(val reflect.Value, w *encbuf) error { s := val.String() if len(s) == 1 && s[0] <= 0x7f { @@ -499,19 +497,39 @@ func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } - writer := func(val reflect.Value, w *encbuf) error { - if !ts.tail { - defer w.listEnd(w.list()) + + var wfn writer + if ts.tail { + // This is for struct tail slices. + // w.list is not called for them. + wfn = func(val reflect.Value, w *encbuf) error { + vlen := val.Len() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + return nil } - vlen := val.Len() - for i := 0; i < vlen; i++ { - if err := etypeinfo.writer(val.Index(i), w); err != nil { - return err + } else { + // This is for regular slices and arrays. + wfn = func(val reflect.Value, w *encbuf) error { + vlen := val.Len() + if vlen == 0 { + w.str = append(w.str, 0xC0) + return nil } + listOffset := w.list() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + w.listEnd(listOffset) + return nil } - return nil } - return writer, nil + return wfn, nil } func makeStructWriter(typ reflect.Type) (writer, error) { @@ -562,12 +580,8 @@ func makeStructWriter(typ reflect.Type) (writer, error) { return writer, nil } -func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) - if etypeinfo.writerErr != nil { - return nil, etypeinfo.writerErr - } - // Determine how to encode nil pointers. +// nilEncoding returns the encoded value of a nil pointer. +func nilEncoding(typ reflect.Type, ts tags) uint8 { var nilKind Kind if ts.nilOK { nilKind = ts.nilKind // use struct tag if provided @@ -575,16 +589,29 @@ func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { nilKind = defaultNilKind(typ.Elem()) } + switch nilKind { + case String: + return 0x80 + case List: + return 0xC0 + default: + panic(fmt.Errorf("rlp: invalid nil kind %d", nilKind)) + } +} + +func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) + if etypeinfo.writerErr != nil { + return nil, etypeinfo.writerErr + } + nilEncoding := nilEncoding(typ, ts) + writer := func(val reflect.Value, w *encbuf) error { - if val.IsNil() { - if nilKind == String { - w.str = append(w.str, 0x80) - } else { - w.listEnd(w.list()) - } - return nil + if ev := val.Elem(); ev.IsValid() { + return etypeinfo.writer(ev, w) } - return etypeinfo.writer(val.Elem(), w) + w.str = append(w.str, nilEncoding) + return nil } return writer, nil } diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 25d4aac267..a63743440d 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -540,3 +540,31 @@ func BenchmarkEncodeByteArrayStruct(b *testing.B) { } } } + +type structSliceElem struct { + X uint64 + Y uint64 + Z uint64 +} + +type structPtrSlice []*structSliceElem + +func BenchmarkEncodeStructPtrSlice(b *testing.B) { + var out bytes.Buffer + var value = structPtrSlice{ + &structSliceElem{1, 1, 1}, + &structSliceElem{2, 2, 2}, + &structSliceElem{3, 3, 3}, + &structSliceElem{5, 5, 5}, + &structSliceElem{6, 6, 6}, + &structSliceElem{7, 7, 7}, + } + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(&out, &value); err != nil { + b.Fatal(err) + } + } +} diff --git a/rlp/safe.go b/rlp/safe.go index 6f3b151ffa..3c910337b6 100644 --- a/rlp/safe.go +++ b/rlp/safe.go @@ -22,6 +22,6 @@ package rlp import "reflect" // byteArrayBytes returns a slice of the byte array v. -func byteArrayBytes(v reflect.Value) []byte { - return v.Slice(0, v.Len()).Bytes() +func byteArrayBytes(v reflect.Value, length int) []byte { + return v.Slice(0, length).Bytes() } diff --git a/rlp/unsafe.go b/rlp/unsafe.go index 30bb9a6595..2152ba35fc 100644 --- a/rlp/unsafe.go +++ b/rlp/unsafe.go @@ -25,12 +25,11 @@ import ( ) // byteArrayBytes returns a slice of the byte array v. -func byteArrayBytes(v reflect.Value) []byte { - len := v.Len() +func byteArrayBytes(v reflect.Value, length int) []byte { var s []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) hdr.Data = v.UnsafeAddr() - hdr.Cap = len - hdr.Len = len + hdr.Cap = length + hdr.Len = length return s } From 154b525ce8908dfa8e4fa19b9da7aee78ecba156 Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Wed, 25 Aug 2021 19:06:42 +0200 Subject: [PATCH 25/30] cmd/devp2p/internal/ethtest: remove WriteTo method (#23379) WriteTo is unused, and go vet warns about it because it doesn't match the usual signature of the io.WriterTo method. --- cmd/devp2p/internal/ethtest/chain.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index 34a20c515b..7dcb412b53 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -39,16 +39,6 @@ type Chain struct { chainConfig *params.ChainConfig } -func (c *Chain) WriteTo(writer io.Writer) error { - for _, block := range c.blocks { - if err := rlp.Encode(writer, block); err != nil { - return err - } - } - - return nil -} - // Len returns the length of the chain. func (c *Chain) Len() int { return len(c.blocks) From 7c4cad064cf7a9f48f9d69c6a4f459ebe8bb19ab Mon Sep 17 00:00:00 2001 From: Dmitry Zenovich Date: Wed, 25 Aug 2021 20:30:29 +0300 Subject: [PATCH 26/30] rpc: add BlockNumber.MarshalText (#23324) Currently rpc.BlockNumber is marshalled to JSON as a numeric value, which is wrong because BlockNumber.UnmarshalJSON() wants it to either be hex-encoded or string "earliest"/"latest"/"pending". As a result, the call chain rpc.BlockNumberOrHashWithNumber(123) -> json.Marshal() -> json.Unmarshal() fails with error "cannot unmarshal object into Go value of type string". --- rpc/types.go | 16 ++++++++++++++++ rpc/types_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/rpc/types.go b/rpc/types.go index ad068defa8..d9c2317a78 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -98,6 +98,22 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { return nil } +// MarshalText implements encoding.TextMarshaler. It marshals: +// - "latest", "earliest" or "pending" as strings +// - other numbers as hex +func (bn BlockNumber) MarshalText() ([]byte, error) { + switch bn { + case EarliestBlockNumber: + return []byte("earliest"), nil + case LatestBlockNumber: + return []byte("latest"), nil + case PendingBlockNumber: + return []byte("pending"), nil + default: + return hexutil.Uint64(bn).MarshalText() + } +} + func (bn BlockNumber) Int64() int64 { return (int64)(bn) } diff --git a/rpc/types_test.go b/rpc/types_test.go index 89b0c9171a..f110dee7c6 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -18,6 +18,7 @@ package rpc import ( "encoding/json" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" @@ -122,3 +123,33 @@ func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { } } } + +func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { + tests := []struct { + name string + number int64 + }{ + {"max", math.MaxInt64}, + {"pending", int64(PendingBlockNumber)}, + {"latest", int64(LatestBlockNumber)}, + {"earliest", int64(EarliestBlockNumber)}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + bnh := BlockNumberOrHashWithNumber(BlockNumber(test.number)) + marshalled, err := json.Marshal(bnh) + if err != nil { + t.Fatal("cannot marshal:", err) + } + var unmarshalled BlockNumberOrHash + err = json.Unmarshal(marshalled, &unmarshalled) + if err != nil { + t.Fatal("cannot unmarshal:", err) + } + if !reflect.DeepEqual(bnh, unmarshalled) { + t.Fatalf("wrong result: expected %v, got %v", bnh, unmarshalled) + } + }) + } +} From d584e3986251902bce78369ab4b0f32da70bc82f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 21:57:31 +0200 Subject: [PATCH 27/30] build: stop publishing deb packages for Ubuntu 20.10 Groovy Gorilla (#23470) Support for this Ubuntu release has ended on July 22 2021 and Launchpad no longer accepts uploads for it. --- build/ci.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/build/ci.go b/build/ci.go index 3de1351205..6f1e975a51 100644 --- a/build/ci.go +++ b/build/ci.go @@ -129,19 +129,13 @@ var ( // Distros for which packages are created. // Note: vivid is unsupported because there is no golang-1.6 package for it. - // Note: wily is unsupported because it was officially deprecated on Launchpad. - // Note: yakkety is unsupported because it was officially deprecated on Launchpad. - // Note: zesty is unsupported because it was officially deprecated on Launchpad. - // Note: artful is unsupported because it was officially deprecated on Launchpad. - // Note: cosmic is unsupported because it was officially deprecated on Launchpad. - // Note: disco is unsupported because it was officially deprecated on Launchpad. - // Note: eoan is unsupported because it was officially deprecated on Launchpad. + // Note: the following Ubuntu releases have been officially deprecated on Launchpad: + // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy debDistroGoBoots = map[string]string{ "trusty": "golang-1.11", "xenial": "golang-go", "bionic": "golang-go", "focal": "golang-go", - "groovy": "golang-go", "hirsute": "golang-go", } From 108eec3feeca2e05d3daf1a198b8286623f56755 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Wed, 25 Aug 2021 22:34:22 +0200 Subject: [PATCH 28/30] node: remove dependency on wallet backend packages (#23019) * accounts: new AddBackends method in manager * node,cmd/geth: mv accman backend init to cmd/geth * node,cmd/geth: mv scrypt config downstreawm from node * accounts: use static buffer size for accman sub chan minor fix * accounts,cmd/geth: update accman backends through its event loop * accounts,node: add comments * accounts: un-export newBackendEvent * accounts: use chan instead of wg in newBlockEvent * node: rename isKeyDirEphem * accounts,cmd: AddBackends->AddBackend * accounts: fix potential blocking when adding backend --- accounts/manager.go | 64 ++++++++++++++++++++++++++------- cmd/geth/accountcmd.go | 9 +++-- cmd/geth/config.go | 68 +++++++++++++++++++++++++++++++++++ node/config.go | 82 ++++++++---------------------------------- node/node.go | 23 +++++++----- 5 files changed, 156 insertions(+), 90 deletions(-) diff --git a/accounts/manager.go b/accounts/manager.go index acf41ed8e9..1e111d1948 100644 --- a/accounts/manager.go +++ b/accounts/manager.go @@ -25,6 +25,10 @@ import ( "github.com/ethereum/go-ethereum/event" ) +// managerSubBufferSize determines how many incoming wallet events +// the manager will buffer in its channel. +const managerSubBufferSize = 50 + // Config contains the settings of the global account manager. // // TODO(rjl493456442, karalabe, holiman): Get rid of this when account management @@ -33,18 +37,27 @@ type Config struct { InsecureUnlockAllowed bool // Whether account unlocking in insecure environment is allowed } +// newBackendEvent lets the manager know it should +// track the given backend for wallet updates. +type newBackendEvent struct { + backend Backend + processed chan struct{} // Informs event emitter that backend has been integrated +} + // Manager is an overarching account manager that can communicate with various // backends for signing transactions. type Manager struct { - config *Config // Global account manager configurations - backends map[reflect.Type][]Backend // Index of backends currently registered - updaters []event.Subscription // Wallet update subscriptions for all backends - updates chan WalletEvent // Subscription sink for backend wallet changes - wallets []Wallet // Cache of all wallets from all registered backends + config *Config // Global account manager configurations + backends map[reflect.Type][]Backend // Index of backends currently registered + updaters []event.Subscription // Wallet update subscriptions for all backends + updates chan WalletEvent // Subscription sink for backend wallet changes + newBackends chan newBackendEvent // Incoming backends to be tracked by the manager + wallets []Wallet // Cache of all wallets from all registered backends feed event.Feed // Wallet feed notifying of arrivals/departures quit chan chan error + term chan struct{} // Channel is closed upon termination of the update loop lock sync.RWMutex } @@ -57,7 +70,7 @@ func NewManager(config *Config, backends ...Backend) *Manager { wallets = merge(wallets, backend.Wallets()...) } // Subscribe to wallet notifications from all backends - updates := make(chan WalletEvent, 4*len(backends)) + updates := make(chan WalletEvent, managerSubBufferSize) subs := make([]event.Subscription, len(backends)) for i, backend := range backends { @@ -65,12 +78,14 @@ func NewManager(config *Config, backends ...Backend) *Manager { } // Assemble the account manager and return am := &Manager{ - config: config, - backends: make(map[reflect.Type][]Backend), - updaters: subs, - updates: updates, - wallets: wallets, - quit: make(chan chan error), + config: config, + backends: make(map[reflect.Type][]Backend), + updaters: subs, + updates: updates, + newBackends: make(chan newBackendEvent), + wallets: wallets, + quit: make(chan chan error), + term: make(chan struct{}), } for _, backend := range backends { kind := reflect.TypeOf(backend) @@ -93,6 +108,14 @@ func (am *Manager) Config() *Config { return am.config } +// AddBackend starts the tracking of an additional backend for wallet updates. +// cmd/geth assumes once this func returns the backends have been already integrated. +func (am *Manager) AddBackend(backend Backend) { + done := make(chan struct{}) + am.newBackends <- newBackendEvent{backend, done} + <-done +} + // update is the wallet event loop listening for notifications from the backends // and updating the cache of wallets. func (am *Manager) update() { @@ -122,10 +145,22 @@ func (am *Manager) update() { // Notify any listeners of the event am.feed.Send(event) - + case event := <-am.newBackends: + am.lock.Lock() + // Update caches + backend := event.backend + am.wallets = merge(am.wallets, backend.Wallets()...) + am.updaters = append(am.updaters, backend.Subscribe(am.updates)) + kind := reflect.TypeOf(backend) + am.backends[kind] = append(am.backends[kind], backend) + am.lock.Unlock() + close(event.processed) case errc := <-am.quit: // Manager terminating, return errc <- nil + // Signals event emitters the loop is not receiving values + // to prevent them from getting stuck. + close(am.term) return } } @@ -133,6 +168,9 @@ func (am *Manager) update() { // Backends retrieves the backend(s) with the given type from the account manager. func (am *Manager) Backends(kind reflect.Type) []Backend { + am.lock.RLock() + defer am.lock.RUnlock() + return am.backends[kind] } diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 6d45c88763..e33b9eb0fb 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -268,11 +268,16 @@ func accountCreate(ctx *cli.Context) error { } } utils.SetNodeConfig(ctx, &cfg.Node) - scryptN, scryptP, keydir, err := cfg.Node.AccountConfig() - + keydir, err := cfg.Node.KeyDirConfig() if err != nil { utils.Fatalf("Failed to read configuration: %v", err) } + scryptN := keystore.StandardScryptN + scryptP := keystore.StandardScryptP + if cfg.Node.UseLightweightKDF { + scryptN = keystore.LightScryptN + scryptP = keystore.LightScryptP + } password := utils.GetPassPhraseWithList("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx)) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 604f907b7f..c97a64f173 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -27,6 +27,10 @@ import ( "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/accounts/external" + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/accounts/scwallet" + "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" @@ -135,6 +139,11 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { if err != nil { utils.Fatalf("Failed to create the protocol stack: %v", err) } + // Node doesn't by default populate account manager backends + if err := setAccountManagerBackends(stack); err != nil { + utils.Fatalf("Failed to set account manager backends: %v", err) + } + utils.SetEthConfig(ctx, stack, &cfg.Eth) if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) { cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name) @@ -257,3 +266,62 @@ func deprecated(field string) bool { return false } } + +func setAccountManagerBackends(stack *node.Node) error { + conf := stack.Config() + am := stack.AccountManager() + keydir := stack.KeyStoreDir() + scryptN := keystore.StandardScryptN + scryptP := keystore.StandardScryptP + if conf.UseLightweightKDF { + scryptN = keystore.LightScryptN + scryptP = keystore.LightScryptP + } + + // Assemble the supported backends + if len(conf.ExternalSigner) > 0 { + log.Info("Using external signer", "url", conf.ExternalSigner) + if extapi, err := external.NewExternalBackend(conf.ExternalSigner); err == nil { + am.AddBackend(extapi) + return nil + } else { + return fmt.Errorf("error connecting to external signer: %v", err) + } + } + + // For now, we're using EITHER external signer OR local signers. + // If/when we implement some form of lockfile for USB and keystore wallets, + // we can have both, but it's very confusing for the user to see the same + // accounts in both externally and locally, plus very racey. + am.AddBackend(keystore.NewKeyStore(keydir, scryptN, scryptP)) + if conf.USB { + // Start a USB hub for Ledger hardware wallets + if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { + log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) + } else { + am.AddBackend(ledgerhub) + } + // Start a USB hub for Trezor hardware wallets (HID version) + if trezorhub, err := usbwallet.NewTrezorHubWithHID(); err != nil { + log.Warn(fmt.Sprintf("Failed to start HID Trezor hub, disabling: %v", err)) + } else { + am.AddBackend(trezorhub) + } + // Start a USB hub for Trezor hardware wallets (WebUSB version) + if trezorhub, err := usbwallet.NewTrezorHubWithWebUSB(); err != nil { + log.Warn(fmt.Sprintf("Failed to start WebUSB Trezor hub, disabling: %v", err)) + } else { + am.AddBackend(trezorhub) + } + } + if len(conf.SmartCardDaemonPath) > 0 { + // Start a smart card hub + if schub, err := scwallet.NewHub(conf.SmartCardDaemonPath, scwallet.Scheme, keydir); err != nil { + log.Warn(fmt.Sprintf("Failed to start smart card hub, disabling: %v", err)) + } else { + am.AddBackend(schub) + } + } + + return nil +} diff --git a/node/config.go b/node/config.go index b7e99eb7bf..26f00cd678 100644 --- a/node/config.go +++ b/node/config.go @@ -26,11 +26,6 @@ import ( "strings" "sync" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/external" - "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/accounts/scwallet" - "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -430,15 +425,8 @@ func (c *Config) parsePersistentNodes(w *bool, path string) []*enode.Node { return nodes } -// AccountConfig determines the settings for scrypt and keydirectory -func (c *Config) AccountConfig() (int, int, string, error) { - scryptN := keystore.StandardScryptN - scryptP := keystore.StandardScryptP - if c.UseLightweightKDF { - scryptN = keystore.LightScryptN - scryptP = keystore.LightScryptP - } - +// KeyDirConfig determines the settings for keydirectory +func (c *Config) KeyDirConfig() (string, error) { var ( keydir string err error @@ -455,71 +443,31 @@ func (c *Config) AccountConfig() (int, int, string, error) { case c.KeyStoreDir != "": keydir, err = filepath.Abs(c.KeyStoreDir) } - return scryptN, scryptP, keydir, err + return keydir, err } -func makeAccountManager(conf *Config) (*accounts.Manager, string, error) { - scryptN, scryptP, keydir, err := conf.AccountConfig() - var ephemeral string +// getKeyStoreDir retrieves the key directory and will create +// and ephemeral one if necessary. +func getKeyStoreDir(conf *Config) (string, bool, error) { + keydir, err := conf.KeyDirConfig() + if err != nil { + return "", false, err + } + isEphemeral := false if keydir == "" { // There is no datadir. keydir, err = ioutil.TempDir("", "go-ethereum-keystore") - ephemeral = keydir + isEphemeral = true } if err != nil { - return nil, "", err + return "", false, err } if err := os.MkdirAll(keydir, 0700); err != nil { - return nil, "", err - } - // Assemble the account manager and supported backends - var backends []accounts.Backend - if len(conf.ExternalSigner) > 0 { - log.Info("Using external signer", "url", conf.ExternalSigner) - if extapi, err := external.NewExternalBackend(conf.ExternalSigner); err == nil { - backends = append(backends, extapi) - } else { - return nil, "", fmt.Errorf("error connecting to external signer: %v", err) - } - } - if len(backends) == 0 { - // For now, we're using EITHER external signer OR local signers. - // If/when we implement some form of lockfile for USB and keystore wallets, - // we can have both, but it's very confusing for the user to see the same - // accounts in both externally and locally, plus very racey. - backends = append(backends, keystore.NewKeyStore(keydir, scryptN, scryptP)) - if conf.USB { - // Start a USB hub for Ledger hardware wallets - if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { - log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) - } else { - backends = append(backends, ledgerhub) - } - // Start a USB hub for Trezor hardware wallets (HID version) - if trezorhub, err := usbwallet.NewTrezorHubWithHID(); err != nil { - log.Warn(fmt.Sprintf("Failed to start HID Trezor hub, disabling: %v", err)) - } else { - backends = append(backends, trezorhub) - } - // Start a USB hub for Trezor hardware wallets (WebUSB version) - if trezorhub, err := usbwallet.NewTrezorHubWithWebUSB(); err != nil { - log.Warn(fmt.Sprintf("Failed to start WebUSB Trezor hub, disabling: %v", err)) - } else { - backends = append(backends, trezorhub) - } - } - if len(conf.SmartCardDaemonPath) > 0 { - // Start a smart card hub - if schub, err := scwallet.NewHub(conf.SmartCardDaemonPath, scwallet.Scheme, keydir); err != nil { - log.Warn(fmt.Sprintf("Failed to start smart card hub, disabling: %v", err)) - } else { - backends = append(backends, schub) - } - } + return "", false, err } - return accounts.NewManager(&accounts.Config{InsecureUnlockAllowed: conf.InsecureUnlockAllowed}, backends...), ephemeral, nil + return keydir, isEphemeral, nil } var warnLock sync.Mutex diff --git a/node/node.go b/node/node.go index 1e65fff1c2..ceab1c9090 100644 --- a/node/node.go +++ b/node/node.go @@ -42,7 +42,8 @@ type Node struct { config *Config accman *accounts.Manager log log.Logger - ephemKeystore string // if non-empty, the key directory that will be removed by Stop + keyDir string // key store directory + keyDirTemp bool // If true, key directory will be removed by Stop dirLock fileutil.Releaser // prevents concurrent use of instance directory stop chan struct{} // Channel to wait for termination notifications server *p2p.Server // Currently running P2P networking layer @@ -112,14 +113,15 @@ func New(conf *Config) (*Node, error) { if err := node.openDataDir(); err != nil { return nil, err } - // Ensure that the AccountManager method works before the node has started. We rely on - // this in cmd/geth. - am, ephemeralKeystore, err := makeAccountManager(conf) + keyDir, isEphem, err := getKeyStoreDir(conf) if err != nil { return nil, err } - node.accman = am - node.ephemKeystore = ephemeralKeystore + node.keyDir = keyDir + node.keyDirTemp = isEphem + // Creates an empty AccountManager with no backends. Callers (e.g. cmd/geth) + // are required to add the backends later on. + node.accman = accounts.NewManager(&accounts.Config{InsecureUnlockAllowed: conf.InsecureUnlockAllowed}) // Initialize the p2p server. This creates the node key and discovery databases. node.server.Config.PrivateKey = node.config.NodeKey() @@ -233,8 +235,8 @@ func (n *Node) doClose(errs []error) error { if err := n.accman.Close(); err != nil { errs = append(errs, err) } - if n.ephemKeystore != "" { - if err := os.RemoveAll(n.ephemKeystore); err != nil { + if n.keyDirTemp { + if err := os.RemoveAll(n.keyDir); err != nil { errs = append(errs, err) } } @@ -514,6 +516,11 @@ func (n *Node) InstanceDir() string { return n.config.instanceDir() } +// KeyStoreDir retrieves the key directory +func (n *Node) KeyStoreDir() string { + return n.keyDir +} + // AccountManager retrieves the account manager used by the protocol stack. func (n *Node) AccountManager() *accounts.Manager { return n.accman From 62ad17fb0046243255048fbf8cb0882f48d8d850 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 26 Aug 2021 08:35:37 +0200 Subject: [PATCH 29/30] Revert "eth, internal/ethapi: make RPC block miner field show block sealer correctly (#23312)" (#23466) This reverts commit 57feabea663496109e59df669238398239438fb1. --- eth/api.go | 2 +- internal/ethapi/api.go | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/eth/api.go b/eth/api.go index 0f57128d75..8b96d1f316 100644 --- a/eth/api.go +++ b/eth/api.go @@ -342,7 +342,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, } else { blockRlp = fmt.Sprintf("0x%x", rlpBytes) } - if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.engine); err != nil { + if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} } results = append(results, &BadBlockArgs{ diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1af98e1071..9a82824ada 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/misc" @@ -1176,8 +1175,7 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { } // RPCMarshalHeader converts the given header to the RPC output . -func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]interface{} { - miner, _ := engine.Author(head) +func RPCMarshalHeader(head *types.Header) map[string]interface{} { result := map[string]interface{}{ "number": (*hexutil.Big)(head.Number), "hash": head.Hash(), @@ -1187,7 +1185,7 @@ func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]in "sha3Uncles": head.UncleHash, "logsBloom": head.Bloom, "stateRoot": head.Root, - "miner": miner, + "miner": head.Coinbase, "difficulty": (*hexutil.Big)(head.Difficulty), "extraData": hexutil.Bytes(head.Extra), "size": hexutil.Uint64(head.Size()), @@ -1208,8 +1206,8 @@ func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]in // RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are // returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain // transaction hashes. -func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, engine consensus.Engine) (map[string]interface{}, error) { - fields := RPCMarshalHeader(block.Header(), engine) +func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields := RPCMarshalHeader(block.Header()) fields["size"] = hexutil.Uint64(block.Size()) if inclTx { @@ -1244,7 +1242,7 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, engine consen // rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { - fields := RPCMarshalHeader(header, s.b.Engine()) + fields := RPCMarshalHeader(header) fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, header.Hash())) return fields } @@ -1252,7 +1250,7 @@ func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *type // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields, err := RPCMarshalBlock(b, inclTx, fullTx, s.b.Engine()) + fields, err := RPCMarshalBlock(b, inclTx, fullTx) if err != nil { return nil, err } From ffae2043f0482e81ac9f5459bf9fceafd74f52e9 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Fri, 27 Aug 2021 06:11:15 -0600 Subject: [PATCH 30/30] internal/ethapi: support both input and data for personal_sendTransaction (#23476) Currently, setDefaults overwrites the transaction input value if only input is provided. This causes personal_sendTransaction to estimate the gas based on a transaction with empty data. eth_estimateGas never calls setDefaults so it was unaffected by this. --- internal/ethapi/transaction_args.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 52811b2a9a..2d08d3008f 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -146,6 +146,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { if args.Gas == nil { // These fields are immutable during the estimation, safe to // pass the pointer directly. + data := args.data() callArgs := TransactionArgs{ From: args.From, To: args.To, @@ -153,7 +154,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { MaxFeePerGas: args.MaxFeePerGas, MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, Value: args.Value, - Data: args.Data, + Data: (*hexutil.Bytes)(&data), AccessList: args.AccessList, } pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber)