diff --git a/.golangci.yml b/.golangci.yml index f5d4a5c678ca..9d8164e42c16 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,15 +30,16 @@ linters: - durationcheck - exportloopref # - gosec + - whitespace - #- structcheck # lots of false positives - #- errcheck #lot of false positives - # - contextcheck - # - errchkjson # lots of false positives - # - errorlint # this check crashes - # - exhaustive # silly check - # - makezero # false positives - # - nilerr # several intentional + # - structcheck # lots of false positives + # - errcheck #lot of false positives + # - contextcheck + # - errchkjson # lots of false positives + # - errorlint # this check crashes + # - exhaustive # silly check + # - makezero # false positives + # - nilerr # several intentional linters-settings: gofmt: @@ -48,9 +49,9 @@ linters-settings: min-occurrences: 6 # minimum number of occurrences gosec: excludes: - - G404 # Use of weak random number generator - lots of FP - - G107 # Potential http request -- those are intentional - - G306 # G306: Expect WriteFile permissions to be 0600 or less + - G404 # Use of weak random number generator - lots of FP + - G107 # Potential http request -- those are intentional + - G306 # G306: Expect WriteFile permissions to be 0600 or less issues: exclude-rules: diff --git a/Dockerfile b/Dockerfile index 5246f4d1bad3..9265e597b84f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth +RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth # Pull Geth into a second stage deploy alpine container FROM alpine:latest diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 73cb661b7fba..ec41c0d51c50 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install +RUN cd /go-ethereum && go run build/ci.go install -static # Pull all binaries into a second stage deploy alpine container FROM alpine:latest diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index ed5b6e92ef9c..81bbee2f2b4a 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -95,7 +95,7 @@ func (abi ABI) getArguments(name string, data []byte) (Arguments, error) { args = event.Inputs } if args == nil { - return nil, errors.New("abi: could not locate named method or event") + return nil, fmt.Errorf("abi: could not locate named method or event: %s", name) } return args, nil } diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index c5326d5700a6..ed204e0a81dd 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -18,6 +18,7 @@ package abi import ( "encoding/json" + "errors" "fmt" "reflect" "strings" @@ -79,7 +80,7 @@ func (arguments Arguments) isTuple() bool { func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) { if len(data) == 0 { if len(arguments.NonIndexed()) != 0 { - return nil, fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected") + return nil, errors.New("abi: attempting to unmarshall an empty string while arguments are expected") } return make([]interface{}, 0), nil } @@ -90,11 +91,11 @@ func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) { func (arguments Arguments) UnpackIntoMap(v map[string]interface{}, data []byte) error { // Make sure map is not nil if v == nil { - return fmt.Errorf("abi: cannot unpack into a nil map") + return errors.New("abi: cannot unpack into a nil map") } if len(data) == 0 { if len(arguments.NonIndexed()) != 0 { - return fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected") + return errors.New("abi: attempting to unmarshall an empty string while arguments are expected") } return nil // Nothing to unmarshal, return } @@ -116,7 +117,7 @@ func (arguments Arguments) Copy(v interface{}, values []interface{}) error { } if len(values) == 0 { if len(arguments.NonIndexed()) != 0 { - return fmt.Errorf("abi: attempting to copy no values while arguments are expected") + return errors.New("abi: attempting to copy no values while arguments are expected") } return nil // Nothing to copy, return } diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index d505752c236c..f6ccc3a73d05 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -68,7 +68,8 @@ type SimulatedBackend struct { pendingState *state.StateDB // Currently pending state that will be the active on request pendingReceipts types.Receipts // Currently receipts for the pending block - events *filters.EventSystem // Event system for filtering log events live + events *filters.EventSystem // for filtering log events live + filterSystem *filters.FilterSystem // for filtering database logs config *params.ChainConfig } @@ -86,7 +87,11 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.Genesis blockchain: blockchain, config: genesis.Config, } - backend.events = filters.NewEventSystem(&filterBackend{database, blockchain, backend}, false) + + filterBackend := &filterBackend{database, blockchain, backend} + backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{}) + backend.events = filters.NewEventSystem(backend.filterSystem, false) + backend.rollback(blockchain.CurrentBlock()) return backend } @@ -106,16 +111,20 @@ func (b *SimulatedBackend) Close() error { // Commit imports all the pending transactions as a single block and starts a // fresh new state. -func (b *SimulatedBackend) Commit() { +func (b *SimulatedBackend) Commit() common.Hash { b.mu.Lock() defer b.mu.Unlock() if _, err := b.blockchain.InsertChain([]*types.Block{b.pendingBlock}); err != nil { panic(err) // This cannot happen unless the simulator is wrong, fail in that case } + blockHash := b.pendingBlock.Hash() + // Using the last inserted block here makes it possible to build on a side // chain after a fork. b.rollback(b.pendingBlock) + + return blockHash } // Rollback aborts all pending transactions, reverting to the last committed state. @@ -605,7 +614,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM // User specified the legacy gas field, convert to 1559 gas typing call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those if call.GasFeeCap == nil { call.GasFeeCap = new(big.Int) } @@ -685,7 +694,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter var filter *filters.Filter if query.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain, b}, *query.BlockHash, query.Addresses, query.Topics) + filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics) } else { // Initialize unset filter boundaries to run from genesis to chain head from := int64(0) @@ -697,7 +706,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter to = query.ToBlock.Int64() } // Construct the range filter - filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain, b}, from, to, query.Addresses, query.Topics) + filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -826,7 +835,8 @@ type filterBackend struct { backend *SimulatedBackend } -func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } +func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } + func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") } func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) { @@ -852,19 +862,8 @@ func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (typ return rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()), nil } -func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(fb.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()) - if receipts == nil { - return nil, nil - } - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(fb.db, hash, number, fb.bc.Config()) return logs, nil } diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 0f346b7d5055..85f6aab18ecd 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -1338,3 +1338,42 @@ func TestForkResendTx(t *testing.T) { t.Errorf("TX included in wrong block: %d", h) } } + +func TestCommitReturnValue(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + startBlockHeight := sim.blockchain.CurrentBlock().NumberU64() + + // Test if Commit returns the correct block hash + h1 := sim.Commit() + if h1 != sim.blockchain.CurrentBlock().Hash() { + t.Error("Commit did not return the hash of the last block.") + } + + // Create a block in the original chain (containing a transaction to force different block hashes) + head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + _tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) + tx, _ := types.SignTx(_tx, types.HomesteadSigner{}, testKey) + sim.SendTransaction(context.Background(), tx) + h2 := sim.Commit() + + // Create another block in the original chain + sim.Commit() + + // Fork at the first bock + if err := sim.Fork(context.Background(), h1); err != nil { + t.Errorf("forking: %v", err) + } + + // Test if Commit returns the correct block hash after the reorg + h2fork := sim.Commit() + if h2 == h2fork { + t.Error("The block in the fork and the original block are the same block!") + } + if sim.blockchain.GetHeader(h2fork, startBlockHeight+2) == nil { + t.Error("Could not retrieve the just created block (side-chain)") + } +} diff --git a/accounts/abi/bind/base_test.go b/accounts/abi/bind/base_test.go index 25b2f8a865f2..2307b9874b18 100644 --- a/accounts/abi/bind/base_test.go +++ b/accounts/abi/bind/base_test.go @@ -115,7 +115,6 @@ func (mc *mockPendingCaller) PendingCallContract(ctx context.Context, call ether } func TestPassingBlockNumber(t *testing.T) { - mc := &mockPendingCaller{ mockCaller: &mockCaller{ codeAtBytes: []byte{1, 2, 3}, diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index a938e7dfcd85..dac43f70e234 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -43,6 +43,43 @@ const ( LangObjC ) +func isKeyWord(arg string) bool { + switch arg { + case "break": + case "case": + case "chan": + case "const": + case "continue": + case "default": + case "defer": + case "else": + case "fallthrough": + case "for": + case "func": + case "go": + case "goto": + case "if": + case "import": + case "interface": + case "iota": + case "map": + case "make": + case "new": + case "package": + case "range": + case "return": + case "select": + case "struct": + case "switch": + case "type": + case "var": + default: + return false + } + + return true +} + // Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant // to be used as is in client code, but rather as an intermediate struct which // enforces compile time type safety and naming convention opposed to having to @@ -114,7 +151,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } if hasStruct(input.Type) { @@ -158,7 +195,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } // Event is a bit special, we need to define event struct in binding, diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 6af7502f6dcc..c8d22e25b322 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1971,7 +1971,7 @@ var bindTests = []struct { } } `, - bytecode: []string{`0x608060405234801561001057600080fd5b5061042b806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063c2bb515f1461003b578063cce7b04814610059575b600080fd5b610043610075565b60405161005091906101af565b60405180910390f35b610073600480360381019061006e91906103ac565b6100b5565b005b61007d6100b8565b604051806040016040528060405180602001604052806000815250815260200160405180602001604052806000815250815250905090565b50565b604051806040016040528060608152602001606081525090565b600081519050919050565b600082825260208201905092915050565b60005b8381101561010c5780820151818401526020810190506100f1565b8381111561011b576000848401525b50505050565b6000601f19601f8301169050919050565b600061013d826100d2565b61014781856100dd565b93506101578185602086016100ee565b61016081610121565b840191505092915050565b600060408301600083015184820360008601526101888282610132565b915050602083015184820360208601526101a28282610132565b9150508091505092915050565b600060208201905081810360008301526101c9818461016b565b905092915050565b6000604051905090565b600080fd5b600080fd5b600080fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61022282610121565b810181811067ffffffffffffffff82111715610241576102406101ea565b5b80604052505050565b60006102546101d1565b90506102608282610219565b919050565b600080fd5b600080fd5b600080fd5b600067ffffffffffffffff82111561028f5761028e6101ea565b5b61029882610121565b9050602081019050919050565b82818337600083830152505050565b60006102c76102c284610274565b61024a565b9050828152602081018484840111156102e3576102e261026f565b5b6102ee8482856102a5565b509392505050565b600082601f83011261030b5761030a61026a565b5b813561031b8482602086016102b4565b91505092915050565b60006040828403121561033a576103396101e5565b5b610344604061024a565b9050600082013567ffffffffffffffff81111561036457610363610265565b5b610370848285016102f6565b600083015250602082013567ffffffffffffffff81111561039457610393610265565b5b6103a0848285016102f6565b60208301525092915050565b6000602082840312156103c2576103c16101db565b5b600082013567ffffffffffffffff8111156103e0576103df6101e0565b5b6103ec84828501610324565b9150509291505056fea264697066735822122033bca1606af9b6aeba1673f98c52003cec19338539fb44b86690ce82c51483b564736f6c634300080e0033`}, + bytecode: []string{"0x608060405234801561001057600080fd5b5061042b806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063c2bb515f1461003b578063cce7b04814610059575b600080fd5b610043610075565b60405161005091906101af565b60405180910390f35b610073600480360381019061006e91906103ac565b6100b5565b005b61007d6100b8565b604051806040016040528060405180602001604052806000815250815260200160405180602001604052806000815250815250905090565b50565b604051806040016040528060608152602001606081525090565b600081519050919050565b600082825260208201905092915050565b60005b8381101561010c5780820151818401526020810190506100f1565b8381111561011b576000848401525b50505050565b6000601f19601f8301169050919050565b600061013d826100d2565b61014781856100dd565b93506101578185602086016100ee565b61016081610121565b840191505092915050565b600060408301600083015184820360008601526101888282610132565b915050602083015184820360208601526101a28282610132565b9150508091505092915050565b600060208201905081810360008301526101c9818461016b565b905092915050565b6000604051905090565b600080fd5b600080fd5b600080fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61022282610121565b810181811067ffffffffffffffff82111715610241576102406101ea565b5b80604052505050565b60006102546101d1565b90506102608282610219565b919050565b600080fd5b600080fd5b600080fd5b600067ffffffffffffffff82111561028f5761028e6101ea565b5b61029882610121565b9050602081019050919050565b82818337600083830152505050565b60006102c76102c284610274565b61024a565b9050828152602081018484840111156102e3576102e261026f565b5b6102ee8482856102a5565b509392505050565b600082601f83011261030b5761030a61026a565b5b813561031b8482602086016102b4565b91505092915050565b60006040828403121561033a576103396101e5565b5b610344604061024a565b9050600082013567ffffffffffffffff81111561036457610363610265565b5b610370848285016102f6565b600083015250602082013567ffffffffffffffff81111561039457610393610265565b5b6103a0848285016102f6565b60208301525092915050565b6000602082840312156103c2576103c16101db565b5b600082013567ffffffffffffffff8111156103e0576103df6101e0565b5b6103ec84828501610324565b9150509291505056fea264697066735822122033bca1606af9b6aeba1673f98c52003cec19338539fb44b86690ce82c51483b564736f6c634300080e0033"}, abi: []string{`[ { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "int256", "name": "msg", "type": "int256" }, { "indexed": false, "internalType": "int256", "name": "_msg", "type": "int256" } ], "name": "log", "type": "event" }, { "inputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "req", "type": "tuple" } ], "name": "addRequest", "outputs": [], "stateMutability": "pure", "type": "function" }, { "inputs": [], "name": "getRequest", "outputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "", "type": "tuple" } ], "stateMutability": "pure", "type": "function" } ]`}, imports: ` "math/big" @@ -2002,6 +2002,43 @@ var bindTests = []struct { } `, }, + { + name: "RangeKeyword", + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract keywordcontract { + function functionWithKeywordParameter(range uint256) public pure {} + } + `, + bytecode: []string{"0x608060405234801561001057600080fd5b5060dc8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063527a119f14602d575b600080fd5b60436004803603810190603f9190605b565b6045565b005b50565b6000813590506055816092565b92915050565b600060208284031215606e57606d608d565b5b6000607a848285016048565b91505092915050565b6000819050919050565b600080fd5b6099816083565b811460a357600080fd5b5056fea2646970667358221220d4f4525e2615516394055d369fb17df41c359e5e962734f27fd683ea81fd9db164736f6c63430008070033"}, + abi: []string{`[{"inputs":[{"internalType":"uint256","name":"range","type":"uint256"}],"name":"functionWithKeywordParameter","outputs":[],"stateMutability":"pure","type":"function"}]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + _, tx, _, err := DeployRangeKeyword(user, sim) + if err != nil { + t.Fatalf("error deploying contract: %v", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Errorf("error deploying the contract: %v", err) + } + `, + }, } // Tests that packages generated by the binder can be successfully compiled and diff --git a/accounts/abi/error_handling.go b/accounts/abi/error_handling.go index f0f71b6c9164..7add7072925e 100644 --- a/accounts/abi/error_handling.go +++ b/accounts/abi/error_handling.go @@ -73,7 +73,6 @@ func typeCheck(t Type, value reflect.Value) error { } else { return nil } - } // typeErr returns a formatted type casting error. diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 3332f8a07216..8f73419496ba 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -161,7 +161,6 @@ func TestEventMultiValueWithArrayUnpack(t *testing.T) { } func TestEventTupleUnpack(t *testing.T) { - type EventTransfer struct { Value *big.Int } diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 285d34944032..731eefac8080 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -101,7 +101,7 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value { func set(dst, src reflect.Value) error { dstType, srcType := dst.Type(), src.Type() switch { - case dstType.Kind() == reflect.Interface && dst.Elem().IsValid(): + case dstType.Kind() == reflect.Interface && dst.Elem().IsValid() && (dst.Elem().Type().Kind() == reflect.Ptr || dst.Elem().CanSet()): return set(dst.Elem(), src) case dstType.Kind() == reflect.Ptr && dstType.Elem() != reflect.TypeOf(big.Int{}): return set(dst.Elem(), src) @@ -226,7 +226,6 @@ func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[stri // second round ~~~ for _, argName := range argNames { - structFieldName := ToCamelCase(argName) if structFieldName == "" { diff --git a/accounts/abi/reflect_test.go b/accounts/abi/reflect_test.go index cf13a79da84e..76ef1ad2aa39 100644 --- a/accounts/abi/reflect_test.go +++ b/accounts/abi/reflect_test.go @@ -32,7 +32,7 @@ type reflectTest struct { var reflectTests = []reflectTest{ { - name: "OneToOneCorrespondance", + name: "OneToOneCorrespondence", args: []string{"fieldA"}, struc: struct { FieldA int `abi:"fieldA"` diff --git a/accounts/abi/selector_parser.go b/accounts/abi/selector_parser.go index 88114e288eb3..d5472e374f5d 100644 --- a/accounts/abi/selector_parser.go +++ b/accounts/abi/selector_parser.go @@ -166,7 +166,7 @@ func ParseSelector(unescapedSelector string) (SelectorMarshaling, error) { return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': unexpected string '%s'", unescapedSelector, rest) } - // Reassemble the fake ABI and constuct the JSON + // Reassemble the fake ABI and construct the JSON fakeArgs, err := assembleArgs(args) if err != nil { return SelectorMarshaling{}, fmt.Errorf("failed to parse selector: %v", err) diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 28c5c82bb3d5..800789295c19 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -115,7 +115,6 @@ func ReadFixedBytes(t Type, word []byte) (interface{}, error) { reflect.Copy(array, reflect.ValueOf(word[0:t.Size])) return array.Interface(), nil - } // forEachUnpack iteratively unpack elements. diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index ae3565c71e29..363e0cd5943e 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -352,6 +352,11 @@ func TestMethodMultiReturn(t *testing.T) { &[]interface{}{&expected.Int, &expected.String}, "", "Can unpack into a slice", + }, { + &[]interface{}{&bigint, ""}, + &[]interface{}{&expected.Int, expected.String}, + "", + "Can unpack into a slice without indirection", }, { &[2]interface{}{&bigint, new(string)}, &[2]interface{}{&expected.Int, &expected.String}, diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 324b31c243ce..3b99e4a6c392 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -330,7 +330,7 @@ func TestUpdatedKeyfileContents(t *testing.T) { t.Parallel() - // Create a temporary kesytore to test with + // Create a temporary keystore to test with rand.Seed(time.Now().UnixNano()) dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int())) ks := NewKeyStore(dir, LightScryptN, LightScryptP) diff --git a/accounts/keystore/file_cache.go b/accounts/keystore/file_cache.go index b3ecf8946b53..79f9a2963743 100644 --- a/accounts/keystore/file_cache.go +++ b/accounts/keystore/file_cache.go @@ -39,7 +39,7 @@ type fileCache struct { func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) { t0 := time.Now() - // List all the failes from the keystore folder + // List all the files from the keystore folder files, err := os.ReadDir(keyDir) if err != nil { return nil, nil, nil, err @@ -61,7 +61,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er log.Trace("Ignoring file on account scan", "path", path) continue } - // Gather the set of all and fresly modified files + // Gather the set of all and freshly modified files all.Add(path) info, err := fi.Info() diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index 37edca826159..6261bdb6b8d7 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -218,7 +218,7 @@ func TestWalletNotifierLifecycle(t *testing.T) { t.Skip("skipping test in short mode") } - // Create a temporary kesytore to test with + // Create a temporary keystore to test with _, ks := tmpKeyStore(t, false) // Ensure that the notification updater is not running yet @@ -381,7 +381,6 @@ func TestImportExport(t *testing.T) { if _, err = ks2.Import(json, "new", "new"); err == nil { t.Errorf("importing a key twice succeeded") } - } // TestImportRace tests the keystore on races. @@ -406,7 +405,6 @@ func TestImportRace(t *testing.T) { if _, err := ks2.Import(json, "new", "new"); err != nil { atomic.AddUint32(&atom, 1) } - }() } wg.Wait() diff --git a/accounts/keystore/passphrase.go b/accounts/keystore/passphrase.go index 22772e93102f..1701fbf53634 100644 --- a/accounts/keystore/passphrase.go +++ b/accounts/keystore/passphrase.go @@ -138,7 +138,6 @@ func (ks keyStorePassphrase) JoinPath(filename string) string { // Encryptdata encrypts the data given as 'data' with the password 'auth'. func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) { - salt := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, salt); err != nil { panic("reading from crypto/rand failed: " + err.Error()) @@ -341,7 +340,6 @@ func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) { r := ensureInt(cryptoJSON.KDFParams["r"]) p := ensureInt(cryptoJSON.KDFParams["p"]) return scrypt.Key(authArray, salt, n, r, p, dkLen) - } else if cryptoJSON.KDF == "pbkdf2" { c := ensureInt(cryptoJSON.KDFParams["c"]) prf := cryptoJSON.KDFParams["prf"].(string) diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index 10887a8b43d0..b1b533eb7243 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -178,7 +178,7 @@ func (s *SecureChannelSession) mutuallyAuthenticate() error { return err } if response.Sw1 != 0x90 || response.Sw2 != 0x00 { - return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: 0x%x%x", response.Sw1, response.Sw2) + return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: %#x%x", response.Sw1, response.Sw2) } if len(response.Data) != scSecretLength { @@ -261,7 +261,7 @@ func (s *SecureChannelSession) transmitEncrypted(cla, ins, p1, p2 byte, data []b rapdu.deserialize(plainData) if rapdu.Sw1 != sw1Ok { - return nil, fmt.Errorf("unexpected response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) + return nil, fmt.Errorf("unexpected response status Cla=%#x, Ins=%#x, Sw=%#x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) } return rapdu, nil diff --git a/accounts/scwallet/wallet.go b/accounts/scwallet/wallet.go index 582e067b1777..b682608d7b46 100644 --- a/accounts/scwallet/wallet.go +++ b/accounts/scwallet/wallet.go @@ -167,7 +167,7 @@ func transmit(card *pcsc.Card, command *commandAPDU) (*responseAPDU, error) { } if response.Sw1 != sw1Ok { - return nil, fmt.Errorf("unexpected insecure response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) + return nil, fmt.Errorf("unexpected insecure response status Cla=%#x, Ins=%#x, Sw=%#x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) } return response, nil @@ -699,7 +699,6 @@ func (w *Wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) // the needed details via SignTxWithPassphrase, or by other means (e.g. unlock // the account in a keystore). func (w *Wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { - // fee delegation if tx.Type() == types.FeeDelegateDynamicFeeTxType { signer := types.NewFeeDelegateSigner(chainID) diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index cf0a7cb15b97..878e314d0194 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -212,10 +212,10 @@ func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, er if _, err := w.trezorExchange(&trezor.EthereumGetAddress{AddressN: derivationPath}, address); err != nil { return common.Address{}, err } - if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary fomats + if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary formats return common.BytesToAddress(addr), nil } - if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal fomats + if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal formats return common.HexToAddress(addr), nil } return common.Address{}, errors.New("missing derived address") diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 3351f13db0fc..171ba3e8cfa1 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -388,7 +388,7 @@ func (w *wallet) selfDerive() { // of legacy-ledger, the first account on the legacy-path will // be shown to the user, even if we don't actively track it if i < len(nextAddrs)-1 { - w.log.Info("Skipping trakcking first account on legacy path, use personal.deriveAccount(,, false) to track", + w.log.Info("Skipping tracking first account on legacy path, use personal.deriveAccount(,, false) to track", "path", path, "address", nextAddrs[i]) break } @@ -534,7 +534,6 @@ func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { - // Unless we are doing 712 signing, simply dispatch to signHash if !(mimeType == accounts.MimetypeTypedData && len(data) == 66 && data[0] == 0x19 && data[1] == 0x01) { return w.signHash(account, crypto.Keccak256(data)) diff --git a/build/ci.go b/build/ci.go index b639a317c72c..dad6e6659177 100644 --- a/build/ci.go +++ b/build/ci.go @@ -199,10 +199,11 @@ func main() { func doInstall(cmdline []string) { var ( - dlgo = flag.Bool("dlgo", false, "Download Go and build with it") - arch = flag.String("arch", "", "Architecture to cross build for") - cc = flag.String("cc", "", "C compiler to cross build with") - tags = flag.String("tags", "", "Build tags") + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Architecture to cross build for") + cc = flag.String("cc", "", "C compiler to cross build with") + tags = flag.String("tags", "", "Build tags") + staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) @@ -213,9 +214,12 @@ func doInstall(cmdline []string) { tc.Root = build.DownloadGo(csdb, dlgoVersion) } + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + // Configure the build. env := build.Env() - gobuild := tc.Go("build", buildFlags(env)...) + gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, // better disable it. This check isn't the best, it should probably @@ -224,9 +228,6 @@ func doInstall(cmdline []string) { gobuild.Args = append(gobuild.Args, "-p", "1") } - // Disable CLI markdown doc generation in release builds. - gobuild.Args = append(gobuild.Args, "-tags", "urfave_cli_no_docs") - // We use -trimpath to avoid leaking local paths into the built executables. gobuild.Args = append(gobuild.Args, "-trimpath") @@ -256,7 +257,7 @@ func doInstall(cmdline []string) { } // buildFlags returns the go tool flags for building. -func buildFlags(env build.Environment) (flags []string) { +func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { var ld []string if env.Commit != "" { ld = append(ld, "-X", "main.gitCommit="+env.Commit) @@ -267,14 +268,24 @@ func buildFlags(env build.Environment) (flags []string) { if runtime.GOOS == "darwin" { ld = append(ld, "-s") } - // Enforce the stacksize to 8M, which is the case on most platforms apart from - // alpine Linux. if runtime.GOOS == "linux" { - ld = append(ld, "-extldflags", "-Wl,-z,stack-size=0x800000") + // Enforce the stacksize to 8M, which is the case on most platforms apart from + // alpine Linux. + extld := []string{"-Wl,-z,stack-size=0x800000"} + if staticLinking { + extld = append(extld, "-static") + // Under static linking, use of certain glibc features must be + // disabled to avoid shared library dependencies. + buildTags = append(buildTags, "osusergo", "netgo") + } + ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + if len(buildTags) > 0 { + flags = append(flags, "-tags", strings.Join(buildTags, ",")) + } return flags } @@ -352,10 +363,10 @@ func downloadLinter(cachedir string) string { arch := runtime.GOARCH ext := ".tar.gz" - if runtime.GOOS == "windows" { - ext = ".zip" - } - if arch == "arm" { + if runtime.GOOS == "windows" { + ext = ".zip" + } + if arch == "arm" { arch += "v" + os.Getenv("GOARM") } base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) @@ -607,7 +618,7 @@ func doDocker(cmdline []string) { } if mismatch { // Build numbers mismatching, retry in a short time to - // avoid concurrent failes in both publisher images. If + // avoid concurrent fails in both publisher images. If // however the retry failed too, it means the concurrent // builder is still crunching, let that do the publish. if i == 0 { diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 7a3413811f80..05290f52feb8 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -591,7 +591,7 @@ func signer(c *cli.Context) error { { Namespace: "account", Service: api, - Version: "1.0"}, + }, } if c.Bool(utils.HTTPEnabledFlag.Name) { vhosts := utils.SplitAndTrim(c.String(utils.HTTPVirtualHostsFlag.Name)) @@ -759,7 +759,6 @@ func confirm(text string) bool { } func testExternalUI(api *core.SignerAPI) { - ctx := context.WithValue(context.Background(), "remote", "clef binary") ctx = context.WithValue(ctx, "scheme", "in-proc") ctx = context.WithValue(ctx, "local", "main") @@ -859,7 +858,6 @@ func testExternalUI(api *core.SignerAPI) { expectDeny("signdata - text", err) } { // Sign transaction - api.UI.ShowInfo("Please reject next transaction") time.Sleep(delay) data := hexutil.Bytes([]byte{}) @@ -902,7 +900,6 @@ func testExternalUI(api *core.SignerAPI) { } result := fmt.Sprintf("Tests completed. %d errors:\n%s\n", len(errs), strings.Join(errs, "\n")) api.UI.ShowInfo(result) - } type encryptedSeedStorage struct { @@ -939,7 +936,6 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) { // GenDoc outputs examples of all structures used in json-rpc communication func GenDoc(ctx *cli.Context) error { - var ( a = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef") b = common.HexToAddress("0x1111111122222222222233333333334444444444") @@ -1049,7 +1045,6 @@ func GenDoc(ctx *cli.Context) error { var tx types.Transaction tx.UnmarshalBinary(rlpdata) add("OnApproved - SignTransactionResult", desc, ðapi.SignTransactionResult{Raw: rlpdata, Tx: &tx}) - } { // User input add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)", diff --git a/cmd/dbbench/main.go b/cmd/dbbench/main.go index a6efd001b14e..ce8e23ece8e2 100644 --- a/cmd/dbbench/main.go +++ b/cmd/dbbench/main.go @@ -77,7 +77,6 @@ func setMinMax(db ethdb.Database, prefix string, min, max int) error { } func read(db ethdb.Database, prefix string, start, end, numThreads int, verbose bool) error { - doRead := func(idx int) error { ks := fmt.Sprintf("%s-%d", prefix, idx) k := sha_256([]byte(ks)) @@ -179,7 +178,6 @@ func genVal(key []byte, sz int) []byte { } func write(db ethdb.Database, prefix string, start, end, numThreads, batchCount, valueSize int) error { - flush := func(dbb ethdb.Batch) error { var err error if dbb.ValueSize() > 0 { diff --git a/cmd/devp2p/discv4cmd.go b/cmd/devp2p/discv4cmd.go index 892a02b6591e..9d35880b128b 100644 --- a/cmd/devp2p/discv4cmd.go +++ b/cmd/devp2p/discv4cmd.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" @@ -49,32 +50,34 @@ var ( Usage: "Sends ping to a node", Action: discv4Ping, ArgsUsage: "", + Flags: v4NodeFlags, } discv4RequestRecordCommand = &cli.Command{ Name: "requestenr", Usage: "Requests a node record using EIP-868 enrRequest", Action: discv4RequestRecord, ArgsUsage: "", + Flags: v4NodeFlags, } discv4ResolveCommand = &cli.Command{ Name: "resolve", Usage: "Finds a node in the DHT", Action: discv4Resolve, ArgsUsage: "", - Flags: []cli.Flag{bootnodesFlag}, + Flags: v4NodeFlags, } discv4ResolveJSONCommand = &cli.Command{ Name: "resolve-json", Usage: "Re-resolves nodes in a nodes.json file", Action: discv4ResolveJSON, - Flags: []cli.Flag{bootnodesFlag}, + Flags: v4NodeFlags, ArgsUsage: "", } discv4CrawlCommand = &cli.Command{ Name: "crawl", Usage: "Updates a nodes.json file with random nodes found in the DHT", Action: discv4Crawl, - Flags: []cli.Flag{bootnodesFlag, crawlTimeoutFlag}, + Flags: flags.Merge(v4NodeFlags, []cli.Flag{crawlTimeoutFlag}), } discv4TestCommand = &cli.Command{ Name: "test", @@ -119,6 +122,13 @@ var ( } ) +var v4NodeFlags = []cli.Flag{ + bootnodesFlag, + nodekeyFlag, + nodedbFlag, + listenAddrFlag, +} + func discv4Ping(ctx *cli.Context) error { n := getNodeArg(ctx) disc := startV4(ctx) diff --git a/cmd/devp2p/dns_cloudflare.go b/cmd/devp2p/dns_cloudflare.go index 73ecc13bc32d..92c6faf272ec 100644 --- a/cmd/devp2p/dns_cloudflare.go +++ b/cmd/devp2p/dns_cloudflare.go @@ -134,7 +134,6 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) ttl := rootTTL if path != name { ttl = treeNodeTTLCloudflare // Max TTL permitted by Cloudflare - } record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl} _, err = c.CreateDNSRecord(context.Background(), c.zoneID, record) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index 0a17252a3503..83ceb2a4f2c5 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -96,12 +96,12 @@ func (c *Chain) Head() *types.Block { return c.blocks[c.Len()-1] } -func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { +func (c *Chain) GetHeaders(req *GetBlockHeaders) ([]*types.Header, error) { if req.Amount < 1 { return nil, fmt.Errorf("no block headers requested") } - headers := make(BlockHeaders, req.Amount) + headers := make([]*types.Header, req.Amount) var blockNumber uint64 // range over blocks to check if our chain has the requested header @@ -119,7 +119,6 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { for i := 1; i < int(req.Amount); i++ { blockNumber -= (1 - req.Skip) headers[i] = c.blocks[blockNumber].Header() - } return headers, nil @@ -140,7 +139,7 @@ func loadChain(chainfile string, genesis string) (*Chain, error) { if err != nil { return nil, err } - gblock := gen.ToBlock(nil) + gblock := gen.ToBlock() blocks, err := blocksFromFile(chainfile, gblock) if err != nil { diff --git a/cmd/devp2p/internal/ethtest/chain_test.go b/cmd/devp2p/internal/ethtest/chain_test.go index b4689d0e6982..9cffb1742bc4 100644 --- a/cmd/devp2p/internal/ethtest/chain_test.go +++ b/cmd/devp2p/internal/ethtest/chain_test.go @@ -21,6 +21,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/p2p" "github.com/stretchr/testify/assert" @@ -140,18 +141,18 @@ func TestChain_GetHeaders(t *testing.T) { var tests = []struct { req GetBlockHeaders - expected BlockHeaders + expected []*types.Header }{ { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(2), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(2)}, + Amount: uint64(5), + Skip: 1, + Reverse: false, }, - Amount: uint64(5), - Skip: 1, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[2].Header(), chain.blocks[4].Header(), chain.blocks[6].Header(), @@ -161,14 +162,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(chain.Len() - 1), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(chain.Len() - 1)}, + Amount: uint64(3), + Skip: 0, + Reverse: true, }, - Amount: uint64(3), - Skip: 0, - Reverse: true, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[chain.Len()-1].Header(), chain.blocks[chain.Len()-2].Header(), chain.blocks[chain.Len()-3].Header(), @@ -176,14 +177,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: chain.Head().Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: chain.Head().Hash()}, + Amount: uint64(1), + Skip: 0, + Reverse: false, }, - Amount: uint64(1), - Skip: 0, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.Head().Header(), }, }, @@ -191,7 +192,7 @@ func TestChain_GetHeaders(t *testing.T) { for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - headers, err := chain.GetHeaders(tt.req) + headers, err := chain.GetHeaders(&tt.req) if err != nil { t.Fatal(err) } diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index eca1eb2c1a3a..50e8d7479eea 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -43,21 +43,6 @@ var ( timeout = 20 * time.Second ) -// Is_66 checks if the node supports the eth66 protocol version, -// and if not, exists the test suite -func (s *Suite) Is_66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err := conn.handshake(); err != nil { - t.Fatalf("handshake failed: %v", err) - } - if conn.negotiatedProtoVersion < 66 { - t.Fail() - } -} - // dial attempts to dial the given node and perform a handshake, // returning the created Conn if successful. func (s *Suite) dial() (*Conn, error) { @@ -76,31 +61,16 @@ func (s *Suite) dial() (*Conn, error) { } // set default p2p capabilities conn.caps = []p2p.Cap{ - {Name: "mir", Version: 64}, - {Name: "mir", Version: 65}, + {Name: "mir", Version: 66}, + {Name: "mir", Version: 67}, } - conn.ourHighestProtoVersion = 65 + conn.ourHighestProtoVersion = 67 return &conn, nil } -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional eth66 capabilities if -// successful -func (s *Suite) dial66() (*Conn, error) { - conn, err := s.dial() - if err != nil { - return nil, fmt.Errorf("dial failed: %v", err) - } - conn.caps = append(conn.caps, p2p.Cap{Name: "mir", Version: 66}) - conn.ourHighestProtoVersion = 66 - return conn, nil -} - -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional snap/1 capabilities if -// successful. +// dialSnap creates a connection with snap/1 capability. func (s *Suite) dialSnap() (*Conn, error) { - conn, err := s.dial66() + conn, err := s.dial() if err != nil { return nil, fmt.Errorf("dial failed: %v", err) } @@ -235,117 +205,68 @@ loop: // createSendAndRecvConns creates two connections, one for sending messages to the // node, and one for receiving messages from the node. -func (s *Suite) createSendAndRecvConns(isEth66 bool) (*Conn, *Conn, error) { - var ( - sendConn *Conn - recvConn *Conn - err error - ) - if isEth66 { - sendConn, err = s.dial66() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial66() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - } else { - sendConn, err = s.dial() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) createSendAndRecvConns() (*Conn, *Conn, error) { + sendConn, err := s.dial() + if err != nil { + return nil, nil, fmt.Errorf("dial failed: %v", err) } - return sendConn, recvConn, nil -} - -func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { - if c.negotiatedProtoVersion == 66 { - _, msg := c.readAndServe66(chain, timeout) - return msg + recvConn, err := s.dial() + if err != nil { + sendConn.Close() + return nil, nil, fmt.Errorf("dial failed: %v", err) } - return c.readAndServe65(chain, timeout) + return sendConn, recvConn, nil } // readAndServe serves GetBlockHeaders requests while waiting // on another message from the node. -func (c *Conn) readAndServe65(chain *Chain, timeout time.Duration) Message { - start := time.Now() - for time.Since(start) < timeout { - c.SetReadDeadline(time.Now().Add(5 * time.Second)) - switch msg := c.Read().(type) { - case *Ping: - c.Write(&Pong{}) - case *GetBlockHeaders: - req := *msg - headers, err := chain.GetHeaders(req) - if err != nil { - return errorf("could not get headers for inbound header request: %v", err) - } - if err := c.Write(headers); err != nil { - return errorf("could not write to connection: %v", err) - } - default: - return msg - } - } - return errorf("no message received within %v", timeout) -} - -// readAndServe66 serves eth66 GetBlockHeaders requests while waiting -// on another message from the node. -func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { +func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { start := time.Now() for time.Since(start) < timeout { c.SetReadDeadline(time.Now().Add(10 * time.Second)) - reqID, msg := c.Read66() - + msg := c.Read() switch msg := msg.(type) { case *Ping: c.Write(&Pong{}) - case GetBlockHeaders: + case *GetBlockHeaders: headers, err := chain.GetHeaders(msg) if err != nil { - return 0, errorf("could not get headers for inbound header request: %v", err) + return errorf("could not get headers for inbound header request: %v", err) } - resp := ð.BlockHeadersPacket66{ - RequestId: reqID, + resp := &BlockHeaders{ + RequestId: msg.ReqID(), BlockHeadersPacket: eth.BlockHeadersPacket(headers), } - if err := c.Write66(resp, BlockHeaders{}.Code()); err != nil { - return 0, errorf("could not write to connection: %v", err) + if err := c.Write(resp); err != nil { + return errorf("could not write to connection: %v", err) } default: - return reqID, msg + return msg } } - return 0, errorf("no message received within %v", timeout) + return errorf("no message received within %v", timeout) } // headersRequest executes the given `GetBlockHeaders` request. -func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, isEth66 bool, reqID uint64) (BlockHeaders, error) { +func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, reqID uint64) ([]*types.Header, error) { defer c.SetReadDeadline(time.Time{}) c.SetReadDeadline(time.Now().Add(20 * time.Second)) - // if on eth66 connection, perform eth66 GetBlockHeaders request - if isEth66 { - return getBlockHeaders66(chain, c, request, reqID) - } + + // write request + request.RequestId = reqID if err := c.Write(request); err != nil { - return nil, err + return nil, fmt.Errorf("could not write to connection: %v", err) } - switch msg := c.readAndServe(chain, timeout).(type) { - case *BlockHeaders: - return *msg, nil - default: - return nil, fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) + + // wait for response + msg := c.waitForResponse(chain, timeout, request.RequestId) + resp, ok := msg.(*BlockHeaders) + if !ok { + return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) } + headers := []*types.Header(resp.BlockHeadersPacket) + return headers, nil } func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error) { @@ -357,28 +278,8 @@ func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error return c.ReadSnap(id) } -// getBlockHeaders66 executes the given `GetBlockHeaders` request over the eth66 protocol. -func getBlockHeaders66(chain *Chain, conn *Conn, request *GetBlockHeaders, id uint64) (BlockHeaders, error) { - // write request - packet := eth.GetBlockHeadersPacket(*request) - req := ð.GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &packet, - } - if err := conn.Write66(req, GetBlockHeaders{}.Code()); err != nil { - return nil, fmt.Errorf("could not write to connection: %v", err) - } - // wait for response - msg := conn.waitForResponse(chain, timeout, req.RequestId) - headers, ok := msg.(BlockHeaders) - if !ok { - return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) - } - return headers, nil -} - // headersMatch returns whether the received headers match the given request -func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { +func headersMatch(expected []*types.Header, headers []*types.Header) bool { return reflect.DeepEqual(expected, headers) } @@ -386,8 +287,8 @@ func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { // request ID is received. func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message { for { - id, msg := c.readAndServe66(chain, timeout) - if id == requestID { + msg := c.readAndServe(chain, timeout) + if msg.ReqID() == requestID { return msg } } @@ -395,9 +296,9 @@ func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID ui // sendNextBlock broadcasts the next block in the chain and waits // for the node to propagate the block and import it into its chain. -func (s *Suite) sendNextBlock(isEth66 bool) error { +func (s *Suite) sendNextBlock() error { // set up sending and receiving connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -420,7 +321,7 @@ func (s *Suite) sendNextBlock(isEth66 bool) error { return fmt.Errorf("failed to announce block: %v", err) } // wait for client to update its chain - if err = s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err = s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("failed to receive confirmation of block import: %v", err) } // update test suite chain @@ -465,29 +366,22 @@ func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error { } } -func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) error { +func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block) error { defer conn.SetReadDeadline(time.Time{}) conn.SetReadDeadline(time.Now().Add(20 * time.Second)) // create request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: block.Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: block.Hash()}, + Amount: 1, }, - Amount: 1, } + // loop until BlockHeaders response contains desired block, confirming the // node imported the block for { - var ( - headers BlockHeaders - err error - ) - if isEth66 { - requestID := uint64(54) - headers, err = conn.headersRequest(req, s.chain, eth66, requestID) - } else { - headers, err = conn.headersRequest(req, s.chain, eth65, 0) - } + requestID := uint64(54) + headers, err := conn.headersRequest(req, s.chain, requestID) if err != nil { return fmt.Errorf("GetBlockHeader request failed: %v", err) } @@ -503,8 +397,8 @@ func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) } } -func (s *Suite) oldAnnounce(isEth66 bool) error { - sendConn, receiveConn, err := s.createSendAndRecvConns(isEth66) +func (s *Suite) oldAnnounce() error { + sendConn, receiveConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -550,23 +444,13 @@ func (s *Suite) oldAnnounce(isEth66 bool) error { return nil } -func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) maliciousHandshakes(t *utesting.T) error { + conn, err := s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } defer conn.Close() + // write hello to client pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] handshakes := []*Hello{ @@ -627,16 +511,9 @@ func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { } } // dial for the next round - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } + conn, err = s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } } return nil @@ -654,6 +531,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { Genesis: s.chain.blocks[0].Hash(), ForkID: s.chain.ForkID(), } + // get status msg, err := conn.statusExchange(s.chain, status) if err != nil { @@ -664,6 +542,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { default: return fmt.Errorf("expected status, got: %#v ", msg) } + // wait for disconnect switch msg := conn.readAndServe(s.chain, timeout).(type) { case *Disconnect: @@ -675,9 +554,9 @@ func (s *Suite) maliciousStatus(conn *Conn) error { } } -func (s *Suite) hashAnnounce(isEth66 bool) error { +func (s *Suite) hashAnnounce() error { // create connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return fmt.Errorf("failed to create connections: %v", err) } @@ -689,6 +568,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // create NewBlockHashes announcement type anno struct { Hash common.Hash // Hash of one particular block being announced @@ -700,56 +580,29 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := sendConn.Write(newBlockHash); err != nil { return fmt.Errorf("failed to write to connection: %v", err) } + // Announcement sent, now wait for a header request - var ( - id uint64 - msg Message - blockHeaderReq GetBlockHeaders - ) - if isEth66 { - id, msg = sendConn.Read66() - switch msg := msg.(type) { - case GetBlockHeaders: - blockHeaderReq = msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write66(ð.BlockHeadersPacket66{ - RequestId: id, - BlockHeadersPacket: eth.BlockHeadersPacket{ - nextBlock.Header(), - }, - }, BlockHeaders{}.Code()); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } - } else { - msg = sendConn.Read() - switch msg := msg.(type) { - case *GetBlockHeaders: - blockHeaderReq = *msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write(&BlockHeaders{nextBlock.Header()}); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } + msg := sendConn.Read() + blockHeaderReq, ok := msg.(*GetBlockHeaders) + if !ok { + return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) } + if blockHeaderReq.Amount != 1 { + return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) + } + if blockHeaderReq.Origin.Hash != announcement.Hash { + return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", + pretty.Sdump(announcement), + pretty.Sdump(blockHeaderReq)) + } + err = sendConn.Write(&BlockHeaders{ + RequestId: blockHeaderReq.ReqID(), + BlockHeadersPacket: eth.BlockHeadersPacket{nextBlock.Header()}, + }) + if err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + // wait for block announcement msg = recvConn.readAndServe(s.chain, timeout) switch msg := msg.(type) { @@ -762,6 +615,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected block hash announcement, wanted %v, got %v", nextBlock.Hash(), hashes[0].Hash) } + case *NewBlock: // node should only propagate NewBlock without having requested the body if the body is empty nextBlockBody := nextBlock.Body() @@ -780,7 +634,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } // confirm node imported block - if err := s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err := s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("error waiting for node to import new block: %v", err) } // update the chain diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index 9c3e88f9cb66..032afeafcdad 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -350,7 +350,6 @@ func hexToCompact(hex []byte) []byte { // TestSnapTrieNodes various forms of GetTrieNodes requests. func (s *Suite) TestSnapTrieNodes(t *utesting.T) { - key := common.FromHex("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a") // helper function to iterate the key, and generate the compact-encoded // trie paths along the way. @@ -496,10 +495,10 @@ func (s *Suite) snapGetAccountRange(t *utesting.T, tc *accRangeTest) error { } if len(hashes) > 0 { if exp, got := tc.expFirst, res.Accounts[0].Hash; exp != got { - return fmt.Errorf("expected first account 0x%x, got 0x%x", exp, got) + return fmt.Errorf("expected first account %#x, got %#x", exp, got) } if exp, got := tc.expLast, res.Accounts[len(res.Accounts)-1].Hash; exp != got { - return fmt.Errorf("expected last account 0x%x, got 0x%x", exp, got) + return fmt.Errorf("expected last account %#x, got %#x", exp, got) } } // Reconstruct a partial trie from the response and verify it diff --git a/cmd/devp2p/internal/ethtest/snapTypes.go b/cmd/devp2p/internal/ethtest/snapTypes.go index e585f5f79be5..61db63d20154 100644 --- a/cmd/devp2p/internal/ethtest/snapTypes.go +++ b/cmd/devp2p/internal/ethtest/snapTypes.go @@ -21,32 +21,40 @@ import "github.com/ethereum/go-ethereum/eth/protocols/snap" // GetAccountRange represents an account range query. type GetAccountRange snap.GetAccountRangePacket -func (g GetAccountRange) Code() int { return 39 } +func (g GetAccountRange) Code() int { return 39 } +func (msg GetAccountRange) ReqID() uint64 { return msg.ID } type AccountRange snap.AccountRangePacket -func (g AccountRange) Code() int { return 40 } +func (g AccountRange) Code() int { return 40 } +func (msg AccountRange) ReqID() uint64 { return msg.ID } type GetStorageRanges snap.GetStorageRangesPacket -func (g GetStorageRanges) Code() int { return 41 } +func (g GetStorageRanges) Code() int { return 41 } +func (msg GetStorageRanges) ReqID() uint64 { return msg.ID } type StorageRanges snap.StorageRangesPacket -func (g StorageRanges) Code() int { return 42 } +func (g StorageRanges) Code() int { return 42 } +func (msg StorageRanges) ReqID() uint64 { return msg.ID } type GetByteCodes snap.GetByteCodesPacket -func (g GetByteCodes) Code() int { return 43 } +func (g GetByteCodes) Code() int { return 43 } +func (msg GetByteCodes) ReqID() uint64 { return msg.ID } type ByteCodes snap.ByteCodesPacket -func (g ByteCodes) Code() int { return 44 } +func (g ByteCodes) Code() int { return 44 } +func (msg ByteCodes) ReqID() uint64 { return msg.ID } type GetTrieNodes snap.GetTrieNodesPacket -func (g GetTrieNodes) Code() int { return 45 } +func (g GetTrieNodes) Code() int { return 45 } +func (msg GetTrieNodes) ReqID() uint64 { return msg.ID } type TrieNodes snap.TrieNodesPacket -func (g TrieNodes) Code() int { return 46 } +func (g TrieNodes) Code() int { return 46 } +func (msg TrieNodes) ReqID() uint64 { return msg.ID } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index a3541df7fc35..135250a57aa5 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -50,79 +50,30 @@ func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, e }, nil } -func (s *Suite) AllEthTests() []utesting.Test { +func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ // status - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestStatus66", Fn: s.TestStatus66}, + {Name: "TestStatus", Fn: s.TestStatus}, // get block headers - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, + {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "TestSimultaneousRequests", Fn: s.TestSimultaneousRequests}, + {Name: "TestSameRequestID", Fn: s.TestSameRequestID}, + {Name: "TestZeroRequestID", Fn: s.TestZeroRequestID}, // get block bodies - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, + {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, // broadcast - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, + {Name: "TestBroadcast", Fn: s.TestBroadcast}, + {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, + {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, + {Name: "TestBlockHashAnnounce", Fn: s.TestBlockHashAnnounce}, // malicious handshakes + status - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, + {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, + {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, // test transactions - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, - } -} - -func (s *Suite) EthTests() []utesting.Test { - return []utesting.Test{ - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - } -} - -func (s *Suite) Eth66Tests() []utesting.Test { - return []utesting.Test{ - // only proceed with eth66 test suite if node supports eth 66 protocol - {Name: "TestStatus66", Fn: s.TestStatus66}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, + {Name: "TestTransaction", Fn: s.TestTransaction}, + {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, + {Name: "TestLargeTxRequest", Fn: s.TestLargeTxRequest}, + {Name: "TestNewPooledTxs", Fn: s.TestNewPooledTxs}, } } @@ -136,14 +87,9 @@ func (s *Suite) SnapTests() []utesting.Test { } } -var ( - eth66 = true // indicates whether suite should negotiate eth66 connection - eth65 = false // indicates whether suite should negotiate eth65 connection or below. -) - -// TestStatus65 attempts to connect to the given node and exchange -// a status message with it. -func (s *Suite) TestStatus65(t *utesting.T) { +// TestStatus attempts to connect to the given node and exchange +// a status message with it on the eth protocol. +func (s *Suite) TestStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -154,79 +100,32 @@ func (s *Suite) TestStatus65(t *utesting.T) { } } -// TestStatus66 attempts to connect to the given node and exchange -// a status message with it on the eth66 protocol. -func (s *Suite) TestStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } -} - -// TestGetBlockHeaders65 tests whether the given node can respond to -// a `GetBlockHeaders` request accurately. -func (s *Suite) TestGetBlockHeaders65(t *utesting.T) { +// TestGetBlockHeaders tests whether the given node can respond to +// an eth `GetBlockHeaders` request and that the response is accurate. +func (s *Suite) TestGetBlockHeaders(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("handshake(s) failed: %v", err) - } - // write request - req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), - }, - Amount: 2, - Skip: 1, - Reverse: false, - } - headers, err := conn.headersRequest(req, s.chain, eth65, 0) - if err != nil { - t.Fatalf("GetBlockHeaders request failed: %v", err) - } - // check for correct headers - expected, err := s.chain.GetHeaders(*req) - if err != nil { - t.Fatalf("failed to get headers for given request: %v", err) - } - if !headersMatch(expected, headers) { - t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) - } -} - -// TestGetBlockHeaders66 tests whether the given node can respond to -// an eth66 `GetBlockHeaders` request and that the response is accurate. -func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() if err = conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } // write request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: s.chain.blocks[1].Hash()}, + Amount: 2, + Skip: 1, + Reverse: false, }, - Amount: 2, - Skip: 1, - Reverse: false, } - headers, err := conn.headersRequest(req, s.chain, eth66, 33) + headers, err := conn.headersRequest(req, s.chain, 33) if err != nil { t.Fatalf("could not get block headers: %v", err) } // check for correct headers - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get headers for given request: %v", err) } @@ -235,12 +134,12 @@ func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { } } -// TestSimultaneousRequests66 sends two simultaneous `GetBlockHeader` requests from +// TestSimultaneousRequests sends two simultaneous `GetBlockHeader` requests from // the same connection with different request IDs and checks to make sure the node // responds with the correct headers per request. -func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { +func (s *Suite) TestSimultaneousRequests(t *utesting.T) { // create a connection - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -248,8 +147,9 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { if err := conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } + // create two requests - req1 := ð.GetBlockHeadersPacket66{ + req1 := &GetBlockHeaders{ RequestId: uint64(111), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -260,7 +160,7 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } - req2 := ð.GetBlockHeadersPacket66{ + req2 := &GetBlockHeaders{ RequestId: uint64(222), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -271,46 +171,49 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } + // write the first request - if err := conn.Write66(req1, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req1); err != nil { t.Fatalf("failed to write to connection: %v", err) } // write the second request - if err := conn.Write66(req2, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, req1.RequestId) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, req2.RequestId) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check received headers for accuracy - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*req1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(req1) if err != nil { t.Fatalf("failed to get expected headers for request 1: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*req2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(req2) if err != nil { t.Fatalf("failed to get expected headers for request 2: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestSameRequestID66 sends two requests with the same request ID to a +// TestSameRequestID sends two requests with the same request ID to a // single node. -func (s *Suite) TestSameRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestSameRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -320,7 +223,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { } // create requests reqID := uint64(1234) - request1 := ð.GetBlockHeadersPacket66{ + request1 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -329,7 +232,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } - request2 := ð.GetBlockHeadersPacket66{ + request2 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -338,45 +241,48 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } + // write the requests - if err = conn.Write66(request1, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request1); err != nil { t.Fatalf("failed to write to connection: %v", err) } - if err = conn.Write66(request2, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, reqID) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, reqID) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check if headers match - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*request1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(request1) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*request2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(request2) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestZeroRequestID_66 checks that a message with a request ID of zero is still handled +// TestZeroRequestID checks that a message with a request ID of zero is still handled // by the node. -func (s *Suite) TestZeroRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestZeroRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -385,16 +291,16 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { t.Fatalf("peering failed: %v", err) } req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: 0, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: 0}, + Amount: 2, }, - Amount: 2, } - headers, err := conn.headersRequest(req, s.chain, eth66, 0) + headers, err := conn.headersRequest(req, s.chain, 0) if err != nil { t.Fatalf("failed to get block headers: %v", err) } - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } @@ -403,9 +309,9 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { } } -// TestGetBlockBodies65 tests whether the given node can respond to +// TestGetBlockBodies tests whether the given node can respond to // a `GetBlockBodies` request and that the response is accurate. -func (s *Suite) TestGetBlockBodies65(t *utesting.T) { +func (s *Suite) TestGetBlockBodies(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -416,126 +322,39 @@ func (s *Suite) TestGetBlockBodies65(t *utesting.T) { } // create block bodies request req := &GetBlockBodies{ - s.chain.blocks[54].Hash(), - s.chain.blocks[75].Hash(), - } - if err := conn.Write(req); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // wait for response - switch msg := conn.readAndServe(s.chain, timeout).(type) { - case *BlockBodies: - t.Logf("received %d block bodies", len(*msg)) - if len(*msg) != len(*req) { - t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(*req), len(*msg)) - } - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } -} - -// TestGetBlockBodies66 tests whether the given node can respond to -// a `GetBlockBodies` request and that the response is accurate over -// the eth66 protocol. -func (s *Suite) TestGetBlockBodies66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - // create block bodies request - req := ð.GetBlockBodiesPacket66{ RequestId: uint64(55), GetBlockBodiesPacket: eth.GetBlockBodiesPacket{ s.chain.blocks[54].Hash(), s.chain.blocks[75].Hash(), }, } - if err := conn.Write66(req, GetBlockBodies{}.Code()); err != nil { + if err := conn.Write(req); err != nil { t.Fatalf("could not write to connection: %v", err) } // wait for block bodies response msg := conn.waitForResponse(s.chain, timeout, req.RequestId) - blockBodies, ok := msg.(BlockBodies) + resp, ok := msg.(*BlockBodies) if !ok { t.Fatalf("unexpected: %s", pretty.Sdump(msg)) } - t.Logf("received %d block bodies", len(blockBodies)) - if len(blockBodies) != len(req.GetBlockBodiesPacket) { + bodies := resp.BlockBodiesPacket + t.Logf("received %d block bodies", len(bodies)) + if len(bodies) != len(req.GetBlockBodiesPacket) { t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(req.GetBlockBodiesPacket), len(blockBodies)) + "got %d", len(req.GetBlockBodiesPacket), len(bodies)) } } -// TestBroadcast65 tests whether a block announcement is correctly -// propagated to the given node's peer(s). -func (s *Suite) TestBroadcast65(t *utesting.T) { - if err := s.sendNextBlock(eth65); err != nil { +// TestBroadcast tests whether a block announcement is correctly +// propagated to the node's peers. +func (s *Suite) TestBroadcast(t *utesting.T) { + if err := s.sendNextBlock(); err != nil { t.Fatalf("block broadcast failed: %v", err) } } -// TestBroadcast66 tests whether a block announcement is correctly -// propagated to the given node's peer(s) on the eth66 protocol. -func (s *Suite) TestBroadcast66(t *utesting.T) { - if err := s.sendNextBlock(eth66); err != nil { - t.Fatalf("block broadcast failed: %v", err) - } -} - -// TestLargeAnnounce65 tests the announcement mechanism with a large block. -func (s *Suite) TestLargeAnnounce65(t *utesting.T) { - nextBlock := len(s.chain.blocks) - blocks := []*NewBlock{ - { - Block: largeBlock(), - TD: s.fullChain.TotalDifficultyAt(nextBlock), - }, - { - Block: s.fullChain.blocks[nextBlock], - TD: largeNumber(2), - }, - { - Block: largeBlock(), - TD: largeNumber(2), - }, - } - - for i, blockAnnouncement := range blocks { - t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err = conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - if err = conn.Write(blockAnnouncement); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { - case *Disconnect: - case *Error: - break - default: - t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) - } - conn.Close() - } - // Test the last block as a valid block - if err := s.sendNextBlock(eth65); err != nil { - t.Fatalf("failed to broadcast next block: %v", err) - } -} - -// TestLargeAnnounce66 tests the announcement mechanism with a large -// block over the eth66 protocol. -func (s *Suite) TestLargeAnnounce66(t *utesting.T) { +// TestLargeAnnounce tests the announcement mechanism with a large block. +func (s *Suite) TestLargeAnnounce(t *utesting.T) { nextBlock := len(s.chain.blocks) blocks := []*NewBlock{ { @@ -554,7 +373,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { for i, blockAnnouncement := range blocks[0:3] { t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -565,7 +384,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { + switch msg := conn.readAndServe(s.chain, 8*time.Second).(type) { case *Disconnect: case *Error: break @@ -575,68 +394,44 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { conn.Close() } // Test the last block as a valid block - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to broadcast next block: %v", err) } } -// TestOldAnnounce65 tests the announcement mechanism with an old block. -func (s *Suite) TestOldAnnounce65(t *utesting.T) { +// TestOldAnnounce tests the announcement mechanism with an old block. +func (s *Suite) TestOldAnnounce(t *utesting.T) { if testing.Short() { t.Log("skipping test in short mode") return } - - if err := s.oldAnnounce(eth65); err != nil { + if err := s.oldAnnounce(); err != nil { t.Fatal(err) } } -// TestOldAnnounce66 tests the announcement mechanism with an old block, -// over the eth66 protocol. -func (s *Suite) TestOldAnnounce66(t *utesting.T) { +// TestBlockHashAnnounce sends a new block hash announcement and expects +// the node to perform a `GetBlockHeaders` request. +func (s *Suite) TestBlockHashAnnounce(t *utesting.T) { if testing.Short() { t.Log("skipping test in short mode") return } - if err := s.oldAnnounce(eth66); err != nil { - t.Fatal(err) - } -} - -// TestBlockHashAnnounce65 sends a new block hash announcement and expects -// the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce65(t *utesting.T) { - if err := s.hashAnnounce(eth65); err != nil { - t.Fatalf("block hash announcement failed: %v", err) - } -} - -// TestBlockHashAnnounce66 sends a new block hash announcement and expects -// the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce66(t *utesting.T) { - if err := s.hashAnnounce(eth66); err != nil { + if err := s.hashAnnounce(); err != nil { t.Fatalf("block hash announcement failed: %v", err) } } -// TestMaliciousHandshake65 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake65(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth65); err != nil { - t.Fatal(err) - } -} - -// TestMaliciousHandshake66 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake66(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth66); err != nil { +// TestMaliciousHandshake tries to send malicious data during the handshake. +func (s *Suite) TestMaliciousHandshake(t *utesting.T) { + if err := s.maliciousHandshakes(t); err != nil { t.Fatal(err) } } -// TestMaliciousStatus65 sends a status package with a large total difficulty. -func (s *Suite) TestMaliciousStatus65(t *utesting.T) { +// TestMaliciousStatus sends a status package with a large total difficulty. +func (s *Suite) TestMaliciousStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -648,58 +443,28 @@ func (s *Suite) TestMaliciousStatus65(t *utesting.T) { } } -// TestMaliciousStatus66 sends a status package with a large total -// difficulty over the eth66 protocol. -func (s *Suite) TestMaliciousStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - - if err := s.maliciousStatus(conn); err != nil { - t.Fatal(err) - } -} - -// TestTransaction65 sends a valid transaction to the node and +// TestTransaction sends a valid transaction to the node and // checks if the transaction gets propagated. -func (s *Suite) TestTransaction65(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth65); err != nil { +func (s *Suite) TestTransaction(t *utesting.T) { + if err := s.sendSuccessfulTxs(t); err != nil { t.Fatal(err) } } -// TestTransaction66 sends a valid transaction to the node and -// checks if the transaction gets propagated. -func (s *Suite) TestTransaction66(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestMaliciousTx65 sends several invalid transactions and tests whether +// TestMaliciousTx sends several invalid transactions and tests whether // the node will propagate them. -func (s *Suite) TestMaliciousTx65(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth65); err != nil { +func (s *Suite) TestMaliciousTx(t *utesting.T) { + if err := s.sendMaliciousTxs(t); err != nil { t.Fatal(err) } } -// TestMaliciousTx66 sends several invalid transactions and tests whether -// the node will propagate them. -func (s *Suite) TestMaliciousTx66(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestLargeTxRequest66 tests whether a node can fulfill a large GetPooledTransactions +// TestLargeTxRequest tests whether a node can fulfill a large GetPooledTransactions // request. -func (s *Suite) TestLargeTxRequest66(t *utesting.T) { +func (s *Suite) TestLargeTxRequest(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } // send 2000 transactions to the node @@ -712,7 +477,7 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } // set up connection to receive to ensure node is peered with the receiving connection // before tx request is sent - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -725,17 +490,17 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { for _, hash := range hashMap { hashes = append(hashes, hash) } - getTxReq := ð.GetPooledTransactionsPacket66{ + getTxReq := &GetPooledTransactions{ RequestId: 1234, GetPooledTransactionsPacket: hashes, } - if err = conn.Write66(getTxReq, GetPooledTransactions{}.Code()); err != nil { + if err = conn.Write(getTxReq); err != nil { t.Fatalf("could not write to conn: %v", err) } // check that all received transactions match those that were sent to node switch msg := conn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) { - case PooledTransactions: - for _, gotTx := range msg { + case *PooledTransactions: + for _, gotTx := range msg.PooledTransactionsPacket { if _, exists := hashMap[gotTx.Hash()]; !exists { t.Fatalf("unexpected tx received: %v", gotTx.Hash()) } @@ -745,12 +510,12 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } } -// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions +// TestNewPooledTxs tests whether a node will do a GetPooledTransactions // request upon receiving a NewPooledTransactionHashes announcement. -func (s *Suite) TestNewPooledTxs66(t *utesting.T) { +func (s *Suite) TestNewPooledTxs(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } @@ -768,7 +533,7 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { announce := NewPooledTransactionHashes(hashes) // send announcement - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -782,11 +547,11 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { // wait for GetPooledTxs request for { - _, msg := conn.readAndServe66(s.chain, timeout) + msg := conn.readAndServe(s.chain, timeout) switch msg := msg.(type) { - case GetPooledTransactions: - if len(msg) != len(hashes) { - t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) + case *GetPooledTransactions: + if len(msg.GetPooledTransactionsPacket) != len(hashes) { + t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsPacket)) } return // ignore propagated txs from previous tests diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 9dcd1a79abb1..51fc0c19d316 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -49,7 +49,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.Eth66Tests() { + for _, test := range suite.EthTests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index e47399813742..1ff9dff14284 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -32,7 +32,7 @@ import ( // var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") -func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendSuccessfulTxs(t *utesting.T) error { tests := []*types.Transaction{ getNextTxFromChain(s), unknownTx(s), @@ -48,15 +48,15 @@ func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { prevTx = tests[i-1] } // write tx to connection - if err := sendSuccessfulTx(s, tx, prevTx, isEth66); err != nil { + if err := sendSuccessfulTx(s, tx, prevTx); err != nil { return fmt.Errorf("send successful tx test failed: %v", err) } } return nil } -func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction, isEth66 bool) error { - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) +func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction) error { + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -73,8 +73,10 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // update last nonce seen nonce = tx.Nonce() + // Wait for the transaction announcement for { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { @@ -114,7 +116,7 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction } } -func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendMaliciousTxs(t *utesting.T) error { badTxs := []*types.Transaction{ getOldTxFromChain(s), invalidNonceTx(s), @@ -122,16 +124,9 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { hugeGasPrice(s), hugeData(s), } + // setup receiving connection before sending malicious txs - var ( - recvConn *Conn - err error - ) - if isEth66 { - recvConn, err = s.dial66() - } else { - recvConn, err = s.dial() - } + recvConn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -139,9 +134,10 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + for i, tx := range badTxs { t.Logf("Testing malicious tx propagation: %v\n", i) - if err = sendMaliciousTx(s, tx, isEth66); err != nil { + if err = sendMaliciousTx(s, tx); err != nil { return fmt.Errorf("malicious tx test failed:\ntx: %v\nerror: %v", tx, err) } } @@ -149,17 +145,8 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { return checkMaliciousTxPropagation(s, badTxs, recvConn) } -func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { - // setup connection - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - } else { - conn, err = s.dial() - } +func sendMaliciousTx(s *Suite, tx *types.Transaction) error { + conn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -167,6 +154,7 @@ func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { if err = conn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // write malicious tx if err = conn.Write(&Transactions{tx}); err != nil { return fmt.Errorf("failed to write to connection: %v", err) @@ -182,7 +170,7 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction txMsg := Transactions(txs) t.Logf("sending %d txs\n", len(txs)) - sendConn, recvConn, err := s.createSendAndRecvConns(true) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -194,15 +182,19 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // Send the transactions if err = sendConn.Write(&txMsg); err != nil { return fmt.Errorf("failed to write message to connection: %v", err) } + // update nonce nonce = txs[len(txs)-1].Nonce() - // Wait for the transaction announcement(s) and make sure all sent txs are being propagated + + // Wait for the transaction announcement(s) and make sure all sent txs are being propagated. + // all txs should be announced within 3 announcements. recvHashes := make([]common.Hash, 0) - // all txs should be announced within 3 announcements + for i := 0; i < 3; i++ { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { case *Transactions: diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index e92b54394067..2c5cb94c699f 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -29,6 +29,7 @@ import ( type Message interface { Code() int + ReqID() uint64 } type Error struct { @@ -37,9 +38,11 @@ type Error struct { func (e *Error) Unwrap() error { return e.err } func (e *Error) Error() string { return e.err.Error() } -func (e *Error) Code() int { return -1 } func (e *Error) String() string { return e.Error() } +func (e *Error) Code() int { return -1 } +func (e *Error) ReqID() uint64 { return 0 } + func errorf(format string, args ...interface{}) *Error { return &Error{fmt.Errorf(format, args...)} } @@ -56,73 +59,88 @@ type Hello struct { Rest []rlp.RawValue `rlp:"tail"` } -func (h Hello) Code() int { return 0x00 } +func (msg Hello) Code() int { return 0x00 } +func (msg Hello) ReqID() uint64 { return 0 } // Disconnect is the RLP structure for a disconnect message. type Disconnect struct { Reason p2p.DiscReason } -func (d Disconnect) Code() int { return 0x01 } +func (msg Disconnect) Code() int { return 0x01 } +func (msg Disconnect) ReqID() uint64 { return 0 } type Ping struct{} -func (p Ping) Code() int { return 0x02 } +func (msg Ping) Code() int { return 0x02 } +func (msg Ping) ReqID() uint64 { return 0 } type Pong struct{} -func (p Pong) Code() int { return 0x03 } +func (msg Pong) Code() int { return 0x03 } +func (msg Pong) ReqID() uint64 { return 0 } // Status is the network packet for the status message for eth/64 and later. type Status eth.StatusPacket -func (s Status) Code() int { return 16 } +func (msg Status) Code() int { return 16 } +func (msg Status) ReqID() uint64 { return 0 } // NewBlockHashes is the network packet for the block announcements. type NewBlockHashes eth.NewBlockHashesPacket -func (nbh NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) ReqID() uint64 { return 0 } type Transactions eth.TransactionsPacket -func (t Transactions) Code() int { return 18 } +func (msg Transactions) Code() int { return 18 } +func (msg Transactions) ReqID() uint64 { return 18 } // GetBlockHeaders represents a block header query. -type GetBlockHeaders eth.GetBlockHeadersPacket +type GetBlockHeaders eth.GetBlockHeadersPacket66 -func (g GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) ReqID() uint64 { return msg.RequestId } -type BlockHeaders eth.BlockHeadersPacket +type BlockHeaders eth.BlockHeadersPacket66 -func (bh BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) ReqID() uint64 { return msg.RequestId } // GetBlockBodies represents a GetBlockBodies request -type GetBlockBodies eth.GetBlockBodiesPacket +type GetBlockBodies eth.GetBlockBodiesPacket66 -func (gbb GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) ReqID() uint64 { return msg.RequestId } // BlockBodies is the network packet for block content distribution. -type BlockBodies eth.BlockBodiesPacket +type BlockBodies eth.BlockBodiesPacket66 -func (bb BlockBodies) Code() int { return 22 } +func (msg BlockBodies) Code() int { return 22 } +func (msg BlockBodies) ReqID() uint64 { return msg.RequestId } // NewBlock is the network packet for the block propagation message. type NewBlock eth.NewBlockPacket -func (nb NewBlock) Code() int { return 23 } +func (msg NewBlock) Code() int { return 23 } +func (msg NewBlock) ReqID() uint64 { return 0 } // NewPooledTransactionHashes is the network packet for the tx hash propagation message. type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket -func (nb NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) ReqID() uint64 { return 0 } -type GetPooledTransactions eth.GetPooledTransactionsPacket +type GetPooledTransactions eth.GetPooledTransactionsPacket66 -func (gpt GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) ReqID() uint64 { return msg.RequestId } -type PooledTransactions eth.PooledTransactionsPacket +type PooledTransactions eth.PooledTransactionsPacket66 -func (pt PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) ReqID() uint64 { return msg.RequestId } // Conn represents an individual connection with a peer type Conn struct { @@ -135,62 +153,13 @@ type Conn struct { caps []p2p.Cap } -// Read reads an eth packet from the connection. +// Read reads an eth66 packet from the connection. func (c *Conn) Read() Message { code, rawData, _, err := c.Conn.Read() if err != nil { return errorf("could not read from connection: %v", err) } - var msg Message - switch int(code) { - case (Hello{}).Code(): - msg = new(Hello) - case (Ping{}).Code(): - msg = new(Ping) - case (Pong{}).Code(): - msg = new(Pong) - case (Disconnect{}).Code(): - msg = new(Disconnect) - case (Status{}).Code(): - msg = new(Status) - case (GetBlockHeaders{}).Code(): - msg = new(GetBlockHeaders) - case (BlockHeaders{}).Code(): - msg = new(BlockHeaders) - case (GetBlockBodies{}).Code(): - msg = new(GetBlockBodies) - case (BlockBodies{}).Code(): - msg = new(BlockBodies) - case (NewBlock{}).Code(): - msg = new(NewBlock) - case (NewBlockHashes{}).Code(): - msg = new(NewBlockHashes) - case (Transactions{}).Code(): - msg = new(Transactions) - case (NewPooledTransactionHashes{}).Code(): - msg = new(NewPooledTransactionHashes) - case (GetPooledTransactions{}.Code()): - msg = new(GetPooledTransactions) - case (PooledTransactions{}.Code()): - msg = new(PooledTransactions) - default: - return errorf("invalid message code: %d", code) - } - // if message is devp2p, decode here - if err := rlp.DecodeBytes(rawData, msg); err != nil { - return errorf("could not rlp decode message: %v", err) - } - return msg -} - -// Read66 reads an eth66 packet from the connection. -func (c *Conn) Read66() (uint64, Message) { - code, rawData, _, err := c.Conn.Read() - if err != nil { - return 0, errorf("could not read from connection: %v", err) - } - var msg Message switch int(code) { case (Hello{}).Code(): @@ -206,27 +175,27 @@ func (c *Conn) Read66() (uint64, Message) { case (GetBlockHeaders{}).Code(): ethMsg := new(eth.GetBlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket) + return (*GetBlockHeaders)(ethMsg) case (BlockHeaders{}).Code(): ethMsg := new(eth.BlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket) + return (*BlockHeaders)(ethMsg) case (GetBlockBodies{}).Code(): ethMsg := new(eth.GetBlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket) + return (*GetBlockBodies)(ethMsg) case (BlockBodies{}).Code(): ethMsg := new(eth.BlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket) + return (*BlockBodies)(ethMsg) case (NewBlock{}).Code(): msg = new(NewBlock) case (NewBlockHashes{}).Code(): @@ -238,26 +207,26 @@ func (c *Conn) Read66() (uint64, Message) { case (GetPooledTransactions{}.Code()): ethMsg := new(eth.GetPooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket) + return (*GetPooledTransactions)(ethMsg) case (PooledTransactions{}.Code()): ethMsg := new(eth.PooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket) + return (*PooledTransactions)(ethMsg) default: msg = errorf("invalid message code: %d", code) } if msg != nil { if err := rlp.DecodeBytes(rawData, msg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return 0, msg + return msg } - return 0, errorf("invalid message: %s", string(rawData)) + return errorf("invalid message: %s", string(rawData)) } // Write writes a eth packet to the connection. @@ -270,16 +239,6 @@ func (c *Conn) Write(msg Message) error { return err } -// Write66 writes an eth66 packet to the connection. -func (c *Conn) Write66(req eth.Packet, code int) error { - payload, err := rlp.EncodeToBytes(req) - if err != nil { - return err - } - _, err = c.Conn.Write(uint64(code), payload) - return err -} - // ReadSnap reads a snap/1 response with the given id from the connection. func (c *Conn) ReadSnap(id uint64) (Message, error) { respId := id + 1 @@ -315,7 +274,6 @@ func (c *Conn) ReadSnap(id uint64) (Message, error) { return nil, fmt.Errorf("could not rlp decode message: %v", err) } return snpMsg.(Message), nil - } return nil, fmt.Errorf("request timed out") } diff --git a/cmd/devp2p/rlpxcmd.go b/cmd/devp2p/rlpxcmd.go index 07978e4f8861..42b38120c475 100644 --- a/cmd/devp2p/rlpxcmd.go +++ b/cmd/devp2p/rlpxcmd.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/rlpx" "github.com/ethereum/go-ethereum/rlp" @@ -110,12 +109,7 @@ func rlpxEthTest(ctx *cli.Context) error { if err != nil { exit(err) } - // check if given node supports eth66, and if so, run eth66 protocol tests as well - is66Failed, _ := utesting.Run(utesting.Test{Name: "Is_66", Fn: suite.Is_66}) - if is66Failed { - return runTests(ctx, suite.EthTests()) - } - return runTests(ctx, suite.AllEthTests()) + return runTests(ctx, suite.EthTests()) } // rlpxSnapTest runs the snap protocol test suite. diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 241b57f55ea9..77f6ec37158b 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -100,7 +100,6 @@ type rejectedTx struct { func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txs types.Transactions, miningReward int64, getTracerFn func(txIndex int, txHash common.Hash) (tracer vm.EVMLogger, err error)) (*state.StateDB, *ExecutionResult, error) { - // Capture errors for BLOCKHASH operation, if we haven't been supplied the // required blockhashes var hashError error @@ -269,7 +268,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB { - sdb := state.NewDatabase(db) + sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) statedb, _ := state.New(common.Hash{}, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 756d762b5348..9b1975c0500e 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -138,7 +138,7 @@ func runCmd(ctx *cli.Context) error { gen := readGenesis(ctx.String(GenesisFlag.Name)) genesisConfig = gen db := rawdb.NewMemoryDatabase() - genesis := gen.ToBlock(db) + genesis := gen.MustCommit(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil) chainConfig = gen.Config } else { @@ -309,7 +309,7 @@ allocated bytes: %d `, initialGas-leftOverGas, stats.time, stats.allocs, stats.bytesAllocated) } if tracer == nil { - fmt.Printf("0x%x\n", output) + fmt.Printf("%#x\n", output) if err != nil { fmt.Printf(" error: %v\n", err) } diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index 805eea90c45e..72c062e8d923 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -244,7 +244,6 @@ func TestT8n(t *testing.T) { expExitCode: 3, }, } { - args := []string{"t8n"} args = append(args, tc.output.get()...) args = append(args, tc.input.get(tc.base)...) @@ -355,7 +354,6 @@ func TestT9n(t *testing.T) { expExitCode: t8ntool.ErrorIO, }, } { - args := []string{"t9n"} args = append(args, tc.input.get(tc.base)...) @@ -475,7 +473,6 @@ func TestB11r(t *testing.T) { expOut: "exp.json", }, } { - args := []string{"b11r"} args = append(args, tc.input.get(tc.base)...) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 6c8796076c07..dfb7d326dc49 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -248,7 +248,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network ui cfg.SyncMode = downloader.LightSync cfg.NetworkId = network cfg.Genesis = genesis - utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock(nil).Hash()) + utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock().Hash()) lesBackend, err := les.New(stack, &cfg) if err != nil { @@ -709,7 +709,7 @@ func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, c case tokenV2 != "": return authTwitterWithTokenV2(tweetID, tokenV2) } - // Twiter API token isn't provided so we just load the public posts + // Twitter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6914e1aa2da9..a3016c4b091f 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" @@ -69,7 +70,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to Name: "import", Usage: "Import a blockchain file", ArgsUsage: " ( ... ) ", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, utils.GCModeFlag, @@ -91,7 +92,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, - }, utils.DatabasePathFlags...), + }, utils.DatabasePathFlags), Description: ` The import command imports blocks from an RLP-encoded form. The form can be one file with several RLP-encoded blocks, or several files can be used. @@ -104,10 +105,10 @@ processing will proceed even if an individual RLP-file import failure occurs.`, Name: "export", Usage: "Export blockchain into file", ArgsUsage: " [ ]", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), + }, utils.DatabasePathFlags), Description: ` Requires a first argument of the file to write to. Optional second and third arguments control the first and @@ -120,10 +121,10 @@ be gzipped.`, Name: "import-preimages", Usage: "Import the preimage database from an RLP stream", ArgsUsage: "", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), + }, utils.DatabasePathFlags), Description: ` The import-preimages command imports hash preimages from an RLP encoded stream. It's deprecated, please use "geth db import" instead. @@ -134,10 +135,10 @@ It's deprecated, please use "geth db import" instead. Name: "export-preimages", Usage: "Export the preimage database into an RLP stream", ArgsUsage: "", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), + }, utils.DatabasePathFlags), Description: ` The export-preimages command exports hash preimages to an RLP encoded stream. It's deprecated, please use "geth db export" instead. @@ -148,7 +149,7 @@ It's deprecated, please use "geth db export" instead. Name: "dump", Usage: "Dump a specific block from storage", ArgsUsage: "[? | ]", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.IterativeOutputFlag, utils.ExcludeCodeFlag, @@ -156,7 +157,7 @@ It's deprecated, please use "geth db export" instead. utils.IncludeIncompletesFlag, utils.StartKeyFlag, utils.DumpLimitFlag, - }, utils.DatabasePathFlags...), + }, utils.DatabasePathFlags), Description: ` This command dumps out the state for a given block (or latest, if none provided). `, @@ -166,10 +167,12 @@ This command dumps out the state for a given block (or latest, if none provided) // initGenesis will initialise the given JSON format genesis file and writes it as // the zero'd block (i.e. genesis) or will fail hard if it can't succeed. func initGenesis(ctx *cli.Context) error { - // Make sure we have a valid genesis JSON + if ctx.Args().Len() != 1 { + utils.Fatalf("need genesis.json file as the only argument") + } genesisPath := ctx.Args().First() if len(genesisPath) == 0 { - utils.Fatalf("Must supply path to genesis JSON file") + utils.Fatalf("invalid path to genesis file") } file, err := os.Open(genesisPath) if err != nil { @@ -381,12 +384,12 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { - number, err := strconv.Atoi(arg) + number, err := strconv.ParseUint(arg, 10, 64) if err != nil { return nil, nil, common.Hash{}, err } - if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) { - header = rawdb.ReadHeader(db, hash, uint64(number)) + if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, number) } else { return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index d14477272b1c..4c79efe97431 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -20,7 +20,6 @@ import ( "bufio" "errors" "fmt" - "math/big" "os" "reflect" "unicode" @@ -49,7 +48,7 @@ var ( Name: "dumpconfig", Usage: "Show configuration values", ArgsUsage: "", - Flags: utils.GroupFlags(nodeFlags, rpcFlags), + Flags: flags.Merge(nodeFlags, rpcFlags), Description: `The dumpconfig command shows configuration values.`, } @@ -158,13 +157,16 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) - if ctx.IsSet(utils.OverrideGrayGlacierFlag.Name) { - cfg.Eth.OverrideGrayGlacier = new(big.Int).SetUint64(ctx.Uint64(utils.OverrideGrayGlacierFlag.Name)) - } if ctx.IsSet(utils.OverrideTerminalTotalDifficulty.Name) { cfg.Eth.OverrideTerminalTotalDifficulty = flags.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name) } + if ctx.IsSet(utils.OverrideTerminalTotalDifficultyPassed.Name) { + override := ctx.Bool(utils.OverrideTerminalTotalDifficultyPassed.Name) + cfg.Eth.OverrideTerminalTotalDifficultyPassed = &override + } + backend, eth := utils.RegisterEthService(stack, &cfg.Eth) + // Warn users to migrate if they have a legacy freezer format. if eth != nil && !ctx.IsSet(utils.IgnoreLegacyReceiptsFlag.Name) { firstIdx := uint64(0) @@ -183,10 +185,14 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } } - // Configure GraphQL if requested + // Configure log filter RPC API. + filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) + + // Configure GraphQL if requested. if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { - utils.RegisterGraphQLService(stack, backend, cfg.Node) + utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) } + // Add the Ethereum Stats daemon if requested. if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index a62b6a6ad592..87bbe24b977a 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/console" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/urfave/cli/v2" @@ -34,7 +35,7 @@ var ( Action: localConsole, Name: "console", Usage: "Start an interactive JavaScript environment", - Flags: utils.GroupFlags(nodeFlags, rpcFlags, consoleFlags), + Flags: flags.Merge(nodeFlags, rpcFlags, consoleFlags), Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. @@ -46,7 +47,7 @@ See https://geth.ethereum.org/docs/interface/javascript-console.`, Name: "attach", Usage: "Start an interactive JavaScript environment (connect to node)", ArgsUsage: "[endpoint]", - Flags: utils.GroupFlags([]cli.Flag{utils.DataDirFlag}, consoleFlags), + Flags: flags.Merge([]cli.Flag{utils.DataDirFlag}, consoleFlags), Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. @@ -59,7 +60,7 @@ This command allows to open a console on a running geth node.`, Name: "js", Usage: "(DEPRECATED) Execute the specified JavaScript files", ArgsUsage: " [jsfile...]", - Flags: utils.GroupFlags(nodeFlags, consoleFlags), + Flags: flags.Merge(nodeFlags, consoleFlags), Description: ` The JavaScript VM exposes a node admin interface as well as the Ðapp JavaScript API. See https://geth.ethereum.org/docs/interface/javascript-console`, @@ -114,6 +115,10 @@ func localConsole(ctx *cli.Context) error { // remoteConsole will connect to a remote geth instance, attaching a JavaScript // console to it. func remoteConsole(ctx *cli.Context) error { + if ctx.Args().Len() > 1 { + utils.Fatalf("invalid command-line: too many arguments") + } + endpoint := ctx.Args().First() if endpoint == "" { cfg := defaultNodeConfig() diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index e9841a731357..0df3e0876457 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -41,7 +41,7 @@ func runMinimalGeth(t *testing.T, args ...string) *testgeth { // --ropsten to make the 'writing genesis to disk' faster (no accounts) // --networkid=1337 to avoid cache bump // --syncmode=full to avoid allocating fast sync bloom - allArgs := []string{"--ropsten", "--networkid", "1337", "--syncmode=full", "--port", "0", + allArgs := []string{"--ropsten", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0", "--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64", "--datadir.minfreedisk", "0"} return runGeth(t, append(allArgs, args...)...) @@ -155,7 +155,7 @@ To exit, press ctrl-d or type exit } // trulyRandInt generates a crypto random integer used by the console tests to -// not clash network ports with other tests running cocurrently. +// not clash network ports with other tests running concurrently. func trulyRandInt(lo, hi int) int { num, _ := rand.Int(rand.Reader, big.NewInt(int64(hi-lo))) return int(num.Int64()) + lo diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index be994def34d7..ab74277123d7 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strconv" "strings" "syscall" @@ -37,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/olekukonko/tablewriter" @@ -77,7 +77,7 @@ Remove blockchain and state databases`, Action: inspect, Name: "inspect", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Usage: "Inspect the storage size for each type of data in the database", @@ -87,7 +87,7 @@ Remove blockchain and state databases`, Action: checkStateContent, Name: "check-state-content", ArgsUsage: "", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Usage: "Verify that state data is cryptographically correct", Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes. For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates @@ -97,7 +97,7 @@ a data corruption.`, Action: dbStats, Name: "stats", Usage: "Print leveldb statistics", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), } @@ -105,7 +105,7 @@ a data corruption.`, Action: dbCompact, Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, @@ -119,7 +119,7 @@ corruption if it is aborted during execution'!`, Name: "get", Usage: "Show the value of a database key", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command looks up the specified database key from the database.", @@ -129,7 +129,7 @@ corruption if it is aborted during execution'!`, Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `This command deletes the specified database key from the database. @@ -140,7 +140,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `This command sets a given database key to the given value. @@ -151,7 +151,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, Name: "dumptrie", Usage: "Show the storage key/values of a given storage trie", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command looks up the specified database key from the database.", @@ -159,9 +159,9 @@ WARNING: This is a low-level operation which may cause database corruption!`, dbDumpFreezerIndex = &cli.Command{ Action: freezerInspect, Name: "freezer-index", - Usage: "Dump out the index of a given freezer type", - ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Usage: "Dump out the index of a specific freezer table", + ArgsUsage: " ", + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command displays information about the freezer index.", @@ -171,7 +171,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, Name: "import", Usage: "Imports leveldb-data from an exported RLP dump.", ArgsUsage: " has .gz suffix, gzip compression will be used.", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", @@ -190,7 +190,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, Action: showMetaData, Name: "metadata", Usage: "Shows metadata about the chain status.", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "Shows metadata about the chain status.", @@ -200,7 +200,7 @@ WARNING: This is a low-level operation which may cause database corruption!`, Name: "freezer-migrate", Usage: "Migrate legacy parts of the freezer. (WARNING: may take a long time)", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those. @@ -274,7 +274,7 @@ func inspect(ctx *cli.Context) error { start []byte ) if ctx.NArg() > 2 { - return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage) + return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage) } if ctx.NArg() >= 1 { if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil { @@ -337,9 +337,9 @@ func checkStateContent(ctx *cli.Context) error { hasher.Read(got) if !bytes.Equal(k, got) { errs++ - fmt.Printf("Error at 0x%x\n", k) - fmt.Printf(" Hash: 0x%x\n", got) - fmt.Printf(" Data: 0x%x\n", v) + fmt.Printf("Error at %#x\n", k) + fmt.Printf(" Hash: %#x\n", got) + fmt.Printf(" Data: %#x\n", v) } if time.Since(lastLog) > 8*time.Second { log.Info("Iterating the database", "at", fmt.Sprintf("%#x", k), "elapsed", common.PrettyDuration(time.Since(startTime))) @@ -416,7 +416,7 @@ func dbGet(ctx *cli.Context) error { data, err := db.Get(key) if err != nil { - log.Info("Get operation failed", "key", fmt.Sprintf("0x%#x", key), "error", err) + log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err) return err } fmt.Printf("key %#x: %#x\n", key, data) @@ -444,7 +444,7 @@ func dbDelete(ctx *cli.Context) error { fmt.Printf("Previous value: %#x\n", data) } if err = db.Delete(key); err != nil { - log.Info("Delete operation returned an error", "key", fmt.Sprintf("0x%#x", key), "error", err) + log.Info("Delete operation returned an error", "key", fmt.Sprintf("%#x", key), "error", err) return err } return nil @@ -535,43 +535,35 @@ func dbDumpTrie(ctx *cli.Context) error { } func freezerInspect(ctx *cli.Context) error { - var ( - start, end int64 - disableSnappy bool - err error - ) - if ctx.NArg() < 3 { + if ctx.NArg() < 4 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } - kind := ctx.Args().Get(0) - if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok { - var options []string - for opt := range rawdb.FreezerNoSnappy { - options = append(options, opt) - } - sort.Strings(options) - return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options) - } else { - disableSnappy = noSnap - } - if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil { - log.Info("Could read start-param", "error", err) + var ( + freezer = ctx.Args().Get(0) + table = ctx.Args().Get(1) + ) + start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64) + if err != nil { + log.Info("Could not read start-param", "err", err) return err } - if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { - log.Info("Could read count param", "error", err) + end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64) + if err != nil { + log.Info("Could not read count param", "err", err) return err } stack, _ := makeConfigNode(ctx) defer stack.Close() - path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") - log.Info("Opening freezer", "location", path, "name", kind) - if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil { + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + ancient, err := db.AncientDatadir() + if err != nil { + log.Info("Failed to retrieve ancient root", "err", err) return err - } else { - f.DumpIndex(start, end) } - return nil + return rawdb.InspectFreezerTable(ancient, freezer, table, start, end) } func importLDBdata(ctx *cli.Context) error { @@ -716,7 +708,7 @@ func showMetaData(ctx *cli.Context) error { if val == nil { return "" } - return fmt.Sprintf("%d (0x%x)", *val, *val) + return fmt.Sprintf("%d (%#x)", *val, *val) } data := [][]string{ {"databaseVersion", pp(rawdb.ReadDatabaseVersion(db))}, @@ -726,7 +718,7 @@ func showMetaData(ctx *cli.Context) error { if b := rawdb.ReadHeadBlock(db); b != nil { data = append(data, []string{"headBlock.Hash", fmt.Sprintf("%v", b.Hash())}) data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())}) - data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (0x%x)", b.Number(), b.Number())}) + data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (%#x)", b.Number(), b.Number())}) } if b := rawdb.ReadSkeletonSyncStatus(db); b != nil { data = append(data, []string{"SkeletonSyncStatus", string(b)}) @@ -734,7 +726,7 @@ func showMetaData(ctx *cli.Context) error { if h := rawdb.ReadHeadHeader(db); h != nil { data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())}) data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) - data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (0x%x)", h.Number, h.Number)}) + data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } data = append(data, [][]string{{"frozen", fmt.Sprintf("%d items", ancients)}, {"lastPivotNumber", pp(rawdb.ReadLastPivotNumber(db))}, diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index c95755f2d919..7667a8581158 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -83,7 +83,7 @@ func TestCustomGenesis(t *testing.T) { // Query the custom genesis block geth := runGeth(t, "--networkid", "1337", "--syncmode=full", "--cache", "16", - "--datadir", datadir, "--maxpeers", "0", "--port", "0", + "--datadir", datadir, "--maxpeers", "0", "--port", "0", "--authrpc.port", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--exec", tt.query, "console") geth.ExpectRegexp(tt.result) diff --git a/cmd/geth/les_test.go b/cmd/geth/les_test.go index bc33c69027d8..7e2408bc3e64 100644 --- a/cmd/geth/les_test.go +++ b/cmd/geth/les_test.go @@ -111,7 +111,7 @@ var nextIPC = uint32(0) func startGethWithIpc(t *testing.T, name string, args ...string) *gethrpc { ipcName := fmt.Sprintf("geth-%d.ipc", atomic.AddUint32(&nextIPC, 1)) - args = append([]string{"--networkid=42", "--port=0", "--ipcpath", ipcName}, args...) + args = append([]string{"--networkid=42", "--port=0", "--authrpc.port", "0", "--ipcpath", ipcName}, args...) t.Logf("Starting %v with rpc: %v", name, args) g := &gethrpc{ diff --git a/cmd/geth/main.go b/cmd/geth/main.go index dae346384758..d2cccc5a004b 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -63,7 +63,7 @@ var ( // The app that holds all commands and flags. app = flags.NewApp(gitCommit, gitDate, "the go-wemix command line interface") // flags that configure the node - nodeFlags = utils.GroupFlags([]cli.Flag{ + nodeFlags = flags.Merge([]cli.Flag{ utils.IdentityFlag, utils.UnlockedAccountFlag, utils.PasswordFileFlag, @@ -74,8 +74,8 @@ var ( utils.NoUSBFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, - utils.OverrideGrayGlacierFlag, utils.OverrideTerminalTotalDifficulty, + utils.OverrideTerminalTotalDifficultyPassed, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, @@ -123,6 +123,7 @@ var ( utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, utils.TriesInMemoryFlag, + utils.CacheLogSizeFlag, utils.FDLimitFlag, utils.ListenPortFlag, utils.DiscoveryPortFlag, @@ -269,7 +270,7 @@ func init() { } sort.Sort(cli.CommandsByName(app.Commands)) - app.Flags = utils.GroupFlags( + app.Flags = flags.Merge( nodeFlags, rpcFlags, consoleFlags, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 82206b58b8ea..39bef1f2d352 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -56,7 +57,7 @@ var ( Usage: "Prune stale ethereum state data based on the snapshot", ArgsUsage: "", Action: pruneState, - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheTrieJournalFlag, utils.BloomFilterSizeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), @@ -80,7 +81,7 @@ the trie clean cache with default directory will be deleted. Usage: "Recalculate state hash based on the snapshot for verification", ArgsUsage: "", Action: verifyState, - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot verify-state will traverse the whole accounts and storages set based on the specified @@ -93,7 +94,7 @@ In other words, this command does the snapshot to trie conversion. Usage: "Check that there is no 'dangling' snap storage", ArgsUsage: "", Action: checkDanglingStorage, - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot check-dangling-storage traverses the snap storage data, and verifies that all snapshot storage data has a corresponding account. @@ -104,7 +105,7 @@ data, and verifies that all snapshot storage data has a corresponding account. Usage: "Check all snapshot layers for the a specific account", ArgsUsage: "
", Action: checkAccount, - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot inspect-account
checks all snapshot layers and prints out information about the specified address. @@ -115,7 +116,7 @@ information about the specified address. Usage: "Traverse the state with given root hash and perform quick verification", ArgsUsage: "", Action: traverseState, - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-state will traverse the whole state from the given state root and will abort if any @@ -130,7 +131,7 @@ It's also usable without snapshot enabled. Usage: "Traverse the state with given root hash and perform detailed verification", ArgsUsage: "", Action: traverseRawState, - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced @@ -146,7 +147,7 @@ It's also usable without snapshot enabled. Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", ArgsUsage: "[? | ]", Action: dumpState, - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.ExcludeCodeFlag, utils.ExcludeStorageFlag, utils.StartKeyFlag, @@ -270,7 +271,7 @@ func traverseState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -291,7 +292,7 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.Key), acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.Key), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err @@ -359,7 +360,7 @@ func traverseRawState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -405,7 +406,7 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") diff --git a/cmd/geth/version_check_test.go b/cmd/geth/version_check_test.go index b841ace5b2f9..bd4d820a7901 100644 --- a/cmd/geth/version_check_test.go +++ b/cmd/geth/version_check_test.go @@ -118,7 +118,6 @@ func TestMatching(t *testing.T) { version, vuln.Introduced, vuln.Fixed, vuln.Name, vulnIntro, current, vulnFixed) } } - } } for major := 1; major < 2; major++ { diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go deleted file mode 100644 index ef1f977bf09f..000000000000 --- a/cmd/puppeth/genesis.go +++ /dev/null @@ -1,626 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "errors" - "math" - "math/big" - "strings" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - math2 "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" -) - -// alethGenesisSpec represents the genesis specification format used by the -// C++ Ethereum implementation. -type alethGenesisSpec struct { - SealEngine string `json:"sealEngine"` - Params struct { - AccountStartNonce math2.HexOrDecimal64 `json:"accountStartNonce"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - HomesteadForkBlock *hexutil.Big `json:"homesteadForkBlock,omitempty"` - DaoHardforkBlock math2.HexOrDecimal64 `json:"daoHardforkBlock"` - EIP150ForkBlock *hexutil.Big `json:"EIP150ForkBlock,omitempty"` - EIP158ForkBlock *hexutil.Big `json:"EIP158ForkBlock,omitempty"` - ByzantiumForkBlock *hexutil.Big `json:"byzantiumForkBlock,omitempty"` - ConstantinopleForkBlock *hexutil.Big `json:"constantinopleForkBlock,omitempty"` - ConstantinopleFixForkBlock *hexutil.Big `json:"constantinopleFixForkBlock,omitempty"` - IstanbulForkBlock *hexutil.Big `json:"istanbulForkBlock,omitempty"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` - TieBreakingGas bool `json:"tieBreakingGas"` - GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"` - DurationLimit *math2.HexOrDecimal256 `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - AllowFutureBlocks bool `json:"allowFutureBlocks"` - } `json:"params"` - - Genesis struct { - Nonce types.BlockNonce `json:"nonce"` - Difficulty *hexutil.Big `json:"difficulty"` - MixHash common.Hash `json:"mixHash"` - Author common.Address `json:"author"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ParentHash common.Hash `json:"parentHash"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - } `json:"genesis"` - - Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"` -} - -// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled -// contract definition. -type alethGenesisSpecAccount struct { - Balance *math2.HexOrDecimal256 `json:"balance,omitempty"` - Nonce uint64 `json:"nonce,omitempty"` - Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"` -} - -// alethGenesisSpecBuiltin is the precompiled contract definition. -type alethGenesisSpecBuiltin struct { - Name string `json:"name,omitempty"` - StartingBlock *hexutil.Big `json:"startingBlock,omitempty"` - Linear *alethGenesisSpecLinearPricing `json:"linear,omitempty"` -} - -type alethGenesisSpecLinearPricing struct { - Base uint64 `json:"base"` - Word uint64 `json:"word"` -} - -// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific -// chain specification format. -func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and aleth - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - // Reconstruct the chain spec in Aleth format - spec := &alethGenesisSpec{ - SealEngine: "Ethash", - } - // Some defaults - spec.Params.AccountStartNonce = 0 - spec.Params.TieBreakingGas = false - spec.Params.AllowFutureBlocks = false - - // Dao hardfork block is a special one. The fork block is listed as 0 in the - // config but aleth will sync with ETC clients up until the actual dao hard - // fork block. - spec.Params.DaoHardforkBlock = 0 - - if num := genesis.Config.HomesteadBlock; num != nil { - spec.Params.HomesteadForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.EIP150Block; num != nil { - spec.Params.EIP150ForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.EIP158Block; num != nil { - spec.Params.EIP158ForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.ByzantiumBlock; num != nil { - spec.Params.ByzantiumForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.ConstantinopleBlock; num != nil { - spec.Params.ConstantinopleForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.PetersburgBlock; num != nil { - spec.Params.ConstantinopleFixForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.IstanbulBlock; num != nil { - spec.Params.IstanbulForkBlock = (*hexutil.Big)(num) - } - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) - spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64) - spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor) - spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) - spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit) - spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) - - spec.Genesis.Nonce = types.EncodeNonce(genesis.Nonce) - spec.Genesis.MixHash = genesis.Mixhash - spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty) - spec.Genesis.Author = genesis.Coinbase - spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp) - spec.Genesis.ParentHash = genesis.ParentHash - spec.Genesis.ExtraData = genesis.ExtraData - spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - - for address, account := range genesis.Alloc { - spec.setAccount(address, account) - } - - spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover", - Linear: &alethGenesisSpecLinearPricing{Base: 3000}}) - spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256", - Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}}) - spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160", - Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}}) - spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity", - Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}}) - if genesis.Config.ByzantiumBlock != nil { - spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock)}) - spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Linear: &alethGenesisSpecLinearPricing{Base: 500}}) - spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Linear: &alethGenesisSpecLinearPricing{Base: 40000}}) - spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock)}) - } - if genesis.Config.IstanbulBlock != nil { - if genesis.Config.ByzantiumBlock == nil { - return nil, errors.New("invalid genesis, istanbul fork is enabled while byzantium is not") - } - spec.setPrecompile(6, &alethGenesisSpecBuiltin{ - Name: "alt_bn128_G1_add", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - }) // Aleth hardcoded the gas policy - spec.setPrecompile(7, &alethGenesisSpecBuiltin{ - Name: "alt_bn128_G1_mul", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - }) // Aleth hardcoded the gas policy - spec.setPrecompile(9, &alethGenesisSpecBuiltin{ - Name: "blake2_compression", - StartingBlock: (*hexutil.Big)(genesis.Config.IstanbulBlock), - }) - } - return spec, nil -} - -func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) - } - addr := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) - if _, exist := spec.Accounts[addr]; !exist { - spec.Accounts[addr] = &alethGenesisSpecAccount{} - } - spec.Accounts[addr].Precompiled = data -} - -func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) - } - - a, exist := spec.Accounts[common.UnprefixedAddress(address)] - if !exist { - a = &alethGenesisSpecAccount{} - spec.Accounts[common.UnprefixedAddress(address)] = a - } - a.Balance = (*math2.HexOrDecimal256)(account.Balance) - a.Nonce = account.Nonce - -} - -// parityChainSpec is the chain specification format used by Parity. -type parityChainSpec struct { - Name string `json:"name"` - Datadir string `json:"dataDir"` - Engine struct { - Ethash struct { - Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward map[string]string `json:"blockReward"` - DifficultyBombDelays map[string]string `json:"difficultyBombDelays"` - HomesteadTransition hexutil.Uint64 `json:"homesteadTransition"` - EIP100bTransition hexutil.Uint64 `json:"eip100bTransition"` - } `json:"params"` - } `json:"Ethash"` - } `json:"engine"` - - Params struct { - AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` - MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` - EIP98Transition hexutil.Uint64 `json:"eip98Transition"` - EIP150Transition hexutil.Uint64 `json:"eip150Transition"` - EIP160Transition hexutil.Uint64 `json:"eip160Transition"` - EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` - EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` - EIP155Transition hexutil.Uint64 `json:"eip155Transition"` - EIP140Transition hexutil.Uint64 `json:"eip140Transition"` - EIP211Transition hexutil.Uint64 `json:"eip211Transition"` - EIP214Transition hexutil.Uint64 `json:"eip214Transition"` - EIP658Transition hexutil.Uint64 `json:"eip658Transition"` - EIP145Transition hexutil.Uint64 `json:"eip145Transition"` - EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` - EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` - EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` - EIP1283DisableTransition hexutil.Uint64 `json:"eip1283DisableTransition"` - EIP1283ReenableTransition hexutil.Uint64 `json:"eip1283ReenableTransition"` - EIP1344Transition hexutil.Uint64 `json:"eip1344Transition"` - EIP1884Transition hexutil.Uint64 `json:"eip1884Transition"` - EIP2028Transition hexutil.Uint64 `json:"eip2028Transition"` - } `json:"params"` - - Genesis struct { - Seal struct { - Ethereum struct { - Nonce types.BlockNonce `json:"nonce"` - MixHash hexutil.Bytes `json:"mixHash"` - } `json:"ethereum"` - } `json:"seal"` - - Difficulty *hexutil.Big `json:"difficulty"` - Author common.Address `json:"author"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ParentHash common.Hash `json:"parentHash"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - } `json:"genesis"` - - Nodes []string `json:"nodes"` - Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"` -} - -// parityChainSpecAccount is the prefunded genesis account and/or precompiled -// contract definition. -type parityChainSpecAccount struct { - Balance math2.HexOrDecimal256 `json:"balance"` - Nonce math2.HexOrDecimal64 `json:"nonce,omitempty"` - Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"` -} - -// parityChainSpecBuiltin is the precompiled contract definition. -type parityChainSpecBuiltin struct { - Name string `json:"name"` // Each builtin should has it own name - Pricing interface{} `json:"pricing"` // Each builtin should has it own price strategy - ActivateAt *hexutil.Big `json:"activate_at,omitempty"` // ActivateAt can't be omitted if empty, default means no fork -} - -// parityChainSpecPricing represents the different pricing models that builtin -// contracts might advertise using. -type parityChainSpecPricing struct { - Linear *parityChainSpecLinearPricing `json:"linear,omitempty"` - ModExp *parityChainSpecModExpPricing `json:"modexp,omitempty"` - - // Before the https://github.com/paritytech/parity-ethereum/pull/11039, - // Parity uses this format to config bn pairing price policy. - AltBnPairing *parityChainSepcAltBnPairingPricing `json:"alt_bn128_pairing,omitempty"` - - // Blake2F is the price per round of Blake2 compression - Blake2F *parityChainSpecBlakePricing `json:"blake2_f,omitempty"` -} - -type parityChainSpecLinearPricing struct { - Base uint64 `json:"base"` - Word uint64 `json:"word"` -} - -type parityChainSpecModExpPricing struct { - Divisor uint64 `json:"divisor"` -} - -// parityChainSpecAltBnConstOperationPricing defines the price -// policy for bn const operation(used after istanbul) -type parityChainSpecAltBnConstOperationPricing struct { - Price uint64 `json:"price"` -} - -// parityChainSepcAltBnPairingPricing defines the price policy -// for bn pairing. -type parityChainSepcAltBnPairingPricing struct { - Base uint64 `json:"base"` - Pair uint64 `json:"pair"` -} - -// parityChainSpecBlakePricing defines the price policy for blake2 f -// compression. -type parityChainSpecBlakePricing struct { - GasPerRound uint64 `json:"gas_per_round"` -} - -type parityChainSpecAlternativePrice struct { - AltBnConstOperationPrice *parityChainSpecAltBnConstOperationPricing `json:"alt_bn128_const_operations,omitempty"` - AltBnPairingPrice *parityChainSepcAltBnPairingPricing `json:"alt_bn128_pairing,omitempty"` -} - -// parityChainSpecVersionedPricing represents a single version price policy. -type parityChainSpecVersionedPricing struct { - Price *parityChainSpecAlternativePrice `json:"price,omitempty"` - Info string `json:"info,omitempty"` -} - -// newParityChainSpec converts a go-ethereum genesis block into a Parity specific -// chain specification format. -func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []string) (*parityChainSpec, error) { - // Only ethash is currently supported between go-ethereum and Parity - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - // Reconstruct the chain spec in Parity's format - spec := &parityChainSpec{ - Name: network, - Nodes: bootnodes, - Datadir: strings.ToLower(network), - } - spec.Engine.Ethash.Params.BlockReward = make(map[string]string) - spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string) - // Frontier - spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) - spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward) - - // Homestead - spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64()) - - // Tangerine Whistle : 150 - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md - spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64()) - - // Spurious Dragon: 155, 160, 161, 170 - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md - spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) - spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) - spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) - spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) - - // Byzantium - if num := genesis.Config.ByzantiumBlock; num != nil { - spec.setByzantium(num) - } - // Constantinople - if num := genesis.Config.ConstantinopleBlock; num != nil { - spec.setConstantinople(num) - } - // ConstantinopleFix (remove eip-1283) - if num := genesis.Config.PetersburgBlock; num != nil { - spec.setConstantinopleFix(num) - } - // Istanbul - if num := genesis.Config.IstanbulBlock; num != nil { - spec.setIstanbul(num) - } - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) - spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaxCodeSize = params.MaxCodeSize - // geth has it set from zero - spec.Params.MaxCodeSizeTransition = 0 - - // Disable this one - spec.Params.EIP98Transition = math.MaxInt64 - - spec.Genesis.Seal.Ethereum.Nonce = types.EncodeNonce(genesis.Nonce) - spec.Genesis.Seal.Ethereum.MixHash = genesis.Mixhash[:] - spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty) - spec.Genesis.Author = genesis.Coinbase - spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp) - spec.Genesis.ParentHash = genesis.ParentHash - spec.Genesis.ExtraData = genesis.ExtraData - spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - - spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) - for address, account := range genesis.Alloc { - bal := math2.HexOrDecimal256(*account.Balance) - - spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{ - Balance: bal, - Nonce: math2.HexOrDecimal64(account.Nonce), - } - } - spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover", - Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}}) - - spec.setPrecompile(2, &parityChainSpecBuiltin{ - Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}}, - }) - spec.setPrecompile(3, &parityChainSpecBuiltin{ - Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}}, - }) - spec.setPrecompile(4, &parityChainSpecBuiltin{ - Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}}, - }) - if genesis.Config.ByzantiumBlock != nil { - spec.setPrecompile(5, &parityChainSpecBuiltin{ - Name: "modexp", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - ModExp: &parityChainSpecModExpPricing{Divisor: 20}, - }, - }) - spec.setPrecompile(6, &parityChainSpecBuiltin{ - Name: "alt_bn128_add", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - Linear: &parityChainSpecLinearPricing{Base: 500, Word: 0}, - }, - }) - spec.setPrecompile(7, &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - Linear: &parityChainSpecLinearPricing{Base: 40000, Word: 0}, - }, - }) - spec.setPrecompile(8, &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - AltBnPairing: &parityChainSepcAltBnPairingPricing{Base: 100000, Pair: 80000}, - }, - }) - } - if genesis.Config.IstanbulBlock != nil { - if genesis.Config.ByzantiumBlock == nil { - return nil, errors.New("invalid genesis, istanbul fork is enabled while byzantium is not") - } - spec.setPrecompile(6, &parityChainSpecBuiltin{ - Name: "alt_bn128_add", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 500}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 150}, - }, - }, - }, - }) - spec.setPrecompile(7, &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 40000}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 6000}, - }, - }, - }, - }) - spec.setPrecompile(8, &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnPairingPrice: &parityChainSepcAltBnPairingPricing{Base: 100000, Pair: 80000}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnPairingPrice: &parityChainSepcAltBnPairingPricing{Base: 45000, Pair: 34000}, - }, - }, - }, - }) - spec.setPrecompile(9, &parityChainSpecBuiltin{ - Name: "blake2_f", - ActivateAt: (*hexutil.Big)(genesis.Config.IstanbulBlock), - Pricing: &parityChainSpecPricing{ - Blake2F: &parityChainSpecBlakePricing{GasPerRound: 1}, - }, - }) - } - return spec, nil -} - -func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) - } - a := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) - if _, exist := spec.Accounts[a]; !exist { - spec.Accounts[a] = &parityChainSpecAccount{} - } - spec.Accounts[a].Builtin = data -} - -func (spec *parityChainSpec) setByzantium(num *big.Int) { - spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward) - spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000) - n := hexutil.Uint64(num.Uint64()) - spec.Engine.Ethash.Params.EIP100bTransition = n - spec.Params.EIP140Transition = n - spec.Params.EIP211Transition = n - spec.Params.EIP214Transition = n - spec.Params.EIP658Transition = n -} - -func (spec *parityChainSpec) setConstantinople(num *big.Int) { - spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward) - spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000) - n := hexutil.Uint64(num.Uint64()) - spec.Params.EIP145Transition = n - spec.Params.EIP1014Transition = n - spec.Params.EIP1052Transition = n - spec.Params.EIP1283Transition = n -} - -func (spec *parityChainSpec) setConstantinopleFix(num *big.Int) { - spec.Params.EIP1283DisableTransition = hexutil.Uint64(num.Uint64()) -} - -func (spec *parityChainSpec) setIstanbul(num *big.Int) { - spec.Params.EIP1344Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP1884Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP2028Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP1283ReenableTransition = hexutil.Uint64(num.Uint64()) -} - -// pyEthereumGenesisSpec represents the genesis specification format used by the -// Python Ethereum implementation. -type pyEthereumGenesisSpec struct { - Nonce types.BlockNonce `json:"nonce"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - Difficulty *hexutil.Big `json:"difficulty"` - Mixhash common.Hash `json:"mixhash"` - Coinbase common.Address `json:"coinbase"` - Alloc core.GenesisAlloc `json:"alloc"` - ParentHash common.Hash `json:"parentHash"` -} - -// newPyEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific -// chain specification format. -func newPyEthereumGenesisSpec(network string, genesis *core.Genesis) (*pyEthereumGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and pyethereum - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - spec := &pyEthereumGenesisSpec{ - Nonce: types.EncodeNonce(genesis.Nonce), - Timestamp: (hexutil.Uint64)(genesis.Timestamp), - ExtraData: genesis.ExtraData, - GasLimit: (hexutil.Uint64)(genesis.GasLimit), - Difficulty: (*hexutil.Big)(genesis.Difficulty), - Mixhash: genesis.Mixhash, - Coinbase: genesis.Coinbase, - Alloc: genesis.Alloc, - ParentHash: genesis.ParentHash, - } - return spec, nil -} diff --git a/cmd/puppeth/genesis_test.go b/cmd/puppeth/genesis_test.go deleted file mode 100644 index 605c1070a80c..000000000000 --- a/cmd/puppeth/genesis_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "bytes" - "encoding/json" - "os" - "reflect" - "strings" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/ethereum/go-ethereum/core" -) - -// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet. -func TestAlethSturebyConverter(t *testing.T) { - blob, err := os.ReadFile("testdata/stureby_geth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - spec, err := newAlethGenesisSpec("stureby", &genesis) - if err != nil { - t.Fatalf("failed creating chainspec: %v", err) - } - - expBlob, err := os.ReadFile("testdata/stureby_aleth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - expspec := &alethGenesisSpec{} - if err := json.Unmarshal(expBlob, expspec); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - if !reflect.DeepEqual(expspec, spec) { - t.Errorf("chainspec mismatch") - c := spew.ConfigState{ - DisablePointerAddresses: true, - SortKeys: true, - } - exp := strings.Split(c.Sdump(expspec), "\n") - got := strings.Split(c.Sdump(spec), "\n") - for i := 0; i < len(exp) && i < len(got); i++ { - if exp[i] != got[i] { - t.Logf("got: %v\nexp: %v\n", exp[i], got[i]) - } - } - } -} - -// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet. -func TestParitySturebyConverter(t *testing.T) { - blob, err := os.ReadFile("testdata/stureby_geth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - spec, err := newParityChainSpec("stureby", &genesis, []string{}) - if err != nil { - t.Fatalf("failed creating chainspec: %v", err) - } - enc, err := json.MarshalIndent(spec, "", " ") - if err != nil { - t.Fatalf("failed encoding chainspec: %v", err) - } - expBlob, err := os.ReadFile("testdata/stureby_parity.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - if !bytes.Equal(expBlob, enc) { - t.Fatalf("chainspec mismatch") - } -} diff --git a/cmd/puppeth/module.go b/cmd/puppeth/module.go index b6a029a01a48..771ae38058bc 100644 --- a/cmd/puppeth/module.go +++ b/cmd/puppeth/module.go @@ -150,3 +150,12 @@ func checkPort(host string, port int) error { conn.Close() return nil } + +// getEthName gets the Ethereum Name from ethstats +func getEthName(s string) string { + n := strings.Index(s, ":") + if n >= 0 { + return s[:n] + } + return s +} diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 35cfada66fd3..fbbbb66501a7 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -18,7 +18,6 @@ package main import ( "bytes" - "encoding/json" "fmt" "html/template" "math/rand" @@ -582,36 +581,6 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da // Marshal the genesis spec files for go-ethereum and all the other clients genesis, _ := conf.Genesis.MarshalJSON() files[filepath.Join(workdir, network+".json")] = genesis - - if conf.Genesis.Config.Ethash != nil { - cppSpec, err := newAlethGenesisSpec(network, conf.Genesis) - if err != nil { - return nil, err - } - cppSpecJSON, _ := json.Marshal(cppSpec) - files[filepath.Join(workdir, network+"-cpp.json")] = cppSpecJSON - - harmonySpecJSON, _ := conf.Genesis.MarshalJSON() - files[filepath.Join(workdir, network+"-harmony.json")] = harmonySpecJSON - - paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootnodes) - if err != nil { - return nil, err - } - paritySpecJSON, _ := json.Marshal(paritySpec) - files[filepath.Join(workdir, network+"-parity.json")] = paritySpecJSON - - pyethSpec, err := newPyEthereumGenesisSpec(network, conf.Genesis) - if err != nil { - return nil, err - } - pyethSpecJSON, _ := json.Marshal(pyethSpec) - files[filepath.Join(workdir, network+"-python.json")] = pyethSpecJSON - } else { - for _, client := range []string{"cpp", "harmony", "parity", "python"} { - files[filepath.Join(workdir, network+"-"+client+".json")] = []byte{} - } - } files[filepath.Join(workdir, "puppeth.png")] = dashboardMascot // Upload the deployment files to the remote server (and clean up afterwards) diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index 1165f70fcf51..3812f9fdb963 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -104,7 +104,7 @@ func deployExplorer(client *sshClient, network string, bootnodes []string, confi "Datadir": config.node.datadir, "DBDir": config.dbdir, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "WebPort": config.port, "Transformer": transformer, }) diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 88cb80ae4c42..a4f6e65694df 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -116,7 +116,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "VHost": config.host, "ApiPort": config.port, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "CaptchaToken": config.captchaToken, "CaptchaSecret": config.captchaSecret, "FaucetAmount": config.amount, diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 3ea96870d4f5..b8aa30db39a8 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -123,7 +123,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "TotalPeers": config.peersTotal, "Light": config.peersLight > 0, "LightPeers": config.peersLight, - "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], + "Ethstats": getEthName(config.ethstats), "Etherbase": config.etherbase, "GasTarget": config.gasTarget, "GasLimit": config.gasLimit, diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 0c23ab556228..a20b3bfda209 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -163,7 +163,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { return nil } // We have a mismatch, forbid connecting - return errors.New("ssh key mismatch, readd the machine to update") + return errors.New("ssh key mismatch, re-add the machine to update") } client, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{User: username, Auth: auths, HostKeyCallback: keycheck}) if err != nil { diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index cb056ab13356..ac17bc7b271c 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -250,8 +250,8 @@ func (w *wizard) manageGenesis() { case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which folder to save the genesis specs into? (default = current)\n") - fmt.Printf(" Will create %s.json, %s-aleth.json, %s-harmony.json, %s-parity.json\n", w.network, w.network, w.network, w.network) + fmt.Printf("Which folder to save the genesis spec into? (default = current)\n") + fmt.Printf(" Will create %s.json\n", w.network) folder := w.readDefaultString(".") if err := os.MkdirAll(folder, 0755); err != nil { @@ -268,21 +268,6 @@ func (w *wizard) manageGenesis() { } log.Info("Saved native genesis chain spec", "path", gethJson) - // Export the genesis spec used by Aleth (formerly C++ Ethereum) - if spec, err := newAlethGenesisSpec(w.network, w.conf.Genesis); err != nil { - log.Error("Failed to create Aleth chain spec", "err", err) - } else { - saveGenesis(folder, w.network, "aleth", spec) - } - // Export the genesis spec used by Parity - if spec, err := newParityChainSpec(w.network, w.conf.Genesis, []string{}); err != nil { - log.Error("Failed to create Parity chain spec", "err", err) - } else { - saveGenesis(folder, w.network, "parity", spec) - } - // Export the genesis spec used by Harmony (formerly EthereumJ) - saveGenesis(folder, w.network, "harmony", w.conf.Genesis) - case "3": // Make sure we don't have any services running if len(w.conf.servers()) > 0 { @@ -298,15 +283,3 @@ func (w *wizard) manageGenesis() { return } } - -// saveGenesis JSON encodes an arbitrary genesis spec into a pre-defined file. -func saveGenesis(folder, network, client string, spec interface{}) { - path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client)) - - out, _ := json.MarshalIndent(spec, "", " ") - if err := os.WriteFile(path, out, 0644); err != nil { - log.Error("Failed to save genesis file", "client", client, "err", err) - return - } - log.Info("Saved genesis chain spec", "client", client, "path", path) -} diff --git a/cmd/rlpdump/main.go b/cmd/rlpdump/main.go index 9c0af012480f..70337749aea3 100644 --- a/cmd/rlpdump/main.go +++ b/cmd/rlpdump/main.go @@ -83,7 +83,7 @@ func main() { if err != nil { die(err) } - fmt.Printf("0x%x\n", data) + fmt.Printf("%#x\n", data) return } else { err := rlpToText(r, out) diff --git a/cmd/rlpdump/rlpdump_test.go b/cmd/rlpdump/rlpdump_test.go index 899beef32f4a..a9ab57fdb880 100644 --- a/cmd/rlpdump/rlpdump_test.go +++ b/cmd/rlpdump/rlpdump_test.go @@ -43,7 +43,7 @@ func TestRoundtrip(t *testing.T) { t.Errorf("test %d: error %v", i, err) continue } - have := fmt.Sprintf("0x%x", rlpBytes) + have := fmt.Sprintf("%#x", rlpBytes) if have != want { t.Errorf("test %d: have\n%v\nwant:\n%v\n", i, have, want) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7c5fd99b1b74..625e4fbbd167 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -43,6 +43,7 @@ import ( ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -64,6 +65,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" "github.com/urfave/cli/v2" @@ -91,7 +93,7 @@ var ( } AncientFlag = &flags.DirectoryFlag{ Name: "datadir.ancient", - Usage: "Data directory for ancient chain segments (default = inside chaindata)", + Usage: "Root directory for ancient data (default = inside chaindata)", Category: flags.EthCategory, } MinFreeDiskSpaceFlag = &flags.DirectoryFlag{ @@ -267,17 +269,16 @@ var ( Value: 2048, Category: flags.EthCategory, } - OverrideGrayGlacierFlag = &cli.Uint64Flag{ - Name: "override.grayglacier", - Usage: "Manually specify Gray Glacier fork-block, overriding the bundled setting", - Category: flags.EthCategory, - } OverrideTerminalTotalDifficulty = &flags.BigFlag{ Name: "override.terminaltotaldifficulty", Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting", Category: flags.EthCategory, } - + OverrideTerminalTotalDifficultyPassed = &cli.BoolFlag{ + Name: "override.terminaltotaldifficultypassed", + Usage: "Manually specify TerminalTotalDifficultyPassed, overriding the bundled setting", + Category: flags.EthCategory, + } // Light server and client settings LightServeFlag = &cli.IntFlag{ Name: "light.serve", @@ -503,6 +504,12 @@ var ( Value: 128, Category: flags.PerfCategory, } + CacheLogSizeFlag = &cli.IntFlag{ + Name: "cache.blocklogs", + Usage: "Size (in number of blocks) of the log cache for filtering", + Category: flags.PerfCategory, + Value: ethconfig.Defaults.FilterLogCacheSize, + } FDLimitFlag = &cli.IntFlag{ Name: "fdlimit", Usage: "Raise the open file descriptor resource limit (default = system fd limit)", @@ -1104,15 +1111,6 @@ var ( } ) -// GroupFlags combines the given flag slices together and returns the merged one. -func GroupFlags(groups ...[]cli.Flag) []cli.Flag { - var ret []cli.Flag - for _, group := range groups { - ret = append(ret, group...) - } - return ret -} - // MakeDataDir retrieves the currently requested data directory, terminating // if none (or the empty string) is specified. If the node is starting a testnet, // then a subdirectory of the specified datadir will be used. @@ -1933,6 +1931,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(TriesInMemoryFlag.Name) { cfg.TriesInMemory = ctx.Uint64(TriesInMemoryFlag.Name) } + if ctx.IsSet(CacheLogSizeFlag.Name) { + cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name) + } if !ctx.Bool(SnapshotFlag.Name) { // If snap-sync is requested, this flag is also required @@ -2171,10 +2172,8 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) - if backend.BlockChain().Config().TerminalTotalDifficulty != nil { - if err := lescatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the catalyst service: %v", err) - } + if err := lescatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) } return backend.ApiBackend, nil } @@ -2188,30 +2187,41 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to create the LES server: %v", err) } } - if backend.BlockChain().Config().TerminalTotalDifficulty != nil { - if err := ethcatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the catalyst service: %v", err) - } + if err := ethcatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend, backend } -// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to -// the given node. +// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to the node. func RegisterEthStatsService(stack *node.Node, backend ethapi.Backend, url string) { if err := ethstats.New(stack, backend, backend.Engine(), url); err != nil { Fatalf("Failed to register the Ethereum Stats service: %v", err) } } -// RegisterGraphQLService is a utility function to construct a new service and register it against a node. -func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.Config) { - if err := graphql.New(stack, backend, cfg.GraphQLCors, cfg.GraphQLVirtualHosts); err != nil { +// RegisterGraphQLService adds the GraphQL API to the node. +func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cfg *node.Config) { + err := graphql.New(stack, backend, filterSystem, cfg.GraphQLCors, cfg.GraphQLVirtualHosts) + if err != nil { Fatalf("Failed to register the GraphQL service: %v", err) } } +// RegisterFilterAPI adds the eth log filtering RPC API to the node. +func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconfig.Config) *filters.FilterSystem { + isLightClient := ethcfg.SyncMode == downloader.LightSync + filterSystem := filters.NewFilterSystem(backend, filters.Config{ + LogCacheSize: ethcfg.FilterLogCacheSize, + }) + stack.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, isLightClient), + }}) + return filterSystem +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index ad8a44aa04ae..9de94017c2ed 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -66,13 +66,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin contracts := make(map[string]*Contract) for name, info := range output.Contracts { // Parse the individual compilation results. - var abi interface{} + var abi, userdoc, devdoc interface{} if err := json.Unmarshal([]byte(info.Abi), &abi); err != nil { return nil, fmt.Errorf("solc: error reading abi definition (%v)", err) } - var userdoc, devdoc interface{} - json.Unmarshal([]byte(info.Userdoc), &userdoc) - json.Unmarshal([]byte(info.Devdoc), &devdoc) + if err := json.Unmarshal([]byte(info.Userdoc), &userdoc); err != nil { + return nil, fmt.Errorf("solc: error reading userdoc definition (%v)", err) + } + if err := json.Unmarshal([]byte(info.Devdoc), &devdoc); err != nil { + return nil, fmt.Errorf("solc: error reading devdoc definition (%v)", err) + } contracts[name] = &Contract{ Code: "0x" + info.Bin, diff --git a/common/math/big_test.go b/common/math/big_test.go index f896ec65becf..803b5e1cc617 100644 --- a/common/math/big_test.go +++ b/common/math/big_test.go @@ -171,7 +171,6 @@ func BenchmarkByteAt(b *testing.B) { } func BenchmarkByteAtOld(b *testing.B) { - bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") for i := 0; i < b.N; i++ { PaddedBigBytes(bigint, 32) @@ -244,7 +243,6 @@ func TestBigEndianByteAt(t *testing.T) { if actual != test.exp { t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) } - } } func TestLittleEndianByteAt(t *testing.T) { @@ -277,7 +275,6 @@ func TestLittleEndianByteAt(t *testing.T) { if actual != test.exp { t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) } - } } diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go index 766ca0f8736c..f5ad3f8bc0aa 100644 --- a/common/mclock/simclock.go +++ b/common/mclock/simclock.go @@ -58,7 +58,7 @@ func (s *Simulated) Run(d time.Duration) { s.mu.Lock() s.init() - end := s.now + AbsTime(d) + end := s.now.Add(d) var do []func() for len(s.scheduled) > 0 && s.scheduled[0].at <= end { ev := heap.Pop(&s.scheduled).(*simTimer) @@ -134,7 +134,7 @@ func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { s.init() - at := s.now + AbsTime(d) + at := s.now.Add(d) ev := &simTimer{do: fn, at: at, s: s} heap.Push(&s.scheduled, ev) s.cond.Broadcast() diff --git a/common/prque/lazyqueue.go b/common/prque/lazyqueue.go index e9f7f900b94e..13ef3ed2cdbf 100644 --- a/common/prque/lazyqueue.go +++ b/common/prque/lazyqueue.go @@ -88,13 +88,13 @@ func (q *LazyQueue) Refresh() { // refresh re-evaluates items in the older queue and swaps the two queues func (q *LazyQueue) refresh(now mclock.AbsTime) { - q.maxUntil = now + mclock.AbsTime(q.period) + q.maxUntil = now.Add(q.period) for q.queue[0].Len() != 0 { q.Push(heap.Pop(q.queue[0]).(*item).value) } q.queue[0], q.queue[1] = q.queue[1], q.queue[0] q.indexOffset = 1 - q.indexOffset - q.maxUntil += mclock.AbsTime(q.period) + q.maxUntil = q.maxUntil.Add(q.period) } // Push adds an item to the queue @@ -164,7 +164,7 @@ func (q *LazyQueue) PopItem() interface{} { return i } -// Remove removes removes the item with the given index. +// Remove removes the item with the given index. func (q *LazyQueue) Remove(index int) interface{} { if index < 0 { return nil diff --git a/common/prque/prque.go b/common/prque/prque.go index 54c78b5fc2ba..fb02e3418c28 100755 --- a/common/prque/prque.go +++ b/common/prque/prque.go @@ -41,13 +41,13 @@ func (p *Prque) Push(data interface{}, priority int64) { heap.Push(p.cont, &item{data, priority}) } -// Peek returns the value with the greates priority but does not pop it off. +// Peek returns the value with the greatest priority but does not pop it off. func (p *Prque) Peek() (interface{}, int64) { item := p.cont.blocks[0][0] return item.value, item.priority } -// Pops the value with the greates priority off the stack and returns it. +// Pops the value with the greatest priority off the stack and returns it. // Currently no shrinking is done. func (p *Prque) Pop() (interface{}, int64) { item := heap.Pop(p.cont).(*item) diff --git a/common/types_test.go b/common/types_test.go index 318e985f870b..94492278d84a 100644 --- a/common/types_test.go +++ b/common/types_test.go @@ -155,7 +155,6 @@ func BenchmarkAddressHex(b *testing.B) { } func TestMixedcaseAccount_Address(t *testing.T) { - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md // Note: 0X{checksum_addr} is not valid according to spec above @@ -192,9 +191,7 @@ func TestMixedcaseAccount_Address(t *testing.T) { if err := json.Unmarshal([]byte(r), &r2); err == nil { t.Errorf("Expected failure, input %v", r) } - } - } func TestHash_Scan(t *testing.T) { diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 4cc42af8c267..ab098911f058 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -45,6 +45,7 @@ var ( errTooManyUncles = errors.New("too many uncles") errInvalidNonce = errors.New("invalid nonce") errInvalidUncleHash = errors.New("invalid uncle hash") + errInvalidTimestamp = errors.New("invalid timestamp") ) // Beacon is a consensus engine that combines the eth1 consensus and proof-of-stake @@ -113,8 +114,16 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ } } - // All the headers have passed the transition point, use new rules. if len(preHeaders) == 0 { + // All the headers are pos headers. Verify that the parent block reached total terminal difficulty. + if reached, _ := IsTTDReached(chain, headers[0].ParentHash, headers[0].Number.Uint64()-1); !reached { + // TTD not reached for the first block, mark subsequent with invalid terminal block + results := make(chan error, len(headers)) + for i := 0; i < len(headers); i++ { + results <- consensus.ErrInvalidTerminalBlock + } + return make(chan struct{}), results + } return beacon.verifyHeaders(chain, headers, nil) } @@ -166,7 +175,7 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ return abort, results } -// verifyTerminalPoWBlock verifies that the preHeaders confirm to the specification +// verifyTerminalPoWBlock verifies that the preHeaders conform to the specification // wrt. their total difficulty. // It expects: // - preHeaders to be at least 1 element @@ -178,6 +187,7 @@ func verifyTerminalPoWBlock(chain consensus.ChainHeaderReader, preHeaders []*typ if td == nil { return 0, consensus.ErrUnknownAncestor } + td = new(big.Int).Set(td) // Check that all blocks before the last one are below the TTD for i, head := range preHeaders { if td.Cmp(chain.Config().TerminalTotalDifficulty) >= 0 { @@ -213,7 +223,7 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo // - unclehash is expected to be Hash(emptyHeader) // to be the desired constants // -// (b) the timestamp is not verified anymore +// (b) we don't verify if a block is in the future anymore // (c) the extradata is limited to 32 bytes func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header) error { // Ensure that the header's extra-data section is of a reasonable size @@ -227,6 +237,10 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if header.UncleHash != types.EmptyUncleHash { return errInvalidUncleHash } + // Verify the timestamp + if header.Time <= parent.Time { + return errInvalidTimestamp + } // Verify the block's difficulty to ensure it's the default constant if beaconDifficulty.Cmp(header.Difficulty) != 0 { return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, beaconDifficulty) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 038227aaaaa7..67ab23236627 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -698,7 +698,6 @@ func (c *Clique) Close() error { func (c *Clique) APIs(chain consensus.ChainHeaderReader) []rpc.API { return []rpc.API{{ Namespace: "clique", - Version: "1.0", Service: &API{chain: chain, clique: c}, }} } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 094868ca744d..4a067c62554a 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -305,7 +305,7 @@ func TestClique(t *testing.T) { }, { // Ensure that pending votes don't survive authorization status changes. This // corner case can only appear if a signer is quickly added, removed and then - // readded (or the inverse), while one of the original voters dropped. If a + // re-added (or the inverse), while one of the original voters dropped. If a // past vote is left cached in the system somewhere, this will interfere with // the final signer outcome. signers: []string{"A", "B", "C", "D", "E"}, @@ -344,7 +344,7 @@ func TestClique(t *testing.T) { }, failure: errUnauthorizedSigner, }, { - // An authorized signer that signed recenty should not be able to sign again + // An authorized signer that signed recently should not be able to sign again signers: []string{"A", "B"}, votes: []testerVote{ {signer: "A"}, @@ -403,7 +403,7 @@ func TestClique(t *testing.T) { } // Create a pristine blockchain with the genesis injected db := rawdb.NewMemoryDatabase() - genesis.Commit(db) + genesisBlock := genesis.MustCommit(db) // Assemble a chain of headers from the cast votes config := *params.TestChainConfig @@ -414,7 +414,7 @@ func TestClique(t *testing.T) { engine := New(config.Clique, db) engine.fakeDiff = true - blocks, _ := core.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { + blocks, _ := core.GenerateChain(&config, genesisBlock, engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { // Cast the vote contained in this block gen.SetCoinbase(accounts.address(tt.votes[j].voted)) if tt.votes[j].auth { diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 902949202250..dfe00d4b93c0 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -278,8 +278,11 @@ func (c *cache) generate(dir string, limit int, lock bool, test bool) { // Iterate over all previous instances and delete old ones for ep := int(c.epoch) - limit; ep >= 0; ep-- { seed := seedHash(uint64(ep)*epochLength + 1) - path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) - os.Remove(path) + path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s*", algorithmRevision, seed[:8], endian)) + files, _ := filepath.Glob(path) // find also the temp files that are generated. + for _, file := range files { + os.Remove(file) + } } }) } @@ -678,12 +681,10 @@ func (ethash *Ethash) APIs(chain consensus.ChainHeaderReader) []rpc.API { return []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: &API{ethash}, }, { Namespace: "ethash", - Version: "1.0", Service: &API{ethash}, }, } diff --git a/consensus/misc/forks.go b/consensus/misc/forks.go index 4a5e7c37e03c..a6f3303ea6fa 100644 --- a/consensus/misc/forks.go +++ b/consensus/misc/forks.go @@ -35,7 +35,7 @@ func VerifyForkHashes(config *params.ChainConfig, header *types.Header, uncle bo // If the homestead reprice hash is set, validate it if config.EIP150Block != nil && config.EIP150Block.Cmp(header.Number) == 0 { if config.EIP150Hash != (common.Hash{}) && config.EIP150Hash != header.Hash() { - return fmt.Errorf("homestead gas reprice fork: have 0x%x, want 0x%x", header.Hash(), config.EIP150Hash) + return fmt.Errorf("homestead gas reprice fork: have %#x, want %#x", header.Hash(), config.EIP150Hash) } } // All ok, return diff --git a/console/console.go b/console/console.go index f79c03ba3d71..ae633044282e 100644 --- a/console/console.go +++ b/console/console.go @@ -293,7 +293,7 @@ func (c *Console) AutoCompleteInput(line string, pos int) (string, []string, str if len(line) == 0 || pos == 0 { return "", nil, "" } - // Chunck data to relevant part for autocompletion + // Chunk data to relevant part for autocompletion // E.g. in case of nested lines eth.getBalance(eth.coinb start := pos - 1 for ; start > 0; start-- { @@ -410,7 +410,7 @@ func (c *Console) StopInteractive() { } } -// Interactive starts an interactive user session, where in.put is propted from +// Interactive starts an interactive user session, where input is prompted from // the configured user prompter. func (c *Console) Interactive() { var ( @@ -500,7 +500,7 @@ func (c *Console) readLines(input chan<- string, errc chan<- error, prompt <-cha } } -// countIndents returns the number of identations for the given input. +// countIndents returns the number of indentations for the given input. // In case of invalid input such as var a = } the result can be negative. func countIndents(input string) int { var ( diff --git a/core/asm/asm.go b/core/asm/asm.go index f3f129714d31..7c1e14ec01ea 100644 --- a/core/asm/asm.go +++ b/core/asm/asm.go @@ -109,7 +109,7 @@ func PrintDisassembled(code string) error { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - fmt.Printf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg()) + fmt.Printf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg()) } else { fmt.Printf("%05x: %v\n", it.PC(), it.Op()) } @@ -124,7 +124,7 @@ func Disassemble(script []byte) ([]string, error) { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg())) + instrs = append(instrs, fmt.Sprintf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg())) } else { instrs = append(instrs, fmt.Sprintf("%05x: %v\n", it.PC(), it.Op())) } diff --git a/core/beacon/types.go b/core/beacon/types.go index c3b841d7516c..e06ab5c692d9 100644 --- a/core/beacon/types.go +++ b/core/beacon/types.go @@ -42,7 +42,7 @@ type payloadAttributesMarshaling struct { //go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go -// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/src/engine/specification.md +// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/tree/main/src/engine/specification.md type ExecutableDataV1 struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` diff --git a/core/blockchain.go b/core/blockchain.go index b90a273593d6..844c33d6a292 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -52,6 +52,7 @@ var ( headHeaderGauge = metrics.NewRegisteredGauge("chain/head/header", nil) headFastBlockGauge = metrics.NewRegisteredGauge("chain/head/receipt", nil) headFinalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil) + headSafeBlockGauge = metrics.NewRegisteredGauge("chain/head/safe", nil) accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) @@ -193,6 +194,7 @@ type BlockChain struct { currentBlock atomic.Value // Current head of the block chain currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) currentFinalizedBlock atomic.Value // Current finalized head + currentSafeBlock atomic.Value // Current safe head stateCache state.Database // State database to reuse between imports (contains state cache) bodyCache *lru.Cache // Cache for the most recent block bodies @@ -274,6 +276,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.currentBlock.Store(nilBlock) bc.currentFastBlock.Store(nilBlock) bc.currentFinalizedBlock.Store(nilBlock) + bc.currentSafeBlock.Store(nilBlock) // Initialize the chain with ancient data if it isn't empty. var txIndexBlock uint64 @@ -471,11 +474,15 @@ func (bc *BlockChain) loadLastState() error { } } - // Restore the last known finalized block + // Restore the last known finalized block and safe block + // Note: the safe block is not stored on disk and it is set to the last + // known finalized block on startup if head := rawdb.ReadFinalizedBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { bc.currentFinalizedBlock.Store(block) headFinalizedBlockGauge.Update(int64(block.NumberU64())) + bc.currentSafeBlock.Store(block) + headSafeBlockGauge.Update(int64(block.NumberU64())) } } // Issue a status log for the user @@ -511,8 +518,23 @@ func (bc *BlockChain) SetHead(head uint64) error { // SetFinalized sets the finalized block. func (bc *BlockChain) SetFinalized(block *types.Block) { bc.currentFinalizedBlock.Store(block) - rawdb.WriteFinalizedBlockHash(bc.db, block.Hash()) - headFinalizedBlockGauge.Update(int64(block.NumberU64())) + if block != nil { + rawdb.WriteFinalizedBlockHash(bc.db, block.Hash()) + headFinalizedBlockGauge.Update(int64(block.NumberU64())) + } else { + rawdb.WriteFinalizedBlockHash(bc.db, common.Hash{}) + headFinalizedBlockGauge.Update(0) + } +} + +// SetSafe sets the safe block. +func (bc *BlockChain) SetSafe(block *types.Block) { + bc.currentSafeBlock.Store(block) + if block != nil { + headSafeBlockGauge.Update(int64(block.NumberU64())) + } else { + headSafeBlockGauge.Update(0) + } } // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition @@ -670,6 +692,16 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo bc.txLookupCache.Purge() bc.futureBlocks.Purge() + // Clear safe block, finalized block if needed + if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.NumberU64() { + log.Warn("SetHead invalidated safe block") + bc.SetSafe(nil) + } + if finalized := bc.CurrentFinalizedBlock(); finalized != nil && head < finalized.NumberU64() { + log.Error("SetHead invalidated finalized block") + bc.SetFinalized(nil) + } + return rootNumber, bc.loadLastState() } @@ -681,7 +713,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } - if _, err := trie.NewSecure(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil { + if _, err := trie.NewStateTrie(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil { return err } @@ -865,6 +897,10 @@ func (bc *BlockChain) Stop() { log.Error("Dangling trie nodes after full cleanup") } } + // Flush the collected preimages to disk + if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil { + log.Error("Failed to commit trie preimages", "err", err) + } // Ensure all live cached entries be saved into disk, so that we can skip // cache warmup when node restarts. if bc.cacheConfig.TrieCleanJournal != "" { @@ -1216,7 +1252,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { // writeBlockWithState writes block, metadata and corresponding state data to the // database. -func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB) error { +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) error { // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { @@ -1311,7 +1347,7 @@ func (bc *BlockChain) WriteBlockAndSetHead(block *types.Block, receipts []*types // writeBlockAndSetHead is the internal implementation of WriteBlockAndSetHead. // This function expects the chain mutex to be held. func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { - if err := bc.writeBlockWithState(block, receipts, logs, state); err != nil { + if err := bc.writeBlockWithState(block, receipts, state); err != nil { return NonStatTy, err } currentBlock := bc.CurrentBlock() @@ -1343,7 +1379,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types } // In theory we should fire a ChainHeadEvent when we inject // a canonical block, but sometimes we can insert a batch of - // canonicial blocks. Avoid firing too many ChainHeadEvents, + // canonical blocks. Avoid firing too many ChainHeadEvents, // we will fire an accumulated ChainHeadEvent and disable fire // event here. if emitHeadEvent { @@ -1580,7 +1616,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) // block in the middle. It can only happen in the clique chain. Whenever // we insert blocks via `insertSideChain`, we only commit `td`, `header` // and `body` if it's non-existent. Since we don't have receipts without - // reexecution, so nothing to commit. But if the sidechain will be adpoted + // reexecution, so nothing to commit. But if the sidechain will be adopted // as the canonical chain eventually, it needs to be reexecuted for missing // state, but if it's this special case here(skip reexecution) we will lose // the empty receipt entry. @@ -1692,7 +1728,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) var status WriteStatus if !setHead { // Don't set the head, only insert the block - err = bc.writeBlockWithState(block, receipts, logs, statedb) + err = bc.writeBlockWithState(block, receipts, statedb) } else { status, err = bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) } @@ -2366,7 +2402,7 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e Chain config: %v Number: %v -Hash: 0x%x +Hash: %#x %v Error: %v @@ -2398,3 +2434,11 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i _, err := bc.hc.InsertHeaderChain(chain, start, bc.forker) return 0, err } + +// SetBlockValidatorAndProcessorForTesting sets the current validator and processor. +// This method can be used to force an invalid blockchain to be verified for tests. +// This method is unsafe and should only be used before block import starts. +func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Processor) { + bc.validator = v + bc.processor = p +} diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index b8d4233c6ecd..96e9f80b6aac 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -55,6 +55,12 @@ func (bc *BlockChain) CurrentFinalizedBlock() *types.Block { return bc.currentFinalizedBlock.Load().(*types.Block) } +// CurrentSafeBlock retrieves the current safe block of the canonical +// chain. The block is retrieved from the blockchain's internal cache. +func (bc *BlockChain) CurrentSafeBlock() *types.Block { + return bc.currentSafeBlock.Load().(*types.Block) +} + // HasHeader checks if a block header is present in the database or not, caching // it if present. func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool { diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 24309405d2b3..feed8a177789 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -564,7 +564,7 @@ func testShortReorgedSnapSyncingRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - newer than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads kept as fast sync data. +// committed block, with everything afterwards kept as fast sync data. func TestLongShallowRepair(t *testing.T) { testLongShallowRepair(t, false) } func TestLongShallowRepairWithSnapshots(t *testing.T) { testLongShallowRepair(t, true) } @@ -609,7 +609,7 @@ func testLongShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - older than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads deleted. +// committed block, with everything afterwards deleted. func TestLongDeepRepair(t *testing.T) { testLongDeepRepair(t, false) } func TestLongDeepRepairWithSnapshots(t *testing.T) { testLongDeepRepair(t, true) } @@ -653,7 +653,7 @@ func testLongDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - newer than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads kept as fast sync data. +// to the committed block, with everything afterwards kept as fast sync data. func TestLongSnapSyncedShallowRepair(t *testing.T) { testLongSnapSyncedShallowRepair(t, false) } @@ -702,7 +702,7 @@ func testLongSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - older than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads deleted. +// to the committed block, with everything afterwards deleted. func TestLongSnapSyncedDeepRepair(t *testing.T) { testLongSnapSyncedDeepRepair(t, false) } func TestLongSnapSyncedDeepRepairWithSnapshots(t *testing.T) { testLongSnapSyncedDeepRepair(t, true) } @@ -843,7 +843,7 @@ func testLongSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongOldForkedShallowRepair(t *testing.T) { testLongOldForkedShallowRepair(t, false) @@ -895,7 +895,7 @@ func testLongOldForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedDeepRepair(t *testing.T) { testLongOldForkedDeepRepair(t, false) } func TestLongOldForkedDeepRepairWithSnapshots(t *testing.T) { testLongOldForkedDeepRepair(t, true) } @@ -942,7 +942,7 @@ func testLongOldForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as +// to be rolled back to the committed block, with everything afterwards kept as // fast sync data; the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedShallowRepair(t *testing.T) { testLongOldForkedSnapSyncedShallowRepair(t, false) @@ -994,7 +994,7 @@ func testLongOldForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedDeepRepair(t *testing.T) { testLongOldForkedSnapSyncedDeepRepair(t, false) @@ -1149,7 +1149,7 @@ func testLongOldForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedShallowRepair(t *testing.T) { testLongNewerForkedShallowRepair(t, false) @@ -1201,7 +1201,7 @@ func testLongNewerForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedDeepRepair(t *testing.T) { testLongNewerForkedDeepRepair(t, false) } func TestLongNewerForkedDeepRepairWithSnapshots(t *testing.T) { testLongNewerForkedDeepRepair(t, true) } @@ -1248,7 +1248,7 @@ func testLongNewerForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as fast +// to be rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedShallowRepair(t *testing.T) { testLongNewerForkedSnapSyncedShallowRepair(t, false) @@ -1300,7 +1300,7 @@ func testLongNewerForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedDeepRepair(t *testing.T) { testLongNewerForkedSnapSyncedDeepRepair(t, false) @@ -1454,7 +1454,7 @@ func testLongNewerForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - newer than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast sync +// rolled back to the committed block, with everything afterwards kept as fast sync // data. The side chain completely nuked by the freezer. func TestLongReorgedShallowRepair(t *testing.T) { testLongReorgedShallowRepair(t, false) } func TestLongReorgedShallowRepairWithSnapshots(t *testing.T) { testLongReorgedShallowRepair(t, true) } @@ -1501,7 +1501,7 @@ func testLongReorgedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - older than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the canonical chains -// to be rolled back to the committed block, with everything afterwads deleted. The +// to be rolled back to the committed block, with everything afterwards deleted. The // side chain completely nuked by the freezer. func TestLongReorgedDeepRepair(t *testing.T) { testLongReorgedDeepRepair(t, false) } func TestLongReorgedDeepRepairWithSnapshots(t *testing.T) { testLongReorgedDeepRepair(t, true) } @@ -1548,7 +1548,7 @@ func testLongReorgedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the chain to be rolled back to the committed block, with everything -// afterwads kept as fast sync data. The side chain completely nuked by the +// afterwards kept as fast sync data. The side chain completely nuked by the // freezer. func TestLongReorgedSnapSyncedShallowRepair(t *testing.T) { testLongReorgedSnapSyncedShallowRepair(t, false) @@ -1600,7 +1600,7 @@ func testLongReorgedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the canonical chains to be rolled back to the committed block, with -// everything afterwads deleted. The side chain completely nuked by the freezer. +// everything afterwards deleted. The side chain completely nuked by the freezer. func TestLongReorgedSnapSyncedDeepRepair(t *testing.T) { testLongReorgedSnapSyncedDeepRepair(t, false) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 9c8da60ba696..e03da3260050 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -759,9 +759,9 @@ func TestFastVsFullChains(t *testing.T) { block.AddTx(tx) } } - // If the block number is a multiple of 5, add a few bonus uncles to the block - if i%5 == 5 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))}) + // If the block number is a multiple of 5, add an uncle to the block + if i%5 == 4 { + block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i))}) } }) // Import the chain as an archive node for the comparison baseline @@ -1239,7 +1239,6 @@ func TestSideLogRebirth(t *testing.T) { chain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) { if i == 1 { gen.OffsetTime(-9) // higher block difficulty - } }) if _, err := blockchain.InsertChain(chain); err != nil { @@ -1368,7 +1367,6 @@ done: t.Errorf("unexpected event fired: %v", e) case <-time.After(250 * time.Millisecond): } - } // Tests if the canonical block can be fetched from the database during chain insertion. @@ -1947,8 +1945,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - signer = types.LatestSigner(gspec.Config) - genesis, _ = gspec.Commit(db) + signer = types.LatestSigner(gspec.Config) + genesis = gspec.MustCommit(db) ) // Generate and import the canonical chain diskdb := rawdb.NewMemoryDatabase() @@ -2766,7 +2764,6 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in b.StopTimer() if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks { b.Fatalf("Transactions were not included, expected %d, got %d", numTxs*numBlocks, got) - } } } @@ -3534,7 +3531,6 @@ func TestEIP2718Transition(t *testing.T) { vm.GasQuickStep*2 + params.WarmStorageReadCostEIP2929 + params.ColdSloadCostEIP2929 if block.GasUsed() != expected { t.Fatalf("incorrect amount of gas spent: expected %d, got %d", expected, block.GasUsed()) - } } diff --git a/core/bloom_indexer.go b/core/bloom_indexer.go index 856746a1c088..68a35d811e41 100644 --- a/core/bloom_indexer.go +++ b/core/bloom_indexer.go @@ -75,7 +75,7 @@ func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error // Commit implements core.ChainIndexerBackend, finalizing the bloom section and // writing it out into the database. func (b *BloomIndexer) Commit() error { - batch := b.db.NewBatch() + batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength) for i := 0; i < types.BloomBitLength; i++ { bits, err := b.gen.Bitset(uint(i)) if err != nil { diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index ca698c47171d..2a0fb167d516 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -138,6 +138,16 @@ func TestCreation(t *testing.T) { {6000000, ID{Hash: checksumToBytes(0xB8C6299D), Next: 0}}, // Future London block }, }, + // Sepolia test cases + { + params.SepoliaChainConfig, + params.SepoliaGenesisHash, + []testcase{ + {0, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin and first London block + {1735370, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Last London block + {1735371, ID{Hash: checksumToBytes(0xb96cbd13), Next: 0}}, // First MergeNetsplit block + }, + }, // Merge test cases { &mergeConfig, diff --git a/core/genesis.go b/core/genesis.go index 199fbedd2c2c..9cc6c3007da0 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -83,10 +83,12 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { return nil } -// flush adds allocated genesis accounts into a fresh new statedb and -// commit the state changes into the given database handler. -func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { - statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) +// deriveHash computes the state root according to the genesis specification. +func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { + // Create an ephemeral in-memory database for computing hash, + // all the derived states will be discarded to not pollute disk. + db := state.NewDatabase(rawdb.NewMemoryDatabase()) + statedb, err := state.New(common.Hash{}, db, nil) if err != nil { return common.Hash{}, err } @@ -98,25 +100,39 @@ func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { statedb.SetState(addr, key, value) } } + return statedb.Commit(false) +} + +// flush is very similar with deriveHash, but the main difference is +// all the generated states will be persisted into the given database. +// Also, the genesis state specification will be flushed as well. +func (ga *GenesisAlloc) flush(db ethdb.Database) error { + statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) + if err != nil { + return err + } + for addr, account := range *ga { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } root, err := statedb.Commit(false) if err != nil { - return common.Hash{}, err + return err } err = statedb.Database().TrieDB().Commit(root, true, nil) if err != nil { - return common.Hash{}, err + return err } - return root, nil -} - -// write writes the json marshaled genesis state into database -// with the given block hash as the unique identifier. -func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { + // Marshal the genesis state specification and persist. blob, err := json.Marshal(ga) if err != nil { return err } - rawdb.WriteGenesisState(db, hash, blob) + rawdb.WriteGenesisStateSpec(db, root, blob) return nil } @@ -124,7 +140,7 @@ func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { // hash and commits them into the given database handler. func CommitGenesisState(db ethdb.Database, hash common.Hash) error { var alloc GenesisAlloc - blob := rawdb.ReadGenesisState(db, hash) + blob := rawdb.ReadGenesisStateSpec(db, hash) if len(blob) != 0 { if err := alloc.UnmarshalJSON(blob); err != nil { return err @@ -154,8 +170,7 @@ func CommitGenesisState(db ethdb.Database, hash common.Hash) error { return errors.New("not found") } } - _, err := alloc.flush(db) - return err + return alloc.flush(db) } // GenesisAccount is an account in the state of the genesis block. @@ -236,10 +251,22 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig return SetupGenesisBlockWithOverride(db, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideGrayGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideTerminalTotalDifficulty *big.Int, overrideTerminalTotalDifficultyPassed *bool) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } + + applyOverrides := func(config *params.ChainConfig) { + if config != nil { + if overrideTerminalTotalDifficulty != nil { + config.TerminalTotalDifficulty = overrideTerminalTotalDifficulty + } + if overrideTerminalTotalDifficultyPassed != nil { + config.TerminalTotalDifficultyPassed = *overrideTerminalTotalDifficultyPassed + } + } + } + // Just commit the new block if there is no stored genesis block. stored := rawdb.ReadCanonicalHash(db, 0) if (stored == common.Hash{}) { @@ -253,6 +280,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if err != nil { return genesis.Config, common.Hash{}, err } + applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // We have the genesis block in database(perhaps in ancient database) @@ -263,7 +291,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override genesis = DefaultGenesisBlock() } // Ensure the stored genesis matches with the given one. - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } @@ -271,23 +299,19 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if err != nil { return genesis.Config, hash, err } + applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // Check whether the genesis block is already written. if genesis != nil { - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } } // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - if overrideGrayGlacier != nil { - newcfg.GrayGlacierBlock = overrideGrayGlacier - } - if overrideTerminalTotalDifficulty != nil { - newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty - } + applyOverrides(newcfg) if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err } @@ -304,12 +328,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override // apply the overrides. if genesis == nil && !(stored == params.MainnetGenesisHash || stored == params.WemixMainnetGenesisHash) { newcfg = storedcfg - if overrideGrayGlacier != nil { - newcfg.GrayGlacierBlock = overrideGrayGlacier - } - if overrideTerminalTotalDifficulty != nil { - newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty - } + applyOverrides(newcfg) } // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. @@ -350,13 +369,9 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { } } -// ToBlock creates the genesis block and writes state of a genesis specification -// to the given database (or discards it if nil). -func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { - if db == nil { - db = rawdb.NewMemoryDatabase() - } - root, err := g.Alloc.flush(db) +// ToBlock returns the genesis block according to genesis specification. +func (g *Genesis) ToBlock() *types.Block { + root, err := g.Alloc.deriveHash() if err != nil { panic(err) } @@ -393,7 +408,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { - block := g.ToBlock(db) + block := g.ToBlock() if block.Number().Sign() != 0 { return nil, errors.New("can't commit genesis block with number > 0") } @@ -407,7 +422,10 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - if err := g.Alloc.write(db, block.Hash()); err != nil { + // All the checks has passed, flush the states derived from the genesis + // specification as well as the specification itself into the provided + // database. + if err := g.Alloc.flush(db); err != nil { return nil, err } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) @@ -431,15 +449,6 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { return block } -// GenesisBlockForTesting creates and writes a block in which addr has the given wei balance. -func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { - g := Genesis{ - Alloc: GenesisAlloc{addr: {Balance: balance}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - return g.MustCommit(db) -} - func loadDefaultGenesisFile() (*Genesis, error) { genesis := new(Genesis) if _, err := os.Stat(params.WemixGenesisFile); err == nil { @@ -558,6 +567,7 @@ func DefaultSepoliaGenesisBlock() *Genesis { } } +// DefaultKilnGenesisBlock returns the kiln network genesis block. func DefaultKilnGenesisBlock() *Genesis { g := new(Genesis) reader := strings.NewReader(KilnAllocData) diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 041c55424238..16df390575c2 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -25,7 +25,6 @@ const mainnetAllocData = "\xfa\x04]X\u0793\r\x83b\x011\x8e\u0189\x9agT\x06\x908' const ropstenAllocData = "\xf9\x03\xa4\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x80\xc2\v\x80\xc2\f\x80\xc2\r\x80\xc2\x0e\x80\xc2\x0f\x80\xc2\x10\x80\xc2\x11\x80\xc2\x12\x80\xc2\x13\x80\xc2\x14\x80\xc2\x15\x80\xc2\x16\x80\xc2\x17\x80\xc2\x18\x80\xc2\x19\x80\xc2\x1a\x80\xc2\x1b\x80\xc2\x1c\x80\xc2\x1d\x80\xc2\x1e\x80\xc2\x1f\x80\xc2 \x80\xc2!\x80\xc2\"\x80\xc2#\x80\xc2$\x80\xc2%\x80\xc2&\x80\xc2'\x80\xc2(\x80\xc2)\x80\xc2*\x80\xc2+\x80\xc2,\x80\xc2-\x80\xc2.\x80\xc2/\x80\xc20\x80\xc21\x80\xc22\x80\xc23\x80\xc24\x80\xc25\x80\xc26\x80\xc27\x80\xc28\x80\xc29\x80\xc2:\x80\xc2;\x80\xc2<\x80\xc2=\x80\xc2>\x80\xc2?\x80\xc2@\x80\xc2A\x80\xc2B\x80\xc2C\x80\xc2D\x80\xc2E\x80\xc2F\x80\xc2G\x80\xc2H\x80\xc2I\x80\xc2J\x80\xc2K\x80\xc2L\x80\xc2M\x80\xc2N\x80\xc2O\x80\xc2P\x80\xc2Q\x80\xc2R\x80\xc2S\x80\xc2T\x80\xc2U\x80\xc2V\x80\xc2W\x80\xc2X\x80\xc2Y\x80\xc2Z\x80\xc2[\x80\xc2\\\x80\xc2]\x80\xc2^\x80\xc2_\x80\xc2`\x80\xc2a\x80\xc2b\x80\xc2c\x80\xc2d\x80\xc2e\x80\xc2f\x80\xc2g\x80\xc2h\x80\xc2i\x80\xc2j\x80\xc2k\x80\xc2l\x80\xc2m\x80\xc2n\x80\xc2o\x80\xc2p\x80\xc2q\x80\xc2r\x80\xc2s\x80\xc2t\x80\xc2u\x80\xc2v\x80\xc2w\x80\xc2x\x80\xc2y\x80\xc2z\x80\xc2{\x80\xc2|\x80\xc2}\x80\xc2~\x80\xc2\u007f\x80\u00c1\x80\x80\u00c1\x81\x80\u00c1\x82\x80\u00c1\x83\x80\u00c1\x84\x80\u00c1\x85\x80\u00c1\x86\x80\u00c1\x87\x80\u00c1\x88\x80\u00c1\x89\x80\u00c1\x8a\x80\u00c1\x8b\x80\u00c1\x8c\x80\u00c1\x8d\x80\u00c1\x8e\x80\u00c1\x8f\x80\u00c1\x90\x80\u00c1\x91\x80\u00c1\x92\x80\u00c1\x93\x80\u00c1\x94\x80\u00c1\x95\x80\u00c1\x96\x80\u00c1\x97\x80\u00c1\x98\x80\u00c1\x99\x80\u00c1\x9a\x80\u00c1\x9b\x80\u00c1\x9c\x80\u00c1\x9d\x80\u00c1\x9e\x80\u00c1\x9f\x80\u00c1\xa0\x80\u00c1\xa1\x80\u00c1\xa2\x80\u00c1\xa3\x80\u00c1\xa4\x80\u00c1\xa5\x80\u00c1\xa6\x80\u00c1\xa7\x80\u00c1\xa8\x80\u00c1\xa9\x80\u00c1\xaa\x80\u00c1\xab\x80\u00c1\xac\x80\u00c1\xad\x80\u00c1\xae\x80\u00c1\xaf\x80\u00c1\xb0\x80\u00c1\xb1\x80\u00c1\xb2\x80\u00c1\xb3\x80\u00c1\xb4\x80\u00c1\xb5\x80\u00c1\xb6\x80\u00c1\xb7\x80\u00c1\xb8\x80\u00c1\xb9\x80\u00c1\xba\x80\u00c1\xbb\x80\u00c1\xbc\x80\u00c1\xbd\x80\u00c1\xbe\x80\u00c1\xbf\x80\u00c1\xc0\x80\u00c1\xc1\x80\u00c1\u0080\u00c1\u00c0\u00c1\u0100\u00c1\u0140\u00c1\u0180\u00c1\u01c0\u00c1\u0200\u00c1\u0240\u00c1\u0280\u00c1\u02c0\u00c1\u0300\u00c1\u0340\u00c1\u0380\u00c1\u03c0\u00c1\u0400\u00c1\u0440\u00c1\u0480\u00c1\u04c0\u00c1\u0500\u00c1\u0540\u00c1\u0580\u00c1\u05c0\u00c1\u0600\u00c1\u0640\u00c1\u0680\u00c1\u06c0\u00c1\u0700\u00c1\u0740\u00c1\u0780\u00c1\u07c0\u00c1\xe0\x80\u00c1\xe1\x80\u00c1\xe2\x80\u00c1\xe3\x80\u00c1\xe4\x80\u00c1\xe5\x80\u00c1\xe6\x80\u00c1\xe7\x80\u00c1\xe8\x80\u00c1\xe9\x80\u00c1\xea\x80\u00c1\xeb\x80\u00c1\xec\x80\u00c1\xed\x80\u00c1\xee\x80\u00c1\xef\x80\u00c1\xf0\x80\u00c1\xf1\x80\u00c1\xf2\x80\u00c1\xf3\x80\u00c1\xf4\x80\u00c1\xf5\x80\u00c1\xf6\x80\u00c1\xf7\x80\u00c1\xf8\x80\u00c1\xf9\x80\u00c1\xfa\x80\u00c1\xfb\x80\u00c1\xfc\x80\u00c1\xfd\x80\u00c1\xfe\x80\u00c1\xff\x80\u3507KT\xa8\xbd\x15)f\xd6?pk\xae\x1f\xfe\xb0A\x19!\xe5\x8d\f\x9f,\x9c\xd0Ft\xed\xea@\x00\x00\x00" const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x941\xb9\x8d\x14\x00{\xde\xe67)\x80\x86\x98\x8a\v\xbd1\x18E#\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" -const calaverasAllocData = "\xf9\x06\x14\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x10A\xaf\xbc\xb3Y\u0568\xdcX\xc1[/\xf5\x13T\xff\x8a!}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94#o\xf1\xe9t\x19\xae\x93\xad\x80\xca\xfb\xaa!\"\f]x\xfb}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xba\xdc\r\xe9\xe0yK\x04\x9b^\xa6<>\x1ei\x8a4v\xc1r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xf00\v\ue24a\xe2r\xeb4~\x83i\xac\fv\xdfB\xc9?\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xfe;U~\x8f\xb6+\x89\xf4\x91kr\x1b\xe5\\\ub08d\xbds\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const sepoliaAllocData = "\xf9\x01\xee\u0791i\x16\xa8{\x823?BE\x04f#\xb27\x94\xc6\\\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\x10\xf5\xd4XT\xe08\a\x14\x85\xac\x9e@#\b\u03c0\xd2\xd2\xfe\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\u0794y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\x88\r\u0db3\xa7d\x00\x00\xe0\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x8b\u007f\tw\xbbO\x0f\xbepv\xfa\"\xbc$\xac\xa0CX?^\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xa2\xa6\xd949\x14O\xfeM'\xc9\xe0\x88\xdc\u0637\x83\x94bc\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xaa\xec\x869DA\xf9\x15\xbc\xe3\xe6\xab9\x99w\xe9\x90o;i\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\u1532\x1c3\xde\x1f\xab?\xa1T\x99\xc6+Y\xfe\f\xc3%\x00 \u044bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xbc\x11)Y6\xaay\u0554\x13\x9d\xe1\xb2\xe1&)AO;\u06ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xbe\xef2\xca[\x9a\x19\x8d'\xb4\xe0/LpC\x9f\xe6\x03V\u03ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe1\x94\xd7\xd7lX\xb3\xa5\x19\xe9\xfal\xc4\xd2-\xc0\x17%\x9b\u011f\x1e\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xd7\xed\xdbx\xed)[<\x96)$\x0e\x89$\xfb\x8d\x88t\xdd\u060a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xe2\xe2e\x90(\x147\x84\xd5W\xbc\xeco\xf3\xa0r\x10H\x88\n\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xf4|\xae\x1c\xf7\x9c\xa6u\x8b\xfcx}\xbd!\u6f7eq\x12\xb8\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00" const KilnAllocData = `{ "config": { diff --git a/core/genesis_test.go b/core/genesis_test.go index e8010e3d4ebd..ba3423e32a08 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -178,7 +178,7 @@ func TestGenesisHashes(t *testing.T) { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock - if have := c.genesis.ToBlock(nil).Hash(); have != c.want { + if have := c.genesis.ToBlock().Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } } @@ -192,11 +192,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock, err := genesis.Commit(db) - if err != nil { - t.Fatal(err) - } - + genesisBlock := genesis.MustCommit(db) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") } @@ -221,12 +217,12 @@ func TestReadWriteGenesisAlloc(t *testing.T) { {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash = common.HexToHash("0xdeadbeef") + hash, _ = alloc.deriveHash() ) - alloc.write(db, hash) + alloc.flush(db) var reload GenesisAlloc - err := reload.UnmarshalJSON(rawdb.ReadGenesisState(db, hash)) + err := reload.UnmarshalJSON(rawdb.ReadGenesisStateSpec(db, hash)) if err != nil { t.Fatalf("Failed to load genesis state %v", err) } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 8ea2e2ca7273..aeba3690d228 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -37,7 +37,7 @@ import ( func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerHashTable, number) + data, _ = reader.Ancient(chainFreezerHashTable, number) if len(data) == 0 { // Get it by hash from leveldb data, _ = db.Get(headerHashKey(number)) @@ -335,7 +335,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu } // read remaining from ancients max := count * 700 - data, err := db.AncientRange(freezerHeaderTable, i+1-count, count, max) + data, err := db.AncientRange(chainFreezerHeaderTable, i+1-count, count, max) if err == nil && uint64(len(data)) == count { // the data is on the order [h, h+1, .., n] -- reordering needed for i := range data { @@ -352,7 +352,7 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu // First try to look up the data in ancient database. Extra hash // comparison is necessary since ancient database only maintains // the canonical data. - data, _ = reader.Ancient(freezerHeaderTable, number) + data, _ = reader.Ancient(chainFreezerHeaderTable, number) if len(data) > 0 && crypto.Keccak256Hash(data) == hash { return nil } @@ -428,7 +428,7 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number // isCanon is an internal utility method, to check whether the given number/hash // is part of the ancient (canon) set. func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool { - h, err := reader.Ancient(freezerHashTable, number) + h, err := reader.Ancient(chainFreezerHashTable, number) if err != nil { return false } @@ -444,7 +444,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) return nil } // If not, try reading from leveldb @@ -459,7 +459,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) if len(data) > 0 { return nil } @@ -527,7 +527,7 @@ func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerDifficultyTable, number) + data, _ = reader.Ancient(chainFreezerDifficultyTable, number) return nil } // If not, try reading from leveldb @@ -587,7 +587,7 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerReceiptTable, number) + data, _ = reader.Ancient(chainFreezerReceiptTable, number) return nil } // If not, try reading from leveldb @@ -819,19 +819,19 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error { num := block.NumberU64() - if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, num, block.Hash().Bytes()); err != nil { return fmt.Errorf("can't add block %d hash: %v", num, err) } - if err := op.Append(freezerHeaderTable, num, header); err != nil { + if err := op.Append(chainFreezerHeaderTable, num, header); err != nil { return fmt.Errorf("can't append block header %d: %v", num, err) } - if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil { + if err := op.Append(chainFreezerBodiesTable, num, block.Body()); err != nil { return fmt.Errorf("can't append block body %d: %v", num, err) } - if err := op.Append(freezerReceiptTable, num, receipts); err != nil { + if err := op.Append(chainFreezerReceiptTable, num, receipts); err != nil { return fmt.Errorf("can't append block %d receipts: %v", num, err) } - if err := op.Append(freezerDifficultyTable, num, td); err != nil { + if err := op.Append(chainFreezerDifficultyTable, num, td); err != nil { return fmt.Errorf("can't append block %d total difficulty: %v", num, err) } return nil diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index dbb13caa416c..21d23e1f0c8b 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -285,7 +285,7 @@ func TestTdStorage(t *testing.T) { func TestCanonicalMappingStorage(t *testing.T) { db := NewMemoryDatabase() - // Create a test canonical number and assinged hash to move around + // Create a test canonical number and assigned hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Non existent canonical mapping returned: %v", entry) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index f5a161adb688..7a9e6442f011 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -81,15 +81,16 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha } } -// ReadGenesisState retrieves the genesis state based on the given genesis hash. -func ReadGenesisState(db ethdb.KeyValueReader, hash common.Hash) []byte { - data, _ := db.Get(genesisKey(hash)) +// ReadGenesisStateSpec retrieves the genesis state specification based on the +// given genesis hash. +func ReadGenesisStateSpec(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(genesisStateSpecKey(hash)) return data } -// WriteGenesisState writes the genesis state into the disk. -func WriteGenesisState(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { - if err := db.Put(genesisKey(hash), data); err != nil { +// WriteGenesisStateSpec writes the genesis state specification into the disk. +func WriteGenesisStateSpec(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { + if err := db.Put(genesisStateSpecKey(hash), data); err != nil { log.Crit("Failed to store genesis state", "err", err) } } diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go new file mode 100644 index 000000000000..3da061cbd977 --- /dev/null +++ b/core/rawdb/ancient_scheme.go @@ -0,0 +1,86 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import "fmt" + +// The list of table names of chain freezer. +const ( + // chainFreezerHeaderTable indicates the name of the freezer header table. + chainFreezerHeaderTable = "headers" + + // chainFreezerHashTable indicates the name of the freezer canonical hash table. + chainFreezerHashTable = "hashes" + + // chainFreezerBodiesTable indicates the name of the freezer block body table. + chainFreezerBodiesTable = "bodies" + + // chainFreezerReceiptTable indicates the name of the freezer receipts table. + chainFreezerReceiptTable = "receipts" + + // chainFreezerDifficultyTable indicates the name of the freezer total difficulty table. + chainFreezerDifficultyTable = "diffs" +) + +// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables. +// Hashes and difficulties don't compress well. +var chainFreezerNoSnappy = map[string]bool{ + chainFreezerHeaderTable: false, + chainFreezerHashTable: true, + chainFreezerBodiesTable: false, + chainFreezerReceiptTable: false, + chainFreezerDifficultyTable: true, +} + +// The list of identifiers of ancient stores. +var ( + chainFreezerName = "chain" // the folder name of chain segment ancient store. +) + +// freezers the collections of all builtin freezers. +var freezers = []string{chainFreezerName} + +// InspectFreezerTable dumps out the index of a specific freezer table. The passed +// ancient indicates the path of root ancient directory where the chain freezer can +// be opened. Start and end specify the range for dumping out indexes. +// Note this function can only be used for debugging purposes. +func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error { + var ( + path string + tables map[string]bool + ) + switch freezerName { + case chainFreezerName: + path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy + default: + return fmt.Errorf("unknown freezer, supported ones: %v", freezers) + } + noSnappy, exist := tables[tableName] + if !exist { + var names []string + for name := range tables { + names = append(names, name) + } + return fmt.Errorf("unknown table, supported ones: %v", names) + } + table, err := newFreezerTable(path, tableName, noSnappy, true) + if err != nil { + return err + } + table.dumpIndexStdout(start, end) + return nil +} diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 4c49db2748b2..7d9c9c015649 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -241,7 +241,7 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { if n := len(ancients); n > 0 { context = append(context, []interface{}{"hash", ancients[n-1]}...) } - log.Info("Deep froze chain segment", context...) + log.Debug("Deep froze chain segment", context...) // Avoid database thrashing with tiny writes if frozen-first < freezerBatchLimit { @@ -278,19 +278,19 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash } // Write to the batch. - if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, number, hash[:]); err != nil { return fmt.Errorf("can't write hash to Freezer: %v", err) } - if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil { + if err := op.AppendRaw(chainFreezerHeaderTable, number, header); err != nil { return fmt.Errorf("can't write header to Freezer: %v", err) } - if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil { + if err := op.AppendRaw(chainFreezerBodiesTable, number, body); err != nil { return fmt.Errorf("can't write body to Freezer: %v", err) } - if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil { + if err := op.AppendRaw(chainFreezerReceiptTable, number, receipts); err != nil { return fmt.Errorf("can't write receipts to Freezer: %v", err) } - if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil { + if err := op.AppendRaw(chainFreezerDifficultyTable, number, td); err != nil { return fmt.Errorf("can't write td to Freezer: %v", err) } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 21e42f42d43a..867fed63ad92 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -50,7 +50,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) { if i+count > frozen { count = frozen - i } - data, err := db.AncientRange(freezerHashTable, i, count, 32*count) + data, err := db.AncientRange(chainFreezerHashTable, i, count, 32*count) if err != nil { log.Crit("Failed to init database from freezer", "err", err) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 43bd5009d7e2..8f5d70dd9f6a 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -39,10 +39,16 @@ import ( // freezerdb is a database wrapper that enabled freezer data retrievals. type freezerdb struct { + ancientRoot string ethdb.KeyValueStore ethdb.AncientStore } +// AncientDatadir returns the path of root ancient directory. +func (frdb *freezerdb) AncientDatadir() (string, error) { + return frdb.ancientRoot, nil +} + // Close implements io.Closer, closing both the fast key-value store as well as // the slow ancient tables. func (frdb *freezerdb) Close() error { @@ -167,12 +173,36 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { return &nofreezedb{KeyValueStore: db} } +// resolveChainFreezerDir is a helper function which resolves the absolute path +// of chain freezer by considering backward compatibility. +func resolveChainFreezerDir(ancient string) string { + // Check if the chain freezer is already present in the specified + // sub folder, if not then two possibilities: + // - chain freezer is not initialized + // - chain freezer exists in legacy location (root ancient folder) + freezer := path.Join(ancient, chainFreezerName) + if !common.FileExist(freezer) { + if !common.FileExist(ancient) { + // The entire ancient store is not initialized, still use the sub + // folder for initialization. + } else { + // Ancient root is already initialized, then we hold the assumption + // that chain freezer is also initialized and located in root folder. + // In this case fallback to legacy location. + freezer = ancient + log.Info("Found legacy ancient chain path", "location", ancient) + } + } + return freezer +} + // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold -// storage. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// storage. The passed ancient indicates the path of root ancient directory +// where the chain freezer can be opened. +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly bool) (ethdb.Database, error) { // Create the idle freezer instance - frdb, err := newChainFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy) + frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, freezerTableSize, chainFreezerNoSnappy) if err != nil { return nil, err } @@ -203,7 +233,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both // the freezer and the key-value store. - frgenesis, err := frdb.Ancient(freezerHashTable, 0) + frgenesis, err := frdb.Ancient(chainFreezerHashTable, 0) if err != nil { return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) } else if !bytes.Equal(kvgenesis, frgenesis) { @@ -234,7 +264,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } - // Block #1 is still in the database, we're allowed to init a new feezer + // Block #1 is still in the database, we're allowed to init a new freezer } // Otherwise, the head header is still the genesis, we're allowed to init a new // freezer. @@ -249,6 +279,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st }() } return &freezerdb{ + ancientRoot: ancient, KeyValueStore: db, AncientStore: frdb, }, nil @@ -278,13 +309,15 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r } // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a -// freezer moving immutable chain segments into cold storage. -func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// freezer moving immutable chain segments into cold storage. The passed ancient +// indicates the path of root ancient directory where the chain freezer can be +// opened. +func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) if err != nil { return nil, err } - frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly) + frdb, err := NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly) if err != nil { kvdb.Close() return nil, err @@ -510,7 +543,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { } // Inspect append-only file store then. ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize} - for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { + for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} { if size, err := db.AncientSize(category); err == nil { *ancientSizes[i] += common.StorageSize(size) total += common.StorageSize(size) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 284d90777455..53bd989a482d 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -68,8 +68,6 @@ type Freezer struct { frozen uint64 // Number of blocks already frozen tail uint64 // Number of the first stored item in the freezer - datadir string // Path of root directory of ancient store - // This lock synchronizes writers and the truncate operation, as well as // the "atomic" (batched) read operations. writeLock sync.RWMutex @@ -111,7 +109,6 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui readonly: readonly, tables: make(map[string]*freezerTable), instanceLock: lock, - datadir: datadir, } // Create the tables. @@ -429,7 +426,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { // Set up new dir for the migrated table, the content of which // we'll at the end move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") - newTable, err := NewFreezerTable(migrationPath, kind, table.noCompression, false) + newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false) if err != nil { return err } @@ -486,11 +483,5 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { if err := os.Remove(migrationPath); err != nil { return err } - return nil } - -// AncientDatadir returns the root directory path of the ancient store. -func (f *Freezer) AncientDatadir() (string, error) { - return f.datadir, nil -} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index dd4a80efcbc5..3fe691cf6d2a 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -46,7 +46,7 @@ var ( errNotSupported = errors.New("this operation is not supported") ) -// indexEntry contains the number/id of the file that the data resides in, aswell as the +// indexEntry contains the number/id of the file that the data resides in, as well as the // offset within the file to the end of the data. // In serialized form, the filenum is stored as uint16. type indexEntry struct { @@ -123,8 +123,8 @@ type freezerTable struct { lock sync.RWMutex // Mutex protecting the data file descriptors } -// NewFreezerTable opens the given path as a freezer table. -func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { +// newFreezerTable opens the given path as a freezer table. +func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly) } @@ -884,9 +884,7 @@ func (t *freezerTable) Sync() error { return t.head.Sync() } -// DumpIndex is a debug print utility function, mainly for testing. It can also -// be used to analyse a live freezer table index. -func (t *freezerTable) DumpIndex(start, stop int64) { +func (t *freezerTable) dumpIndexStdout(start, stop int64) { t.dumpIndex(os.Stdout, start, stop) } diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 8401fd025049..7c9cd70da2bb 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -906,7 +906,7 @@ func TestSequentialRead(t *testing.T) { } // Write 15 bytes 30 times writeChunks(t, f, 30, 15) - f.DumpIndex(0, 30) + f.dumpIndexStdout(0, 30) f.Close() } { // Open it, iterate, verify iteration diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 041c9f044967..d5f751da3a13 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -111,33 +111,6 @@ var ( preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) -const ( - // freezerHeaderTable indicates the name of the freezer header table. - freezerHeaderTable = "headers" - - // freezerHashTable indicates the name of the freezer canonical hash table. - freezerHashTable = "hashes" - - // freezerBodiesTable indicates the name of the freezer block body table. - freezerBodiesTable = "bodies" - - // freezerReceiptTable indicates the name of the freezer receipts table. - freezerReceiptTable = "receipts" - - // freezerDifficultyTable indicates the name of the freezer total difficulty table. - freezerDifficultyTable = "diffs" -) - -// FreezerNoSnappy configures whether compression is disabled for the ancient-tables. -// Hashes and difficulties don't compress well. -var FreezerNoSnappy = map[string]bool{ - freezerHeaderTable: false, - freezerHashTable: true, - freezerBodiesTable: false, - freezerReceiptTable: false, - freezerDifficultyTable: true, -} - // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary // fields. type LegacyTxLookupEntry struct { @@ -247,7 +220,7 @@ func configKey(hash common.Hash) []byte { return append(configPrefix, hash.Bytes()...) } -// genesisKey = genesisPrefix + hash -func genesisKey(hash common.Hash) []byte { +// genesisStateSpecKey = genesisPrefix + hash +func genesisStateSpecKey(hash common.Hash) []byte { return append(genesisPrefix, hash.Bytes()...) } diff --git a/core/state/database.go b/core/state/database.go index ce5d8d731715..96b6bcfe6551 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -63,7 +63,7 @@ type Trie interface { // GetKey returns the sha3 preimage of a hashed key that was previously used // to store a value. // - // TODO(fjl): remove this when SecureTrie is removed + // TODO(fjl): remove this when StateTrie is removed GetKey([]byte) []byte // TryGet returns the value for key stored in the trie. The value bytes must @@ -71,8 +71,8 @@ type Trie interface { // trie.MissingNodeError is returned. TryGet(key []byte) ([]byte, error) - // TryUpdateAccount abstract an account write in the trie. - TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryGetAccount abstract an account read from the trie. + TryGetAccount(key []byte) (*types.StateAccount, error) // TryUpdate associates key with value in the trie. If value has length zero, any // existing value is deleted from the trie. The value bytes must not be modified @@ -80,17 +80,27 @@ type Trie interface { // database, a trie.MissingNodeError is returned. TryUpdate(key, value []byte) error + // TryUpdateAccount abstract an account write to the trie. + TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryDelete removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. TryDelete(key []byte) error + // TryDeleteAccount abstracts an account deletion from the trie. + TryDeleteAccount(key []byte) error + // Hash returns the root hash of the trie. It does not write to the database and // can be used even if the trie doesn't have one. Hash() common.Hash - // Commit writes all nodes to the trie's memory database, tracking the internal - // and external (for account tries) references. - Commit(onleaf trie.LeafCallback) (common.Hash, int, error) + // Commit collects all dirty nodes in the trie and replace them with the + // corresponding node hash. All collected nodes(including dirty leaves if + // collectLeaf is true) will be encapsulated into a nodeset for return. + // The returned nodeset can be nil if the trie is clean(nothing to commit). + // Once the trie is committed, it's not usable anymore. A new trie must + // be created with new root and updated trie database for following usage + Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. @@ -133,7 +143,7 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(common.Hash{}, root, db.db) + tr, err := trie.NewStateTrie(common.Hash{}, root, db.db) if err != nil { return nil, err } @@ -142,7 +152,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(addrHash, root, db.db) + tr, err := trie.NewStateTrie(addrHash, root, db.db) if err != nil { return nil, err } @@ -152,7 +162,7 @@ func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { // CopyTrie returns an independent copy of the given trie. func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { - case *trie.SecureTrie: + case *trie.StateTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) diff --git a/core/state/metrics.go b/core/state/metrics.go index 7b40ff37aff0..35d2df92dda4 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -19,10 +19,10 @@ package state import "github.com/ethereum/go-ethereum/metrics" var ( - accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) - storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) - accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) - storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) - accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil) - storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil) + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil) + storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil) ) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 98f36c74906b..87bc357a5c10 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewSecure(common.Hash{}, genesis.Root(), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(common.Hash{}, genesis.Root(), trie.NewDatabase(db)) if err != nil { return err } @@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) if err != nil { return err } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index e15c1d5049b0..59db920481b0 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -332,7 +332,6 @@ func BenchmarkFlatten(b *testing.B) { value := make([]byte, 32) rand.Read(value) accStorage[randomHash()] = value - } storage[accountKey] = accStorage } @@ -382,7 +381,6 @@ func BenchmarkJournal(b *testing.B) { value := make([]byte, 32) rand.Read(value) accStorage[randomHash()] = value - } storage[accountKey] = accStorage } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 3b3ba32d0928..bf714db4c2d0 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -367,7 +367,10 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, _, _ := snapTrie.Commit(nil) + root, nodes, _ := snapTrie.Commit(false) + if nodes != nil { + snapTrieDb.Update(trie.NewWithNodeSet(nodes)) + } snapTrieDb.Commit(root, false, nil) } // Construct the trie for state iteration, reuse the trie @@ -615,8 +618,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er // If the iterated account is the contract, create a further loop to // verify or regenerate the contract storage. if acc.Root == emptyRoot { - log.Debug("removeStorageAt skip for rocksdb") - // ctx.removeStorageAt(account) + ctx.removeStorageAt(account) } else { var storeMarker []byte if accMarker != nil && bytes.Equal(account[:], accMarker) && len(dl.genMarker) > common.HashLength { diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 21c062f9c65a..1c15e43f2179 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -142,17 +142,19 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { type testHelper struct { diskdb ethdb.Database triedb *trie.Database - accTrie *trie.SecureTrie + accTrie *trie.StateTrie + nodes *trie.MergedNodeSet } func newHelper() *testHelper { diskdb := rawdb.NewMemoryDatabase() triedb := trie.NewDatabase(diskdb) - accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb) + accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, triedb) return &testHelper{ diskdb: diskdb, triedb: triedb, accTrie: accTrie, + nodes: trie.NewMergedNodeSet(), } } @@ -180,21 +182,26 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) } func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { - stTrie, _ := trie.NewSecure(owner, common.Hash{}, t.triedb) + stTrie, _ := trie.NewStateTrie(owner, common.Hash{}, t.triedb) for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } - var root common.Hash if !commit { - root = stTrie.Hash() - } else { - root, _, _ = stTrie.Commit(nil) + return stTrie.Hash().Bytes() + } + root, nodes, _ := stTrie.Commit(false) + if nodes != nil { + t.nodes.Merge(nodes) } return root.Bytes() } func (t *testHelper) Commit() common.Hash { - root, _, _ := t.accTrie.Commit(nil) + root, nodes, _ := t.accTrie.Commit(true) + if nodes != nil { + t.nodes.Merge(nodes) + } + t.triedb.Update(t.nodes) t.triedb.Commit(root, false, nil) return root } @@ -384,7 +391,7 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - root, _, _ := helper.accTrie.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 + root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 // Delete an account trie leaf and ensure the generator chokes helper.triedb.Commit(root, false, nil) @@ -423,18 +430,8 @@ func TestGenerateMissingStorageTrie(t *testing.T) { helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _, _ := helper.accTrie.Commit(nil) - - // We can only corrupt the disk database, so flush the tries out - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - helper.triedb.Commit(root, false, nil) + + root := helper.Commit() // Delete a storage trie root and ensure the generator chokes helper.diskdb.Delete(stRoot) @@ -472,18 +469,7 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _, _ := helper.accTrie.Commit(nil) - - // We can only corrupt the disk database, so flush the tries out - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - helper.triedb.Commit(root, false, nil) + root := helper.Commit() // Delete a storage trie leaf and ensure the generator chokes helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) @@ -839,10 +825,12 @@ func populateDangling(disk ethdb.KeyValueStore) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) @@ -872,10 +860,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) populateDangling(helper.diskdb) diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 48069b8fcf5c..435c28e96f9e 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -319,7 +319,7 @@ func (fi *fastIterator) Slot() []byte { } // Release iterates over all the remaining live layer iterators and releases each -// of thme individually. +// of them individually. func (fi *fastIterator) Release() { for _, it := range fi.iterators { it.it.Release() @@ -327,7 +327,7 @@ func (fi *fastIterator) Release() { fi.iterators = nil } -// Debug is a convencience helper during testing +// Debug is a convenience helper during testing func (fi *fastIterator) Debug() { for _, it := range fi.iterators { fmt.Printf("[p=%v v=%v] ", it.priority, it.it.Hash()[0]) diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index bc4e5cbd0462..7c8077b652ed 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -265,7 +265,7 @@ func TestPostCapBasicDataAccess(t *testing.T) { snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) - // checkExist verifies if an account exiss in a snapshot + // checkExist verifies if an account exists in a snapshot checkExist := func(layer *diffLayer, key string) error { if data, _ := layer.Account(common.HexToHash(key)); data == nil { return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key)) diff --git a/core/state/state_object.go b/core/state/state_object.go index 1ffb7eb40228..a23df895458c 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) var emptyCodeHash = crypto.Keccak256(nil) @@ -49,7 +50,7 @@ func (s Storage) String() (str string) { } func (s Storage) Copy() Storage { - cpy := make(Storage) + cpy := make(Storage, len(s)) for key, value := range s { cpy[key] = value } @@ -375,23 +376,23 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. -func (s *stateObject) CommitTrie(db Database) (int, error) { +func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) { // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { - return 0, nil + return nil, nil } if s.dbErr != nil { - return 0, s.dbErr + return nil, s.dbErr } // Track the amount of time wasted on committing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, committed, err := s.trie.Commit(nil) + root, nodes, err := s.trie.Commit(false) if err == nil { s.data.Root = root } - return committed, err + return nodes, err } // AddBalance adds amount to s's balance. diff --git a/core/state/state_test.go b/core/state/state_test.go index 0a55d7781fd1..b6b46e446fba 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) type stateTest struct { @@ -40,7 +41,7 @@ func newStateTest() *stateTest { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, nil), nil) + sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) s := &stateTest{db: db, state: sdb} // generate a few entries diff --git a/core/state/statedb.go b/core/state/statedb.go index 9c61a8594d41..aa957d48d74a 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -484,7 +484,7 @@ func (s *StateDB) deleteStateObject(obj *stateObject) { } // Delete the account from the trie addr := obj.Address() - if err := s.trie.TryDelete(addr[:]); err != nil { + if err := s.trie.TryDeleteAccount(addr[:]); err != nil { s.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err)) } } @@ -537,20 +537,16 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { // If snapshot unavailable or reading from it failed, load from the database if data == nil { start := time.Now() - enc, err := s.trie.TryGet(addr.Bytes()) + var err error + data, err = s.trie.TryGetAccount(addr.Bytes()) if metrics.EnabledExpensive { s.AccountReads += time.Since(start) } if err != nil { - s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %v", addr.Bytes(), err)) + s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) return nil } - if len(enc) == 0 { - return nil - } - data = new(types.StateAccount) - if err := rlp.DecodeBytes(enc, data); err != nil { - log.Error("Failed to decode state object", "addr", addr, "err", err) + if data == nil { return nil } } @@ -774,7 +770,7 @@ func (s *StateDB) GetRefund() uint64 { return s.refund } -// Finalise finalises the state by removing the s destructed objects and clears +// Finalise finalises the state by removing the destructed objects and clears // the journal as well as the refunds. Finalise, however, will not push any updates // into the tries just yet. Only IntermediateRoot or Commit will do that. func (s *StateDB) Finalise(deleteEmptyObjects bool) { @@ -796,7 +792,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // If state snapshotting is active, also mark the destruction there. // Note, we can't do this only at the end of a block because multiple // transactions within the same block might self destruct and then - // ressurrect an account; but the snapshotter needs both events. + // resurrect an account; but the snapshotter needs both events. if s.snap != nil { s.snapDestructs[obj.addrHash] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely) delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a ressurrect) @@ -844,7 +840,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Although naively it makes sense to retrieve the account trie and then do // the contract storage and account updates sequentially, that short circuits // the account prefetcher. Instead, let's process all the storage updates - // first, giving the account prefeches just a few more milliseconds of time + // first, giving the account prefetches just a few more milliseconds of time // to pull useful data from disk. for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; !obj.deleted { @@ -895,7 +891,7 @@ func (s *StateDB) clearJournalAndRefund() { s.journal = newJournal() s.refund = 0 } - s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entires + s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries } // Commit writes the state to the underlying in-memory trie database. @@ -907,7 +903,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time - var storageCommitted int + var ( + accountTrieNodes int + storageTrieNodes int + nodes = trie.NewMergedNodeSet() + ) codeWriter := s.db.TrieDB().DiskDB().NewBatch() for addr := range s.stateObjectsDirty { if obj := s.stateObjects[addr]; !obj.deleted { @@ -917,11 +917,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - committed, err := obj.CommitTrie(s.db) + set, err := obj.CommitTrie(s.db) if err != nil { return common.Hash{}, err } - storageCommitted += committed + // Merge the dirty nodes of storage trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + storageTrieNodes += set.Len() + } } } if len(s.stateObjectsDirty) > 0 { @@ -932,26 +938,22 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { log.Crit("Failed to commit dirty codes", "error", err) } } - // Write the account trie changes, measuing the amount of wasted time + // Write the account trie changes, measuring the amount of wasted time var start time.Time if metrics.EnabledExpensive { start = time.Now() } - // The onleaf func is called _serially_, so we can reuse the same account - // for unmarshalling every time. - var account types.StateAccount - root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { - return nil - } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) - } - return nil - }) + root, set, err := s.trie.Commit(true) if err != nil { return common.Hash{}, err } + // Merge the dirty nodes of account trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + accountTrieNodes = set.Len() + } if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) @@ -959,8 +961,8 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { storageUpdatedMeter.Mark(int64(s.StorageUpdated)) accountDeletedMeter.Mark(int64(s.AccountDeleted)) storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountCommittedMeter.Mark(int64(accountCommitted)) - storageCommittedMeter.Mark(int64(storageCommitted)) + accountTrieCommittedMeter.Mark(int64(accountTrieNodes)) + storageTriesCommittedMeter.Mark(int64(storageTrieNodes)) s.AccountUpdated, s.AccountDeleted = 0, 0 s.StorageUpdated, s.StorageDeleted = 0, 0 } @@ -984,6 +986,9 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil } + if err := s.db.TrieDB().Update(nodes); err != nil { + return common.Hash{}, err + } s.originalRoot = root return root, err } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 822139cbeeaa..053cbfea331d 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -703,7 +703,6 @@ func TestDeleteCreateRevert(t *testing.T) { // the Commit operation fails with an error // If we are missing trie nodes, we should not continue writing to the trie func TestMissingTrieNodes(t *testing.T) { - // Create an initial state with a few accounts memDb := rawdb.NewMemoryDatabase() db := NewDatabase(memDb) @@ -776,7 +775,7 @@ func TestStateDBAccessList(t *testing.T) { t.Fatalf("expected %x to be in access list", address) } } - // Check that only the expected addresses are present in the acesslist + // Check that only the expected addresses are present in the access list for address := range state.accessList.addresses { if _, exist := addressMap[address]; !exist { t.Fatalf("extra address %x in access list", address) diff --git a/core/state/sync.go b/core/state/sync.go index cc7d01a2188d..00a4c67aa3cb 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -27,20 +27,20 @@ import ( ) // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(paths [][]byte, leaf []byte) error) *trie.Sync { +func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error) *trie.Sync { // Register the storage slot callback if the external callback is specified. - var onSlot func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error + var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error if onLeaf != nil { - onSlot = func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { - return onLeaf(paths, leaf) + onSlot = func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { + return onLeaf(keys, leaf) } } // Register the account callback to connect the state trie and the storage // trie belongs to the contract. var syncer *trie.Sync - onAccount := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { + onAccount := func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { if onLeaf != nil { - if err := onLeaf(paths, leaf); err != nil { + if err := onLeaf(keys, leaf); err != nil { return err } } @@ -48,8 +48,8 @@ func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(p if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } - syncer.AddSubTrie(obj.Root, hexpath, parent, onSlot) - syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), hexpath, parent) + syncer.AddSubTrie(obj.Root, path, parent, parentPath, onSlot) + syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent, parentPath) return nil } syncer = trie.NewSync(root, database, onAccount) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 83c5aa2df7a8..3d9fe556d2ad 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -134,8 +134,8 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), nil) - if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { - t.Errorf(" content requested for empty state: %v, %v, %v", nodes, paths, codes) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes) } } @@ -160,6 +160,14 @@ func TestIterativeStateSyncBatchedByPath(t *testing.T) { testIterativeStateSync(t, 100, false, true) } +// stateElement represents the element in the state trie(bytecode or trie node). +type stateElement struct { + path string + hash common.Hash + code common.Hash + syncPath trie.SyncPath +} + func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { // Create a random state to copy srcDb, srcRoot, srcAccounts := makeTestState() @@ -172,54 +180,73 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - nodes, paths, codes := sched.Missing(count) var ( - hashQueue []common.Hash - pathQueue []trie.SyncPath + nodeElements []stateElement + codeElements []stateElement ) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, codes := sched.Missing(count) + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) } - for len(hashQueue)+len(pathQueue) > 0 { - results := make([]trie.SyncResult, len(hashQueue)+len(pathQueue)) - for i, hash := range hashQueue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) - } + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } + for len(nodeElements)+len(codeElements) > 0 { + var ( + nodeResults = make([]trie.NodeSyncResult, len(nodeElements)) + codeResults = make([]trie.CodeSyncResult, len(codeElements)) + ) + for i, element := range codeElements { + data, err := srcDb.ContractCode(common.Hash{}, element.code) if err != nil { - t.Fatalf("failed to retrieve node data for hash %x", hash) + t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } - results[i] = trie.SyncResult{Hash: hash, Data: data} + codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} } - for i, path := range pathQueue { - if len(path) == 1 { - data, _, err := srcTrie.TryGetNode(path[0]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) + for i, node := range nodeElements { + if bypath { + if len(node.syncPath) == 1 { + data, _, err := srcTrie.TryGetNode(node.syncPath[0]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[0], err) + } + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} + } else { + var acc types.StateAccount + if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil { + t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) + } + stTrie, err := trie.New(common.BytesToHash(node.syncPath[0]), acc.Root, srcDb.TrieDB()) + if err != nil { + t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) + } + data, _, err := stTrie.TryGetNode(node.syncPath[1]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[1], err) + } + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } - results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data} } else { - var acc types.StateAccount - if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil { - t.Fatalf("failed to decode account on path %x: %v", path, err) - } - stTrie, err := trie.New(common.BytesToHash(path[0]), acc.Root, srcDb.TrieDB()) + data, err := srcDb.TrieDB().Node(node.hash) if err != nil { - t.Fatalf("failed to retriev storage trie for path %x: %v", path, err) + t.Fatalf("failed to retrieve node data for key %v", []byte(node.path)) } - data, _, err := stTrie.TryGetNode(path[1]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) - } - results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data} + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } } - for _, result := range results { - if err := sched.Process(result); err != nil { + for _, result := range codeResults { + if err := sched.ProcessCode(result); err != nil { + t.Errorf("failed to process result %v", err) + } + } + for _, result := range nodeResults { + if err := sched.ProcessNode(result); err != nil { t.Errorf("failed to process result %v", err) } } @@ -229,12 +256,20 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { } batch.Write() - nodes, paths, codes = sched.Missing(count) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, codes = sched.Missing(count) + nodeElements = nodeElements[:0] + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + codeElements = codeElements[:0] + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) } } // Cross check that the two states are in sync @@ -251,26 +286,58 @@ func TestIterativeDelayedStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - nodes, _, codes := sched.Missing(0) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + var ( + nodeElements []stateElement + codeElements []stateElement + ) + paths, nodes, codes := sched.Missing(0) + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } + for len(nodeElements)+len(codeElements) > 0 { // Sync only half of the scheduled nodes - results := make([]trie.SyncResult, len(queue)/2+1) - for i, hash := range queue[:len(results)] { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + var nodeProcessed int + var codeProcessed int + if len(codeElements) > 0 { + codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) + for i, element := range codeElements[:len(codeResults)] { + data, err := srcDb.ContractCode(common.Hash{}, element.code) + if err != nil { + t.Fatalf("failed to retrieve contract bytecode for %x", element.code) + } + codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range codeResults { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results[i] = trie.SyncResult{Hash: hash, Data: data} + codeProcessed = len(codeResults) } - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + if len(nodeElements) > 0 { + nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1) + for i, element := range nodeElements[:len(nodeResults)] { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve contract bytecode for %x", element.code) + } + nodeResults[i] = trie.NodeSyncResult{Path: element.path, Data: data} + } + for _, result := range nodeResults { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } + nodeProcessed = len(nodeResults) } batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { @@ -278,8 +345,21 @@ func TestIterativeDelayedStateSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(0) - queue = append(append(queue[len(results):], nodes...), codes...) + paths, nodes, codes = sched.Missing(0) + nodeElements = nodeElements[nodeProcessed:] + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + codeElements = codeElements[codeProcessed:] + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } } // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) @@ -299,40 +379,70 @@ func testIterativeRandomStateSync(t *testing.T, count int) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(count) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } } - for len(queue) > 0 { + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } + for len(nodeQueue)+len(codeQueue) > 0 { // Fetch all the queued nodes in a random order - results := make([]trie.SyncResult, 0, len(queue)) - for hash := range queue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)) + for hash := range codeQueue { + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results = append(results, trie.SyncResult{Hash: hash, Data: data}) } - // Feed the retrieved results back and queue new tasks - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) + for path, element := range nodeQueue { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path) + } + results = append(results, trie.NodeSyncResult{Path: path, Data: data}) + } + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } + // Feed the retrieved results back and queue new tasks batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { t.Fatalf("failed to commit data: %v", err) } batch.Write() - queue = make(map[common.Hash]struct{}) - nodes, _, codes = sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue = make(map[string]stateElement) + codeQueue = make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(count) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} } } // Cross check that the two states are in sync @@ -349,34 +459,62 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(0) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(0) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} } - for len(queue) > 0 { + for len(nodeQueue)+len(codeQueue) > 0 { // Sync only half of the scheduled nodes, even those in random order - results := make([]trie.SyncResult, 0, len(queue)/2+1) - for hash := range queue { - delete(queue, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)/2+1) + for hash := range codeQueue { + delete(codeQueue, hash) - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) + + if len(results) >= cap(results) { + break + } } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results = append(results, trie.SyncResult{Hash: hash, Data: data}) + } + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)/2+1) + for path, element := range nodeQueue { + delete(nodeQueue, path) - if len(results) >= cap(results) { - break + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", element.hash) + } + results = append(results, trie.NodeSyncResult{Path: path, Data: data}) + + if len(results) >= cap(results) { + break + } } - } - // Feed the retrieved results back and queue new tasks - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + // Feed the retrieved results back and queue new tasks + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } batch := dstDb.NewBatch() @@ -384,12 +522,17 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() - for _, result := range results { - delete(queue, result.Hash) + + paths, nodes, codes := sched.Missing(0) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } } - nodes, _, codes = sched.Missing(0) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + for _, hash := range codes { + codeQueue[hash] = struct{}{} } } // Cross check that the two states are in sync @@ -416,28 +559,62 @@ func TestIncompleteStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - var added []common.Hash - - nodes, _, codes := sched.Missing(1) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + var ( + addedCodes []common.Hash + addedNodes []common.Hash + ) + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(1) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } + for len(nodeQueue)+len(codeQueue) > 0 { // Fetch a batch of state nodes - results := make([]trie.SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)) + for hash := range codeQueue { + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) + addedCodes = append(addedCodes, hash) } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + // Process each of the state nodes + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results[i] = trie.SyncResult{Hash: hash, Data: data} } - // Process each of the state nodes - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + var nodehashes []common.Hash + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) + for key, element := range nodeQueue { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", element.hash) + } + results = append(results, trie.NodeSyncResult{Path: key, Data: data}) + + if element.hash != srcRoot { + addedNodes = append(addedNodes, element.hash) + } + nodehashes = append(nodehashes, element.hash) + } + // Process each of the state nodes + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } batch := dstDb.NewBatch() @@ -445,43 +622,44 @@ func TestIncompleteStateSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() - for _, result := range results { - added = append(added, result.Hash) - // Check that all known sub-tries added so far are complete or missing entirely. - if _, ok := isCode[result.Hash]; ok { - continue - } + + for _, root := range nodehashes { // Can't use checkStateConsistency here because subtrie keys may have odd // length and crash in LeafKey. - if err := checkTrieConsistency(dstDb, result.Hash); err != nil { + if err := checkTrieConsistency(dstDb, root); err != nil { t.Fatalf("state inconsistent: %v", err) } } // Fetch the next batch to retrieve - nodes, _, codes = sched.Missing(1) - queue = append(append(queue[:0], nodes...), codes...) + nodeQueue = make(map[string]stateElement) + codeQueue = make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(1) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } } // Sanity check that removing any node from the database is detected - for _, node := range added[1:] { - var ( - key = node.Bytes() - _, code = isCode[node] - val []byte - ) - if code { - val = rawdb.ReadCode(dstDb, node) - rawdb.DeleteCode(dstDb, node) - } else { - val = rawdb.ReadTrieNode(dstDb, node) - rawdb.DeleteTrieNode(dstDb, node) + for _, node := range addedCodes { + val := rawdb.ReadCode(dstDb, node) + rawdb.DeleteCode(dstDb, node) + if err := checkStateConsistency(dstDb, srcRoot); err == nil { + t.Errorf("trie inconsistency not caught, missing: %x", node) } - if err := checkStateConsistency(dstDb, added[0]); err == nil { - t.Fatalf("trie inconsistency not caught, missing: %x", key) - } - if code { - rawdb.WriteCode(dstDb, node, val) - } else { - rawdb.WriteTrieNode(dstDb, node, val) + rawdb.WriteCode(dstDb, node, val) + } + for _, node := range addedNodes { + val := rawdb.ReadTrieNode(dstDb, node) + rawdb.DeleteTrieNode(dstDb, node) + if err := checkStateConsistency(dstDb, srcRoot); err == nil { + t.Errorf("trie inconsistency not caught, missing: %v", node.Hex()) } + rawdb.WriteTrieNode(dstDb, node, val) } } diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 4c817b1bc6fb..83e8966d4c9f 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -212,7 +212,7 @@ type subfetcher struct { wake chan struct{} // Wake channel if a new task is scheduled stop chan struct{} // Channel to interrupt processing - term chan struct{} // Channel to signal iterruption + term chan struct{} // Channel to signal interruption copy chan chan Trie // Channel to request a copy of the current trie seen map[string]struct{} // Tracks the entries already loaded @@ -331,7 +331,11 @@ func (sf *subfetcher) loop() { if _, ok := sf.seen[string(task)]; ok { sf.dups++ } else { - sf.trie.TryGet(task) + if len(task) == len(common.Address{}) { + sf.trie.TryGetAccount(task) + } else { + sf.trie.TryGet(task) + } sf.seen[string(task)] = struct{}{} } } diff --git a/core/state_processor.go b/core/state_processor.go index e2b2c1fc5efc..f8ce38ea340d 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -80,7 +80,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, nil, 0, big.NewInt(0), fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.Prepare(tx.Hash(), i) - receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, fees, vmenv) + receipt, err := applyTransaction(msg, p.config, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, fees, vmenv) if err != nil { return nil, nil, 0, big.NewInt(0), fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -93,7 +93,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return receipts, allLogs, *usedGas, fees, nil } -func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, fees *big.Int, evm *vm.EVM) (*types.Receipt, error) { +func applyTransaction(msg types.Message, config *params.ChainConfig, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, fees *big.Int, evm *vm.EVM) (*types.Receipt, error) { // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) evm.Reset(txContext, statedb) @@ -151,5 +151,5 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo // Create a new context to be used in the EVM environment blockContext := NewEVMBlockContext(header, bc, author) vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, config, cfg) - return applyTransaction(msg, config, bc, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, fees, vmenv) + return applyTransaction(msg, config, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, fees, vmenv) } diff --git a/core/state_transition.go b/core/state_transition.go index 8dc521b9c141..2d1c681ade26 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -383,7 +383,15 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if rules.IsLondon { effectiveTip = cmath.BigMin(st.gasTipCap, new(big.Int).Sub(st.gasFeeCap, st.evm.Context.BaseFee)) } - bigFee := new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), effectiveTip) + + bigFee := big.NewInt(0) + if st.evm.Config.NoBaseFee && st.gasFeeCap.Sign() == 0 && st.gasTipCap.Sign() == 0 { + // Skip fee payment when NoBaseFee is set and the fee fields + // are 0. This avoids a negative effectiveTip being applied to + // the coinbase when simulating calls. + } else { + bigFee = new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), effectiveTip) + } // In wemix, block reward and fees are combined and distributed as // agreed in the governance contract diff --git a/core/tx_journal.go b/core/tx_journal.go index 5453ee191658..62344f564676 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -19,6 +19,7 @@ package core import ( "errors" "io" + "io/fs" "os" "github.com/ethereum/go-ethereum/common" @@ -57,12 +58,12 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. func (journal *txJournal) load(add func([]*types.Transaction) []error) error { - // Skip the parsing if the journal file doesn't exist at all - if !common.FileExist(journal.path) { - return nil - } // Open the journal for loading any past transactions input, err := os.Open(journal.path) + if errors.Is(err, fs.ErrNotExist) { + // Skip the parsing if the journal file doesn't exist at all + return nil + } if err != nil { return err } diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 56ea45473772..4b122347f04e 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -676,7 +676,6 @@ func TestTransactionPostponing(t *testing.T) { // Add a batch consecutive pending transactions for validation txs := []*types.Transaction{} for i, key := range keys { - for j := 0; j < 100; j++ { var tx *types.Transaction if (i+j)%2 == 0 { diff --git a/core/types/block.go b/core/types/block.go index 18b5bff1cef6..724cfcd75d0d 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -394,7 +394,7 @@ func (b *Block) Header() *Header { return CopyHeader(b.header) } func (b *Block) Body() *Body { return &Body{b.transactions, b.uncles} } // Size returns the true RLP encoded storage size of the block, either by encoding -// and returning it, or returning a previsouly cached value. +// and returning it, or returning a previously cached value. func (b *Block) Size() common.StorageSize { if size := b.size.Load(); size != nil { return size.(common.StorageSize) diff --git a/core/types/block_test.go b/core/types/block_test.go index aa1db2f4faad..9e7f581b1dc4 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -314,7 +314,7 @@ func TestRlpDecodeParentHash(t *testing.T) { } // Also test a very very large header. { - // The rlp-encoding of the heder belowCauses _total_ length of 65540, + // The rlp-encoding of the header belowCauses _total_ length of 65540, // which is the first to blow the fast-path. h := &Header{ ParentHash: want, diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 1793c2adc73c..a560a20724fd 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -154,7 +154,7 @@ func bloomValues(data []byte, hashbuf []byte) (uint, byte, uint, byte, uint, byt return i1, v1, i2, v2, i3, v3 } -// BloomLookup is a convenience-method to check presence int he bloom filter +// BloomLookup is a convenience-method to check presence in the bloom filter func BloomLookup(bin Bloom, topic bytesBacked) bool { return bin.Test(topic.Bytes()) } diff --git a/core/types/bloom9_test.go b/core/types/bloom9_test.go index 893df486dd1b..d3178d112efb 100644 --- a/core/types/bloom9_test.go +++ b/core/types/bloom9_test.go @@ -92,7 +92,6 @@ func BenchmarkBloom9Lookup(b *testing.B) { } func BenchmarkCreateBloom(b *testing.B) { - var txs = Transactions{ NewContractCreation(1, big.NewInt(1), 1, big.NewInt(1), nil), NewTransaction(2, common.HexToAddress("0x2"), big.NewInt(2), 2, big.NewInt(2), nil), diff --git a/core/types/hashing.go b/core/types/hashing.go index a115a8842ec3..3df75432a4b4 100644 --- a/core/types/hashing.go +++ b/core/types/hashing.go @@ -31,7 +31,7 @@ var hasherPool = sync.Pool{ New: func() interface{} { return sha3.NewLegacyKeccak256() }, } -// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. +// encodeBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. var encodeBufferPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, } diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index 44726c9cbb9f..294a3977d03b 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -197,7 +197,7 @@ func printList(l types.DerivableList) { for i := 0; i < l.Len(); i++ { var buf bytes.Buffer l.EncodeIndex(i, &buf) - fmt.Printf("\"0x%x\",\n", buf.Bytes()) + fmt.Printf("\"%#x\",\n", buf.Bytes()) } fmt.Printf("},\n") } diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 689fc38a9b66..1c775f129d65 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -111,7 +111,6 @@ func TestEIP155SigningVitalik(t *testing.T) { if from != addr { t.Errorf("%d: expected %x got %x", i, addr, from) } - } } diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 2e418b230986..67e5b3cce3f5 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -114,7 +114,6 @@ func TestEIP2718TransactionSigHash(t *testing.T) { // This test checks signature operations on access list transactions. func TestEIP2930Signer(t *testing.T) { - var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") keyAddr = crypto.PubkeyToAddress(key.PublicKey) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index fb0fcc1da49d..602cde51015e 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -46,7 +46,6 @@ var commonParams []*twoOperandParams var twoOpMethods map[string]executionFunc func init() { - // Params is a list of common edgecases that should be used for some common tests params := []string{ "0000000000000000000000000000000000000000000000000000000000000000", // 0 @@ -92,7 +91,6 @@ func init() { } func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) { - var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) stack = newstack() @@ -641,7 +639,6 @@ func TestCreate2Addreses(t *testing.T) { expected: "0xE33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0", }, } { - origin := common.BytesToAddress(common.FromHex(tt.origin)) salt := common.BytesToHash(common.FromHex(tt.salt)) code := common.FromHex(tt.code) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 4f1ebc43a229..40fe23dc516c 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -114,7 +114,6 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { - // Increment the call depth which is restricted to 1024 in.evm.depth++ defer func() { in.evm.depth-- }() diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index dfae0f2e2af7..31ee9922dbac 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -73,5 +73,4 @@ func TestLoopInterrupt(t *testing.T) { } } } - } diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index eef3b53d8c66..707b52e79daf 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -63,7 +63,7 @@ type JumpTable [256]*operation func validate(jt JumpTable) JumpTable { for i, op := range jt { if op == nil { - panic(fmt.Sprintf("op 0x%x is not set", i)) + panic(fmt.Sprintf("op %#x is not set", i)) } // The interpreter has an assumption that if the memorySize function is // set, then the dynamicGas function is also set. This is a somewhat @@ -198,7 +198,6 @@ func newSpuriousDragonInstructionSet() JumpTable { instructionSet := newTangerineWhistleInstructionSet() instructionSet[EXP].dynamicGas = gasExpEIP158 return validate(instructionSet) - } // EIP 150 a.k.a Tangerine Whistle diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 19252b01f256..77d619abb9c1 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -392,7 +392,7 @@ var opCodeToString = map[OpCode]string{ func (op OpCode) String() string { str := opCodeToString[op] if len(str) == 0 { - return fmt.Sprintf("opcode 0x%x not defined", int(op)) + return fmt.Sprintf("opcode %#x not defined", int(op)) } return str diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index ca4e64843695..ab77e284df35 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -333,7 +333,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) cfg.GasLimit = gas if len(tracerCode) > 0 { - tracer, err := tracers.New(tracerCode, new(tracers.Context)) + tracer, err := tracers.New(tracerCode, new(tracers.Context), nil) if err != nil { b.Fatal(err) } @@ -379,7 +379,6 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode // BenchmarkSimpleLoop test a pretty simple loop which loops until OOG // 55 ms func BenchmarkSimpleLoop(b *testing.B) { - staticCallIdentity := []byte{ byte(vm.JUMPDEST), // [ count ] // push args for the call @@ -458,7 +457,7 @@ func BenchmarkSimpleLoop(b *testing.B) { byte(vm.JUMP), } - calllRevertingContractWithInput := []byte{ + callRevertingContractWithInput := []byte{ byte(vm.JUMPDEST), // // push args for the call byte(vm.PUSH1), 0, // out size @@ -486,7 +485,7 @@ func BenchmarkSimpleLoop(b *testing.B) { benchmarkNonModifyingCode(100000000, loopingCode, "loop-100M", "", b) benchmarkNonModifyingCode(100000000, callInexistant, "call-nonexist-100M", "", b) benchmarkNonModifyingCode(100000000, callEOA, "call-EOA-100M", "", b) - benchmarkNonModifyingCode(100000000, calllRevertingContractWithInput, "call-reverting-100M", "", b) + benchmarkNonModifyingCode(100000000, callRevertingContractWithInput, "call-reverting-100M", "", b) //benchmarkNonModifyingCode(10000000, staticCallIdentity, "staticcall-identity-10M", b) //benchmarkNonModifyingCode(10000000, loopingCode, "loop-10M", b) @@ -498,12 +497,11 @@ func TestEip2929Cases(t *testing.T) { t.Skip("Test only useful for generating documentation") id := 1 prettyPrint := func(comment string, code []byte) { - instrs := make([]string, 0) it := asm.NewInstructionIterator(code) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%v 0x%x", it.Op(), it.Arg())) + instrs = append(instrs, fmt.Sprintf("%v %#x", it.Op(), it.Arg())) } else { instrs = append(instrs, fmt.Sprintf("%v", it.Op())) } @@ -511,7 +509,7 @@ func TestEip2929Cases(t *testing.T) { ops := strings.Join(instrs, ", ") fmt.Printf("### Case %d\n\n", id) id++ - fmt.Printf("%v\n\nBytecode: \n```\n0x%x\n```\nOperations: \n```\n%v\n```\n\n", + fmt.Printf("%v\n\nBytecode: \n```\n%#x\n```\nOperations: \n```\n%v\n```\n\n", comment, code, ops) Execute(code, nil, &Config{ @@ -834,7 +832,7 @@ func TestRuntimeJSTracer(t *testing.T) { statedb.SetCode(common.HexToAddress("0xee"), calleeCode) statedb.SetCode(common.HexToAddress("0xff"), depressedCode) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -870,7 +868,7 @@ func TestJSTracerCreateTx(t *testing.T) { code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } diff --git a/crypto/bls12381/field_element_test.go b/crypto/bls12381/field_element_test.go index 0f6abd280cbb..70bbe5cfe5e7 100644 --- a/crypto/bls12381/field_element_test.go +++ b/crypto/bls12381/field_element_test.go @@ -102,7 +102,6 @@ func TestFieldElementEquality(t *testing.T) { if a12.equal(b12) { t.Fatal("a != a + 1") } - } func TestFieldElementHelpers(t *testing.T) { diff --git a/crypto/bls12381/fp12.go b/crypto/bls12381/fp12.go index 3141c76c3995..51e949fe5f04 100644 --- a/crypto/bls12381/fp12.go +++ b/crypto/bls12381/fp12.go @@ -96,7 +96,6 @@ func (e *fp12) add(c, a, b *fe12) { fp6 := e.fp6 fp6.add(&c[0], &a[0], &b[0]) fp6.add(&c[1], &a[1], &b[1]) - } func (e *fp12) double(c, a *fe12) { @@ -109,7 +108,6 @@ func (e *fp12) sub(c, a, b *fe12) { fp6 := e.fp6 fp6.sub(&c[0], &a[0], &b[0]) fp6.sub(&c[1], &a[1], &b[1]) - } func (e *fp12) neg(c, a *fe12) { diff --git a/crypto/bls12381/fp_test.go b/crypto/bls12381/fp_test.go index 97528d9db32e..0bad35de1630 100644 --- a/crypto/bls12381/fp_test.go +++ b/crypto/bls12381/fp_test.go @@ -465,7 +465,6 @@ func TestFpNonResidue(t *testing.T) { i -= 1 } } - } func TestFp2Serialization(t *testing.T) { diff --git a/crypto/bls12381/g2.go b/crypto/bls12381/g2.go index fa110e3edfc5..c2ca959bcca1 100644 --- a/crypto/bls12381/g2.go +++ b/crypto/bls12381/g2.go @@ -41,7 +41,6 @@ func (p *PointG2) Zero() *PointG2 { p[1].one() p[2].zero() return p - } type tempG2 struct { diff --git a/crypto/bls12381/isogeny.go b/crypto/bls12381/isogeny.go index c3cb0a6f7bf0..a63f585dd00a 100644 --- a/crypto/bls12381/isogeny.go +++ b/crypto/bls12381/isogeny.go @@ -19,7 +19,7 @@ package bls12381 // isogenyMapG1 applies 11-isogeny map for BLS12-381 G1 defined at draft-irtf-cfrg-hash-to-curve-06. func isogenyMapG1(x, y *fe) { // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06#appendix-C.2 - params := isogenyConstansG1 + params := isogenyConstantsG1 degree := 15 xNum, xDen, yNum, yDen := new(fe), new(fe), new(fe), new(fe) xNum.set(params[0][degree]) @@ -76,7 +76,7 @@ func isogenyMapG2(e *fp2, x, y *fe2) { y.set(yNum) } -var isogenyConstansG1 = [4][16]*fe{ +var isogenyConstantsG1 = [4][16]*fe{ { {0x4d18b6f3af00131c, 0x19fa219793fee28c, 0x3f2885f1467f19ae, 0x23dcea34f2ffb304, 0xd15b58d2ffc00054, 0x0913be200a20bef4}, {0x898985385cdbbd8b, 0x3c79e43cc7d966aa, 0x1597e193f4cd233a, 0x8637ef1e4d6623ad, 0x11b22deed20d827b, 0x07097bc5998784ad}, diff --git a/crypto/ecies/ecies_test.go b/crypto/ecies/ecies_test.go index 96e33da006fb..8ca42c9c8ee6 100644 --- a/crypto/ecies/ecies_test.go +++ b/crypto/ecies/ecies_test.go @@ -334,7 +334,6 @@ func testParamSelection(t *testing.T, c testCase) { if err == nil { t.Fatalf("ecies: encryption should not have succeeded (%s)\n", c.Name) } - } // Ensure that the basic public key validation in the decryption operation diff --git a/eth/api.go b/eth/api.go index a70cd06cb989..082c4cf413e3 100644 --- a/eth/api.go +++ b/eth/api.go @@ -171,7 +171,7 @@ func (api *AdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool } if _, err := os.Stat(file); err == nil { // File already exists. Allowing overwrite could be a DoS vector, - // since the 'file' may point to arbitrary paths on the drive + // since the 'file' may point to arbitrary paths on the drive. return false, errors.New("location would overwrite an existing file") } // Make sure we can create the file to export into @@ -339,6 +339,8 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { block = api.eth.blockchain.CurrentBlock() } else if blockNr == rpc.FinalizedBlockNumber { block = api.eth.blockchain.CurrentFinalizedBlock() + } else if blockNr == rpc.SafeBlockNumber { + block = api.eth.blockchain.CurrentSafeBlock() } else { block = api.eth.blockchain.GetBlockByNumber(uint64(blockNr)) } @@ -383,7 +385,7 @@ func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) if rlpBytes, err := rlp.EncodeToBytes(block); err != nil { blockRlp = err.Error() // Hacky, but hey, it works } else { - blockRlp = fmt.Sprintf("0x%x", rlpBytes) + blockRlp = fmt.Sprintf("%#x", rlpBytes) } if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.APIBackend.ChainConfig()); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} @@ -417,6 +419,8 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex block = api.eth.blockchain.CurrentBlock() } else if number == rpc.FinalizedBlockNumber { block = api.eth.blockchain.CurrentFinalizedBlock() + } else if number == rpc.SafeBlockNumber { + block = api.eth.blockchain.CurrentSafeBlock() } else { block = api.eth.blockchain.GetBlockByNumber(uint64(number)) } @@ -569,11 +573,11 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c } triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(common.Hash{}, startBlock.Root(), triedb) + oldTrie, err := trie.NewStateTrie(common.Hash{}, startBlock.Root(), triedb) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(common.Hash{}, endBlock.Root(), triedb) + newTrie, err := trie.NewStateTrie(common.Hash{}, endBlock.Root(), triedb) if err != nil { return nil, err } @@ -649,5 +653,5 @@ func (api *DebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error return uint64(i), nil } } - return 0, fmt.Errorf("No state found") + return 0, errors.New("no state found") } diff --git a/eth/api_backend.go b/eth/api_backend.go index 330fb870e3ba..001c14e495a2 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -19,7 +19,6 @@ package eth import ( "context" "errors" - "fmt" "math/big" "time" @@ -76,7 +75,18 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb return b.eth.blockchain.CurrentBlock().Header(), nil } if number == rpc.FinalizedBlockNumber { - return b.eth.blockchain.CurrentFinalizedBlock().Header(), nil + block := b.eth.blockchain.CurrentFinalizedBlock() + if block != nil { + return block.Header(), nil + } + return nil, errors.New("finalized block not found") + } + if number == rpc.SafeBlockNumber { + block := b.eth.blockchain.CurrentSafeBlock() + if block != nil { + return block.Header(), nil + } + return nil, errors.New("safe block not found") } return b.eth.blockchain.GetHeaderByNumber(uint64(number)), nil } @@ -115,6 +125,9 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe if number == rpc.FinalizedBlockNumber { return b.eth.blockchain.CurrentFinalizedBlock(), nil } + if number == rpc.SafeBlockNumber { + return b.eth.blockchain.CurrentSafeBlock(), nil + } return b.eth.blockchain.GetBlockByNumber(uint64(number)), nil } @@ -192,17 +205,8 @@ func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return b.eth.blockchain.GetReceiptsByHash(hash), nil } -func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - db := b.eth.ChainDb() - number := rawdb.ReadHeaderNumber(db, hash) - if number == nil { - return nil, fmt.Errorf("failed to get block number for hash %#x", hash) - } - logs := rawdb.ReadLogs(db, hash, *number, b.eth.blockchain.Config()) - if logs == nil { - return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", *number, hash.TerminalString()) - } - return logs, nil +func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return rawdb.ReadLogs(b.eth.chainDb, hash, number, b.ChainConfig()), nil } func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { diff --git a/eth/api_test.go b/eth/api_test.go index 39a1d5846004..250591c1079b 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/trie" ) var dumper = spew.ConfigState{Indent: " "} @@ -66,7 +67,7 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), nil) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) state, _ = state.New(common.Hash{}, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} @@ -213,7 +214,7 @@ func TestStorageRangeAt(t *testing.T) { t.Error(err) } if !reflect.DeepEqual(result, test.want) { - t.Fatalf("wrong result for range 0x%x.., limit %d:\ngot %s\nwant %s", + t.Fatalf("wrong result for range %#x.., limit %d:\ngot %s\nwant %s", test.start, test.limit, dumper.Sdump(result), dumper.Sdump(&test.want)) } } diff --git a/eth/backend.go b/eth/backend.go index 6f448aa43ab5..37a40a9549e6 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -25,7 +25,6 @@ import ( "strings" "sync" "sync/atomic" - "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -41,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -136,7 +134,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideGrayGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideTerminalTotalDifficulty, config.OverrideTerminalTotalDifficultyPassed) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } @@ -303,31 +301,21 @@ func (s *Ethereum) APIs() []rpc.API { return append(apis, []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: NewEthereumAPI(s), }, { Namespace: "miner", - Version: "1.0", Service: NewMinerAPI(s), }, { Namespace: "eth", - Version: "1.0", Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), - }, { - Namespace: "eth", - Version: "1.0", - Service: filters.NewFilterAPI(s.APIBackend, false, 5*time.Minute), }, { Namespace: "admin", - Version: "1.0", Service: NewAdminAPI(s), }, { Namespace: "debug", - Version: "1.0", Service: NewDebugAPI(s), }, { Namespace: "net", - Version: "1.0", Service: s.netRPCService, }, }...) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index a9f35092b1da..703850d07b8b 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "errors" "fmt" + "math/big" "sync" "time" @@ -31,18 +32,18 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" ) -// Register adds catalyst APIs to the full node. +// Register adds the engine API to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { - log.Warn("Catalyst mode enabled", "protocol", "eth") + log.Warn("Engine API enabled", "protocol", "eth") stack.RegisterAPIs([]rpc.API{ { Namespace: "engine", - Version: "1.0", Service: NewConsensusAPI(backend), Authenticated: true, }, @@ -50,31 +51,100 @@ func Register(stack *node.Node, backend *eth.Ethereum) error { return nil } +const ( + // invalidBlockHitEviction is the number of times an invalid block can be + // referenced in forkchoice update or new payload before it is attempted + // to be reprocessed again. + invalidBlockHitEviction = 128 + + // invalidTipsetsCap is the max number of recent block hashes tracked that + // have lead to some bad ancestor block. It's just an OOM protection. + invalidTipsetsCap = 512 + + // beaconUpdateStartupTimeout is the time to wait for a beacon client to get + // attached before starting to issue warnings. + beaconUpdateStartupTimeout = 30 * time.Second + + // beaconUpdateExchangeTimeout is the max time allowed for a beacon client to + // do a transition config exchange before it's considered offline and the user + // is warned. + beaconUpdateExchangeTimeout = 2 * time.Minute + + // beaconUpdateConsensusTimeout is the max time allowed for a beacon client + // to send a consensus update before it's considered offline and the user is + // warned. + beaconUpdateConsensusTimeout = 30 * time.Second + + // beaconUpdateWarnFrequency is the frequency at which to warn the user that + // the beacon client is offline. + beaconUpdateWarnFrequency = 5 * time.Minute +) + type ConsensusAPI struct { - eth *eth.Ethereum + eth *eth.Ethereum + remoteBlocks *headerQueue // Cache of remote payloads received localBlocks *payloadQueue // Cache of local payloads generated - // Lock for the forkChoiceUpdated method - forkChoiceLock sync.Mutex + + // The forkchoice update and new payload method require us to return the + // latest valid hash in an invalid chain. To support that return, we need + // to track historical bad blocks as well as bad tipsets in case a chain + // is constantly built on it. + // + // There are a few important caveats in this mechanism: + // - The bad block tracking is ephemeral, in-memory only. We must never + // persist any bad block information to disk as a bug in Geth could end + // up blocking a valid chain, even if a later Geth update would accept + // it. + // - Bad blocks will get forgotten after a certain threshold of import + // attempts and will be retried. The rationale is that if the network + // really-really-really tries to feed us a block, we should give it a + // new chance, perhaps us being racey instead of the block being legit + // bad (this happened in Geth at a point with import vs. pending race). + // - Tracking all the blocks built on top of the bad one could be a bit + // problematic, so we will only track the head chain segment of a bad + // chain to allow discarding progressing bad chains and side chains, + // without tracking too much bad data. + invalidBlocksHits map[common.Hash]int // Emhemeral cache to track invalid blocks and their hit count + invalidTipsets map[common.Hash]*types.Header // Ephemeral cache to track invalid tipsets and their bad ancestor + invalidLock sync.Mutex // Protects the invalid maps from concurrent access + + // Geth can appear to be stuck or do strange things if the beacon client is + // offline or is sending us strange data. Stash some update stats away so + // that we can warn the user and not have them open issues on our tracker. + lastTransitionUpdate time.Time + lastTransitionLock sync.Mutex + lastForkchoiceUpdate time.Time + lastForkchoiceLock sync.Mutex + lastNewPayloadUpdate time.Time + lastNewPayloadLock sync.Mutex + + forkchoiceLock sync.Mutex // Lock for the forkChoiceUpdated method } // NewConsensusAPI creates a new consensus api for the given backend. // The underlying blockchain needs to have a valid terminal total difficulty set. func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { if eth.BlockChain().Config().TerminalTotalDifficulty == nil { - panic("Catalyst started without valid total difficulty") + log.Warn("Engine API started but chain not configured for merge yet") } - return &ConsensusAPI{ - eth: eth, - remoteBlocks: newHeaderQueue(), - localBlocks: newPayloadQueue(), + api := &ConsensusAPI{ + eth: eth, + remoteBlocks: newHeaderQueue(), + localBlocks: newPayloadQueue(), + invalidBlocksHits: make(map[common.Hash]int), + invalidTipsets: make(map[common.Hash]*types.Header), } + eth.Downloader().SetBadBlockCallback(api.setInvalidAncestor) + go api.heartbeat() + + return api } // ForkchoiceUpdatedV1 has several responsibilities: // If the method is called with an empty head block: // -// we return success, which can be used to check if the catalyst mode is enabled +// we return success, which can be used to check if the engine API is enabled // // If the total difficulty was not reached: // @@ -89,20 +159,28 @@ func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { // // we try to assemble a block with the payloadAttributes and return its payloadID func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { - api.forkChoiceLock.Lock() - defer api.forkChoiceLock.Unlock() + api.forkchoiceLock.Lock() + defer api.forkchoiceLock.Unlock() log.Trace("Engine API request received", "method", "ForkchoiceUpdated", "head", update.HeadBlockHash, "finalized", update.FinalizedBlockHash, "safe", update.SafeBlockHash) if update.HeadBlockHash == (common.Hash{}) { log.Warn("Forkchoice requested update to zero hash") return beacon.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastForkchoiceLock.Lock() + api.lastForkchoiceUpdate = time.Now() + api.lastForkchoiceLock.Unlock() // Check whether we have the block yet in our database or not. If not, we'll // need to either trigger a sync, or to reject this forkchoice update for a // reason. block := api.eth.BlockChain().GetBlockByHash(update.HeadBlockHash) if block == nil { + // If this block was previously invalidated, keep rejecting it here too + if res := api.checkInvalidAncestor(update.HeadBlockHash, update.HeadBlockHash); res != nil { + return beacon.ForkChoiceResponse{PayloadStatus: *res, PayloadID: nil}, nil + } // If the head hash is unknown (was not given to us in a newPayload request), // we cannot resolve the header, so not much to do. This could be extended in // the future to resolve from the `eth` network, but it's an unexpected case @@ -198,6 +276,8 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa log.Warn("Safe block not in canonical chain") return beacon.STATUS_INVALID, beacon.InvalidForkChoiceState.With(errors.New("safe block not in canonical chain")) } + // Set the safe block + api.eth.BlockChain().SetSafe(safeBlock) } // If payload generation was requested, create a new block to be potentially // sealed by the beacon client. The payload will be requested later, and we @@ -226,15 +306,20 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa // ExchangeTransitionConfigurationV1 checks the given configuration against // the configuration of the node. func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config beacon.TransitionConfigurationV1) (*beacon.TransitionConfigurationV1, error) { + log.Trace("Engine API request received", "method", "ExchangeTransitionConfiguration", "ttd", config.TerminalTotalDifficulty) if config.TerminalTotalDifficulty == nil { return nil, errors.New("invalid terminal total difficulty") } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastTransitionLock.Lock() + api.lastTransitionUpdate = time.Now() + api.lastTransitionLock.Unlock() + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty - if ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { + if ttd == nil || ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { log.Warn("Invalid TTD configured", "geth", ttd, "beacon", config.TerminalTotalDifficulty) return nil, fmt.Errorf("invalid ttd: execution %v consensus %v", ttd, config.TerminalTotalDifficulty) } - if config.TerminalBlockHash != (common.Hash{}) { if hash := api.eth.BlockChain().GetCanonicalHash(uint64(config.TerminalBlockNumber)); hash == config.TerminalBlockHash { return &beacon.TransitionConfigurationV1{ @@ -266,6 +351,11 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa log.Debug("Invalid NewPayload params", "params", params, "error", err) return beacon.PayloadStatusV1{Status: beacon.INVALIDBLOCKHASH}, nil } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastNewPayloadLock.Lock() + api.lastNewPayloadUpdate = time.Now() + api.lastNewPayloadLock.Unlock() + // If we already have the block locally, ignore the entire execution and just // return a fake success. if block := api.eth.BlockChain().GetBlockByHash(params.BlockHash); block != nil { @@ -273,6 +363,10 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa hash := block.Hash() return beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &hash}, nil } + // If this block was rejected previously, keep rejecting it + if res := api.checkInvalidAncestor(block.Hash(), block.Hash()); res != nil { + return *res, nil + } // If the parent is missing, we - in theory - could trigger a sync, but that // would also entail a reorg. That is problematic if multiple sibling blocks // are being fed to us, and even more so, if some semi-distant uncle shortens @@ -281,23 +375,7 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa // update after legit payload executions. parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - // Stash the block away for a potential forced forckchoice update to it - // at a later time. - api.remoteBlocks.put(block.Hash(), block.Header()) - - // Although we don't want to trigger a sync, if there is one already in - // progress, try to extend if with the current payload request to relieve - // some strain from the forkchoice update. - if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { - log.Debug("Payload accepted for sync extension", "number", params.Number, "hash", params.BlockHash) - return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil - } - // Either no beacon sync was started yet, or it rejected the delivered - // payload as non-integratable on top of the existing sync. We'll just - // have to rely on the beacon client to forcefully update the head with - // a forkchoice update request. - log.Warn("Ignoring payload with missing parent", "number", params.Number, "hash", params.BlockHash, "parent", params.ParentHash) - return beacon.PayloadStatusV1{Status: beacon.ACCEPTED}, nil + return api.delayPayloadImport(block) } // We have an existing parent, do some sanity checks to avoid the beacon client // triggering too early @@ -316,7 +394,14 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa } if block.Time() <= parent.Time() { log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) - return api.invalid(errors.New("invalid timestamp"), parent), nil + return api.invalid(errors.New("invalid timestamp"), parent.Header()), nil + } + // Another cornercase: if the node is in snap sync mode, but the CL client + // tries to make it import a block. That should be denied as pushing something + // into the database directly will conflict with the assumptions of snap sync + // that it has an empty db that it can fill itself. + if api.eth.SyncMode() != downloader.FullSync { + return api.delayPayloadImport(block) } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { api.remoteBlocks.put(block.Hash(), block.Header()) @@ -326,7 +411,13 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number) if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil { log.Warn("NewPayloadV1: inserting block failed", "error", err) - return api.invalid(err, parent), nil + + api.invalidLock.Lock() + api.invalidBlocksHits[block.Hash()] = 1 + api.invalidTipsets[block.Hash()] = block.Header() + api.invalidLock.Unlock() + + return api.invalid(err, parent.Header()), nil } // We've accepted a valid payload from the beacon client. Mark the local // chain transitions to notify other subsystems (e.g. downloader) of the @@ -352,14 +443,114 @@ func computePayloadId(headBlockHash common.Hash, params *beacon.PayloadAttribute return out } +// delayPayloadImport stashes the given block away for import at a later time, +// either via a forkchoice update or a sync extension. This method is meant to +// be called by the newpayload command when the block seems to be ok, but some +// prerequisite prevents it from being processed (e.g. no parent, or snap sync). +func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (beacon.PayloadStatusV1, error) { + // Sanity check that this block's parent is not on a previously invalidated + // chain. If it is, mark the block as invalid too. + if res := api.checkInvalidAncestor(block.ParentHash(), block.Hash()); res != nil { + return *res, nil + } + // Stash the block away for a potential forced forkchoice update to it + // at a later time. + api.remoteBlocks.put(block.Hash(), block.Header()) + + // Although we don't want to trigger a sync, if there is one already in + // progress, try to extend if with the current payload request to relieve + // some strain from the forkchoice update. + if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { + log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil + } + // Either no beacon sync was started yet, or it rejected the delivered + // payload as non-integratable on top of the existing sync. We'll just + // have to rely on the beacon client to forcefully update the head with + // a forkchoice update request. + if api.eth.SyncMode() == downloader.FullSync { + // In full sync mode, failure to import a well-formed block can only mean + // that the parent state is missing and the syncer rejected extending the + // current cycle with the new payload. + log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash()) + } else { + // In non-full sync mode (i.e. snap sync) all payloads are rejected until + // snap sync terminates as snap sync relies on direct database injections + // and cannot afford concurrent out-if-band modifications via imports. + log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash()) + } + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil +} + +// setInvalidAncestor is a callback for the downloader to notify us if a bad block +// is encountered during the async sync. +func (api *ConsensusAPI) setInvalidAncestor(invalid *types.Header, origin *types.Header) { + api.invalidLock.Lock() + defer api.invalidLock.Unlock() + + api.invalidTipsets[origin.Hash()] = invalid + api.invalidBlocksHits[invalid.Hash()]++ +} + +// checkInvalidAncestor checks whether the specified chain end links to a known +// bad ancestor. If yes, it constructs the payload failure response to return. +func (api *ConsensusAPI) checkInvalidAncestor(check common.Hash, head common.Hash) *beacon.PayloadStatusV1 { + api.invalidLock.Lock() + defer api.invalidLock.Unlock() + + // If the hash to check is unknown, return valid + invalid, ok := api.invalidTipsets[check] + if !ok { + return nil + } + // If the bad hash was hit too many times, evict it and try to reprocess in + // the hopes that we have a data race that we can exit out of. + badHash := invalid.Hash() + + api.invalidBlocksHits[badHash]++ + if api.invalidBlocksHits[badHash] >= invalidBlockHitEviction { + log.Warn("Too many bad block import attempt, trying", "number", invalid.Number, "hash", badHash) + delete(api.invalidBlocksHits, badHash) + + for descendant, badHeader := range api.invalidTipsets { + if badHeader.Hash() == badHash { + delete(api.invalidTipsets, descendant) + } + } + return nil + } + // Not too many failures yet, mark the head of the invalid chain as invalid + if check != head { + log.Warn("Marked new chain head as invalid", "hash", head, "badnumber", invalid.Number, "badhash", badHash) + for len(api.invalidTipsets) >= invalidTipsetsCap { + for key := range api.invalidTipsets { + delete(api.invalidTipsets, key) + break + } + } + api.invalidTipsets[head] = invalid + } + // If the last valid hash is the terminal pow block, return 0x0 for latest valid hash + lastValid := &invalid.ParentHash + if header := api.eth.BlockChain().GetHeader(invalid.ParentHash, invalid.Number.Uint64()-1); header != nil && header.Difficulty.Sign() != 0 { + lastValid = &common.Hash{} + } + failure := "links to previously rejected block" + return &beacon.PayloadStatusV1{ + Status: beacon.INVALID, + LatestValidHash: lastValid, + ValidationError: &failure, + } +} + // invalid returns a response "INVALID" with the latest valid hash supplied by latest or to the current head // if no latestValid block was provided. -func (api *ConsensusAPI) invalid(err error, latestValid *types.Block) beacon.PayloadStatusV1 { +func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) beacon.PayloadStatusV1 { currentHash := api.eth.BlockChain().CurrentBlock().Hash() if latestValid != nil { // Set latest valid hash to 0x0 if parent is PoW block currentHash = common.Hash{} - if latestValid.Difficulty().BitLen() == 0 { + if latestValid.Difficulty.BitLen() == 0 { // Otherwise set latest valid hash to parent hash currentHash = latestValid.Hash() } @@ -367,3 +558,127 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Block) beacon.Pay errorMsg := err.Error() return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash, ValidationError: &errorMsg} } + +// heartbeat loops indefinitely, and checks if there have been beacon client updates +// received in the last while. If not - or if they but strange ones - it warns the +// user that something might be off with their consensus node. +// +// TODO(karalabe): Spin this goroutine down somehow +func (api *ConsensusAPI) heartbeat() { + // Sleep a bit on startup since there's obviously no beacon client yet + // attached, so no need to print scary warnings to the user. + time.Sleep(beaconUpdateStartupTimeout) + + var ( + offlineLogged time.Time + ) + for { + // Sleep a bit and retrieve the last known consensus updates + time.Sleep(5 * time.Second) + + // If the network is not yet merged/merging, don't bother scaring the user + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty + if ttd == nil { + continue + } + api.lastTransitionLock.Lock() + lastTransitionUpdate := api.lastTransitionUpdate + api.lastTransitionLock.Unlock() + + api.lastForkchoiceLock.Lock() + lastForkchoiceUpdate := api.lastForkchoiceUpdate + api.lastForkchoiceLock.Unlock() + + api.lastNewPayloadLock.Lock() + lastNewPayloadUpdate := api.lastNewPayloadUpdate + api.lastNewPayloadLock.Unlock() + + // If there have been no updates for the past while, warn the user + // that the beacon client is probably offline + if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { + if time.Since(lastForkchoiceUpdate) > beaconUpdateConsensusTimeout && time.Since(lastNewPayloadUpdate) > beaconUpdateConsensusTimeout { + if time.Since(lastTransitionUpdate) > beaconUpdateExchangeTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastTransitionUpdate.IsZero() { + log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") + } else { + log.Warn("Previously seen beacon client is offline. Please ensure it is operational to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { + log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") + } else { + log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } else { + offlineLogged = time.Time{} + } + } else { + if time.Since(lastTransitionUpdate) > beaconUpdateExchangeTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + // Retrieve the last few blocks and make a rough estimate as + // to when the merge transition should happen + var ( + chain = api.eth.BlockChain() + head = chain.CurrentBlock() + htd = chain.GetTd(head.Hash(), head.NumberU64()) + eta time.Duration + ) + if head.NumberU64() > 0 && htd.Cmp(ttd) < 0 { + // Accumulate the last 64 difficulties to estimate the growth + var diff float64 + + block := head + for i := 0; i < 64; i++ { + diff += float64(block.Difficulty().Uint64()) + if parent := chain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent == nil { + break + } else { + block = parent + } + } + // Estimate an ETA based on the block times and the difficulty growth + growth := diff / float64(head.Time()-block.Time()+1) // +1 to avoid div by zero + if growth > 0 { + if left := new(big.Int).Sub(ttd, htd); left.IsUint64() { + eta = time.Duration(float64(left.Uint64())/growth) * time.Second + } else { + eta = time.Duration(new(big.Int).Div(left, big.NewInt(int64(growth))).Uint64()) * time.Second + } + } + } + var message string + if htd.Cmp(ttd) > 0 { + if lastTransitionUpdate.IsZero() { + message = "Merge already reached, but no beacon client seen. Please launch one to follow the chain!" + } else { + message = "Merge already reached, but previously seen beacon client is offline. Please ensure it is operational to follow the chain!" + } + } else { + if lastTransitionUpdate.IsZero() { + message = "Merge is configured, but no beacon client seen. Please ensure you have one available before the transition arrives!" + } else { + message = "Merge is configured, but previously seen beacon client is offline. Please ensure it is operational before the transition arrives!" + } + } + if eta == 0 { + log.Warn(message) + } else { + log.Warn(message, "eta", common.PrettyAge(time.Now().Add(-eta))) // weird hack, but duration formatted doesn't handle days + } + offlineLogged = time.Now() + } + continue + } else { + offlineLogged = time.Time{} + } + } + } +} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index beedc8e4608a..4c9a54f4157b 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -69,7 +69,7 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { g.AddTx(tx) testNonce++ } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) totalDifficulty := big.NewInt(0) @@ -403,7 +403,7 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}, SyncMode: downloader.SnapSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) @@ -663,8 +663,8 @@ func TestEmptyBlocks(t *testing.T) { if err != nil { t.Fatal(err) } - if status.Status != beacon.ACCEPTED { - t.Errorf("invalid status: expected ACCEPTED got: %v", status.Status) + if status.Status != beacon.SYNCING { + t.Errorf("invalid status: expected SYNCING got: %v", status.Status) } if status.LatestValidHash != nil { t.Fatalf("invalid LVH: got %v wanted nil", status.LatestValidHash) @@ -774,8 +774,8 @@ func TestTrickRemoteBlockCache(t *testing.T) { if err != nil { panic(err) } - if status.Status == beacon.INVALID { - panic("success") + if status.Status == beacon.VALID { + t.Error("invalid status: VALID on an invalid chain") } // Now reorg to the head of the invalid chain resp, err := apiB.ForkchoiceUpdatedV1(beacon.ForkchoiceStateV1{HeadBlockHash: payload.BlockHash, SafeBlockHash: payload.BlockHash, FinalizedBlockHash: payload.ParentHash}, nil) @@ -783,7 +783,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { t.Fatal(err) } if resp.PayloadStatus.Status == beacon.VALID { - t.Errorf("invalid status: expected INVALID got: %v", resp.PayloadStatus.Status) + t.Error("invalid status: VALID on an invalid chain") } time.Sleep(100 * time.Millisecond) } diff --git a/eth/downloader/api.go b/eth/downloader/api.go index b36dd6386500..b3f7113bcde9 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 533404f6c9b9..484a4e20de64 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -137,6 +137,13 @@ func (b *beaconBackfiller) setMode(mode SyncMode) { b.resume() } +// SetBadBlockCallback sets the callback to run when a bad block is hit by the +// block processor. This method is not thread safe and should be set only once +// on startup before system events are fired. +func (d *Downloader) SetBadBlockCallback(onBadBlock badBlockFn) { + d.badBlock = onBadBlock +} + // BeaconSync is the post-merge version of the chain synchronization, where the // chain is not downloaded from genesis onward, rather from trusted head announces // backwards. @@ -229,7 +236,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { // Binary search to find the ancestor start, end := beaconTail.Number.Uint64()-1, number if number := beaconHead.Number.Uint64(); end > number { - // This shouldn't really happen in a healty network, but if the consensus + // This shouldn't really happen in a healthy network, but if the consensus // clients feeds us a shorter chain as the canonical, we should not attempt // to access non-existent skeleton items. log.Warn("Beacon head lower than local chain", "beacon", number, "local", end) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 6d812c8ff233..6d91c8677811 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -85,6 +85,10 @@ var ( // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) +// badBlockFn is a callback for the async beacon sync to notify the caller that +// the origin header requested to sync to, produced a chain with a bad block. +type badBlockFn func(invalid *types.Header, origin *types.Header) + // headerTask is a set of downloaded headers to queue along with their precomputed // hashes to avoid constant rehashing. type headerTask struct { @@ -113,6 +117,7 @@ type Downloader struct { // Callbacks dropPeer peerDropFn // Drops a peer for misbehaving + badBlock badBlockFn // Reports a block as rejected by the chain // Status synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing @@ -359,7 +364,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td, ttd *big.Int, // The beacon header syncer is async. It will start this synchronization and // will continue doing other tasks. However, if synchronization needs to be // cancelled, the syncer needs to know if we reached the startup point (and - // inited the cancel cannel) or not yet. Make sure that we'll signal even in + // inited the cancel channel) or not yet. Make sure that we'll signal even in // case of a failure. if beaconPing != nil { defer func() { @@ -1458,7 +1463,7 @@ func (d *Downloader) processHeaders(origin uint64, td, ttd *big.Int, beaconMode } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} { select { case ch <- true: @@ -1530,7 +1535,7 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { return errCancelContentProcessing default: } - // Retrieve the a batch of results to import + // Retrieve a batch of results to import first, last := results[0].Header, results[len(results)-1].Header log.Debug("Inserting downloaded chain", "items", len(results), "firstnum", first.Number, "firsthash", first.Hash(), @@ -1545,7 +1550,17 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { // consensus-layer. if index, err := d.blockchain.InsertChain(blocks); err != nil { if index < len(results) { - log.Error("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) + log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) + + // In post-merge, notify the engine API of encountered bad chains + if d.badBlock != nil { + head, _, err := d.skeleton.Bounds() + if err != nil { + log.Error("Failed to retrieve beacon bounds for bad block reporting", "err", err) + } else { + d.badBlock(blocks[index].Header(), head) + } + } } else { // The InsertChain method in blockchain.go will sometimes return an out-of-bounds index, // when it needs to preprocess blocks to import a sidechain. diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 2b49aaca6de3..dc00af2351be 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -68,7 +68,11 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { t.Cleanup(func() { db.Close() }) - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) + gspec := core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + gspec.MustCommit(db) chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { @@ -356,7 +360,7 @@ func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limi } // RequestStorageRanges fetches a batch of storage slots belonging to one or -// more accounts. If slots from only one accout is requested, an origin marker +// more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { // Create the request and service it @@ -395,7 +399,7 @@ func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, paths []snap.TrieNodePathSet, bytes uint64) error { req := &snap.GetTrieNodesPacket{ ID: id, @@ -571,8 +575,8 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chainB.blocks)) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// currently and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, SnapSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } diff --git a/eth/downloader/fetchers_concurrent.go b/eth/downloader/fetchers_concurrent.go index a0aa197175a3..44e6aa8f8d88 100644 --- a/eth/downloader/fetchers_concurrent.go +++ b/eth/downloader/fetchers_concurrent.go @@ -47,7 +47,7 @@ type typedQueue interface { // capacity is responsible for calculating how many items of the abstracted // type a particular peer is estimated to be able to retrieve within the - // alloted round trip time. + // allotted round trip time. capacity(peer *peerConnection, rtt time.Duration) int // updateCapacity is responsible for updating how many items of the abstracted @@ -58,7 +58,7 @@ type typedQueue interface { // from the download queue to the specified peer. reserve(peer *peerConnection, items int) (*fetchRequest, bool, bool) - // unreserve is resposible for removing the current retrieval allocation + // unreserve is responsible for removing the current retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. unreserve(peer string) int @@ -190,7 +190,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { req, err := queue.request(peer, request, responses) if err != nil { // Sending the request failed, which generally means the peer - // was diconnected in between assignment and network send. + // was disconnected in between assignment and network send. // Although all peer removal operations return allocated tasks // to the queue, that is async, and we can do better here by // immediately pushing the unfulfilled requests. diff --git a/eth/downloader/fetchers_concurrent_bodies.go b/eth/downloader/fetchers_concurrent_bodies.go index a8de410323f3..e84206fe9951 100644 --- a/eth/downloader/fetchers_concurrent_bodies.go +++ b/eth/downloader/fetchers_concurrent_bodies.go @@ -41,7 +41,7 @@ func (q *bodyQueue) pending() int { } // capacity is responsible for calculating how many bodies a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *bodyQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.BodyCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *bodyQueue) reserve(peer *peerConnection, items int) (*fetchRequest, boo return q.queue.ReserveBodies(peer, items) } -// unreserve is resposible for removing the current body retrieval allocation +// unreserve is responsible for removing the current body retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *bodyQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_headers.go b/eth/downloader/fetchers_concurrent_headers.go index bd3bb3e00bf3..84c7f209865a 100644 --- a/eth/downloader/fetchers_concurrent_headers.go +++ b/eth/downloader/fetchers_concurrent_headers.go @@ -41,7 +41,7 @@ func (q *headerQueue) pending() int { } // capacity is responsible for calculating how many headers a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *headerQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.HeaderCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *headerQueue) reserve(peer *peerConnection, items int) (*fetchRequest, b return q.queue.ReserveHeaders(peer, items), false, false } -// unreserve is resposible for removing the current header retrieval allocation +// unreserve is responsible for removing the current header retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *headerQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_receipts.go b/eth/downloader/fetchers_concurrent_receipts.go index fee2c34101d2..1c853c218443 100644 --- a/eth/downloader/fetchers_concurrent_receipts.go +++ b/eth/downloader/fetchers_concurrent_receipts.go @@ -28,7 +28,7 @@ import ( // concurrent fetcher and the downloader. type receiptQueue Downloader -// waker returns a notification channel that gets pinged in case more reecipt +// waker returns a notification channel that gets pinged in case more receipt // fetches have been queued up, so the fetcher might assign it to idle peers. func (q *receiptQueue) waker() chan bool { return q.queue.receiptWakeCh @@ -41,7 +41,7 @@ func (q *receiptQueue) pending() int { } // capacity is responsible for calculating how many receipts a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *receiptQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.ReceiptCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *receiptQueue) reserve(peer *peerConnection, items int) (*fetchRequest, return q.queue.ReserveReceipts(peer, items) } -// unreserve is resposible for removing the current receipt retrieval allocation +// unreserve is responsible for removing the current receipt retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *receiptQueue) unreserve(peer string) int { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index d74d23e74d55..6b8269495948 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -237,6 +237,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 211716e91b1b..26c41711ded5 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -818,7 +818,6 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, reqTimer metrics.Timer, resInMeter metrics.Meter, resDropMeter metrics.Meter, results int, validate func(index int, header *types.Header) error, reconstruct func(index int, result *fetchResult)) (int, error) { - // Short circuit if the data was never requested request := pendPool[id] if request == nil { @@ -861,7 +860,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index 09b18afe5df5..8631b27c9275 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -27,24 +27,18 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -70,10 +64,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -156,7 +150,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -185,7 +179,6 @@ func TestBasics(t *testing.T) { if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { t.Fatalf("expected header %d, got %d", exp, got) } - } if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { t.Errorf("expected block task queue to be %d, got %d", exp, got) @@ -239,7 +232,6 @@ func TestEmptyBlocks(t *testing.T) { if fetchReq != nil { t.Fatal("there should be no body fetch tasks remaining") } - } if q.blockTaskQueue.Size() != numOfBlocks-10 { t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) @@ -253,7 +245,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { @@ -273,14 +265,13 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo world.progress(10) if false { log.Root().SetHandler(log.StdoutHandler) - } q := newQueue(10, 10) var wg sync.WaitGroup @@ -315,7 +306,6 @@ func XTestDelivery(t *testing.T) { fmt.Printf("got %d results, %d tot\n", len(res), tot) // Now we can forget about these world.forget(res[len(res)-1].Header.Number.Uint64()) - } }() wg.Add(1) @@ -396,7 +386,6 @@ func XTestDelivery(t *testing.T) { } for i := 0; i < 50; i++ { time.Sleep(2990 * time.Millisecond) - } }() wg.Add(1) @@ -447,10 +436,8 @@ func (n *network) forget(blocknum uint64) { n.chain = n.chain[index:] n.receipts = n.receipts[index:] n.offset = int(blocknum) - } func (n *network) progress(numBlocks int) { - n.lock.Lock() defer n.lock.Unlock() //fmt.Printf("progressing...\n") @@ -458,7 +445,6 @@ func (n *network) progress(numBlocks int) { n.chain = append(n.chain, newBlocks...) n.receipts = append(n.receipts, newR...) n.cond.Broadcast() - } func (n *network) headers(from int) []*types.Header { diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index be4e8fbfc10c..e627c6ae5a35 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -51,7 +51,7 @@ const requestHeaders = 512 // errSyncLinked is an internal helper error to signal that the current sync // cycle linked up to the genesis block, this the skeleton syncer should ping // the backfiller to resume. Since we already have that logic on sync start, -// piggie-back on that instead of 2 entrypoints. +// piggy-back on that instead of 2 entrypoints. var errSyncLinked = errors.New("sync linked") // errSyncMerged is an internal helper error to signal that the current sync @@ -148,7 +148,7 @@ type backfiller interface { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even - // on initial sartup. + // on initial startup. // // The method should return the last block header that has been successfully // backfilled, or nil if the backfiller was not resumed. @@ -209,7 +209,7 @@ type skeleton struct { headEvents chan *headUpdate // Notification channel for new heads terminate chan chan error // Termination channel to abort sync - terminated chan struct{} // Channel to signal that the syner is dead + terminated chan struct{} // Channel to signal that the syncer is dead // Callback hooks used during testing syncStarting func() // callback triggered after a sync cycle is inited but before started @@ -553,7 +553,7 @@ func (s *skeleton) initSync(head *types.Header) { return } } - // Either we've failed to decode the previus state, or there was none. Start + // Either we've failed to decode the previous state, or there was none. Start // a fresh sync with a single subchain represented by the currently sent // chain head. s.progress = &skeletonProgress{ @@ -823,7 +823,7 @@ func (s *skeleton) executeTask(peer *peerConnection, req *headerRequest) { } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *skeleton) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -871,7 +871,7 @@ func (s *skeleton) revertRequest(req *headerRequest) { delete(s.requests, req.id) // Remove the request from the tracked set and mark the task as not-pending, - // ready for resheduling + // ready for rescheduling s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" } diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index e281078cd190..92eb2ec079c3 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "math/big" - "os" "sync/atomic" "testing" "time" @@ -55,7 +54,7 @@ func newHookedBackfiller() backfiller { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even -// on initial sartup. +// on initial startup. func (hf *hookedBackfiller) suspend() *types.Header { if hf.suspendHook != nil { hf.suspendHook() @@ -113,7 +112,7 @@ func newSkeletonTestPeerWithHook(id string, headers []*types.Header, serve func( // function can be used to retrieve batches of headers from the particular peer. func (p *skeletonTestPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool, sink chan *eth.Response) (*eth.Request, error) { // Since skeleton test peer are in-memory mocks, dropping the does not make - // them inaccepssible. As such, check a local `dropped` field to see if the + // them inaccessible. As such, check a local `dropped` field to see if the // peer has been dropped and should not respond any more. if atomic.LoadUint64(&p.dropped) != 0 { return nil, errors.New("peer already dropped") @@ -206,7 +205,7 @@ func (p *skeletonTestPeer) RequestReceipts([]common.Hash, chan *eth.Response) (* panic("skeleton sync must not request receipts") } -// Tests various sync initialzations based on previous leftovers in the database +// Tests various sync initializations based on previous leftovers in the database // and announced heads. func TestSkeletonSyncInit(t *testing.T) { // Create a few key headers @@ -229,7 +228,7 @@ func TestSkeletonSyncInit(t *testing.T) { newstate: []*subchain{{Head: 50, Tail: 50}}, }, // Empty database with only the genesis set with a leftover empty sync - // progess. This is a synthetic case, just for the sake of covering things. + // progress. This is a synthetic case, just for the sake of covering things. { oldstate: []*subchain{}, head: block50, @@ -521,7 +520,7 @@ func TestSkeletonSyncExtend(t *testing.T) { // Tests that the skeleton sync correctly retrieves headers from one or more // peers without duplicates or other strange side effects. func TestSkeletonSyncRetrievals(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // Since skeleton headers don't need to be meaningful, beyond a parent hash // progression, create a long fake chain to test with. @@ -540,13 +539,13 @@ func TestSkeletonSyncRetrievals(t *testing.T) { peers []*skeletonTestPeer // Initial peer set to start the sync with midstate []*subchain // Expected sync state after initial cycle midserve uint64 // Expected number of header retrievals after initial cycle - middrop uint64 // Expectd number of peers dropped after initial cycle + middrop uint64 // Expected number of peers dropped after initial cycle - newHead *types.Header // New header to annount on top of the old one + newHead *types.Header // New header to anoint on top of the old one newPeer *skeletonTestPeer // New peer to join the skeleton syncer endstate []*subchain // Expected sync state after the post-init event endserve uint64 // Expected number of header retrievals after the post-init event - enddrop uint64 // Expectd number of peers dropped after the post-init event + enddrop uint64 // Expected number of peers dropped after the post-init event }{ // Completely empty database with only the genesis set. The sync is expected // to create a single subchain with the requested head. No peers however, so @@ -797,7 +796,6 @@ func TestSkeletonSyncRetrievals(t *testing.T) { check := func() error { if len(progress.Subchains) != len(tt.midstate) { return fmt.Errorf("test %d, mid state: subchain count mismatch: have %d, want %d", i, len(progress.Subchains), len(tt.midstate)) - } for j := 0; j < len(progress.Subchains); j++ { if progress.Subchains[j].Head != tt.midstate[j].Head { diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 8b873343cac4..785da40b5989 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -37,7 +37,12 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + testGspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = testGspec.MustCommit(testDB) ) // The common prefix of all test chains: @@ -212,7 +217,7 @@ func newTestBlockchain(blocks []*types.Block) *core.BlockChain { panic("Requested chain generation outside of init") } db := rawdb.NewMemoryDatabase() - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) + testGspec.MustCommit(db) chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 6b7f909bd8e5..b8c241ac223b 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -83,6 +83,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, SnapshotCache: 102, + FilterLogCacheSize: 32, Miner: miner.Config{ GasCeil: 30000000, GasPrice: big.NewInt(params.GWei), @@ -172,6 +173,9 @@ type Config struct { Preimages bool TriesInMemory uint64 + // This is the number of blocks for which logs will be cached in the filter system. + FilterLogCacheSize int + // Mining options Miner miner.Config @@ -197,7 +201,7 @@ type Config struct { RPCEVMTimeout time.Duration // RPCTxFeeCap is the global transaction fee(price * gaslimit) cap for - // send-transction variants. The unit is ether. + // send-transaction variants. The unit is ether. RPCTxFeeCap float64 // Checkpoint is a hardcoded checkpoint which can be nil. @@ -206,11 +210,11 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - // Gray Glacier block override (TODO: remove after the fork) - OverrideGrayGlacier *big.Int `toml:",omitempty"` - // OverrideTerminalTotalDifficulty (TODO: remove after the fork) OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + + // OverrideTerminalTotalDifficultyPassed (TODO: remove after the fork) + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index e714dd97ab19..9c7a04364d20 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -18,49 +18,50 @@ import ( // MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId uint64 - SyncMode downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning bool - NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ int `toml:",omitempty"` - LightIngress int `toml:",omitempty"` - LightEgress int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - LightNoPrune bool `toml:",omitempty"` - LightNoSyncServe bool `toml:",omitempty"` - SyncFromCheckpoint bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction int `toml:",omitempty"` - UltraLightOnlyAnnounce bool `toml:",omitempty"` - SkipBcVersionCheck bool `toml:"-"` - DatabaseHandles int `toml:"-"` - DatabaseCache int - DatabaseFreezer string - TrieCleanCache int - TrieCleanCacheJournal string `toml:",omitempty"` - TrieCleanCacheRejournal time.Duration `toml:",omitempty"` - TrieDirtyCache int - TrieTimeout time.Duration - SnapshotCache int - Preimages bool - Miner miner.Config - Ethash ethash.Config - TxPool core.TxPoolConfig - GPO gasprice.Config - EnablePreimageRecording bool - DocRoot string `toml:"-"` - RPCGasCap uint64 - RPCEVMTimeout time.Duration - RPCTxFeeCap float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideGrayGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId uint64 + SyncMode downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning bool + NoPrefetch bool + TxLookupLimit uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ int `toml:",omitempty"` + LightIngress int `toml:",omitempty"` + LightEgress int `toml:",omitempty"` + LightPeers int `toml:",omitempty"` + LightNoPrune bool `toml:",omitempty"` + LightNoSyncServe bool `toml:",omitempty"` + SyncFromCheckpoint bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction int `toml:",omitempty"` + UltraLightOnlyAnnounce bool `toml:",omitempty"` + SkipBcVersionCheck bool `toml:"-"` + DatabaseHandles int `toml:"-"` + DatabaseCache int + DatabaseFreezer string + TrieCleanCache int + TrieCleanCacheJournal string `toml:",omitempty"` + TrieCleanCacheRejournal time.Duration `toml:",omitempty"` + TrieDirtyCache int + TrieTimeout time.Duration + SnapshotCache int + Preimages bool + FilterLogCacheSize int + Miner miner.Config + Ethash ethash.Config + TxPool core.TxPoolConfig + GPO gasprice.Config + EnablePreimageRecording bool + DocRoot string `toml:"-"` + RPCGasCap uint64 + RPCEVMTimeout time.Duration + RPCTxFeeCap float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -93,6 +94,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache enc.Preimages = c.Preimages + enc.FilterLogCacheSize = c.FilterLogCacheSize enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -104,57 +106,58 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle - enc.OverrideGrayGlacier = c.OverrideGrayGlacier enc.OverrideTerminalTotalDifficulty = c.OverrideTerminalTotalDifficulty + enc.OverrideTerminalTotalDifficultyPassed = c.OverrideTerminalTotalDifficultyPassed return &enc, nil } // UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId *uint64 - SyncMode *downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning *bool - NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ *int `toml:",omitempty"` - LightIngress *int `toml:",omitempty"` - LightEgress *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - LightNoPrune *bool `toml:",omitempty"` - LightNoSyncServe *bool `toml:",omitempty"` - SyncFromCheckpoint *bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction *int `toml:",omitempty"` - UltraLightOnlyAnnounce *bool `toml:",omitempty"` - SkipBcVersionCheck *bool `toml:"-"` - DatabaseHandles *int `toml:"-"` - DatabaseCache *int - DatabaseFreezer *string - TrieCleanCache *int - TrieCleanCacheJournal *string `toml:",omitempty"` - TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` - TrieDirtyCache *int - TrieTimeout *time.Duration - SnapshotCache *int - Preimages *bool - Miner *miner.Config - Ethash *ethash.Config - TxPool *core.TxPoolConfig - GPO *gasprice.Config - EnablePreimageRecording *bool - DocRoot *string `toml:"-"` - RPCGasCap *uint64 - RPCEVMTimeout *time.Duration - RPCTxFeeCap *float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideGrayGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId *uint64 + SyncMode *downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning *bool + NoPrefetch *bool + TxLookupLimit *uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ *int `toml:",omitempty"` + LightIngress *int `toml:",omitempty"` + LightEgress *int `toml:",omitempty"` + LightPeers *int `toml:",omitempty"` + LightNoPrune *bool `toml:",omitempty"` + LightNoSyncServe *bool `toml:",omitempty"` + SyncFromCheckpoint *bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction *int `toml:",omitempty"` + UltraLightOnlyAnnounce *bool `toml:",omitempty"` + SkipBcVersionCheck *bool `toml:"-"` + DatabaseHandles *int `toml:"-"` + DatabaseCache *int + DatabaseFreezer *string + TrieCleanCache *int + TrieCleanCacheJournal *string `toml:",omitempty"` + TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` + TrieDirtyCache *int + TrieTimeout *time.Duration + SnapshotCache *int + Preimages *bool + FilterLogCacheSize *int + Miner *miner.Config + Ethash *ethash.Config + TxPool *core.TxPoolConfig + GPO *gasprice.Config + EnablePreimageRecording *bool + DocRoot *string `toml:"-"` + RPCGasCap *uint64 + RPCEVMTimeout *time.Duration + RPCTxFeeCap *float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -250,6 +253,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.Preimages != nil { c.Preimages = *dec.Preimages } + if dec.FilterLogCacheSize != nil { + c.FilterLogCacheSize = *dec.FilterLogCacheSize + } if dec.Miner != nil { c.Miner = *dec.Miner } @@ -283,11 +289,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } - if dec.OverrideGrayGlacier != nil { - c.OverrideGrayGlacier = dec.OverrideGrayGlacier - } if dec.OverrideTerminalTotalDifficulty != nil { c.OverrideTerminalTotalDifficulty = dec.OverrideTerminalTotalDifficulty } + if dec.OverrideTerminalTotalDifficultyPassed != nil { + c.OverrideTerminalTotalDifficultyPassed = dec.OverrideTerminalTotalDifficultyPassed + } return nil } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 98a046a8c7e2..1493c1a7c2c6 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -692,7 +692,6 @@ func (f *BlockFetcher) loop() { } else { f.forgetHash(hash) } - } if matched { task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index bb1bd889f2c8..9c4a584e79c2 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -36,10 +36,14 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index b665ebf28020..85ac8c95de1c 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -120,7 +120,7 @@ type txDelivery struct { direct bool // Whether this is a direct reply or a broadcast } -// txDrop is the notiication that a peer has disconnected. +// txDrop is the notification that a peer has disconnected. type txDrop struct { peer string } @@ -260,7 +260,7 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { // Enqueue imports a batch of received transaction into the transaction pool // and the fetcher. This method may be called by both transaction broadcasts and // direct request replies. The differentiation is important so the fetcher can -// re-shedule missing transactions as soon as possible. +// re-schedule missing transactions as soon as possible. func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { // Keep track of all the propagated transactions if direct { @@ -558,7 +558,7 @@ func (f *TxFetcher) loop() { // In case of a direct delivery, also reschedule anything missing // from the original query if delivery.direct { - // Mark the reqesting successful (independent of individual status) + // Mark the requesting successful (independent of individual status) txRequestDoneMeter.Mark(int64(len(delivery.hashes))) // Make sure something was pending, nuke it @@ -607,7 +607,7 @@ func (f *TxFetcher) loop() { delete(f.alternates, hash) delete(f.fetching, hash) } - // Something was delivered, try to rechedule requests + // Something was delivered, try to reschedule requests f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too } @@ -719,7 +719,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { // should be rescheduled if some request is pending. In practice, a timeout will // cause the timer to be rescheduled every 5 secs (until the peer comes through or // disconnects). This is a limitation of the fetcher code because we don't trac -// pending requests and timed out requests separatey. Without double tracking, if +// pending requests and timed out requests separately. Without double tracking, if // we simply didn't reschedule the timer on all-timeout then the timer would never // be set again since len(request) > 0 => something's running. func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) { diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index b10275c3b7df..5d5ee90d1d6e 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -1011,7 +1011,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { } // Tests that dropping a peer cleans out all internal data structures in all the -// live or danglng stages. +// live or dangling stages. func TestTransactionFetcherDrop(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1121,7 +1121,7 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction timing out and clashing on readd with a concurrently +// dangling transaction timing out and clashing on re-add with a concurrently // announced one. func TestTransactionFetcherFuzzCrash01(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ @@ -1148,7 +1148,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction getting peer-dropped and clashing on readd with a +// dangling transaction getting peer-dropped and clashing on re-add with a // concurrently announced one. func TestTransactionFetcherFuzzCrash02(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ diff --git a/eth/filters/api.go b/eth/filters/api.go index 6463a189b061..43e63d5ba98a 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,7 +36,7 @@ import ( // and associated subscription in the event system. type filter struct { typ Type - deadline *time.Timer // filter is inactiv when deadline triggers + deadline *time.Timer // filter is inactive when deadline triggers hashes []common.Hash crit FilterCriteria logs []*types.Log @@ -46,7 +46,7 @@ type filter struct { // FilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // information related to the Ethereum protocol such als blocks, transactions and logs. type FilterAPI struct { - backend Backend + sys *FilterSystem events *EventSystem filtersMu sync.Mutex filters map[rpc.ID]*filter @@ -54,14 +54,14 @@ type FilterAPI struct { } // NewFilterAPI returns a new FilterAPI instance. -func NewFilterAPI(backend Backend, lightMode bool, timeout time.Duration) *FilterAPI { +func NewFilterAPI(system *FilterSystem, lightMode bool) *FilterAPI { api := &FilterAPI{ - backend: backend, - events: NewEventSystem(backend, lightMode), + sys: system, + events: NewEventSystem(system, lightMode), filters: make(map[rpc.ID]*filter), - timeout: timeout, + timeout: system.cfg.Timeout, } - go api.timeoutLoop(timeout) + go api.timeoutLoop(system.cfg.Timeout) return api } @@ -248,7 +248,6 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc } go func() { - for { select { case logs := <-matchedLogs: @@ -321,7 +320,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type var filter *Filter if crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *crit.BlockHash, crit.Addresses, crit.Topics) + filter = api.sys.NewBlockFilter(*crit.BlockHash, crit.Addresses, crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -333,7 +332,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type end = crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, crit.Addresses, crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -372,7 +371,7 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo var filter *Filter if f.crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewBlockFilter(*f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -384,7 +383,7 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo end = f.crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) diff --git a/eth/filters/api_test.go b/eth/filters/api_test.go index 02229a7549a7..0a80d0f8ddbd 100644 --- a/eth/filters/api_test.go +++ b/eth/filters/api_test.go @@ -56,7 +56,7 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { // from, to block number var test1 FilterCriteria - vector := fmt.Sprintf(`{"fromBlock":"0x%x","toBlock":"0x%x"}`, fromBlock, toBlock) + vector := fmt.Sprintf(`{"fromBlock":"%#x","toBlock":"%#x"}`, fromBlock, toBlock) if err := json.Unmarshal([]byte(vector), &test1); err != nil { t.Fatal(err) } diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 694d73735028..73b96b77af62 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -122,22 +122,27 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { b.Log("Running filter benchmarks...") start = time.Now() - var backend *testBackend + var ( + backend *testBackend + sys *FilterSystem + ) for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) backend = &testBackend{db: db, sections: cnt} + sys = NewFilterSystem(backend, Config{}) } var addr common.Address addr[0] = byte(i) addr[1] = byte(i / 256) - filter := NewRangeFilter(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) + filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) if _, err := filter.Logs(context.Background()); err != nil { - b.Error("filter.Find error:", err) + b.Error("filter.Logs error:", err) } } + d = time.Since(start) b.Log("Finished running filter benchmarks") b.Log(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") @@ -171,10 +176,11 @@ func BenchmarkNoBloomBits(b *testing.B) { clearBloomBits(db) + _, sys := newTestFilterSystem(b, db, Config{}) + b.Log("Running filter benchmarks...") start := time.Now() - backend := &testBackend{db: db} - filter := NewRangeFilter(backend, 0, int64(*headNum), []common.Address{{}}, nil) + filter := sys.NewRangeFilter(0, int64(*headNum), []common.Address{{}}, nil) filter.Logs(context.Background()) d := time.Since(start) b.Log("Finished running filter benchmarks") diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 9ff7ab7f55e1..0a70c9ece1db 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -22,37 +22,15 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" ) -type Backend interface { - ChainDb() ethdb.Database - HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) - HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) - GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) - - SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - - BloomStatus() (uint64, uint64) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) -} - // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend + sys *FilterSystem - db ethdb.Database addresses []common.Address topics [][]common.Hash @@ -64,7 +42,7 @@ type Filter struct { // NewRangeFilter creates a new filter which uses a bloom filter on blocks to // figure out whether a particular block is interesting or not. -func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { // Flatten the address and topic filter clauses into a single bloombits filter // system. Since the bloombits are not positional, nil topics are permitted, // which get flattened into a nil byte slice. @@ -83,10 +61,10 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres } filters = append(filters, filter) } - size, _ := backend.BloomStatus() + size, _ := sys.backend.BloomStatus() // Create a generic filter and convert it into a range filter - filter := newFilter(backend, addresses, topics) + filter := newFilter(sys, addresses, topics) filter.matcher = bloombits.NewMatcher(size, filters) filter.begin = begin @@ -97,21 +75,20 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres // NewBlockFilter creates a new filter which directly inspects the contents of // a block to figure out whether it is interesting or not. -func NewBlockFilter(backend Backend, block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { // Create a generic filter and convert it into a block filter - filter := newFilter(backend, addresses, topics) + filter := newFilter(sys, addresses, topics) filter.block = block return filter } // newFilter creates a generic filter that can either filter based on a block hash, // or based on range queries. The search criteria needs to be explicitly set. -func newFilter(backend Backend, addresses []common.Address, topics [][]common.Hash) *Filter { +func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common.Hash) *Filter { return &Filter{ - backend: backend, + sys: sys, addresses: addresses, topics: topics, - db: backend.ChainDb(), } } @@ -120,14 +97,14 @@ func newFilter(backend Backend, addresses []common.Address, topics [][]common.Ha func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { // If we're doing singleton block filtering, execute and return if f.block != (common.Hash{}) { - header, err := f.backend.HeaderByHash(ctx, f.block) + header, err := f.sys.backend.HeaderByHash(ctx, f.block) if err != nil { return nil, err } if header == nil { return nil, errors.New("unknown block") } - return f.blockLogs(ctx, header) + return f.blockLogs(ctx, header, false) } // Short-cut if all we care about is pending logs if f.begin == rpc.PendingBlockNumber.Int64() { @@ -137,7 +114,7 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return f.pendingLogs() } // Figure out the limits of the filter range - header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) + header, _ := f.sys.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) if header == nil { return nil, nil } @@ -156,7 +133,7 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { var ( logs []*types.Log err error - size, sections = f.backend.BloomStatus() + size, sections = f.sys.backend.BloomStatus() ) if indexed := sections * size; indexed > uint64(f.begin) { if indexed > end { @@ -192,7 +169,7 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err } defer session.Close() - f.backend.ServiceFilter(ctx, session) + f.sys.backend.ServiceFilter(ctx, session) // Iterate over the matches until exhausted or context closed var logs []*types.Log @@ -211,11 +188,11 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err f.begin = int64(number) + 1 // Retrieve the suggested block and pull any truly matching logs - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) if header == nil || err != nil { return logs, err } - found, err := f.checkMatches(ctx, header) + found, err := f.blockLogs(ctx, header, true) if err != nil { return logs, err } @@ -233,11 +210,11 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e var logs []*types.Log for ; f.begin <= int64(end); f.begin++ { - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) if header == nil || err != nil { return logs, err } - found, err := f.blockLogs(ctx, header) + found, err := f.blockLogs(ctx, header, false) if err != nil { return logs, err } @@ -247,34 +224,34 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e } // blockLogs returns the logs matching the filter criteria within a single block. -func (f *Filter) blockLogs(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - if bloomFilter(header.Bloom, f.addresses, f.topics) { - found, err := f.checkMatches(ctx, header) +func (f *Filter) blockLogs(ctx context.Context, header *types.Header, skipBloom bool) ([]*types.Log, error) { + // Fast track: no filtering criteria + if len(f.addresses) == 0 && len(f.topics) == 0 { + list, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { - return logs, err + return nil, err } - logs = append(logs, found...) + return flatten(list), nil + } else if skipBloom || bloomFilter(header.Bloom, f.addresses, f.topics) { + return f.checkMatches(ctx, header) } - return logs, nil + return nil, nil } // checkMatches checks if the receipts belonging to the given header contain any log events that // match the filter criteria. This function is called when the bloom filter signals a potential match. -func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - // Get the logs of the block - logsList, err := f.backend.GetLogs(ctx, header.Hash()) +func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*types.Log, error) { + logsList, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil, err } - var unfiltered []*types.Log - for _, logs := range logsList { - unfiltered = append(unfiltered, logs...) - } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + + unfiltered := flatten(logsList) + logs := filterLogs(unfiltered, nil, nil, f.addresses, f.topics) if len(logs) > 0 { // We have matching logs, check if we need to resolve full logs via the light client if logs[0].TxHash == (common.Hash{}) { - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + receipts, err := f.sys.backend.GetReceipts(ctx, header.Hash()) if err != nil { return nil, err } @@ -291,7 +268,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ // pendingLogs returns the logs matching the filter criteria within the pending block. func (f *Filter) pendingLogs() ([]*types.Log, error) { - block, receipts := f.backend.PendingBlockAndReceipts() + block, receipts := f.sys.backend.PendingBlockAndReceipts() if bloomFilter(block.Bloom(), f.addresses, f.topics) { var unfiltered []*types.Log for _, r := range receipts { @@ -376,3 +353,11 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo } return true } + +func flatten(list [][]*types.Log) []*types.Log { + var flat []*types.Log + for _, logs := range list { + flat = append(flat, logs...) + } + return flat +} diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index c1a1b408b7a7..79a9b089f422 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -27,13 +27,90 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" ) +// Config represents the configuration of the filter system. +type Config struct { + LogCacheSize int // maximum number of cached blocks (default: 32) + Timeout time.Duration // how long filters stay active (default: 5min) +} + +func (cfg Config) withDefaults() Config { + if cfg.Timeout == 0 { + cfg.Timeout = 5 * time.Minute + } + if cfg.LogCacheSize == 0 { + cfg.LogCacheSize = 32 + } + return cfg +} + +type Backend interface { + ChainDb() ethdb.Database + HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) + HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) + GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) + + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription + SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription + + BloomStatus() (uint64, uint64) + ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) +} + +// FilterSystem holds resources shared by all filters. +type FilterSystem struct { + backend Backend + logsCache *lru.Cache + cfg *Config +} + +// NewFilterSystem creates a filter system. +func NewFilterSystem(backend Backend, config Config) *FilterSystem { + config = config.withDefaults() + + cache, err := lru.New(config.LogCacheSize) + if err != nil { + panic(err) + } + return &FilterSystem{ + backend: backend, + logsCache: cache, + cfg: &config, + } +} + +// cachedGetLogs loads block logs from the backend and caches the result. +func (sys *FilterSystem) cachedGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + cached, ok := sys.logsCache.Get(blockHash) + if ok { + return cached.([][]*types.Log), nil + } + + logs, err := sys.backend.GetLogs(ctx, blockHash, number) + if err != nil { + return nil, err + } + if logs == nil { + return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", number, blockHash.TerminalString()) + } + sys.logsCache.Add(blockHash, logs) + return logs, nil +} + // Type determines the kind of filter and is used to put the filter in to // the correct bucket when added. type Type byte @@ -84,6 +161,7 @@ type subscription struct { // subscription which match the subscription criteria. type EventSystem struct { backend Backend + sys *FilterSystem lightMode bool lastHead *types.Header @@ -110,9 +188,10 @@ type EventSystem struct { // // The returned manager has a loop that needs to be stopped with the Stop function // or by stopping the given mux. -func NewEventSystem(backend Backend, lightMode bool) *EventSystem { +func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - backend: backend, + sys: sys, + backend: sys.backend, lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), @@ -405,7 +484,7 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - logsList, err := es.backend.GetLogs(ctx, header.Hash()) + logsList, err := es.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 9aa758b006a3..a9da7c1406ab 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -39,10 +39,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var ( - deadline = 5 * time.Minute -) - type testBackend struct { db ethdb.Database sections uint64 @@ -91,17 +87,8 @@ func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types. return nil, nil } -func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(b.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(b.db, hash, *number, params.TestChainConfig) - - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(b.db, hash, number, params.TestChainConfig) return logs, nil } @@ -160,6 +147,12 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { + backend := &testBackend{db: db} + sys := NewFilterSystem(backend, cfg) + return backend, sys +} + // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // It creates multiple subscriptions: // - one at the start and should receive all posted chain events and a second (blockHashes) @@ -173,12 +166,12 @@ func TestBlockSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) - genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) - chainEvents = []core.ChainEvent{} + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) + genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) + chainEvents = []core.ChainEvent{} ) for _, blk := range chain { @@ -229,9 +222,9 @@ func TestPendingTxFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil), @@ -284,9 +277,9 @@ func TestPendingTxFilter(t *testing.T) { // If not it must return an error. func TestLogFilterCreation(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) testCases = []struct { crit FilterCriteria @@ -331,9 +324,9 @@ func TestInvalidLogFilterCreation(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) ) // different situations where log filter creation should fail. @@ -354,8 +347,8 @@ func TestInvalidLogFilterCreation(t *testing.T) { func TestInvalidGetLogsRequest(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) blockHash = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") ) @@ -382,9 +375,9 @@ func TestLogFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -500,9 +493,9 @@ func TestPendingLogsSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -684,10 +677,10 @@ func TestPendingTxFilterDeadlock(t *testing.T) { timeout := 100 * time.Millisecond var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, timeout) - done = make(chan struct{}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{Timeout: timeout}) + api = NewFilterAPI(sys, false) + done = make(chan struct{}) ) go func() { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index f415046a82aa..2c1f7cadf43a 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -44,16 +44,23 @@ func BenchmarkFilters(b *testing.B) { var ( db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + _, sys = newTestFilterSystem(b, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) addr3 = common.BytesToAddress([]byte("ethereum")) addr4 = common.BytesToAddress([]byte("random addresses please")) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.ToBlock() ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr1, big.NewInt(1000000)) + gspec.MustCommit(db) + chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 100010, func(i int, gen *core.BlockGen) { switch i { case 2403: @@ -72,7 +79,6 @@ func BenchmarkFilters(b *testing.B) { receipt := makeReceipt(addr4) gen.AddUncheckedReceipt(receipt) gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) - } }) for i, block := range chain { @@ -83,7 +89,7 @@ func BenchmarkFilters(b *testing.B) { } b.ResetTimer() - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { logs, _ := filter.Logs(context.Background()) @@ -98,7 +104,7 @@ func TestFilters(t *testing.T) { var ( db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + _, sys = newTestFilterSystem(t, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -106,10 +112,17 @@ func TestFilters(t *testing.T) { hash2 = common.BytesToHash([]byte("topic2")) hash3 = common.BytesToHash([]byte("topic3")) hash4 = common.BytesToHash([]byte("topic4")) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.ToBlock() ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr, big.NewInt(1000000)) + gspec.MustCommit(db) + chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { switch i { case 1: @@ -162,14 +175,14 @@ func TestFilters(t *testing.T) { rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } - filter = NewRangeFilter(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -178,7 +191,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -187,7 +200,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) + filter = sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 2 { @@ -195,7 +208,7 @@ func TestFilters(t *testing.T) { } failHash := common.BytesToHash([]byte("fail")) - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { @@ -203,14 +216,14 @@ func TestFilters(t *testing.T) { } failAddr := common.BytesToAddress([]byte("failmenow")) - filter = NewRangeFilter(backend, 0, -1, []common.Address{failAddr}, nil) + filter = sys.NewRangeFilter(0, -1, []common.Address{failAddr}, nil) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 2c0998702422..6028ef03cf15 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -137,44 +137,68 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, reqEnd rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( - headBlock rpc.BlockNumber + headBlock *types.Header pendingBlock *types.Block pendingReceipts types.Receipts + err error ) - // query either pending block or head header and set headBlock - if lastBlock == rpc.PendingBlockNumber { - if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { - lastBlock = rpc.BlockNumber(pendingBlock.NumberU64()) - headBlock = lastBlock - 1 - } else { - // pending block not supported by backend, process until latest block - lastBlock = rpc.LatestBlockNumber - blocks-- - if blocks == 0 { - return nil, nil, 0, 0, nil + + // Get the chain's current head. + if headBlock, err = oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err != nil { + return nil, nil, 0, 0, err + } + head := rpc.BlockNumber(headBlock.Number.Uint64()) + + // Fail if request block is beyond the chain's current head. + if head < reqEnd { + return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, reqEnd, head) + } + + // Resolve block tag. + if reqEnd < 0 { + var ( + resolved *types.Header + err error + ) + switch reqEnd { + case rpc.PendingBlockNumber: + if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { + resolved = pendingBlock.Header() + } else { + // Pending block not supported by backend, process only until latest block. + resolved = headBlock + + // Update total blocks to return to account for this. + blocks-- } + case rpc.LatestBlockNumber: + // Retrieved above. + resolved = headBlock + case rpc.SafeBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.SafeBlockNumber) + case rpc.FinalizedBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.FinalizedBlockNumber) + case rpc.EarliestBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.EarliestBlockNumber) } - } - if pendingBlock == nil { - // if pending block is not fetched then we retrieve the head header to get the head block number - if latestHeader, err := oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err == nil { - headBlock = rpc.BlockNumber(latestHeader.Number.Uint64()) - } else { + if resolved == nil || err != nil { return nil, nil, 0, 0, err } + // Absolute number resolved. + reqEnd = rpc.BlockNumber(resolved.Number.Uint64()) } - if lastBlock == rpc.LatestBlockNumber { - lastBlock = headBlock - } else if pendingBlock == nil && lastBlock > headBlock { - return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock) + + // If there are no blocks to return, short circuit. + if blocks == 0 { + return nil, nil, 0, 0, nil } - // ensure not trying to retrieve before genesis - if rpc.BlockNumber(blocks) > lastBlock+1 { - blocks = int(lastBlock + 1) + // Ensure not trying to retrieve before genesis. + if int(reqEnd+1) < blocks { + blocks = int(reqEnd + 1) } - return pendingBlock, pendingReceipts, uint64(lastBlock), blocks, nil + return pendingBlock, pendingReceipts, uint64(reqEnd), blocks, nil } // FeeHistory returns data relevant for fee estimation based on the specified range of blocks. diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index c259eb0acf76..deece7f46150 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -50,6 +50,8 @@ func TestFeeHistory(t *testing.T) { {false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, + {false, 1000, 1000, 2, rpc.FinalizedBlockNumber, []float64{0, 10}, 24, 2, nil}, + {false, 1000, 1000, 2, rpc.SafeBlockNumber, []float64{0, 10}, 24, 2, nil}, } for i, c := range cases { config := Config{ diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 95a908fc1e9c..d405188f8194 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -45,6 +45,15 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock().Header(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock().Header(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -62,6 +71,15 @@ func (b *testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -109,12 +127,11 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke config.LondonBlock = londonBlock config.ArrowGlacierBlock = londonBlock config.GrayGlacierBlock = londonBlock + config.TerminalTotalDifficulty = common.Big0 engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis, err := gspec.Commit(db) - if err != nil { - t.Fatal(err) - } + genesis := gspec.MustCommit(db) + // Generate testing blocks blocks, _ := core.GenerateChain(gspec.Config, genesis, engine, db, testHead+1, func(i int, b *core.BlockGen) { b.SetCoinbase(common.Address{1}) @@ -144,12 +161,14 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke }) // Construct testing chain diskdb := rawdb.NewMemoryDatabase() - gspec.Commit(diskdb) + gspec.MustCommit(diskdb) chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec.Config, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create local chain, %v", err) } chain.InsertChain(blocks) + chain.SetFinalized(chain.GetBlockByNumber(25)) + chain.SetSafe(chain.GetBlockByNumber(25)) return &testBackend{chain: chain, pending: pending} } diff --git a/eth/handler.go b/eth/handler.go index 5c881aebee8c..409ff047234d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -193,11 +193,22 @@ func newHandler(config *handlerConfig) (*handler, error) { } } } - // Construct the downloader (long sync) and its backing state bloom if snap - // sync is requested. The downloader is responsible for deallocating the state - // bloom when it's done. + // Construct the downloader (long sync) h.downloader = downloader.New(h.checkpointNumber, config.Database, h.eventMux, h.chain, nil, h.removePeer, success) - + if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { + if h.chain.Config().TerminalTotalDifficultyPassed { + log.Info("Chain post-merge, sync via beacon client") + } else { + head := h.chain.CurrentBlock() + if td := h.chain.GetTd(head.Hash(), head.NumberU64()); td.Cmp(ttd) >= 0 { + log.Info("Chain post-TTD, sync via beacon client") + } else { + log.Warn("Chain pre-merge, sync via PoW (ensure beacon client is ready)") + } + } + } else if h.chain.Config().TerminalTotalDifficultyPassed { + log.Error("Chain configured post-merge, but without TTD. Are you debugging sync?") + } // Construct the fetcher (short sync) validator := func(header *types.Header) error { // All the block fetcher activities should be disabled @@ -251,7 +262,7 @@ func newHandler(config *handlerConfig) (*handler, error) { // out a way yet where nodes can decide unilaterally whether the network is new // or not. This should be fixed if we figure out a solution. if atomic.LoadUint32(&h.snapSync) == 1 { - log.Warn("Fast syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) + log.Warn("Snap syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) return 0, nil } if h.merger.TDDReached() { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 2550ef63a202..e893810082ec 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -494,7 +494,6 @@ func TestCheckpointChallenge(t *testing.T) { } func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpoint bool, timeout bool, empty bool, match bool, drop bool) { - // Reduce the checkpoint handshake challenge timeout defer func(old time.Duration) { syncChallengeTimeout = old }(syncChallengeTimeout) syncChallengeTimeout = 250 * time.Millisecond diff --git a/eth/peerset.go b/eth/peerset.go index c48fba050446..5f5864dc33af 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -43,7 +43,7 @@ var ( errPeerNotRegistered = errors.New("peer not registered") // errSnapWithoutEth is returned if a peer attempts to connect only on the - // snap protocol without advertizing the eth main protocol. + // snap protocol without advertising the eth main protocol. errSnapWithoutEth = errors.New("peer connected on snap without compatible eth support") ) diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 09330cfdf320..6fc15f136bff 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -36,7 +36,7 @@ type blockPropagation struct { td *big.Int } -// broadcastBlocks is a write loop that multiplexes blocks and block accouncements +// broadcastBlocks is a write loop that multiplexes blocks and block announcements // to the remote peer. The goal is to have an async writer that does not lock up // node internals and at the same time rate limits queued data. func (p *Peer) broadcastBlocks() { diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index bf88d400d4a0..65a935d55548 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -224,7 +224,7 @@ func (p *Peer) dispatcher() { switch { case res.Req == nil: // Response arrived with an untracked ID. Since even cancelled - // requests are tracked until fulfilment, a dangling repsponse + // requests are tracked until fulfilment, a dangling response // means the remote peer implements the protocol badly. resOp.fail <- errDanglingResponse diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index bf836e8f5132..2707a420bc6a 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -94,7 +94,7 @@ func (b *testBackend) Chain() *core.BlockChain { return b.chain } func (b *testBackend) TxPool() TxPool { return b.txpool } func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { - // Normally the backend would do peer mainentance and handshakes. All that + // Normally the backend would do peer maintenance and handshakes. All that // is omitted and we will just give control back to the handler. return handler(peer) } diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 46fac17a7954..500b20d27362 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -134,7 +134,7 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `eth` protocol version. +// Version retrieves the peer's negotiated `eth` protocol version. func (p *Peer) Version() uint { return p.version } diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index c03ccdbc8882..6ee1e3d92a6d 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -116,12 +116,10 @@ func TestEth66EmptyMessages(t *testing.T) { t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, msg, have, want) } } - } // TestEth66Messages tests the encoding of all redefined eth66 messages func TestEth66Messages(t *testing.T) { - // Some basic structs used during testing var ( header *types.Header diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 7ecf041e9a54..41380d96f571 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -23,14 +23,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -415,15 +413,15 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB()) + accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } - var acc types.StateAccount - if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { + acc, err := accTrie.TryGetAccountWithPreHashedKey(account[:]) + if err != nil || acc == nil { return nil, nil } - stTrie, err := trie.New(account, acc.Root, chain.StateCache().TrieDB()) + stTrie, err := trie.NewStateTrie(account, acc.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -489,7 +487,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s // Make sure we have the state associated with the request triedb := chain.StateCache().TrieDB() - accTrie, err := trie.NewSecure(common.Hash{}, req.Root, triedb) + accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, triedb) if err != nil { // We don't have the requested state available, bail out return nil, nil @@ -506,7 +504,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s var ( nodes [][]byte bytes uint64 - loads int // Trie hash expansions to cound database reads + loads int // Trie hash expansions to count database reads ) for _, pathset := range req.Paths { switch len(pathset) { @@ -531,7 +529,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if err != nil || account == nil { break } - stTrie, err := trie.NewSecure(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) + stTrie, err := trie.NewStateTrie(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) loads++ // always account database reads, even for failures if err != nil { break diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 87a62d2f8a41..235d499ffdc9 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -61,12 +61,12 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `snap` protocol version. +// Version retrieves the peer's negotiated `snap` protocol version. func (p *Peer) Version() uint { return p.version } -// Log overrides the P2P logget with the higher level one containing only the id. +// Log overrides the P2P logger with the higher level one containing only the id. func (p *Peer) Log() log.Logger { return p.logger } @@ -87,7 +87,7 @@ func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit co } // RequestStorageRange fetches a batch of storage slots belonging to one or more -// accounts. If slots from only one accout is requested, an origin marker may also +// accounts. If slots from only one account is requested, an origin marker may also // be used to retrieve from there. func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { if len(accounts) == 1 && origin != nil { @@ -119,7 +119,7 @@ func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) e } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) diff --git a/eth/protocols/snap/sort_test.go b/eth/protocols/snap/sort_test.go index 49730c886e7f..be0a8c570696 100644 --- a/eth/protocols/snap/sort_test.go +++ b/eth/protocols/snap/sort_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/trie" ) func hexToNibbles(s string) []byte { @@ -38,22 +37,17 @@ func hexToNibbles(s string) []byte { } func TestRequestSorting(t *testing.T) { - // - Path 0x9 -> {0x19} // - Path 0x99 -> {0x0099} // - Path 0x01234567890123456789012345678901012345678901234567890123456789019 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x19} // - Path 0x012345678901234567890123456789010123456789012345678901234567890199 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x0099} - var f = func(path string) (trie.SyncPath, TrieNodePathSet, common.Hash) { + var f = func(path string) string { data := hexToNibbles(path) - sp := trie.NewSyncPath(data) - tnps := TrieNodePathSet([][]byte(sp)) - hash := common.Hash{} - return sp, tnps, hash + return string(data) } var ( - hashes []common.Hash - paths []trie.SyncPath - pathsets []TrieNodePathSet + hashes []common.Hash + paths []string ) for _, x := range []string{ "0x9", @@ -67,15 +61,14 @@ func TestRequestSorting(t *testing.T) { "0x01234567890123456789012345678901012345678901234567890123456789010", "0x01234567890123456789012345678901012345678901234567890123456789011", } { - sp, _, hash := f(x) - hashes = append(hashes, hash) - paths = append(paths, sp) + paths = append(paths, f(x)) + hashes = append(hashes, common.Hash{}) } - _, paths, pathsets = sortByAccountPath(hashes, paths) + _, _, syncPaths, pathsets := sortByAccountPath(paths, hashes) { var b = new(bytes.Buffer) - for i := 0; i < len(paths); i++ { - fmt.Fprintf(b, "\n%d. paths %x", i, paths[i]) + for i := 0; i < len(syncPaths); i++ { + fmt.Fprintf(b, "\n%d. paths %x", i, syncPaths[i]) } want := ` 0. paths [0099] diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index d68e728ff211..deaa4456e0fd 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -230,8 +230,8 @@ type trienodeHealRequest struct { timeout *time.Timer // Timer to track delivery timeout stale chan struct{} // Channel to signal the request was dropped - hashes []common.Hash // Trie node hashes to validate responses - paths []trie.SyncPath // Trie node paths requested for rescheduling + paths []string // Trie node paths for identifying trie node + hashes []common.Hash // Trie node hashes to validate responses task *healTask // Task which this request is filling (only access fields through the runloop!!) } @@ -240,9 +240,9 @@ type trienodeHealRequest struct { type trienodeHealResponse struct { task *healTask // Task which this request is filling - hashes []common.Hash // Hashes of the trie nodes to avoid double hashing - paths []trie.SyncPath // Trie node paths requested for rescheduling missing ones - nodes [][]byte // Actual trie nodes to store into the database (nil = missing) + paths []string // Paths of the trie nodes + hashes []common.Hash // Hashes of the trie nodes to avoid double hashing + nodes [][]byte // Actual trie nodes to store into the database (nil = missing) } // bytecodeHealRequest tracks a pending bytecode request to ensure responses are to @@ -321,8 +321,8 @@ type storageTask struct { type healTask struct { scheduler *trie.Sync // State trie sync scheduler defining the tasks - trieTasks map[common.Hash]trie.SyncPath // Set of trie node tasks currently queued for retrieval - codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval + trieTasks map[string]common.Hash // Set of trie node tasks currently queued for retrieval, indexed by node path + codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval, indexed by code hash } // SyncProgress is a database entry to allow suspending and resuming a snapshot state @@ -365,7 +365,7 @@ type SyncPeer interface { RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error // RequestStorageRanges fetches a batch of storage slots belonging to one or - // more accounts. If slots from only one accout is requested, an origin marker + // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error @@ -373,7 +373,7 @@ type SyncPeer interface { RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in - // a specificstate trie. + // a specific state trie. RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error // Log retrieves the peer's own contextual logger. @@ -540,7 +540,7 @@ func (s *Syncer) Unregister(id string) error { return nil } -// Sync starts (or resumes a previous) sync cycle to iterate over an state trie +// Sync starts (or resumes a previous) sync cycle to iterate over a state trie // with the given root and reconstruct the nodes based on the snapshot leaves. // Previously downloaded segments will not be redownloaded of fixed, rather any // errors will be healed after the leaves are fully accumulated. @@ -551,7 +551,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { s.root = root s.healer = &healTask{ scheduler: state.NewStateSync(root, s.db, s.onHealState), - trieTasks: make(map[common.Hash]trie.SyncPath), + trieTasks: make(map[string]common.Hash), codeTasks: make(map[common.Hash]struct{}), } s.statelessPeers = make(map[string]struct{}) @@ -743,7 +743,7 @@ func (s *Syncer) loadSyncStatus() { return } } - // Either we've failed to decode the previus state, or there was none. + // Either we've failed to decode the previous state, or there was none. // Start a fresh sync by chunking up the account range and scheduling // them for retrieval. s.tasks = nil @@ -1183,10 +1183,10 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st } if subtask == nil { // No large contract required retrieval, but small ones available - for acccount, root := range task.stateTasks { - delete(task.stateTasks, acccount) + for account, root := range task.stateTasks { + delete(task.stateTasks, account) - accounts = append(accounts, acccount) + accounts = append(accounts, account) roots = append(roots, root) if len(accounts) >= storageSets { @@ -1280,9 +1280,9 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai want = maxTrieRequestCount + maxCodeRequestCount ) if have < want { - nodes, paths, codes := s.healer.scheduler.Missing(want - have) - for i, hash := range nodes { - s.healer.trieTasks[hash] = paths[i] + paths, hashes, codes := s.healer.scheduler.Missing(want - have) + for i, path := range paths { + s.healer.trieTasks[path] = hashes[i] } for _, hash := range codes { s.healer.codeTasks[hash] = struct{}{} @@ -1323,21 +1323,20 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai } var ( hashes = make([]common.Hash, 0, cap) - paths = make([]trie.SyncPath, 0, cap) + paths = make([]string, 0, cap) pathsets = make([]TrieNodePathSet, 0, cap) ) - for hash, pathset := range s.healer.trieTasks { - delete(s.healer.trieTasks, hash) + for path, hash := range s.healer.trieTasks { + delete(s.healer.trieTasks, path) + paths = append(paths, path) hashes = append(hashes, hash) - paths = append(paths, pathset) - - if len(hashes) >= cap { + if len(paths) >= cap { break } } // Group requests by account hash - hashes, paths, pathsets = sortByAccountPath(hashes, paths) + paths, hashes, _, pathsets = sortByAccountPath(paths, hashes) req := &trienodeHealRequest{ peer: idle, id: reqid, @@ -1346,8 +1345,8 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai revert: fail, cancel: cancel, stale: make(chan struct{}), - hashes: hashes, paths: paths, + hashes: hashes, task: s.healer, } req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { @@ -1405,9 +1404,9 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai want = maxTrieRequestCount + maxCodeRequestCount ) if have < want { - nodes, paths, codes := s.healer.scheduler.Missing(want - have) - for i, hash := range nodes { - s.healer.trieTasks[hash] = paths[i] + paths, hashes, codes := s.healer.scheduler.Missing(want - have) + for i, path := range paths { + s.healer.trieTasks[path] = hashes[i] } for _, hash := range codes { s.healer.codeTasks[hash] = struct{}{} @@ -1487,7 +1486,7 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *Syncer) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -1576,7 +1575,7 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the account - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.task.req == req { req.task.req = nil @@ -1617,7 +1616,7 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -1658,7 +1657,7 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the storage - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.subTask != nil { req.subTask.req = nil @@ -1703,10 +1702,10 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the trie node - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() - for i, hash := range req.hashes { - req.task.trieTasks[hash] = req.paths[i] + for i, path := range req.paths { + req.task.trieTasks[path] = req.hashes[i] } } @@ -1744,7 +1743,7 @@ func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -2036,7 +2035,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { } tr.Commit() } - // Persist the received storage segements. These flat state maybe + // Persist the received storage segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. for j := 0; j < len(res.hashes[i]); j++ { @@ -2096,14 +2095,14 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { // If the trie node was not delivered, reschedule it if node == nil { - res.task.trieTasks[hash] = res.paths[i] + res.task.trieTasks[res.paths[i]] = res.hashes[i] continue } // Push the trie node into the state syncer s.trienodeHealSynced++ s.trienodeHealBytes += common.StorageSize(len(node)) - err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + err := s.healer.scheduler.ProcessNode(trie.NodeSyncResult{Path: res.paths[i], Data: node}) switch err { case nil: case trie.ErrAlreadyProcessed: @@ -2139,7 +2138,7 @@ func (s *Syncer) processBytecodeHealResponse(res *bytecodeHealResponse) { s.bytecodeHealSynced++ s.bytecodeHealBytes += common.StorageSize(len(node)) - err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + err := s.healer.scheduler.ProcessCode(trie.CodeSyncResult{Hash: hash, Data: node}) switch err { case nil: case trie.ErrAlreadyProcessed: @@ -2171,7 +2170,7 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { } task.res = nil - // Persist the received account segements. These flat state maybe + // Persist the received account segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. oldAccountBytes := s.accountBytes @@ -2666,9 +2665,9 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error } // Response validated, send it to the scheduler for filling response := &trienodeHealResponse{ + paths: req.paths, task: req.task, hashes: req.hashes, - paths: req.paths, nodes: nodes, } select { @@ -2774,7 +2773,7 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e } // onHealState is a callback method to invoke when a flat state(account -// or storage slot) is downloded during the healing stage. The flat states +// or storage slot) is downloaded during the healing stage. The flat states // can be persisted blindly and can be fixed later in the generation stage. // Note it's not concurrent safe, please handle the concurrent issue outside. func (s *Syncer) onHealState(paths [][]byte, value []byte) error { @@ -2913,8 +2912,9 @@ func (s *capacitySort) Swap(i, j int) { // healRequestSort implements the Sort interface, allowing sorting trienode // heal requests, which is a prerequisite for merging storage-requests. type healRequestSort struct { - hashes []common.Hash - paths []trie.SyncPath + paths []string + hashes []common.Hash + syncPaths []trie.SyncPath } func (t *healRequestSort) Len() int { @@ -2922,8 +2922,8 @@ func (t *healRequestSort) Len() int { } func (t *healRequestSort) Less(i, j int) bool { - a := t.paths[i] - b := t.paths[j] + a := t.syncPaths[i] + b := t.syncPaths[j] switch bytes.Compare(a[0], b[0]) { case -1: return true @@ -2944,8 +2944,9 @@ func (t *healRequestSort) Less(i, j int) bool { } func (t *healRequestSort) Swap(i, j int) { - t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i] t.paths[i], t.paths[j] = t.paths[j], t.paths[i] + t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i] + t.syncPaths[i], t.syncPaths[j] = t.syncPaths[j], t.syncPaths[i] } // Merge merges the pathsets, so that several storage requests concerning the @@ -2953,7 +2954,7 @@ func (t *healRequestSort) Swap(i, j int) { // OBS: This operation is moot if t has not first been sorted. func (t *healRequestSort) Merge() []TrieNodePathSet { var result []TrieNodePathSet - for _, path := range t.paths { + for _, path := range t.syncPaths { pathset := TrieNodePathSet([][]byte(path)) if len(path) == 1 { // It's an account reference. @@ -2962,7 +2963,7 @@ func (t *healRequestSort) Merge() []TrieNodePathSet { // It's a storage reference. end := len(result) - 1 if len(result) == 0 || !bytes.Equal(pathset[0], result[end][0]) { - // The account doesn't doesn't match last, create a new entry. + // The account doesn't match last, create a new entry. result = append(result, pathset) } else { // It's the same account as the previous one, add to the storage @@ -2976,9 +2977,13 @@ func (t *healRequestSort) Merge() []TrieNodePathSet { // sortByAccountPath takes hashes and paths, and sorts them. After that, it generates // the TrieNodePaths and merges paths which belongs to the same account path. -func sortByAccountPath(hashes []common.Hash, paths []trie.SyncPath) ([]common.Hash, []trie.SyncPath, []TrieNodePathSet) { - n := &healRequestSort{hashes, paths} +func sortByAccountPath(paths []string, hashes []common.Hash) ([]string, []common.Hash, []trie.SyncPath, []TrieNodePathSet) { + var syncPaths []trie.SyncPath + for _, path := range paths { + syncPaths = append(syncPaths, trie.NewSyncPath([]byte(path))) + } + n := &healRequestSort{paths, hashes, syncPaths} sort.Sort(n) pathsets := n.Merge() - return n.hashes, n.paths, pathsets + return n.paths, n.hashes, n.syncPaths, pathsets } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 5f37bfb3137d..168371e481c3 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1361,9 +1361,11 @@ func getCodeByHash(hash common.Hash) []byte { // makeAccountTrieNoStorage spits out a trie, along with the leafs func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { - db := trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie := trie.NewEmpty(db) - var entries entrySlice + var ( + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie = trie.NewEmpty(db) + entries entrySlice + ) for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, @@ -1377,7 +1379,13 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { entries = append(entries, elem) } sort.Sort(entries) - accTrie.Commit(nil) + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(common.Hash{}, root, db) return accTrie, entries } @@ -1389,8 +1397,8 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { entries entrySlice boundaries []common.Hash - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - trie = trie.NewEmpty(db) + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie = trie.NewEmpty(db) ) // Initialize boundaries var next common.Hash @@ -1417,7 +1425,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(uint64(i)), }) elem := &kv{boundaries[i].Bytes(), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } // Fill other accounts if required @@ -1429,12 +1437,18 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(i), }) elem := &kv{key32(i), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(common.Hash{}, root, db) + return accTrie, entries } // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts @@ -1444,8 +1458,10 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { @@ -1455,9 +1471,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) codehash = getCodeHash(i) } // Create a storage trie - stTrie, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db) - stRoot := stTrie.Hash() - stTrie.Commit(nil) + stRoot, stNodes, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db) + nodes.Merge(stNodes) + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), @@ -1468,12 +1484,25 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) accTrie.Update(elem.k, elem.v) entries = append(entries, elem) - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - accTrie.Commit(nil) + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, _ = trie.New(common.Hash{}, root, db) + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + trie, _ := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } @@ -1483,8 +1512,10 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { @@ -1495,16 +1526,16 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie } // Make a storage trie var ( - stTrie *trie.Trie + stRoot common.Hash + stNodes *trie.NodeSet stEntries entrySlice ) if boundary { - stTrie, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) + stRoot, stNodes, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) } else { - stTrie, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) + stRoot, stNodes, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) } - stRoot := stTrie.Hash() - stTrie.Commit(nil) + nodes.Merge(stNodes) value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, @@ -1515,19 +1546,40 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie elem := &kv{key, value} accTrie.Update(elem.k, elem.v) entries = append(entries, elem) + // we reuse the same one for all accounts - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - accTrie.Commit(nil) + + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, err := trie.New(common.Hash{}, root, db) + if err != nil { + panic(err) + } + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + trie, err := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + if err != nil { + panic(err) + } + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { trie, _ := trie.New(owner, common.Hash{}, db) var entries entrySlice for i := uint64(1); i <= n; i++ { @@ -1543,14 +1595,14 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie.Trie, entrySlice) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { var ( entries entrySlice boundaries []common.Hash @@ -1594,8 +1646,8 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { @@ -1619,7 +1671,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } accounts++ if acc.Root != emptyRoot { - storeTrie, err := trie.NewSecure(common.BytesToHash(accIt.Key), acc.Root, triedb) + storeTrie, err := trie.NewStateTrie(common.BytesToHash(accIt.Key), acc.Root, triedb) if err != nil { t.Fatal(err) } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 98be012f3326..b76fc47d75b9 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -44,7 +44,7 @@ import ( // perform Commit or other 'save-to-disk' changes, this should be set to false to avoid // storing trash persistently // - preferDisk: this arg can be used by the caller to signal that even though the 'base' is provided, -// it would be preferrable to start from a fresh state, if we have it on disk. +// it would be preferable to start from a fresh state, if we have it on disk. func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { var ( current *types.Block diff --git a/eth/sync.go b/eth/sync.go index fb01bf96463a..b813add06e4d 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -164,7 +164,7 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { // An alternative would be to check the local chain for exceeding the TTD and // avoid triggering a sync in that case, but that could also miss sibling or // other family TTD block being accepted. - if cs.handler.merger.TDDReached() { + if cs.handler.chain.Config().TerminalTotalDifficultyPassed || cs.handler.merger.TDDReached() { return nil } // Ensure we're at minimum peer count. diff --git a/eth/tracers/api.go b/eth/tracers/api.go index b0f30567f025..092950e78fa9 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -20,6 +20,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -115,7 +116,7 @@ func (context *chainContext) GetHeader(hash common.Hash, number uint64) *types.H return header } -// chainContext construts the context reader which is used by the evm for reading +// chainContext constructs the context reader which is used by the evm for reading // the necessary chain context. func (api *API) chainContext(ctx context.Context) core.ChainContext { return &chainContext{api: api, ctx: ctx} @@ -169,15 +170,15 @@ type TraceConfig struct { Tracer *string Timeout *string Reexec *uint64 + // Config specific to given tracer. Note struct logger + // config are historically embedded in main object. + TracerConfig json.RawMessage } // TraceCallConfig is the config for traceCall API. It holds one more // field to override the state for tracing. type TraceCallConfig struct { - *logger.Config - Tracer *string - Timeout *string - Reexec *uint64 + TraceConfig StateOverrides *ethapi.StateOverride BlockOverrides *ethapi.BlockOverrides } @@ -201,10 +202,10 @@ type blockTraceTask struct { statedb *state.StateDB // Intermediate state prepped for tracing block *types.Block // Block to trace the transactions from rootref common.Hash // Trie root reference held for this task - results []*txTraceResult // Trace results procudes by the task + results []*txTraceResult // Trace results produced by the task } -// blockTraceResult represets the results of tracing a single block when an entire +// blockTraceResult represents the results of tracing a single block when an entire // chain is being traced. type blockTraceResult struct { Block hexutil.Uint64 `json:"block"` // Block number corresponding to this trace @@ -562,7 +563,7 @@ func (api *API) StandardTraceBadBlockToFile(ctx context.Context, hash common.Has // traceBlock configures a new tracer according to the provided configuration, and // executes all the transactions contained within. The return value will be one item -// per transaction, dependent on the requestd tracer. +// per transaction, dependent on the requested tracer. func (api *API) traceBlock(ctx context.Context, block *types.Block, config *TraceConfig) ([]*txTraceResult, error) { if block.NumberU64() == 0 { return nil, errors.New("genesis is not traceable") @@ -706,7 +707,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } } for i, tx := range block.Transactions() { - // Prepare the trasaction for un-traced execution + // Prepare the transaction for un-traced execution var ( msg, _ = tx.AsMessage(signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) @@ -882,7 +883,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex // Default tracer is the struct logger tracer = logger.NewStructLogger(config.Config) if config.Tracer != nil { - tracer, err = New(*config.Tracer, txctx) + tracer, err = New(*config.Tracer, txctx, config.TracerConfig) if err != nil { return nil, err } @@ -918,7 +919,6 @@ func APIs(backend Backend) []rpc.API { return []rpc.API{ { Namespace: "debug", - Version: "1.0", Service: NewAPI(backend), }, } diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index cbf20ed00c0c..d2c50656d9a8 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -39,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - // Force-load native and js pacakges, to trigger registration + // Force-load native and js packages, to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) @@ -118,10 +118,11 @@ type callTrace struct { // callTracerTest defines a single test to check the call tracer against. type callTracerTest struct { - Genesis *core.Genesis `json:"genesis"` - Context *callContext `json:"context"` - Input string `json:"input"` - Result *callTrace `json:"result"` + Genesis *core.Genesis `json:"genesis"` + Context *callContext `json:"context"` + Input string `json:"input"` + TracerConfig json.RawMessage `json:"tracerConfig"` + Result *callTrace `json:"result"` } // Iterates over all the input-output datasets in the tracer test harness and @@ -179,7 +180,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) ) - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -293,7 +294,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), nil) if err != nil { b.Fatalf("failed to create call tracer: %v", err) } @@ -359,7 +360,7 @@ func TestZeroValueToNotExitCall(t *testing.T) { } _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := tracers.New("callTracer", nil) + tracer, err := tracers.New("callTracer", nil, nil) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json new file mode 100644 index 000000000000..ac1fef44098e --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json @@ -0,0 +1,72 @@ +{ + "context": { + "difficulty": "3502894804", + "gasLimit": "4722976", + "miner": "0x1585936b53834b021f68cc13eeefdec2efc8e724", + "number": "2289806", + "timestamp": "1513601314" + }, + "genesis": { + "alloc": { + "0x0024f658a46fbb89d8ac105e98d7ac7cbbaf27c5": { + "balance": "0x0", + "code": "0x", + "nonce": "22", + "storage": {} + }, + "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe": { + "balance": "0x4d87094125a369d9bd5", + "code": "0x606060405236156100935763ffffffff60e060020a60003504166311ee8382811461009c57806313af4035146100be5780631f5e8f4c146100ee57806324daddc5146101125780634921a91a1461013b57806363e4bff414610157578063764978f91461017f578063893d20e8146101a1578063ba40aaa1146101cd578063cebc9a82146101f4578063e177246e14610216575b61009a5b5b565b005b34156100a457fe5b6100ac61023d565b60408051918252519081900360200190f35b34156100c657fe5b6100da600160a060020a0360043516610244565b604080519115158252519081900360200190f35b34156100f657fe5b6100da610307565b604080519115158252519081900360200190f35b341561011a57fe5b6100da6004351515610318565b604080519115158252519081900360200190f35b6100da6103d6565b604080519115158252519081900360200190f35b6100da600160a060020a0360043516610420565b604080519115158252519081900360200190f35b341561018757fe5b6100ac61046c565b60408051918252519081900360200190f35b34156101a957fe5b6101b1610473565b60408051600160a060020a039092168252519081900360200190f35b34156101d557fe5b6100da600435610483565b604080519115158252519081900360200190f35b34156101fc57fe5b6100ac61050d565b60408051918252519081900360200190f35b341561021e57fe5b6100da600435610514565b604080519115158252519081900360200190f35b6003545b90565b60006000610250610473565b600160a060020a031633600160a060020a03161415156102705760006000fd5b600160a060020a03831615156102865760006000fd5b50600054600160a060020a0390811690831681146102fb57604051600160a060020a0380851691908316907ffcf23a92150d56e85e3a3d33b357493246e55783095eb6a733eb8439ffc752c890600090a360008054600160a060020a031916600160a060020a03851617905560019150610300565b600091505b5b50919050565b60005460a060020a900460ff165b90565b60006000610324610473565b600160a060020a031633600160a060020a03161415156103445760006000fd5b5060005460a060020a900460ff16801515831515146102fb576000546040805160a060020a90920460ff1615158252841515602083015280517fe6cd46a119083b86efc6884b970bfa30c1708f53ba57b86716f15b2f4551a9539281900390910190a16000805460a060020a60ff02191660a060020a8515150217905560019150610300565b600091505b5b50919050565b60006103e0610307565b801561040557506103ef610473565b600160a060020a031633600160a060020a031614155b156104105760006000fd5b610419336105a0565b90505b5b90565b600061042a610307565b801561044f5750610439610473565b600160a060020a031633600160a060020a031614155b1561045a5760006000fd5b610463826105a0565b90505b5b919050565b6001545b90565b600054600160a060020a03165b90565b6000600061048f610473565b600160a060020a031633600160a060020a03161415156104af5760006000fd5b506001548281146102fb57604080518281526020810185905281517f79a3746dde45672c9e8ab3644b8bb9c399a103da2dc94b56ba09777330a83509929181900390910190a160018381559150610300565b600091505b5b50919050565b6002545b90565b60006000610520610473565b600160a060020a031633600160a060020a03161415156105405760006000fd5b506002548281146102fb57604080518281526020810185905281517ff6991a728965fedd6e927fdf16bdad42d8995970b4b31b8a2bf88767516e2494929181900390910190a1600283905560019150610300565b600091505b5b50919050565b60006000426105ad61023d565b116102fb576105c46105bd61050d565b4201610652565b6105cc61046c565b604051909150600160a060020a038416908290600081818185876187965a03f1925050501561063d57604080518281529051600160a060020a038516917f9bca65ce52fdef8a470977b51f247a2295123a4807dfa9e502edf0d30722da3b919081900360200190a260019150610300565b6102fb42610652565b5b600091505b50919050565b60038190555b505600a165627a7a72305820f3c973c8b7ed1f62000b6701bd5b708469e19d0f1d73fde378a56c07fd0b19090029", + "nonce": "1", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000001b436ba50d378d4bbc8660d312a13df6af6e89dfb", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x00000000000000000000000000000000000000000000000006f05b59d3b20000", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x000000000000000000000000000000000000000000000000000000000000003c", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x000000000000000000000000000000000000000000000000000000005a37b834" + } + }, + "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb": { + "balance": "0x1780d77678137ac1b775", + "code": "0x", + "nonce": "29072", + "storage": {} + } + }, + "config": { + "byzantiumBlock": 1700000, + "chainId": 3, + "daoForkSupport": true, + "eip150Block": 0, + "eip150Hash": "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d", + "eip155Block": 10, + "eip158Block": 10, + "ethash": {}, + "homesteadBlock": 0 + }, + "difficulty": "3509749784", + "extraData": "0x4554482e45544846414e532e4f52472d4641313738394444", + "gasLimit": "4727564", + "hash": "0x609948ac3bd3c00b7736b933248891d6c901ee28f066241bddb28f4e00a9f440", + "miner": "0xbbf5029fd710d227630c8b7d338051b8e76d50b3", + "mixHash": "0xb131e4507c93c7377de00e7c271bf409ec7492767142ff0f45c882f8068c2ada", + "nonce": "0x4eb12e19c16d43da", + "number": "2289805", + "stateRoot": "0xc7f10f352bff82fac3c2999d3085093d12652e19c7fd32591de49dc5d91b4f1f", + "timestamp": "1513601261", + "totalDifficulty": "7143276353481064" + }, + "input": "0xf88b8271908506fc23ac0083015f90943b873a919aa0512d5a0f09e6dcceaa4a6727fafe80a463e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c52aa0bdce0b59e8761854e857fe64015f06dd08a4fbb7624f6094893a79a72e6ad6bea01d9dde033cff7bb235a3163f348a6d7ab8d6b52bc0963a95b91612e40ca766a4", + "tracerConfig": { + "onlyTopCall": true + }, + "result": { + "from": "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb", + "gas": "0x10738", + "gasUsed": "0x3ef9", + "input": "0x63e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c5", + "output": "0x0000000000000000000000000000000000000000000000000000000000000001", + "to": "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe", + "type": "CALL", + "value": "0x0" + } +} diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index f54c8010494f..8238bb603dd3 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -125,7 +125,7 @@ type jsTracer struct { // The methods `result` and `fault` are required to be present. // The methods `step`, `enter`, and `exit` are optional, but note that // `enter` and `exit` always go together. -func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { +func newJsTracer(code string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if c, ok := assetTracers[code]; ok { code = c } @@ -177,6 +177,17 @@ func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { t.exit = exit t.result = result t.fault = fault + + // Pass in config + if setup, ok := goja.AssertFunction(obj.Get("setup")); ok { + cfgStr := "{}" + if cfg != nil { + cfgStr = string(cfg) + } + if _, err := setup(obj, vm.ToValue(cfgStr)); err != nil { + return nil, err + } + } // Setup objects carrying data to JS. These are created once and re-used. t.log = &steplog{ vm: vm, @@ -548,10 +559,10 @@ func (mo *memoryObj) slice(begin, end int64) ([]byte, error) { return []byte{}, nil } if end < begin || begin < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound memory: offset %d, end %d", begin, end) + return nil, fmt.Errorf("tracer accessed out of bound memory: offset %d, end %d", begin, end) } if mo.memory.Len() < int(end) { - return nil, fmt.Errorf("Tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), begin, end-begin) + return nil, fmt.Errorf("tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), begin, end-begin) } return mo.memory.GetCopy(begin, end-begin), nil } @@ -573,7 +584,7 @@ func (mo *memoryObj) GetUint(addr int64) goja.Value { // getUint returns the 32 bytes at the specified address interpreted as a uint. func (mo *memoryObj) getUint(addr int64) (*big.Int, error) { if mo.memory.Len() < int(addr)+32 || addr < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), addr, 32) + return nil, fmt.Errorf("tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), addr, 32) } return new(big.Int).SetBytes(mo.memory.GetPtr(addr, 32)), nil } @@ -613,7 +624,7 @@ func (s *stackObj) Peek(idx int) goja.Value { // peek returns the nth-from-the-top element of the stack. func (s *stackObj) peek(idx int) (*big.Int, error) { if len(s.stack.Data()) <= idx || idx < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound stack: size %d, index %d", len(s.stack.Data()), idx) + return nil, fmt.Errorf("tracer accessed out of bound stack: size %d, index %d", len(s.stack.Data()), idx) } return s.stack.Back(idx).ToBig(), nil } @@ -878,7 +889,6 @@ func (r *callframeResult) GetError() goja.Value { return r.vm.ToValue(r.err.Error()) } return goja.Undefined() - } func (r *callframeResult) setupObject() *goja.Object { diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 2863bd4451b8..80a002d5af28 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -85,7 +85,7 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon func TestTracer(t *testing.T) { execTracer := func(code string) ([]byte, string) { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -103,15 +103,15 @@ func TestTracer(t *testing.T) { { // tests that we don't panic on bad arguments to memory access code: "{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(15)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(15)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to stack peeks code: "{depths: [], step: function(log) { this.depths.push(log.stack.peek(-1)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound stack: size 0, index -1 at step (:1:53(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound stack: size 0, index -1 at step (:1:53(13)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to memory getUint code: "{ depths: [], step: function(log, db) { this.depths.push(log.memory.getUint(-64));}, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(13)) in server-side tracer function 'step'", }, { // tests some general counting code: "{count: 0, step: function() { this.count += 1; }, fault: function() {}, result: function() { return this.count; }}", want: `3`, @@ -149,7 +149,7 @@ func TestTracer(t *testing.T) { func TestHalt(t *testing.T) { timeout := errors.New("stahp") - tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil) + tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil, nil) if err != nil { t.Fatal(err) } @@ -163,7 +163,7 @@ func TestHalt(t *testing.T) { } func TestHaltBetweenSteps(t *testing.T) { - tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil) + tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestHaltBetweenSteps(t *testing.T) { func TestNoStepExec(t *testing.T) { execTracer := func(code string) []byte { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -221,7 +221,7 @@ func TestIsPrecompile(t *testing.T) { chaincfg.IstanbulBlock = big.NewInt(200) chaincfg.BerlinBlock = big.NewInt(300) txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} - tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -232,30 +232,30 @@ func TestIsPrecompile(t *testing.T) { t.Error(err) } if string(res) != "false" { - t.Errorf("Tracer should not consider blake2f as precompile in byzantium") + t.Errorf("tracer should not consider blake2f as precompile in byzantium") } - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) blockCtx = vm.BlockContext{BlockNumber: big.NewInt(250)} res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { t.Error(err) } if string(res) != "true" { - t.Errorf("Tracer should consider blake2f as precompile in istanbul") + t.Errorf("tracer should consider blake2f as precompile in istanbul") } } func TestEnterExit(t *testing.T) { // test that either both or none of enter() and exit() are defined - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context)); err == nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context), nil); err == nil { t.Fatal("tracer creation should've failed without exit() definition") } - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context)); err != nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context), nil); err != nil { t.Fatal(err) } // test that the enter and exit method are correctly invoked and the values passed - tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context)) + tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -274,3 +274,33 @@ func TestEnterExit(t *testing.T) { t.Errorf("Number of invocations of enter() and exit() is wrong. Have %s, want %s\n", have, want) } } + +func TestSetup(t *testing.T) { + // Test empty config + _, err := newJsTracer(`{setup: function(cfg) { if (cfg !== "{}") { throw("invalid empty config") } }, fault: function() {}, result: function() {}}`, new(tracers.Context), nil) + if err != nil { + t.Error(err) + } + + cfg, err := json.Marshal(map[string]string{"foo": "bar"}) + if err != nil { + t.Fatal(err) + } + // Test no setup func + _, err = newJsTracer(`{fault: function() {}, result: function() {}}`, new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + // Test config value + tracer, err := newJsTracer("{config: null, setup: function(cfg) { this.config = JSON.parse(cfg) }, step: function() {}, fault: function() {}, result: function() { return this.config.foo }}", new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + have, err := tracer.GetResult() + if err != nil { + t.Fatal(err) + } + if string(have) != `"bar"` { + t.Errorf("tracer returned wrong result. have: %s, want: \"bar\"\n", string(have)) + } +} diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index fe850d6b3e61..07aa2f2b4301 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -40,7 +40,7 @@ type Storage map[common.Hash]common.Hash // Copy duplicates the current storage. func (s Storage) Copy() Storage { - cpy := make(Storage) + cpy := make(Storage, len(s)) for key, value := range s { cpy[key] = value } @@ -224,7 +224,7 @@ func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration l.output = output l.err = err if l.cfg.Debug { - fmt.Printf("0x%x\n", output) + fmt.Printf("%#x\n", output) if err != nil { fmt.Printf(" error: %v\n", err) } @@ -346,11 +346,11 @@ func NewMarkdownLogger(cfg *Config, writer io.Writer) *mdLogger { func (t *mdLogger) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { t.env = env if !create { - fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), input, gas, value) } else { - fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), input, gas, value) } @@ -387,7 +387,7 @@ func (t *mdLogger) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope } func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, err error) { - fmt.Fprintf(t.out, "\nOutput: `0x%x`\nConsumed gas: `%d`\nError: `%v`\n", + fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", output, gasUsed, err) } diff --git a/eth/tracers/native/4byte.go b/eth/tracers/native/4byte.go index cd4af5b3d72a..7fb1c5e6c612 100644 --- a/eth/tracers/native/4byte.go +++ b/eth/tracers/native/4byte.go @@ -56,11 +56,11 @@ type fourByteTracer struct { // newFourByteTracer returns a native go tracer which collects // 4 byte-identifiers of a tx, and implements vm.EVMLogger. -func newFourByteTracer(ctx *tracers.Context) tracers.Tracer { +func newFourByteTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { t := &fourByteTracer{ ids: make(map[string]int), } - return t + return t, nil } // isPrecompiled returns whether the addr is a precompile. Logic borrowed from newJsTracer in eth/tracers/js/tracer.go diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index d334e328a5ff..7af0e658a8bf 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -50,16 +50,27 @@ type callFrame struct { type callTracer struct { env *vm.EVM callstack []callFrame + config callTracerConfig interrupt uint32 // Atomic flag to signal execution interruption reason error // Textual reason for the interruption } +type callTracerConfig struct { + OnlyTopCall bool `json:"onlyTopCall"` // If true, call tracer won't collect any subcalls +} + // newCallTracer returns a native go tracer which tracks // call frames of a tx, and implements vm.EVMLogger. -func newCallTracer(ctx *tracers.Context) tracers.Tracer { +func newCallTracer(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { + var config callTracerConfig + if cfg != nil { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, err + } + } // First callframe contains tx context info // and is populated on start and end. - return &callTracer{callstack: make([]callFrame, 1)} + return &callTracer{callstack: make([]callFrame, 1), config: config}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. @@ -101,6 +112,9 @@ func (t *callTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ * // CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if t.config.OnlyTopCall { + return + } // Skip if tracing was interrupted if atomic.LoadUint32(&t.interrupt) > 0 { t.env.Cancel() @@ -121,6 +135,9 @@ func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. // CaptureExit is called when EVM exits a scope, even if the scope didn't // execute any code. func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if t.config.OnlyTopCall { + return + } size := len(t.callstack) if size <= 1 { return diff --git a/eth/tracers/native/noop.go b/eth/tracers/native/noop.go index 0849fd74e987..c252b2408fc9 100644 --- a/eth/tracers/native/noop.go +++ b/eth/tracers/native/noop.go @@ -35,8 +35,8 @@ func init() { type noopTracer struct{} // newNoopTracer returns a new noop tracer. -func newNoopTracer(ctx *tracers.Context) tracers.Tracer { - return &noopTracer{} +func newNoopTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &noopTracer{}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 4d289ca62210..b513f383b9c2 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -51,10 +51,10 @@ type prestateTracer struct { reason error // Textual reason for the interruption } -func newPrestateTracer(ctx *tracers.Context) tracers.Tracer { +func newPrestateTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { // First callframe contains tx context info // and is populated on start and end. - return &prestateTracer{prestate: prestate{}} + return &prestateTracer{prestate: prestate{}}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/revertreason.go b/eth/tracers/native/revertreason.go new file mode 100644 index 000000000000..d09b86100901 --- /dev/null +++ b/eth/tracers/native/revertreason.go @@ -0,0 +1,108 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package native + +import ( + "bytes" + "encoding/json" + "math/big" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers" +) + +func init() { + register("revertReasonTracer", newRevertReasonTracer) +} + +var revertSelector = crypto.Keccak256([]byte("Error(string)"))[:4] + +// revertReasonTracer is a go implementation of the Tracer interface which +// track the error message or revert reason return by the contract. +type revertReasonTracer struct { + env *vm.EVM + revertReason string // The revert reason return from the tx, if tx success, empty string return + interrupt uint32 // Atomic flag to signal execution interruption + reason error // Textual reason for the interruption +} + +// newRevertReasonTracer returns a new revert reason tracer. +func newRevertReasonTracer(_ *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &revertReasonTracer{}, nil +} + +// CaptureStart implements the EVMLogger interface to initialize the tracing operation. +func (t *revertReasonTracer) CaptureStart(env *vm.EVM, _ common.Address, _ common.Address, _ bool, _ []byte, _ uint64, _ *big.Int) { + t.env = env +} + +// CaptureEnd is called after the call finishes to finalize the tracing. +func (t *revertReasonTracer) CaptureEnd(output []byte, _ uint64, _ time.Duration, err error) { + if err != nil { + if err == vm.ErrExecutionReverted && len(output) > 4 && bytes.Equal(output[:4], revertSelector) { + errMsg, _ := abi.UnpackRevert(output) + t.revertReason = err.Error() + ": " + errMsg + } else { + t.revertReason = err.Error() + } + } +} + +// CaptureState implements the EVMLogger interface to trace a single step of VM execution. +func (t *revertReasonTracer) CaptureState(_ uint64, _ vm.OpCode, _, _ uint64, _ *vm.ScopeContext, _ []byte, _ int, _ error) { +} + +// CaptureFault implements the EVMLogger interface to trace an execution fault. +func (t *revertReasonTracer) CaptureFault(_ uint64, _ vm.OpCode, _, _ uint64, _ *vm.ScopeContext, _ int, _ error) { +} + +// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). +func (t *revertReasonTracer) CaptureEnter(_ vm.OpCode, _ common.Address, _ common.Address, _ []byte, _ uint64, _ *big.Int) { + // Skip if tracing was interrupted + if atomic.LoadUint32(&t.interrupt) > 0 { + t.env.Cancel() + return + } +} + +// CaptureExit is called when EVM exits a scope, even if the scope didn't +// execute any code. +func (t *revertReasonTracer) CaptureExit(_ []byte, _ uint64, _ error) {} + +func (t *revertReasonTracer) CaptureTxStart(_ uint64) {} + +func (t *revertReasonTracer) CaptureTxEnd(_ uint64) {} + +// GetResult returns an error message json object. +func (t *revertReasonTracer) GetResult() (json.RawMessage, error) { + res, err := json.Marshal(t.revertReason) + if err != nil { + return nil, err + } + return res, t.reason +} + +// Stop terminates execution of the tracer at the first opportune moment. +func (t *revertReasonTracer) Stop(err error) { + t.reason = err + atomic.StoreUint32(&t.interrupt, 1) +} diff --git a/eth/tracers/native/tracer.go b/eth/tracers/native/tracer.go index a5b3956c0f26..187d89fa1bc4 100644 --- a/eth/tracers/native/tracer.go +++ b/eth/tracers/native/tracer.go @@ -37,6 +37,7 @@ Example: package native import ( + "encoding/json" "errors" "github.com/ethereum/go-ethereum/eth/tracers" @@ -48,7 +49,7 @@ func init() { } // ctorFn is the constructor signature of a native tracer. -type ctorFn = func(*tracers.Context) tracers.Tracer +type ctorFn = func(*tracers.Context, json.RawMessage) (tracers.Tracer, error) /* ctors is a map of package-local tracer constructors. @@ -73,12 +74,12 @@ func register(name string, ctor ctorFn) { } // lookup returns a tracer, if one can be matched to the given name. -func lookup(name string, ctx *tracers.Context) (tracers.Tracer, error) { +func lookup(name string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if ctors == nil { ctors = make(map[string]ctorFn) } if ctor, ok := ctors[name]; ok { - return ctor(ctx), nil + return ctor(ctx, cfg) } return nil, errors.New("no tracer found") } diff --git a/eth/tracers/tracers.go b/eth/tracers/tracers.go index e7073e7d2edf..3d2d1256c091 100644 --- a/eth/tracers/tracers.go +++ b/eth/tracers/tracers.go @@ -42,7 +42,7 @@ type Tracer interface { Stop(err error) } -type lookupFunc func(string, *Context) (Tracer, error) +type lookupFunc func(string, *Context, json.RawMessage) (Tracer, error) var ( lookups []lookupFunc @@ -62,9 +62,9 @@ func RegisterLookup(wildcard bool, lookup lookupFunc) { // New returns a new instance of a tracer, by iterating through the // registered lookups. -func New(code string, ctx *Context) (Tracer, error) { +func New(code string, ctx *Context, cfg json.RawMessage) (Tracer, error) { for _, lookup := range lookups { - if tracer, err := lookup(code, ctx); err == nil { + if tracer, err := lookup(code, ctx, cfg); err == nil { return tracer, nil } } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 9e08cb21b17b..e009f37ac39b 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -506,6 +506,38 @@ func (ec *Client) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { return (*big.Int)(&hex), nil } +type feeHistoryResultMarshaling struct { + OldestBlock *hexutil.Big `json:"oldestBlock"` + Reward [][]*hexutil.Big `json:"reward,omitempty"` + BaseFee []*hexutil.Big `json:"baseFeePerGas,omitempty"` + GasUsedRatio []float64 `json:"gasUsedRatio"` +} + +// FeeHistory retrieves the fee market history. +func (ec *Client) FeeHistory(ctx context.Context, blockCount uint64, lastBlock *big.Int, rewardPercentiles []float64) (*ethereum.FeeHistory, error) { + var res feeHistoryResultMarshaling + if err := ec.c.CallContext(ctx, &res, "eth_feeHistory", hexutil.Uint(blockCount), toBlockNumArg(lastBlock), rewardPercentiles); err != nil { + return nil, err + } + reward := make([][]*big.Int, len(res.Reward)) + for i, r := range res.Reward { + reward[i] = make([]*big.Int, len(r)) + for j, r := range r { + reward[i][j] = (*big.Int)(r) + } + } + baseFee := make([]*big.Int, len(res.BaseFee)) + for i, b := range res.BaseFee { + baseFee[i] = (*big.Int)(b) + } + return ðereum.FeeHistory{ + OldestBlock: (*big.Int)(res.OldestBlock), + Reward: reward, + BaseFee: baseFee, + GasUsedRatio: res.GasUsedRatio, + }, nil +} + // EstimateGas tries to estimate the gas needed to execute a specific transaction based on // the current pending state of the backend blockchain. There is no guarantee that this is // the true gas limit requirement as other transactions may be added or removed by miners, diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index dd01c08f539e..e0ceec0d6fe7 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -248,7 +248,7 @@ func generateTestChain() []*types.Block { g.AddTx(testTx2) } } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(genesis.Config, gblock, engine, db, 2, generate) blocks = append([]*types.Block{gblock}, blocks...) @@ -512,6 +512,29 @@ func testStatusFunctions(t *testing.T, client *rpc.Client) { if gasTipCap.Cmp(big.NewInt(234375000)) != 0 { t.Fatalf("unexpected gas tip cap: %v", gasTipCap) } + + // FeeHistory + history, err := ec.FeeHistory(context.Background(), 1, big.NewInt(2), []float64{95, 99}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := ðereum.FeeHistory{ + OldestBlock: big.NewInt(2), + Reward: [][]*big.Int{ + { + big.NewInt(234375000), + big.NewInt(234375000), + }, + }, + BaseFee: []*big.Int{ + big.NewInt(765625000), + big.NewInt(671627818), + }, + GasUsedRatio: []float64{0.008912678667376286}, + } + if !reflect.DeepEqual(history, want) { + t.Fatalf("FeeHistory result doesn't match expected: (got: %v, want: %v)", history, want) + } } func testCallContractAtHash(t *testing.T, client *rpc.Client) { @@ -562,7 +585,7 @@ func testCallContract(t *testing.T, client *rpc.Client) { if _, err := ec.CallContract(context.Background(), msg, big.NewInt(1)); err != nil { t.Fatalf("unexpected error: %v", err) } - // PendingCallCOntract + // PendingCallContract if _, err := ec.PendingCallContract(context.Background(), msg); err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 7af2bf45d791..a86f4339f425 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -79,7 +79,6 @@ type StorageResult struct { // GetProof returns the account and storage values of the specified account including the Merkle-proof. // The block number can be nil, in which case the value is taken from the latest known block. func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []string, blockNumber *big.Int) (*AccountResult, error) { - type storageResult struct { Key string `json:"key"` Value *hexutil.Big `json:"value"` diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 718dc14bbcb6..17ba5a60566b 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -64,6 +65,12 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { if err != nil { t.Fatalf("can't create new ethereum service: %v", err) } + filterSystem := filters.NewFilterSystem(ethservice.APIBackend, filters.Config{}) + n.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, false), + }}) + // Import the test chain. if err := n.Start(); err != nil { t.Fatalf("can't start test node: %v", err) @@ -87,7 +94,7 @@ func generateTestChain() (*core.Genesis, []*types.Block) { g.OffsetTime(5) g.SetExtra([]byte("test")) } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, 1, generate) blocks = append([]*types.Block{gblock}, blocks...) @@ -226,7 +233,6 @@ func testGetProof(t *testing.T, client *rpc.Client) { if proof.Key != testSlot.String() { t.Fatalf("invalid storage proof key, want: %v, got: %v", testSlot.String(), proof.Key) } - } func testGCStats(t *testing.T, client *rpc.Client) { diff --git a/ethdb/database.go b/ethdb/database.go index 651aa977d1da..b8f9618c2f45 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -144,7 +144,9 @@ type AncientWriteOp interface { // AncientStater wraps the Stat method of a backing data store. type AncientStater interface { - // AncientDatadir returns the root directory path of the ancient store. + // AncientDatadir returns the path of root ancient directory. Empty string + // will be returned if ancient store is not enabled at all. The returned + // path can be used to construct the path of other freezers. AncientDatadir() (string, error) } @@ -174,7 +176,6 @@ type Stater interface { type AncientStore interface { AncientReader AncientWriter - AncientStater io.Closer } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index e94570cb3f0e..7e4fd7e5e7f0 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -66,7 +66,7 @@ func NewWithCap(size int) *Database { } // Close deallocates the internal map and ensures any consecutive data access op -// failes with an error. +// fails with an error. func (db *Database) Close() error { db.lock.Lock() defer db.lock.Unlock() diff --git a/ethdb/rocksdb/rocksdb.go b/ethdb/rocksdb/rocksdb.go index 9cac3c6b10b9..8b84d48e2cd8 100644 --- a/ethdb/rocksdb/rocksdb.go +++ b/ethdb/rocksdb/rocksdb.go @@ -272,6 +272,11 @@ func (it *RDBIterator) Next() bool { if it.first { it.first = false } else { + // Added conditions to prevent Rocksdb Iterator error. + // Valid() call is a RocksDB requirement. + if C.rocksdb_iter_valid(it.it) == 0 { + return false + } C.rocksdb_iter_next(it.it) } return C.rocksdb_iter_valid(it.it) != 0 diff --git a/ethstats/ethstats_test.go b/ethstats/ethstats_test.go index 0692ecdae9be..60322f765439 100644 --- a/ethstats/ethstats_test.go +++ b/ethstats/ethstats_test.go @@ -79,5 +79,4 @@ func TestParseEthstatsURL(t *testing.T) { t.Errorf("case=%d mismatch host value, got: %v ,want: %v", i, host, c.host) } } - } diff --git a/go.mod b/go.mod index 9a2b37229f1b..be6f482fc755 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.2 github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 - github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef github.com/urfave/cli/v2 v2.10.2 github.com/yahoo/coname v0.0.0-20170609175141-84592ddf8673 diff --git a/go.sum b/go.sum index ce50c67376f3..ee8b16cbcae4 100644 --- a/go.sum +++ b/go.sum @@ -178,9 +178,8 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/ github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= @@ -213,7 +212,6 @@ github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5Nq github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/logger v1.0.0/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= @@ -280,7 +278,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -335,7 +332,6 @@ github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= @@ -454,23 +450,18 @@ github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hz github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -577,8 +568,8 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 h1:m+8fKfQwCAy1QjzINvKe/pYtLjo2dl59x2w9YSEJxuY= github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= -github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= -github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -736,6 +727,7 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -744,10 +736,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -797,13 +787,14 @@ golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -814,9 +805,6 @@ golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -869,7 +857,6 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= diff --git a/graphql/graphql.go b/graphql/graphql.go index 0654fd1af388..97b460c205ce 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -76,14 +76,14 @@ func (b *Long) UnmarshalGraphQL(input interface{}) error { // Account represents an Ethereum account at a particular block. type Account struct { - backend ethapi.Backend + r *Resolver address common.Address blockNrOrHash rpc.BlockNumberOrHash } // getState fetches the StateDB object for an account. func (a *Account) getState(ctx context.Context) (*state.StateDB, error) { - state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) + state, _, err := a.r.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) return state, err } @@ -106,7 +106,7 @@ func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { func (a *Account) TransactionCount(ctx context.Context) (hexutil.Uint64, error) { // Ask transaction pool for the nonce which includes pending transactions if blockNr, ok := a.blockNrOrHash.Number(); ok && blockNr == rpc.PendingBlockNumber { - nonce, err := a.backend.GetPoolNonce(ctx, a.address) + nonce, err := a.r.backend.GetPoolNonce(ctx, a.address) if err != nil { return 0, err } @@ -137,7 +137,7 @@ func (a *Account) Storage(ctx context.Context, args struct{ Slot common.Hash }) // Log represents an individual log message. All arguments are mandatory. type Log struct { - backend ethapi.Backend + r *Resolver transaction *Transaction log *types.Log } @@ -148,7 +148,7 @@ func (l *Log) Transaction(ctx context.Context) *Transaction { func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account { return &Account{ - backend: l.backend, + r: l.r, address: l.log.Address, blockNrOrHash: args.NumberOrLatest(), } @@ -183,30 +183,30 @@ func (at *AccessTuple) StorageKeys(ctx context.Context) []common.Hash { // Transaction represents an Ethereum transaction. // backend and hash are mandatory; all others will be fetched when required. type Transaction struct { - backend ethapi.Backend - hash common.Hash - tx *types.Transaction - block *Block - index uint64 + r *Resolver + hash common.Hash + tx *types.Transaction + block *Block + index uint64 } // resolve returns the internal transaction object, fetching it if needed. func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, error) { if t.tx == nil { // Try to return an already finalized transaction - tx, blockHash, _, index, err := t.backend.GetTransaction(ctx, t.hash) + tx, blockHash, _, index, err := t.r.backend.GetTransaction(ctx, t.hash) if err == nil && tx != nil { t.tx = tx blockNrOrHash := rpc.BlockNumberOrHashWithHash(blockHash, false) t.block = &Block{ - backend: t.backend, + r: t.r, numberOrHash: &blockNrOrHash, } t.index = index return t.tx, nil } // No finalized transaction, try to retrieve it from the pool - t.tx = t.backend.GetPoolTransaction(t.hash) + t.tx = t.r.backend.GetPoolTransaction(t.hash) } return t.tx, nil } @@ -354,7 +354,7 @@ func (t *Transaction) To(ctx context.Context, args BlockNumberArgs) (*Account, e return nil, nil } return &Account{ - backend: t.backend, + r: t.r, address: *to, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -365,10 +365,10 @@ func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account, if err != nil || tx == nil { return nil, err } - signer := types.LatestSigner(t.backend.ChainConfig()) + signer := types.LatestSigner(t.r.backend.ChainConfig()) from, _ := types.Sender(signer, tx) return &Account{ - backend: t.backend, + r: t.r, address: from, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -443,21 +443,45 @@ func (t *Transaction) CreatedContract(ctx context.Context, args BlockNumberArgs) return nil, err } return &Account{ - backend: t.backend, + r: t.r, address: receipt.ContractAddress, blockNrOrHash: args.NumberOrLatest(), }, nil } func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { - receipt, err := t.getReceipt(ctx) - if err != nil || receipt == nil { + if _, err := t.resolve(ctx); err != nil { return nil, err } - ret := make([]*Log, 0, len(receipt.Logs)) - for _, log := range receipt.Logs { + if t.block == nil { + return nil, nil + } + if _, ok := t.block.numberOrHash.Hash(); !ok { + header, err := t.r.backend.HeaderByNumberOrHash(ctx, *t.block.numberOrHash) + if err != nil { + return nil, err + } + hash := header.Hash() + t.block.numberOrHash.BlockHash = &hash + } + return t.getLogs(ctx) +} + +// getLogs returns log objects for the given tx. +// Assumes block hash is resolved. +func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { + var ( + hash, _ = t.block.numberOrHash.Hash() + filter = t.r.filterSystem.NewBlockFilter(hash, nil, nil) + logs, err = filter.Logs(ctx) + ) + if err != nil { + return nil, err + } + ret := make([]*Log, 0, len(logs)) + for _, log := range logs { ret = append(ret, &Log{ - backend: t.backend, + r: t.r, transaction: t, log: log, }) @@ -539,7 +563,7 @@ type BlockType int // backend, and numberOrHash are mandatory. All other fields are lazily fetched // when required. type Block struct { - backend ethapi.Backend + r *Resolver numberOrHash *rpc.BlockNumberOrHash hash common.Hash header *types.Header @@ -558,7 +582,7 @@ func (b *Block) resolve(ctx context.Context) (*types.Block, error) { b.numberOrHash = &latest } var err error - b.block, err = b.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) + b.block, err = b.r.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) if b.block != nil && b.header == nil { b.header = b.block.Header() if hash, ok := b.numberOrHash.Hash(); ok { @@ -578,9 +602,9 @@ func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { var err error if b.header == nil { if b.hash != (common.Hash{}) { - b.header, err = b.backend.HeaderByHash(ctx, b.hash) + b.header, err = b.r.backend.HeaderByHash(ctx, b.hash) } else { - b.header, err = b.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) + b.header, err = b.r.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) } } return b.header, err @@ -598,7 +622,7 @@ func (b *Block) resolveReceipts(ctx context.Context) ([]*types.Receipt, error) { } hash = header.Hash() } - receipts, err := b.backend.GetReceipts(ctx, hash) + receipts, err := b.r.backend.GetReceipts(ctx, hash) if err != nil { return nil, err } @@ -659,7 +683,7 @@ func (b *Block) NextBaseFeePerGas(ctx context.Context) (*hexutil.Big, error) { if err != nil { return nil, err } - chaincfg := b.backend.ChainConfig() + chaincfg := b.r.backend.ChainConfig() if header.BaseFee == nil { // Make sure next block doesn't enable EIP-1559 if !chaincfg.IsLondon(new(big.Int).Add(header.Number, common.Big1)) { @@ -679,7 +703,7 @@ func (b *Block) Parent(ctx context.Context) (*Block, error) { } num := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.header.Number.Uint64() - 1)) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &num, hash: b.header.ParentHash, }, nil @@ -767,7 +791,7 @@ func (b *Block) Ommers(ctx context.Context) (*[]*Block, error) { for _, uncle := range block.Uncles() { blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) ret = append(ret, &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }) @@ -800,7 +824,7 @@ func (b *Block) TotalDifficulty(ctx context.Context) (hexutil.Big, error) { } h = header.Hash() } - td := b.backend.GetTd(ctx, h) + td := b.r.backend.GetTd(ctx, h) if td == nil { return hexutil.Big{}, fmt.Errorf("total difficulty not found %x", b.hash) } @@ -853,7 +877,7 @@ func (b *Block) Miner(ctx context.Context, args BlockNumberArgs) (*Account, erro return nil, err } return &Account{ - backend: b.backend, + r: b.r, address: header.Coinbase, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -876,11 +900,11 @@ func (b *Block) Transactions(ctx context.Context) (*[]*Transaction, error) { ret := make([]*Transaction, 0, len(block.Transactions())) for i, tx := range block.Transactions() { ret = append(ret, &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(i), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(i), }) } return &ret, nil @@ -897,11 +921,11 @@ func (b *Block) TransactionAt(ctx context.Context, args struct{ Index int32 }) ( } tx := txs[args.Index] return &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(args.Index), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(args.Index), }, nil } @@ -917,7 +941,7 @@ func (b *Block) OmmerAt(ctx context.Context, args struct{ Index int32 }) (*Block uncle := uncles[args.Index] blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }, nil @@ -944,7 +968,7 @@ type BlockFilterCriteria struct { // runFilter accepts a filter and executes it, returning all its results as // `Log` objects. -func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ([]*Log, error) { +func runFilter(ctx context.Context, r *Resolver, filter *filters.Filter) ([]*Log, error) { logs, err := filter.Logs(ctx) if err != nil || logs == nil { return nil, err @@ -952,8 +976,8 @@ func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ( ret := make([]*Log, 0, len(logs)) for _, log := range logs { ret = append(ret, &Log{ - backend: be, - transaction: &Transaction{backend: be, hash: log.TxHash}, + r: r, + transaction: &Transaction{r: r, hash: log.TxHash}, log: log, }) } @@ -978,10 +1002,10 @@ func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteri hash = header.Hash() } // Construct the range filter - filter := filters.NewBlockFilter(b.backend, hash, addresses, topics) + filter := b.r.filterSystem.NewBlockFilter(hash, addresses, topics) // Run the filter and return all the logs - return runFilter(ctx, b.backend, filter) + return runFilter(ctx, b.r, filter) } func (b *Block) Account(ctx context.Context, args struct { @@ -994,7 +1018,7 @@ func (b *Block) Account(ctx context.Context, args struct { } } return &Account{ - backend: b.backend, + r: b.r, address: args.Address, blockNrOrHash: *b.numberOrHash, }, nil @@ -1041,7 +1065,7 @@ func (b *Block) Call(ctx context.Context, args struct { return nil, err } } - result, err := ethapi.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, b.backend.RPCEVMTimeout(), b.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, b.r.backend, args.Data, *b.numberOrHash, nil, b.r.backend.RPCEVMTimeout(), b.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1066,31 +1090,31 @@ func (b *Block) EstimateGas(ctx context.Context, args struct { return 0, err } } - gas, err := ethapi.DoEstimateGas(ctx, b.backend, args.Data, *b.numberOrHash, b.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, b.r.backend, args.Data, *b.numberOrHash, b.r.backend.RPCGasCap()) return Long(gas), err } type Pending struct { - backend ethapi.Backend + r *Resolver } func (p *Pending) TransactionCount(ctx context.Context) (int32, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() return int32(len(txs)), err } func (p *Pending) Transactions(ctx context.Context) (*[]*Transaction, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() if err != nil { return nil, err } ret := make([]*Transaction, 0, len(txs)) for i, tx := range txs { ret = append(ret, &Transaction{ - backend: p.backend, - hash: tx.Hash(), - tx: tx, - index: uint64(i), + r: p.r, + hash: tx.Hash(), + tx: tx, + index: uint64(i), }) } return &ret, nil @@ -1101,7 +1125,7 @@ func (p *Pending) Account(ctx context.Context, args struct { }) *Account { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) return &Account{ - backend: p.backend, + r: p.r, address: args.Address, blockNrOrHash: pendingBlockNr, } @@ -1111,7 +1135,7 @@ func (p *Pending) Call(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (*CallResult, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - result, err := ethapi.DoCall(ctx, p.backend, args.Data, pendingBlockNr, nil, p.backend.RPCEVMTimeout(), p.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, p.r.backend, args.Data, pendingBlockNr, nil, p.r.backend.RPCEVMTimeout(), p.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1131,13 +1155,14 @@ func (p *Pending) EstimateGas(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (Long, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - gas, err := ethapi.DoEstimateGas(ctx, p.backend, args.Data, pendingBlockNr, p.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, p.r.backend, args.Data, pendingBlockNr, p.r.backend.RPCGasCap()) return Long(gas), err } // Resolver is the top-level object in the GraphQL hierarchy. type Resolver struct { - backend ethapi.Backend + backend ethapi.Backend + filterSystem *filters.FilterSystem } func (r *Resolver) Block(ctx context.Context, args struct { @@ -1152,19 +1177,19 @@ func (r *Resolver) Block(ctx context.Context, args struct { number := rpc.BlockNumber(*args.Number) numberOrHash := rpc.BlockNumberOrHashWithNumber(number) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else if args.Hash != nil { numberOrHash := rpc.BlockNumberOrHashWithHash(*args.Hash, false) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else { numberOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } @@ -1199,7 +1224,7 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { for i := from; i <= to; i++ { numberOrHash := rpc.BlockNumberOrHashWithNumber(i) block := &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } // Resolve the header to check for existence. @@ -1218,13 +1243,13 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { } func (r *Resolver) Pending(ctx context.Context) *Pending { - return &Pending{r.backend} + return &Pending{r} } func (r *Resolver) Transaction(ctx context.Context, args struct{ Hash common.Hash }) (*Transaction, error) { tx := &Transaction{ - backend: r.backend, - hash: args.Hash, + r: r, + hash: args.Hash, } // Resolve the transaction; if it doesn't exist, return nil. t, err := tx.resolve(ctx) @@ -1284,8 +1309,8 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria topics = *args.Filter.Topics } // Construct the range filter - filter := filters.NewRangeFilter(filters.Backend(r.backend), begin, end, addresses, topics) - return runFilter(ctx, r.backend, filter) + filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics) + return runFilter(ctx, r, filter) } func (r *Resolver) GasPrice(ctx context.Context) (hexutil.Big, error) { diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 4b7f7bf96021..d55f4e063486 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -50,7 +51,7 @@ func TestBuildSchema(t *testing.T) { } defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. - if err := newHandler(stack, nil, []string{}, []string{}); err != nil { + if err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) } } @@ -263,7 +264,8 @@ func createGQLService(t *testing.T, stack *node.Node) { t.Fatalf("could not create import blocks: %v", err) } // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) + filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) + err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } @@ -348,7 +350,8 @@ func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { t.Fatalf("could not create import blocks: %v", err) } // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) + filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) + err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } diff --git a/graphql/service.go b/graphql/service.go index 29d98ad74683..019026bc7ea7 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -20,6 +20,7 @@ import ( "encoding/json" "net/http" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/node" "github.com/graph-gophers/graphql-go" @@ -52,22 +53,17 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(responseJSON) - } // New constructs a new GraphQL service instance. -func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - if backend == nil { - panic("missing backend") - } - // check if http server with given endpoint exists and enable graphQL on it - return newHandler(stack, backend, cors, vhosts) +func New(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { + return newHandler(stack, backend, filterSystem, cors, vhosts) } // newHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. -func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - q := Resolver{backend} +func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { + q := Resolver{backend, filterSystem} s, err := graphql.ParseSchema(schema, &q) if err != nil { diff --git a/interfaces.go b/interfaces.go index 53d2df6df559..ac49c6ddf098 100644 --- a/interfaces.go +++ b/interfaces.go @@ -203,6 +203,15 @@ type GasPricer interface { SuggestGasPrice(ctx context.Context) (*big.Int, error) } +// FeeHistory provides recent fee market data that consumers can use to determine +// a reasonable maxPriorityFeePerGas value. +type FeeHistory struct { + OldestBlock *big.Int // block corresponding to first response value + Reward [][]*big.Int // list every txs priority fee per block + BaseFee []*big.Int // list of each block's base fee + GasUsedRatio []float64 // ratio of gas used out of the total available limit +} + // A PendingStateReader provides access to the pending state, which is the result of all // known executable transactions which have not yet been included in the blockchain. It is // commonly used to display the result of ’unconfirmed’ actions (e.g. wallet value diff --git a/internal/build/archive.go b/internal/build/archive.go index 8b3ac23d1d89..c16246070e8c 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "strings" + "time" ) type Archive interface { @@ -159,6 +160,7 @@ func (a *TarballArchive) Directory(name string) error { Name: a.dir, Mode: 0755, Typeflag: tar.TypeDir, + ModTime: time.Now(), }) } diff --git a/internal/build/util.go b/internal/build/util.go index 654349fac307..9a721e9b83b1 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -29,6 +29,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "text/template" "time" @@ -39,7 +40,7 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") // MustRun executes the given command and exits the host process for // any error. func MustRun(cmd *exec.Cmd) { - fmt.Println(">>>", strings.Join(cmd.Args, " ")) + fmt.Println(">>>", printArgs(cmd.Args)) if !*DryRunFlag { cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout @@ -49,6 +50,20 @@ func MustRun(cmd *exec.Cmd) { } } +func printArgs(args []string) string { + var s strings.Builder + for i, arg := range args { + if i > 0 { + s.WriteByte(' ') + } + if strings.IndexByte(arg, ' ') >= 0 { + arg = strconv.QuoteToASCII(arg) + } + s.WriteString(arg) + } + return s.String() +} + func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } @@ -121,7 +136,7 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { sftp.Args = append(sftp.Args, "-i", identityFile) } sftp.Args = append(sftp.Args, host) - fmt.Println(">>>", strings.Join(sftp.Args, " ")) + fmt.Println(">>>", printArgs(sftp.Args)) if *DryRunFlag { return nil } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 89e85d51c189..c672de192e4d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -88,6 +88,7 @@ type feeHistoryResult struct { GasUsedRatio []float64 `json:"gasUsedRatio"` } +// FeeHistory returns the fee market history. func (s *EthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.DecimalOrHex, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { oldest, reward, baseFee, gasUsed, err := s.b.FeeHistory(ctx, int(blockCount), lastBlock, rewardPercentiles) if err != nil { @@ -694,7 +695,6 @@ func (s *BlockChainAPI) BlockNumber() hexutil.Uint64 { // GetBlockReceipts returns all the transaction receipts for the given block hash. func (s *BlockChainAPI) GetReceiptsByHash(ctx context.Context, blockHash common.Hash) ([]map[string]interface{}, error) { - block, err1 := s.b.BlockByHash(ctx, blockHash) if block == nil && err1 == nil { return nil, nil @@ -716,7 +716,6 @@ func (s *BlockChainAPI) GetReceiptsByHash(ctx context.Context, blockHash common. fieldsList := make([]map[string]interface{}, 0, len(receipts)) for index, receipt := range receipts { - bigblock := new(big.Int).SetUint64(block.NumberU64()) signer := types.MakeSigner(s.b.ChainConfig(), bigblock) from, _ := types.Sender(signer, txs[index]) @@ -1031,6 +1030,7 @@ type BlockOverrides struct { GasLimit *hexutil.Uint64 Coinbase *common.Address Random *common.Hash + BaseFee *hexutil.Big } // Apply overrides the given header fields into the given block context. @@ -1056,6 +1056,9 @@ func (diff *BlockOverrides) Apply(blockCtx *vm.BlockContext) { if diff.Random != nil { blockCtx.Random = diff.Random } + if diff.BaseFee != nil { + blockCtx.BaseFee = diff.BaseFee.ToInt() + } } func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { @@ -1125,7 +1128,7 @@ func newRevertError(result *core.ExecutionResult) *revertError { } } -// revertError is an API error that encompassas an EVM revertal with JSON error +// revertError is an API error that encompasses an EVM revertal with JSON error // code and a binary data blob. type revertError struct { error @@ -1437,7 +1440,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber switch tx.Type() { case types.LegacyTxType: // if a legacy transaction has an EIP-155 chain id, include it explicitly - if id := tx.ChainId(); id.Sign() == 0 { + if id := tx.ChainId(); id.Sign() != 0 { result.ChainID = (*hexutil.Big)(id) } case types.AccessListTxType: @@ -1558,9 +1561,11 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, extract this as it will depend on access - // lists and we'll need to reestimate every time - nogas := args.Gas == nil + // If the gas amount is not set, default to RPC gas cap. + if args.Gas == nil { + tmp := hexutil.Uint64(b.RPCGasCap()) + args.Gas = &tmp + } // Ensure any missing fields are filled, extract the recipient and input data if err := args.setDefaults(ctx, b); err != nil { @@ -1586,15 +1591,6 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH accessList := prevTracer.AccessList() log.Trace("Creating access list", "input", accessList) - // If no gas amount was specified, each unique access list needs it's own - // gas calculation. This is quite expensive, but we need to be accurate - // and it's convered by the sender only anyway. - if nogas { - args.Gas = nil - if err := args.setDefaults(ctx, b); err != nil { - return nil, 0, nil, err // shouldn't happen, just in case - } - } // Copy the original db so we don't modify it statedb := db.Copy() // Set the accesslist to the last al @@ -2168,7 +2164,7 @@ func (api *DebugAPI) SeedHash(ctx context.Context, number uint64) (string, error if block == nil { return "", fmt.Errorf("block #%d not found", number) } - return fmt.Sprintf("0x%x", ethash.SeedHash(number)), nil + return fmt.Sprintf("%#x", ethash.SeedHash(number)), nil } // ChaindbProperty returns leveldb properties of the key-value database. diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 5119be859d49..5b4ceb631069 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -27,10 +27,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -84,16 +84,12 @@ type Backend interface { TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - // Filter API - BloomStatus() (uint64, uint64) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - ChainConfig() *params.ChainConfig Engine() consensus.Engine + + // eth/filters needs to be initialized from this backend type, so methods needed by + // it must also be included here. + filters.Backend } func GetAPIs(apiBackend Backend) []rpc.API { @@ -101,31 +97,24 @@ func GetAPIs(apiBackend Backend) []rpc.API { return []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: NewEthereumAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", Service: NewBlockChainAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", Service: NewTransactionAPI(apiBackend, nonceLock), }, { Namespace: "txpool", - Version: "1.0", Service: NewTxPoolAPI(apiBackend), }, { Namespace: "debug", - Version: "1.0", Service: NewDebugAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", Service: NewEthereumAccountAPI(apiBackend.AccountManager()), }, { Namespace: "personal", - Version: "1.0", Service: NewPersonalAccountAPI(apiBackend, nonceLock), }, } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index f03f6700e502..b2277f6ad925 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -82,56 +82,8 @@ func (args *TransactionArgs) data() []byte { // setDefaults fills in default values for unspecified tx fields. func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { - if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { - return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") - } - // After london, default to 1559 unless gasPrice is set - head := b.CurrentHeader() - // If user specifies both maxPriorityfee and maxFee, then we do not - // need to consult the chain for defaults. It's definitely a London tx. - if args.MaxPriorityFeePerGas == nil || args.MaxFeePerGas == nil { - // In this clause, user left some fields unspecified. - if b.ChainConfig().IsLondon(head.Number) && args.GasPrice == nil { - if args.MaxPriorityFeePerGas == nil { - tip, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) - } - if args.MaxFeePerGas == nil { - gasFeeCap := new(big.Int).Add( - (*big.Int)(args.MaxPriorityFeePerGas), - new(big.Int).Mul(head.BaseFee, big.NewInt(2)), - ) - args.MaxFeePerGas = (*hexutil.Big)(gasFeeCap) - } - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } - } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { - return errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") - } - if args.GasPrice == nil { - price, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - if b.ChainConfig().IsLondon(head.Number) { - // The legacy tx gas price suggestion should not add 2x base fee - // because all fees are consumed, so it would result in a spiral - // upwards. - price.Add(price, head.BaseFee) - } - args.GasPrice = (*hexutil.Big)(price) - } - } - } else { - // Both maxPriorityfee and maxFee set by caller. Sanity-check their internal relation - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } + if err := args.setFeeDefaults(ctx, b); err != nil { + return err } if args.Value == nil { args.Value = new(hexutil.Big) @@ -172,9 +124,81 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { args.Gas = &estimated log.Trace("Estimate gas usage automatically", "gas", args.Gas) } - if args.ChainID == nil { - id := (*hexutil.Big)(b.ChainConfig().ChainID) - args.ChainID = id + // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local + // chain id as the default. + want := b.ChainConfig().ChainID + if args.ChainID != nil { + if have := (*big.Int)(args.ChainID); have.Cmp(want) != 0 { + return fmt.Errorf("chainId does not match node's (have=%v, want=%v)", have, want) + } + } else { + args.ChainID = (*hexutil.Big)(want) + } + return nil +} + +// setFeeDefaults fills in default fee values for unspecified tx fields. +func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } + // If the tx has completely specified a fee mechanism, no default is needed. This allows users + // who are not yet synced past London to get defaults for other tx values. See + // https://github.com/ethereum/go-ethereum/pull/23274 for more information. + eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil + if (args.GasPrice != nil && !eip1559ParamsSet) || (args.GasPrice == nil && eip1559ParamsSet) { + // Sanity check the EIP-1559 fee parameters if present. + if args.GasPrice == nil && args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + return nil + } + // Now attempt to fill in default value depending on whether London is active or not. + head := b.CurrentHeader() + if b.ChainConfig().IsLondon(head.Number) { + // London is active, set maxPriorityFeePerGas and maxFeePerGas. + if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { + return err + } + } else { + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") + } + // London not active, set gas price. + price, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.GasPrice = (*hexutil.Big)(price) + } + return nil +} + +// setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. +func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { + // Set maxPriorityFeePerGas if it is missing. + if args.MaxPriorityFeePerGas == nil { + tip, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) + } + // Set maxFeePerGas if it is missing. + if args.MaxFeePerGas == nil { + // Set the max fee to be 2 times larger than the previous block's base fee. + // The additional slack allows the tx to not become invalidated if the base + // fee is rising. + val := new(big.Int).Add( + args.MaxPriorityFeePerGas.ToInt(), + new(big.Int).Mul(head.BaseFee, big.NewInt(2)), + ) + args.MaxFeePerGas = (*hexutil.Big)(val) + } + // Both EIP-1559 fee parameters are now set; sanity check them. + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) } return nil } @@ -221,7 +245,7 @@ func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (t gasPrice = args.GasPrice.ToInt() gasFeeCap, gasTipCap = gasPrice, gasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those gasFeeCap = new(big.Int) if args.MaxFeePerGas != nil { gasFeeCap = args.MaxFeePerGas.ToInt() diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go new file mode 100644 index 000000000000..28dc561c36e4 --- /dev/null +++ b/internal/ethapi/transaction_args_test.go @@ -0,0 +1,342 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "context" + "fmt" + "math/big" + "reflect" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +// TestSetFeeDefaults tests the logic for filling in default fee values works as expected. +func TestSetFeeDefaults(t *testing.T) { + type test struct { + name string + isLondon bool + in *TransactionArgs + want *TransactionArgs + err error + } + + var ( + b = newBackendMock() + fortytwo = (*hexutil.Big)(big.NewInt(42)) + maxFee = (*hexutil.Big)(new(big.Int).Add(new(big.Int).Mul(b.current.BaseFee, big.NewInt(2)), fortytwo.ToInt())) + al = &types.AccessList{types.AccessTuple{Address: common.Address{0xaa}, StorageKeys: []common.Hash{{0x01}}}} + ) + + tests := []test{ + // Legacy txs + { + "legacy tx pre-London", + false, + &TransactionArgs{}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + { + "legacy tx post-London, explicit gas price", + true, + &TransactionArgs{GasPrice: fortytwo}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + + // Access list txs + { + "access list tx pre-London", + false, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London, explicit gas price", + false, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London", + true, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only max fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only priority fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + + // Dynamic fee txs + { + "dynamic tx post-London", + true, + &TransactionArgs{}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only max fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only priority fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic fee tx pre-London, maxFee set", + false, + &TransactionArgs{MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx pre-London, priorityFee set", + false, + &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx, maxFee < priorityFee", + true, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: (*hexutil.Big)(big.NewInt(1000))}, + nil, + fmt.Errorf("maxFeePerGas (0x3e) < maxPriorityFeePerGas (0x3e8)"), + }, + { + "dynamic fee tx, maxFee < priorityFee while setting default", + true, + &TransactionArgs{MaxFeePerGas: (*hexutil.Big)(big.NewInt(7))}, + nil, + fmt.Errorf("maxFeePerGas (0x7) < maxPriorityFeePerGas (0x2a)"), + }, + + // Misc + { + "set all fee parameters", + false, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxPriorityFee", + false, + &TransactionArgs{GasPrice: fortytwo, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxFee", + true, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + } + + ctx := context.Background() + for i, test := range tests { + if test.isLondon { + b.activateLondon() + } else { + b.deactivateLondon() + } + got := test.in + err := got.setFeeDefaults(ctx, b) + if err != nil && err.Error() == test.err.Error() { + // Test threw expected error. + continue + } else if err != nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } + if !reflect.DeepEqual(got, test.want) { + t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) + } + } +} + +type backendMock struct { + current *types.Header + config *params.ChainConfig +} + +func newBackendMock() *backendMock { + config := ¶ms.ChainConfig{ + ChainID: big.NewInt(42), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(1000), + } + return &backendMock{ + current: &types.Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1100), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + BaseFee: big.NewInt(10), + }, + config: config, + } +} + +func (b *backendMock) activateLondon() { + b.current.Number = big.NewInt(1100) +} + +func (b *backendMock) deactivateLondon() { + b.current.Number = big.NewInt(900) +} +func (b *backendMock) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + return big.NewInt(42), nil +} +func (b *backendMock) CurrentHeader() *types.Header { return b.current } +func (b *backendMock) ChainConfig() *params.ChainConfig { return b.config } + +// Other methods needed to implement Backend interface. +func (b *backendMock) SyncProgress() ethereum.SyncProgress { return ethereum.SyncProgress{} } +func (b *backendMock) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { + return nil, nil, nil, nil, nil +} +func (b *backendMock) ChainDb() ethdb.Database { return nil } +func (b *backendMock) AccountManager() *accounts.Manager { return nil } +func (b *backendMock) ExtRPCEnabled() bool { return false } +func (b *backendMock) RPCGasCap() uint64 { return 0 } +func (b *backendMock) RPCEVMTimeout() time.Duration { return time.Second } +func (b *backendMock) RPCTxFeeCap() float64 { return 0 } +func (b *backendMock) UnprotectedAllowed() bool { return false } +func (b *backendMock) SetHead(number uint64) {} +func (b *backendMock) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) CurrentBlock() *types.Block { return nil } +func (b *backendMock) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { return nil, nil } +func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + return nil, nil +} +func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + return nil, nil +} +func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } +func (b *backendMock) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { + return nil, nil, nil +} +func (b *backendMock) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return nil } +func (b *backendMock) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + return nil +} +func (b *backendMock) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { + return nil +} +func (b *backendMock) SendTx(ctx context.Context, signedTx *types.Transaction) error { return nil } +func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + return nil, [32]byte{}, 0, 0, nil +} +func (b *backendMock) GetPoolTransactions() (types.Transactions, error) { return nil, nil } +func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil } +func (b *backendMock) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { + return 0, nil +} +func (b *backendMock) Stats() (pending int, queued int) { return 0, 0 } +func (b *backendMock) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { + return nil, nil +} +func (b *backendMock) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { + return nil, nil +} +func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { return nil } +func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } +func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} +func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } +func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return nil +} +func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { + return nil +} + +func (b *backendMock) Engine() consensus.Engine { return nil } diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index d735d94faa78..de1d29ffd4b9 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -38,6 +38,15 @@ func NewApp(gitCommit, gitDate, usage string) *cli.App { return app } +// Merge merges the given flag slices. +func Merge(groups ...[]cli.Flag) []cli.Flag { + var ret []cli.Flag + for _, group := range groups { + ret = append(ret, group...) + } + return ret +} + var migrationApplied = map[*cli.Command]struct{}{} // MigrateGlobalFlags makes all global flag values available in the @@ -70,6 +79,10 @@ func MigrateGlobalFlags(ctx *cli.Context) { // This iterates over all commands and wraps their action function. iterate(ctx.App.Commands, func(cmd *cli.Command) { + if cmd.Action == nil { + return + } + action := cmd.Action cmd.Action = func(ctx *cli.Context) error { doMigrateFlags(ctx) diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index f07093fe8fc8..f1a6f23bad52 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -3696,7 +3696,7 @@ var outputBigNumberFormatter = function (number) { }; var isPredefinedBlockNumber = function (blockNumber) { - return blockNumber === 'latest' || blockNumber === 'pending' || blockNumber === 'earliest' || blockNumber === 'finalized'; + return blockNumber === 'latest' || blockNumber === 'pending' || blockNumber === 'earliest' || blockNumber === 'finalized' || blockNumber === 'safe'; }; var inputDefaultBlockNumberFormatter = function (blockNumber) { diff --git a/internal/jsre/jsre.go b/internal/jsre/jsre.go index c5946506c7a8..37ef254532de 100644 --- a/internal/jsre/jsre.go +++ b/internal/jsre/jsre.go @@ -329,11 +329,11 @@ func (re *JSRE) loadScript(call Call) (goja.Value, error) { file = common.AbsolutePath(re.assetPath, file) source, err := os.ReadFile(file) if err != nil { - return nil, fmt.Errorf("Could not read file %s: %v", file, err) + return nil, fmt.Errorf("could not read file %s: %v", file, err) } value, err := compileAndRun(re.vm, file, string(source)) if err != nil { - return nil, fmt.Errorf("Error while compiling or running script: %v", err) + return nil, fmt.Errorf("error while compiling or running script: %v", err) } return value, nil } diff --git a/internal/jsre/pretty.go b/internal/jsre/pretty.go index 4171e0090617..bd772b4927c2 100644 --- a/internal/jsre/pretty.go +++ b/internal/jsre/pretty.go @@ -219,7 +219,6 @@ func (ctx ppctx) fields(obj *goja.Object) []string { vals = append(vals, k) } } - } iterOwnAndConstructorKeys(ctx.vm, obj, add) sort.Strings(vals) diff --git a/les/api_backend.go b/les/api_backend.go index 11a9ca128aab..5b4213134b24 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -168,11 +168,8 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return nil, nil } -func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { - return light.GetBlockLogs(ctx, b.eth.odr, hash, *number) - } - return nil, nil +func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return light.GetBlockLogs(ctx, b.eth.odr, hash, number) } func (b *LesApiBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { diff --git a/les/api_test.go b/les/api_test.go index ea6870e35627..3db1c5fd5ec9 100644 --- a/les/api_test.go +++ b/les/api_test.go @@ -340,7 +340,6 @@ func freezeClient(ctx context.Context, t *testing.T, server *rpc.Client, clientI if err := server.CallContext(ctx, nil, "debug_freezeClient", clientID); err != nil { t.Fatalf("Failed to freeze client: %v", err) } - } func setCapacity(ctx context.Context, t *testing.T, server *rpc.Client, clientID enode.ID, cap uint64) { diff --git a/les/catalyst/api.go b/les/catalyst/api.go index ee888f4cdd03..1a59b27788e5 100644 --- a/les/catalyst/api.go +++ b/les/catalyst/api.go @@ -35,7 +35,6 @@ func Register(stack *node.Node, backend *les.LightEthereum) error { stack.RegisterAPIs([]rpc.API{ { Namespace: "engine", - Version: "1.0", Service: NewConsensusAPI(backend), Authenticated: true, }, @@ -51,7 +50,7 @@ type ConsensusAPI struct { // The underlying blockchain needs to have a valid terminal total difficulty set. func NewConsensusAPI(les *les.LightEthereum) *ConsensusAPI { if les.BlockChain().Config().TerminalTotalDifficulty == nil { - panic("Catalyst started without valid total difficulty") + log.Warn("Catalyst started without valid total difficulty") } return &ConsensusAPI{les: les} } diff --git a/les/catalyst/api_test.go b/les/catalyst/api_test.go index 70a6d24719ea..26c49d6ef908 100644 --- a/les/catalyst/api_test.go +++ b/les/catalyst/api_test.go @@ -55,7 +55,7 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Header, []*types.Bloc Timestamp: 9000, BaseFee: big.NewInt(params.InitialBaseFee), } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, n, nil) totalDifficulty := big.NewInt(0) diff --git a/les/client.go b/les/client.go index 97a333ab9b43..6504fe2af8f6 100644 --- a/les/client.go +++ b/les/client.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -93,7 +92,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideGrayGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideTerminalTotalDifficulty, config.OverrideTerminalTotalDifficultyPassed) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } @@ -294,27 +293,18 @@ func (s *LightEthereum) APIs() []rpc.API { return append(apis, []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: &LightDummyAPI{}, }, { Namespace: "eth", - Version: "1.0", Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), - }, { - Namespace: "eth", - Version: "1.0", - Service: filters.NewFilterAPI(s.ApiBackend, true, 5*time.Minute), }, { Namespace: "net", - Version: "1.0", Service: s.netRPCService, }, { Namespace: "les", - Version: "1.0", Service: NewLightAPI(&s.lesCommons), }, { Namespace: "vflux", - Version: "1.0", Service: s.serverPool.API(), }, }...) diff --git a/les/distributor.go b/les/distributor.go index 31150e4d731a..a0319c67f737 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -256,7 +256,7 @@ func (d *requestDistributor) queue(r *distReq) chan distPeer { if r.reqOrder == 0 { d.lastReqOrder++ r.reqOrder = d.lastReqOrder - r.waitForPeers = d.clock.Now() + mclock.AbsTime(waitForPeers) + r.waitForPeers = d.clock.Now().Add(waitForPeers) } // Assign the timestamp when the request is queued no matter it's // a new one or re-queued one. diff --git a/les/downloader/api.go b/les/downloader/api.go index 58e7aa5917d7..20fedfb1a7a2 100644 --- a/les/downloader/api.go +++ b/les/downloader/api.go @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} diff --git a/les/downloader/downloader.go b/les/downloader/downloader.go index e5af6177db84..c3c1b447e2eb 100644 --- a/les/downloader/downloader.go +++ b/les/downloader/downloader.go @@ -1332,7 +1332,6 @@ func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) expire func() map[string]int, pending func() int, inFlight func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, bool), fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, idle func() ([]*peerConnection, int), setIdle func(*peerConnection, int, time.Time), kind string) error { - // Create a ticker to detect expired retrieval tasks ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() @@ -1628,7 +1627,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { log.Warn("Invalid header encountered", "number", chunk[n].Number, "hash", chunk[n].Hash(), "parent", chunk[n].ParentHash, "err", err) return fmt.Errorf("%w: %v", errInvalidChain, err) } - // All verifications passed, track all headers within the alloted limits + // All verifications passed, track all headers within the allotted limits if mode == FastSync { head := chunk[len(chunk)-1].Number.Uint64() if head-rollback > uint64(fsHeaderSafetyNet) { @@ -1666,7 +1665,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { select { case ch <- true: diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index 2a4ebb02a9ce..37b2f78dd4ef 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewSecure(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewStateTrie(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) @@ -625,7 +625,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { t.Fatalf("block synchronization failed: %v", err) } tester.terminate() - } // Tests that simple synchronization against a forked chain works correctly. In @@ -662,8 +661,8 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// correctly and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } diff --git a/les/downloader/peer.go b/les/downloader/peer.go index 5a92e9cf9b87..c2161e2dae42 100644 --- a/les/downloader/peer.go +++ b/les/downloader/peer.go @@ -350,6 +350,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p diff --git a/les/downloader/queue.go b/les/downloader/queue.go index 73c10e42062f..98ebff526e83 100644 --- a/les/downloader/queue.go +++ b/les/downloader/queue.go @@ -834,7 +834,6 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest, reqTimer metrics.Timer, results int, validate func(index int, header *types.Header) error, reconstruct func(index int, result *fetchResult)) (int, error) { - // Short circuit if the data was never requested request := pendPool[id] if request == nil { @@ -874,7 +873,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/les/downloader/queue_test.go b/les/downloader/queue_test.go index 2a884d30aaba..44b2208595ff 100644 --- a/les/downloader/queue_test.go +++ b/les/downloader/queue_test.go @@ -27,23 +27,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -69,10 +63,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -150,7 +144,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -179,7 +173,6 @@ func TestBasics(t *testing.T) { if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { t.Fatalf("expected header %d, got %d", exp, got) } - } if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { t.Errorf("expected block task queue to be %d, got %d", exp, got) @@ -227,7 +220,6 @@ func TestEmptyBlocks(t *testing.T) { if fetchReq != nil { t.Fatal("there should be no body fetch tasks remaining") } - } if q.blockTaskQueue.Size() != numOfBlocks-10 { t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) @@ -241,7 +233,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { @@ -261,14 +253,13 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo world.progress(10) if false { log.Root().SetHandler(log.StdoutHandler) - } q := newQueue(10, 10) var wg sync.WaitGroup @@ -299,7 +290,6 @@ func XTestDelivery(t *testing.T) { fmt.Printf("got %d results, %d tot\n", len(res), tot) // Now we can forget about these world.forget(res[len(res)-1].Header.Number.Uint64()) - } }() wg.Add(1) @@ -362,7 +352,6 @@ func XTestDelivery(t *testing.T) { } for i := 0; i < 50; i++ { time.Sleep(2990 * time.Millisecond) - } }() wg.Add(1) @@ -413,10 +402,8 @@ func (n *network) forget(blocknum uint64) { n.chain = n.chain[index:] n.receipts = n.receipts[index:] n.offset = int(blocknum) - } func (n *network) progress(numBlocks int) { - n.lock.Lock() defer n.lock.Unlock() //fmt.Printf("progressing...\n") @@ -424,7 +411,6 @@ func (n *network) progress(numBlocks int) { n.chain = append(n.chain, newBlocks...) n.receipts = append(n.receipts, newR...) n.cond.Broadcast() - } func (n *network) headers(from int) []*types.Header { diff --git a/les/downloader/statesync.go b/les/downloader/statesync.go index fd24c5150b3c..22f952155f11 100644 --- a/les/downloader/statesync.go +++ b/les/downloader/statesync.go @@ -34,7 +34,7 @@ import ( // a single data retrieval network packet. type stateReq struct { nItems uint16 // Number of items requested for download (max is 384, so uint16 is sufficient) - trieTasks map[common.Hash]*trieTask // Trie node download tasks to track previous attempts + trieTasks map[string]*trieTask // Trie node download tasks to track previous attempts codeTasks map[common.Hash]*codeTask // Byte code download tasks to track previous attempts timeout time.Duration // Maximum round trip time for this to complete timer *time.Timer // Timer to fire when the RTT timeout expires @@ -263,8 +263,8 @@ type stateSync struct { sched *trie.Sync // State trie sync scheduler defining the tasks keccak crypto.KeccakState // Keccak256 hasher to verify deliveries with - trieTasks map[common.Hash]*trieTask // Set of trie node tasks currently queued for retrieval - codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval + trieTasks map[string]*trieTask // Set of trie node tasks currently queued for retrieval, indexed by path + codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval, indexed by hash numUncommitted int bytesUncommitted int @@ -281,6 +281,7 @@ type stateSync struct { // trieTask represents a single trie node download task, containing a set of // peers already attempted retrieval from to detect stalled syncs and abort. type trieTask struct { + hash common.Hash path [][]byte attempts map[string]struct{} } @@ -299,7 +300,7 @@ func newStateSync(d *Downloader, root common.Hash) *stateSync { root: root, sched: state.NewStateSync(root, d.stateDB, nil), keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState), - trieTasks: make(map[common.Hash]*trieTask), + trieTasks: make(map[string]*trieTask), codeTasks: make(map[common.Hash]*codeTask), deliver: make(chan *stateReq), cancel: make(chan struct{}), @@ -455,10 +456,11 @@ func (s *stateSync) assignTasks() { func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths []trie.SyncPath, codes []common.Hash) { // Refill available tasks from the scheduler. if fill := n - (len(s.trieTasks) + len(s.codeTasks)); fill > 0 { - nodes, paths, codes := s.sched.Missing(fill) - for i, hash := range nodes { - s.trieTasks[hash] = &trieTask{ - path: paths[i], + paths, hashes, codes := s.sched.Missing(fill) + for i, path := range paths { + s.trieTasks[path] = &trieTask{ + hash: hashes[i], + path: trie.NewSyncPath([]byte(path)), attempts: make(map[string]struct{}), } } @@ -474,7 +476,7 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths paths = make([]trie.SyncPath, 0, n) codes = make([]common.Hash, 0, n) - req.trieTasks = make(map[common.Hash]*trieTask, n) + req.trieTasks = make(map[string]*trieTask, n) req.codeTasks = make(map[common.Hash]*codeTask, n) for hash, t := range s.codeTasks { @@ -492,7 +494,7 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths req.codeTasks[hash] = t delete(s.codeTasks, hash) } - for hash, t := range s.trieTasks { + for path, t := range s.trieTasks { // Stop when we've gathered enough requests if len(nodes)+len(codes) == n { break @@ -504,11 +506,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths // Assign the request to this peer t.attempts[req.peer.id] = struct{}{} - nodes = append(nodes, hash) + nodes = append(nodes, t.hash) paths = append(paths, t.path) - req.trieTasks[hash] = t - delete(s.trieTasks, hash) + req.trieTasks[path] = t + delete(s.trieTasks, path) } req.nItems = uint16(len(nodes) + len(codes)) return nodes, paths, codes @@ -530,7 +532,7 @@ func (s *stateSync) process(req *stateReq) (int, error) { // Iterate over all the delivered data and inject one-by-one into the trie for _, blob := range req.response { - hash, err := s.processNodeData(blob) + hash, err := s.processNodeData(req.trieTasks, req.codeTasks, blob) switch err { case nil: s.numUncommitted++ @@ -543,13 +545,10 @@ func (s *stateSync) process(req *stateReq) (int, error) { default: return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } - // Delete from both queues (one delivery is enough for the syncer) - delete(req.trieTasks, hash) - delete(req.codeTasks, hash) } // Put unfulfilled tasks back into the retry queue npeers := s.d.peers.Len() - for hash, task := range req.trieTasks { + for path, task := range req.trieTasks { // If the node did deliver something, missing items may be due to a protocol // limit or a previous timeout + delayed delivery. Both cases should permit // the node to retry the missing items (to avoid single-peer stalls). @@ -559,10 +558,10 @@ func (s *stateSync) process(req *stateReq) (int, error) { // If we've requested the node too many times already, it may be a malicious // sync where nobody has the right data. Abort. if len(task.attempts) >= npeers { - return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", task.hash.TerminalString(), len(task.attempts), npeers) } // Missing item, place into the retry queue. - s.trieTasks[hash] = task + s.trieTasks[path] = task } for hash, task := range req.codeTasks { // If the node did deliver something, missing items may be due to a protocol @@ -585,13 +584,35 @@ func (s *stateSync) process(req *stateReq) (int, error) { // processNodeData tries to inject a trie node data blob delivered from a remote // peer into the state trie, returning whether anything useful was written or any // error occurred. -func (s *stateSync) processNodeData(blob []byte) (common.Hash, error) { - res := trie.SyncResult{Data: blob} +// +// If multiple requests correspond to the same hash, this method will inject the +// blob as a result for the first one only, leaving the remaining duplicates to +// be fetched again. +func (s *stateSync) processNodeData(nodeTasks map[string]*trieTask, codeTasks map[common.Hash]*codeTask, blob []byte) (common.Hash, error) { + var hash common.Hash s.keccak.Reset() s.keccak.Write(blob) - s.keccak.Read(res.Hash[:]) - err := s.sched.Process(res) - return res.Hash, err + s.keccak.Read(hash[:]) + + if _, present := codeTasks[hash]; present { + err := s.sched.ProcessCode(trie.CodeSyncResult{ + Hash: hash, + Data: blob, + }) + delete(codeTasks, hash) + return hash, err + } + for path, task := range nodeTasks { + if task.hash == hash { + err := s.sched.ProcessNode(trie.NodeSyncResult{ + Path: path, + Data: blob, + }) + delete(nodeTasks, path) + return hash, err + } + } + return common.Hash{}, trie.ErrNotRequested } // updateStats bumps the various state sync progress counters and displays a log diff --git a/les/downloader/testchain_test.go b/les/downloader/testchain_test.go index b9865f7e032b..400eec94e7c4 100644 --- a/les/downloader/testchain_test.go +++ b/les/downloader/testchain_test.go @@ -35,7 +35,12 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = gspec.MustCommit(testDB) ) // The common prefix of all test chains: diff --git a/les/fetcher.go b/les/fetcher.go index 0e80658edefa..0b6ce8d1e29b 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -253,7 +253,7 @@ func (f *lightFetcher) forEachPeer(check func(id enode.ID, p *fetcherPeer) bool) // request will be made for header retrieval. // // - re-sync trigger -// If the local chain lags too much, then the fetcher will enter "synnchronise" +// If the local chain lags too much, then the fetcher will enter "synchronise" // mode to retrieve missing headers in batch. func (f *lightFetcher) mainloop() { defer f.wg.Done() diff --git a/les/fetcher/block_fetcher.go b/les/fetcher/block_fetcher.go index 283008db0f1e..86b3c552ce27 100644 --- a/les/fetcher/block_fetcher.go +++ b/les/fetcher/block_fetcher.go @@ -641,7 +641,6 @@ func (f *BlockFetcher) loop() { } else { f.forgetHash(hash) } - } if matched { task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) diff --git a/les/fetcher/block_fetcher_test.go b/les/fetcher/block_fetcher_test.go index 58cfed7db590..be6e12b77d6a 100644 --- a/les/fetcher/block_fetcher_test.go +++ b/les/fetcher/block_fetcher_test.go @@ -35,10 +35,15 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index 4f0de8231835..76a241fa5a7f 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -182,7 +182,7 @@ func (node *ClientNode) UpdateParams(params ServerParams) { return } } - node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now + mclock.AbsTime(DecParamDelay), params: params}) + node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now.Add(DecParamDelay), params: params}) } } diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index 4ffbee58f0d0..4367974d632e 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -55,7 +55,7 @@ var ( // ClientManager controls the capacity assigned to the clients of a server. // Since ServerParams guarantee a safe lower estimate for processable requests // even in case of all clients being active, ClientManager calculates a -// corrigated buffer value and usually allows a higher remaining buffer value +// corrugated buffer value and usually allows a higher remaining buffer value // to be returned with each reply. type ClientManager struct { clock mclock.Clock diff --git a/les/flowcontrol/manager_test.go b/les/flowcontrol/manager_test.go index e5321c6742cd..82d98d66fba1 100644 --- a/les/flowcontrol/manager_test.go +++ b/les/flowcontrol/manager_test.go @@ -108,7 +108,6 @@ func testConstantTotalCapacity(t *testing.T, nodeCount, maxCapacityNodes, random if ratio < 0.98 || ratio > 1.02 { t.Errorf("totalCost/totalCapacity/testLength ratio incorrect (expected: 1, got: %f)", ratio) } - } func (n *testNode) send(t *testing.T, now mclock.AbsTime) bool { diff --git a/les/odr.go b/les/odr.go index 10ff0854d385..2643a534787f 100644 --- a/les/odr.go +++ b/les/odr.go @@ -126,7 +126,7 @@ const ( // RetrieveTxStatus retrieves the transaction status from the LES network. // There is no guarantee in the LES protocol that the mined transaction will // be retrieved back for sure because of different reasons(the transaction -// is unindexed, the malicous server doesn't reply it deliberately, etc). +// is unindexed, the malicious server doesn't reply it deliberately, etc). // Therefore, unretrieved transactions(UNKNOWN) will receive a certain number // of retries, thus giving a weak guarantee. func (odr *LesOdr) RetrieveTxStatus(ctx context.Context, req *light.TxStatusRequest) error { diff --git a/les/peer_test.go b/les/peer_test.go index d6551ce6b639..b8a1482a040a 100644 --- a/les/peer_test.go +++ b/les/peer_test.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -100,7 +99,7 @@ type fakeChain struct{} func (f *fakeChain) Config() *params.ChainConfig { return params.MainnetChainConfig } func (f *fakeChain) Genesis() *types.Block { - return core.DefaultGenesisBlock().ToBlock(rawdb.NewMemoryDatabase()) + return core.DefaultGenesisBlock().ToBlock() } func (f *fakeChain) CurrentHeader() *types.Header { return &types.Header{Number: big.NewInt(10000000)} } diff --git a/les/server.go b/les/server.go index 894fc138367f..df453b4819a2 100644 --- a/les/server.go +++ b/les/server.go @@ -159,17 +159,14 @@ func (s *LesServer) APIs() []rpc.API { return []rpc.API{ { Namespace: "les", - Version: "1.0", Service: NewLightAPI(&s.lesCommons), }, { Namespace: "les", - Version: "1.0", Service: NewLightServerAPI(s), }, { Namespace: "debug", - Version: "1.0", Service: NewDebugAPI(s), }, } diff --git a/les/utils/timeutils_test.go b/les/utils/timeutils_test.go index 9f9e1c2dc938..b219d0439dcb 100644 --- a/les/utils/timeutils_test.go +++ b/les/utils/timeutils_test.go @@ -37,7 +37,7 @@ func TestUpdateTimer(t *testing.T) { if updated := timer.Update(func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } - if updated := timer.UpdateAt(sim.Now()+mclock.AbsTime(time.Second), func(diff time.Duration) bool { return true }); !updated { + if updated := timer.UpdateAt(sim.Now().Add(time.Second), func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } timer = NewUpdateTimer(sim, 0) diff --git a/les/vflux/client/fillset_test.go b/les/vflux/client/fillset_test.go index ca5af8f07ecc..ddb12a82f9b3 100644 --- a/les/vflux/client/fillset_test.go +++ b/les/vflux/client/fillset_test.go @@ -104,7 +104,7 @@ func TestFillSet(t *testing.T) { fs.SetTarget(10) expWaiting(4, true) expNotWaiting() - // remove all previosly set flags + // remove all previously set flags ns.ForEach(sfTest1, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { ns.SetState(node, nodestate.Flags{}, sfTest1, 0) }) diff --git a/les/vflux/client/serverpool.go b/les/vflux/client/serverpool.go index e481075f70bd..cf96f0ee3a23 100644 --- a/les/vflux/client/serverpool.go +++ b/les/vflux/client/serverpool.go @@ -222,7 +222,6 @@ func (s *serverPoolIterator) Close() { func (s *ServerPool) AddMetrics( suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge metrics.Gauge, sessionValueMeter, serverDialedMeter metrics.Meter) { - s.suggestedTimeoutGauge = suggestedTimeoutGauge s.totalValueGauge = totalValueGauge s.sessionValueMeter = sessionValueMeter diff --git a/les/vflux/client/serverpool_test.go b/les/vflux/client/serverpool_test.go index 9f83c5f7f2c0..f1fd987d7edb 100644 --- a/les/vflux/client/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -66,7 +66,7 @@ type ServerPoolTest struct { // (accessed from both the main thread and the preNeg callback) preNegLock sync.Mutex queryWg *sync.WaitGroup // a new wait group is created each time the simulation is started - stopping bool // stopping avoid callind queryWg.Add after queryWg.Wait + stopping bool // stopping avoid calling queryWg.Add after queryWg.Wait cycle, conn, servedConn int serviceCycles, dialCount int diff --git a/les/vflux/client/wrsiterator.go b/les/vflux/client/wrsiterator.go index 8a2e39ad4422..1b37cba6e5de 100644 --- a/les/vflux/client/wrsiterator.go +++ b/les/vflux/client/wrsiterator.go @@ -109,7 +109,6 @@ func (w *WrsIterator) chooseNode() *enode.Node { return w.ns.GetNode(id) } } - } // Close ends the iterator. diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index 727ce09a432f..b09f7bb5012b 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -356,7 +356,7 @@ func (n *nodeBalance) estimatePriority(capacity uint64, addBalance int64, future b = n.reducedBalance(b, now, future, capacity, avgReqCost) } if bias > 0 { - b = n.reducedBalance(b, now+mclock.AbsTime(future), bias, capacity, 0) + b = n.reducedBalance(b, now.Add(future), bias, capacity, 0) } pri := n.balanceToPriority(now, b, capacity) // Ensure that biased estimates are always lower than actual priorities, even if @@ -512,7 +512,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { n.updateAfter(0) return } - if n.nextUpdate == 0 || n.nextUpdate > now+mclock.AbsTime(d) { + if n.nextUpdate == 0 || n.nextUpdate > now.Add(d) { if d > time.Second { // Note: if the scheduled update is not in the very near future then we // schedule the update a bit earlier. This way we do need to update a few @@ -520,7 +520,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { // brings the expected firing time a little bit closer. d = ((d - time.Second) * 7 / 8) + time.Second } - n.nextUpdate = now + mclock.AbsTime(d) + n.nextUpdate = now.Add(d) n.updateAfter(d) } } else { @@ -623,13 +623,13 @@ func (n *nodeBalance) priorityToBalance(priority int64, capacity uint64) (uint64 return 0, uint64(-priority) } -// reducedBalance estimates the reduced balance at a given time in the fututre based +// reducedBalance estimates the reduced balance at a given time in the future based // on the given balance, the time factor and an estimated average request cost per time ratio func (n *nodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Duration, capacity uint64, avgReqCost float64) balance { // since the costs are applied continuously during the dt time period we calculate // the expiration offset at the middle of the period var ( - at = start + mclock.AbsTime(dt/2) + at = start.Add(dt / 2) dtf = float64(dt) ) if !b.pos.IsZero() { diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index 9f253cabf48d..7c100aab509f 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -54,7 +54,7 @@ func newBalanceTestSetup(db ethdb.KeyValueStore, posExp, negExp utils.ValueExpir // Initialize and customize the setup for the balance testing clock := &mclock.Simulated{} setup := newServerSetup() - setup.clientField = setup.setup.NewField("balancTestClient", reflect.TypeOf(balanceTestClient{})) + setup.clientField = setup.setup.NewField("balanceTestClient", reflect.TypeOf(balanceTestClient{})) ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) if posExp == nil { @@ -298,7 +298,7 @@ func TestEstimatedPriority(t *testing.T) { } } -func TestPostiveBalanceCounting(t *testing.T) { +func TestPositiveBalanceCounting(t *testing.T) { b := newBalanceTestSetup(nil, nil, nil) defer b.stop() diff --git a/les/vflux/server/clientpool_test.go b/les/vflux/server/clientpool_test.go index 23dde86cb100..baf36d486790 100644 --- a/les/vflux/server/clientpool_test.go +++ b/les/vflux/server/clientpool_test.go @@ -414,7 +414,6 @@ func TestFreeClientKickedOut(t *testing.T) { clock.Run(5 * time.Minute) for i := 0; i < 10; i++ { connect(pool, newPoolTestPeer(i+10, kicked)) - } clock.Run(0) diff --git a/les/vflux/server/status.go b/les/vflux/server/status.go index 469190777b25..2d7e25b68461 100644 --- a/les/vflux/server/status.go +++ b/les/vflux/server/status.go @@ -41,7 +41,7 @@ type serverSetup struct { activeFlag nodestate.Flags // Flag is set if the node is active inactiveFlag nodestate.Flags // Flag is set if the node is inactive capacityField nodestate.Field // Field contains the capacity of the node - queueField nodestate.Field // Field contains the infomration in the priority queue + queueField nodestate.Field // Field contains the information in the priority queue } // newServerSetup initializes the setup for state machine and returns the flags/fields group. diff --git a/light/lightchain.go b/light/lightchain.go index 2a8e3672145a..dca97ce45ce6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -397,7 +397,7 @@ func (lc *LightChain) SetCanonical(header *types.Header) error { // // The verify parameter can be used to fine tune whether nonce verification // should be done or not. The reason behind the optional check is because some -// of the header retrieval mechanisms already need to verfy nonces, as well as +// of the header retrieval mechanisms already need to verify nonces, as well as // because nonces can be verified sparsely, not needing to check each. // // In the case of a light chain, InsertHeaderChain also creates and posts light diff --git a/light/odr_util.go b/light/odr_util.go index bbbcdbce2135..48631139b488 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -272,9 +272,9 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bit uint, sections []uint // GetTransaction retrieves a canonical transaction by hash and also returns // its position in the chain. There is no guarantee in the LES protocol that // the mined transaction will be retrieved back for sure because of different -// reasons(the transaction is unindexed, the malicous server doesn't reply it +// reasons(the transaction is unindexed, the malicious server doesn't reply it // deliberately, etc). Therefore, unretrieved transactions will receive a certain -// number of retrys, thus giving a weak guarantee. +// number of retries, thus giving a weak guarantee. func GetTransaction(ctx context.Context, odr OdrBackend, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { r := &TxStatusRequest{Hashes: []common.Hash{txHash}} if err := odr.RetrieveTxStatus(ctx, r); err != nil || r.Status[0].Status != core.TxStatusIncluded { diff --git a/light/postprocess.go b/light/postprocess.go index c09b00e71c81..3f9da659333e 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -217,7 +217,18 @@ func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) e // Commit implements core.ChainIndexerBackend func (c *ChtIndexerBackend) Commit() error { - root, _, err := c.trie.Commit(nil) + root, nodes, err := c.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := c.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + c.trie, err = trie.New(common.Hash{}, root, c.triedb) if err != nil { return err } @@ -302,7 +313,7 @@ var ( BloomTrieTablePrefix = "blt-" ) -// GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database +// GetBloomTrieRoot reads the BloomTrie root associated to the given section from the database func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -310,7 +321,7 @@ func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.H return common.BytesToHash(data) } -// StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database +// StoreBloomTrieRoot writes the BloomTrie root associated to the given section into the database func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -453,7 +464,18 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.trie.Delete(encKey[:]) } } - root, _, err := b.trie.Commit(nil) + root, nodes, err := b.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := b.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + b.trie, err = trie.New(common.Hash{}, root, b.triedb) if err != nil { return err } diff --git a/light/trie.go b/light/trie.go index 931ba30cb40a..b88265e87d40 100644 --- a/light/trie.go +++ b/light/trie.go @@ -112,6 +112,22 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) { return res, err } +func (t *odrTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + key = crypto.Keccak256(key) + var res types.StateAccount + err := t.do(key, func() (err error) { + value, err := t.trie.TryGet(key) + if err != nil { + return err + } + if value == nil { + return nil + } + return rlp.DecodeBytes(value, &res) + }) + return &res, err +} + func (t *odrTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { key = crypto.Keccak256(key) value, err := rlp.EncodeToBytes(acc) @@ -137,11 +153,19 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } -func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, int, error) { +// TryDeleteAccount abstracts an account deletion from the trie. +func (t *odrTrie) TryDeleteAccount(key []byte) error { + key = crypto.Keccak256(key) + return t.do(key, func() error { + return t.trie.TryDelete(key) + }) +} + +func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) { if t.trie == nil { - return t.id.Root, 0, nil + return t.id.Root, nil, nil } - return t.trie.Commit(onleaf) + return t.trie.Commit(collectLeaf) } func (t *odrTrie) Hash() common.Hash { diff --git a/light/trie_test.go b/light/trie_test.go index e8294cc2a235..63dd9020f20c 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -76,7 +76,7 @@ func diffTries(t1, t2 state.Trie) error { case i1.Err != nil: return fmt.Errorf("full trie iterator error: %v", i1.Err) case i2.Err != nil: - return fmt.Errorf("light trie iterator error: %v", i1.Err) + return fmt.Errorf("light trie iterator error: %v", i2.Err) case i1.Next(): return fmt.Errorf("full trie iterator has more k/v pairs") case i2.Next(): diff --git a/light/txpool.go b/light/txpool.go index c34998252fb5..d7bcf0265a7b 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -73,7 +73,7 @@ type TxPool struct { feedelegation bool // Fork indicator whether we are in the fee delegation stage. } -// TxRelayBackend provides an interface to the mechanism that forwards transacions +// TxRelayBackend provides an interface to the mechanism that forwards transactions // to the ETH network. The implementations of the functions should be non-blocking. // // Send instructs backend to forward new transactions @@ -424,7 +424,7 @@ func (pool *TxPool) add(ctx context.Context, tx *types.Transaction) error { hash := tx.Hash() if pool.pending[hash] != nil { - return fmt.Errorf("Known transaction (%x)", hash[:4]) + return fmt.Errorf("known transaction (%x)", hash[:4]) } err := pool.validateTx(ctx, tx) if err != nil { diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go index 02b75580c4e5..7b854d232ba8 100644 --- a/metrics/gauge_float64_test.go +++ b/metrics/gauge_float64_test.go @@ -2,7 +2,7 @@ package metrics import "testing" -func BenchmarkGuageFloat64(b *testing.B) { +func BenchmarkGaugeFloat64(b *testing.B) { g := NewGaugeFloat64() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go index 3aee143455c3..a98fe985d8c2 100644 --- a/metrics/gauge_test.go +++ b/metrics/gauge_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func BenchmarkGuage(b *testing.B) { +func BenchmarkGauge(b *testing.B) { g := NewGauge() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go index 076faa8fa327..6ee8be133bb2 100644 --- a/metrics/influxdb/influxdbv2.go +++ b/metrics/influxdb/influxdbv2.go @@ -80,7 +80,6 @@ func (r *v2Reporter) run() { } } } - } func (r *v2Reporter) send() { @@ -89,7 +88,6 @@ func (r *v2Reporter) send() { namespace := r.namespace switch metric := i.(type) { - case metrics.Counter: v := metric.Count() l := r.cache[name] diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 9ad5ec7e9929..c8408d8cab85 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -36,7 +36,7 @@ func Handler(reg metrics.Registry) http.Handler { }) sort.Strings(names) - // Aggregate all the metris into a Prometheus collector + // Aggregate all the metrics into a Prometheus collector c := newCollector() for _, name := range names { diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 6cfedfd88f00..d277ae5c3e47 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -307,5 +307,4 @@ func TestWalkRegistries(t *testing.T) { if prefix != "prefix.prefix2." { t.Fatal(prefix) } - } diff --git a/miner/miner_test.go b/miner/miner_test.go index cf619845dd47..5bf344fd7076 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -188,7 +188,6 @@ func TestStartStopMiner(t *testing.T) { waitForMiningState(t, miner, true) miner.Stop() waitForMiningState(t, miner, false) - } func TestCloseMiner(t *testing.T) { diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 439bcc5d10bd..88af84c7fcd3 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -236,7 +236,7 @@ func newNodeManager(genesis *core.Genesis) *nodeManager { return &nodeManager{ close: make(chan struct{}), genesis: genesis, - genesisBlock: genesis.ToBlock(nil), + genesisBlock: genesis.ToBlock(), } } diff --git a/miner/unconfirmed_test.go b/miner/unconfirmed_test.go index dc83cb92652d..60958f658abc 100644 --- a/miner/unconfirmed_test.go +++ b/miner/unconfirmed_test.go @@ -74,7 +74,7 @@ func TestUnconfirmedShifts(t *testing.T) { if n := pool.blocks.Len(); n != int(limit)/2 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit/2) } - // Try to shift all the remaining blocks out and verify emptyness + // Try to shift all the remaining blocks out and verify emptiness pool.Shift(start + 2*uint64(limit)) if n := pool.blocks.Len(); n != 0 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0) diff --git a/miner/worker.go b/miner/worker.go index f26094585f33..53a6d3949d1a 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -1216,7 +1216,6 @@ func (w *worker) throttleMining(ts []int64) (int64, int64) { } func (w *worker) commitTransactionsEx(env *environment, interrupt *int32, tstart time.Time) bool { - // committed transactions committedTxs := map[common.Hash]*types.Transaction{} @@ -1225,7 +1224,6 @@ func (w *worker) commitTransactionsEx(env *environment, interrupt *int32, tstart // Short circuit if there is no available pending transactions if len(pending) != 0 { - // using new simple round-robin ordering instead of old one. if params.PrefetchCount == 0 { // remove processed txs from 'pending' @@ -1255,7 +1253,6 @@ func (w *worker) commitTransactionsEx(env *environment, interrupt *int32, tstart return true } } - } time.Sleep(time.Until(*env.till)) diff --git a/miner/worker_test.go b/miner/worker_test.go index cc6858ced8c9..e3513d98839c 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -511,7 +511,7 @@ func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine co } w.start() - time.Sleep(time.Second) // Ensure two tasks have been summitted due to start opt + time.Sleep(time.Second) // Ensure two tasks have been submitted due to start opt atomic.StoreUint32(&start, 1) w.setRecommitInterval(3 * time.Second) diff --git a/mobile/accounts.go b/mobile/accounts.go index 76c811efd5a4..d3433aaff048 100644 --- a/mobile/accounts.go +++ b/mobile/accounts.go @@ -212,12 +212,12 @@ func (ks *KeyStore) ImportECDSAKey(key []byte, passphrase string) (account *Acco // ImportPreSaleKey decrypts the given Ethereum presale wallet and stores // a key file in the key directory. The key file is encrypted with the same passphrase. -func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (ccount *Account, _ error) { - account, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) +func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (account *Account, _ error) { + acc, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) if err != nil { return nil, err } - return &Account{account}, nil + return &Account{acc}, nil } func (ks *KeyStore) EdPubKey(a accounts.Account) ([]byte, error) { diff --git a/mobile/ethclient.go b/mobile/ethclient.go index 662125c4adeb..00bcb3a2b9bc 100644 --- a/mobile/ethclient.go +++ b/mobile/ethclient.go @@ -94,7 +94,6 @@ func (ec *EthereumClient) GetTransactionCount(ctx *Context, hash *Hash) (count i func (ec *EthereumClient) GetTransactionInBlock(ctx *Context, hash *Hash, index int) (tx *Transaction, _ error) { rawTx, err := ec.client.TransactionInBlock(ctx.context, hash.hash, uint(index)) return &Transaction{rawTx}, err - } // GetTransactionReceipt returns the receipt of a transaction by transaction hash. diff --git a/mobile/init.go b/mobile/init.go index 2025d85edc92..94f5baf28be7 100644 --- a/mobile/init.go +++ b/mobile/init.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Contains initialization code for the mbile library. +// Contains initialization code for the mobile library. package geth diff --git a/mobile/types.go b/mobile/types.go index a224f12ab23a..f3f92e4d4ac3 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -55,7 +55,7 @@ func (n *Nonce) GetBytes() []byte { // GetHex retrieves the hex string representation of the block nonce. func (n *Nonce) GetHex() string { - return fmt.Sprintf("0x%x", n.nonce[:]) + return fmt.Sprintf("%#x", n.nonce[:]) } // String returns a printable representation of the nonce. @@ -75,7 +75,7 @@ func (b *Bloom) GetBytes() []byte { // GetHex retrieves the hex string representation of the bloom filter. func (b *Bloom) GetHex() string { - return fmt.Sprintf("0x%x", b.bloom[:]) + return fmt.Sprintf("%#x", b.bloom[:]) } // String returns a printable representation of the bloom filter. diff --git a/node/api.go b/node/api.go index 5095d693e074..30a1a4c8d806 100644 --- a/node/api.go +++ b/node/api.go @@ -36,15 +36,12 @@ func (n *Node) apis() []rpc.API { return []rpc.API{ { Namespace: "admin", - Version: "1.0", Service: &adminAPI{n}, }, { Namespace: "debug", - Version: "1.0", Service: debug.Handler, }, { Namespace: "web3", - Version: "1.0", Service: &web3API{n}, }, } diff --git a/node/config_test.go b/node/config_test.go index 2a2eecf61cfb..50b2910105fa 100644 --- a/node/config_test.go +++ b/node/config_test.go @@ -115,7 +115,7 @@ func TestNodeKeyPersistency(t *testing.T) { } config := &Config{Name: "unit-test", DataDir: dir, P2P: p2p.Config{PrivateKey: key}} config.NodeKey() - if _, err := os.Stat(filepath.Join(keyfile)); err == nil { + if _, err := os.Stat(keyfile); err == nil { t.Fatalf("one-shot node key persisted to data directory") } @@ -136,7 +136,7 @@ func TestNodeKeyPersistency(t *testing.T) { // Configure a new node and ensure the previously persisted key is loaded config = &Config{Name: "unit-test", DataDir: dir} config.NodeKey() - blob2, err := os.ReadFile(filepath.Join(keyfile)) + blob2, err := os.ReadFile(keyfile) if err != nil { t.Fatalf("failed to read previously persisted node key: %v", err) } diff --git a/node/endpoints.go b/node/endpoints.go index efc311e7e317..14c12fd1f175 100644 --- a/node/endpoints.go +++ b/node/endpoints.go @@ -39,10 +39,11 @@ func StartHTTPEndpoint(endpoint string, timeouts rpc.HTTPTimeouts, handler http. CheckTimeouts(&timeouts) // Bundle and start the HTTP server httpSrv := &http.Server{ - Handler: handler, - ReadTimeout: timeouts.ReadTimeout, - WriteTimeout: timeouts.WriteTimeout, - IdleTimeout: timeouts.IdleTimeout, + Handler: handler, + ReadTimeout: timeouts.ReadTimeout, + ReadHeaderTimeout: timeouts.ReadHeaderTimeout, + WriteTimeout: timeouts.WriteTimeout, + IdleTimeout: timeouts.IdleTimeout, } go httpSrv.Serve(listener) return httpSrv, listener.Addr(), err @@ -75,6 +76,10 @@ func CheckTimeouts(timeouts *rpc.HTTPTimeouts) { log.Warn("Sanitizing invalid HTTP read timeout", "provided", timeouts.ReadTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadTimeout) timeouts.ReadTimeout = rpc.DefaultHTTPTimeouts.ReadTimeout } + if timeouts.ReadHeaderTimeout < time.Second { + log.Warn("Sanitizing invalid HTTP read header timeout", "provided", timeouts.ReadHeaderTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadHeaderTimeout) + timeouts.ReadHeaderTimeout = rpc.DefaultHTTPTimeouts.ReadHeaderTimeout + } if timeouts.WriteTimeout < time.Second { log.Warn("Sanitizing invalid HTTP write timeout", "provided", timeouts.WriteTimeout, "updated", rpc.DefaultHTTPTimeouts.WriteTimeout) timeouts.WriteTimeout = rpc.DefaultHTTPTimeouts.WriteTimeout diff --git a/node/jwt_handler.go b/node/jwt_handler.go index 28d5b87c60bc..363f6b3aad47 100644 --- a/node/jwt_handler.go +++ b/node/jwt_handler.go @@ -24,6 +24,8 @@ import ( "github.com/golang-jwt/jwt/v4" ) +const jwtExpiryTimeout = 60 * time.Second + type jwtHandler struct { keyFunc func(token *jwt.Token) (interface{}, error) next http.Handler @@ -68,9 +70,9 @@ func (handler *jwtHandler) ServeHTTP(out http.ResponseWriter, r *http.Request) { http.Error(out, "token is expired", http.StatusForbidden) case claims.IssuedAt == nil: http.Error(out, "missing issued-at", http.StatusForbidden) - case time.Since(claims.IssuedAt.Time) > 5*time.Second: + case time.Since(claims.IssuedAt.Time) > jwtExpiryTimeout: http.Error(out, "stale token", http.StatusForbidden) - case time.Until(claims.IssuedAt.Time) > 5*time.Second: + case time.Until(claims.IssuedAt.Time) > jwtExpiryTimeout: http.Error(out, "future token", http.StatusForbidden) default: handler.next.ServeHTTP(out, r) diff --git a/node/node.go b/node/node.go index 55b1600d4b0b..66fd22a34f21 100644 --- a/node/node.go +++ b/node/node.go @@ -703,7 +703,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -715,14 +715,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, if n.config.DataDir == "" { db = rawdb.NewMemoryDatabase() } else { - root := n.ResolvePath(name) - switch { - case freezer == "": - freezer = filepath.Join(root, "ancient") - case !filepath.IsAbs(freezer): - freezer = n.ResolvePath(freezer) - } - db, err = rawdb.NewDBWithFreezer(root, cache, handles, freezer, namespace, readonly) + db, err = rawdb.NewDBWithFreezer(n.ResolvePath(name), cache, handles, n.ResolveAncient(name, ancient), namespace, readonly) } if err == nil { @@ -736,6 +729,17 @@ func (n *Node) ResolvePath(x string) string { return n.config.ResolvePath(x) } +// ResolveAncient returns the absolute path of the root ancient directory. +func (n *Node) ResolveAncient(name string, ancient string) string { + switch { + case ancient == "": + ancient = filepath.Join(n.ResolvePath(name), "ancient") + case !filepath.IsAbs(ancient): + ancient = n.ResolvePath(ancient) + } + return ancient +} + // closeTrackingDB wraps the Close method of a database. When the database is closed by the // service, the wrapper removes it from the node's database map. This ensures that Node // won't auto-close the database if it is closed by the service that opened it. diff --git a/node/node_test.go b/node/node_test.go index 9f9febcacbfe..7c76e21f6baf 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -581,7 +581,6 @@ func (test rpcPrefixTest) check(t *testing.T, node *Node) { if err == nil { t.Errorf("Error: %s: WebSocket connection succeeded for path in wantNoWS", path) } - } } @@ -614,7 +613,6 @@ func doHTTPRequest(t *testing.T, req *http.Request) *http.Response { resp, err := client.Do(req) if err != nil { t.Fatalf("could not issue a GET request to the given endpoint: %v", err) - } return resp } diff --git a/node/rpcstack.go b/node/rpcstack.go index 09692c0a0b19..5d411fa61e81 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -27,6 +27,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" @@ -81,6 +82,10 @@ type httpServer struct { handlerNames map[string]string } +const ( + shutdownTimeout = 5 * time.Second +) + func newHTTPServer(log log.Logger, timeouts rpc.HTTPTimeouts) *httpServer { h := &httpServer{log: log, timeouts: timeouts, handlerNames: make(map[string]string)} @@ -129,6 +134,7 @@ func (h *httpServer) start() error { if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout + h.server.ReadHeaderTimeout = h.timeouts.ReadHeaderTimeout h.server.WriteTimeout = h.timeouts.WriteTimeout h.server.IdleTimeout = h.timeouts.IdleTimeout } @@ -261,7 +267,13 @@ func (h *httpServer) doStop() { h.wsHandler.Store((*rpcHandler)(nil)) wsHandler.server.Stop() } - h.server.Shutdown(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + err := h.server.Shutdown(ctx) + if err == ctx.Err() { + h.log.Warn("HTTP server graceful shutdown timed out") + h.server.Close() + } h.listener.Close() h.log.Info("HTTP server stopped", "endpoint", h.listener.Addr()) @@ -427,7 +439,6 @@ func (h *virtualHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // It's an IP address, we can serve that h.next.ServeHTTP(w, r) return - } // Not an IP address, but a hostname. Need to validate if _, exist := h.vhosts["*"]; exist { diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 58a02234025a..09acf7ea0458 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -100,7 +100,7 @@ func TestWebsocketOrigins(t *testing.T) { expFail: []string{ "test", // no scheme, required by spec "http://test", // wrong scheme - "http://test.foo", "https://a.test.x", // subdomain variatoins + "http://test.foo", "https://a.test.x", // subdomain variations "http://testx:8540", "https://xtest:8540"}, }, // ip tests @@ -356,11 +356,11 @@ func TestJWT(t *testing.T) { expFail := []func() string{ // future func() string { - return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 6})) + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + int64(jwtExpiryTimeout.Seconds()) + 1})) }, // stale func() string { - return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 6})) + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - int64(jwtExpiryTimeout.Seconds()) - 1})) }, // wrong algo func() string { diff --git a/node/utils_test.go b/node/utils_test.go index 31dbe911ba81..681f3a8b285c 100644 --- a/node/utils_test.go +++ b/node/utils_test.go @@ -95,15 +95,12 @@ func (f *FullService) APIs() []rpc.API { return []rpc.API{ { Namespace: "admin", - Version: "1.0", }, { Namespace: "debug", - Version: "1.0", }, { Namespace: "net", - Version: "1.0", }, } } diff --git a/p2p/discover/ntp.go b/p2p/discover/ntp.go index 1bb52399fbc5..48ceffe95b8d 100644 --- a/p2p/discover/ntp.go +++ b/p2p/discover/ntp.go @@ -108,7 +108,7 @@ func sntpDrift(measurements int) (time.Duration, error) { // Calculate the drift based on an assumed answer time of RRT/2 drifts = append(drifts, sent.Sub(t)+elapsed/2) } - // Calculate average drif (drop two extremities to avoid outliers) + // Calculate average drift (drop two extremities to avoid outliers) sort.Sort(durationSlice(drifts)) drift := time.Duration(0) diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 47a2e7ac3caf..77e03ca9e7e4 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -134,8 +134,8 @@ func newPingRecorder() *pingRecorder { } } -// setRecord updates a node record. Future calls to ping and -// requestENR will return this record. +// updateRecord updates a node record. Future calls to ping and +// RequestENR will return this record. func (t *pingRecorder) updateRecord(n *enode.Node) { t.mu.Lock() defer t.mu.Unlock() @@ -162,7 +162,7 @@ func (t *pingRecorder) ping(n *enode.Node) (seq uint64, err error) { return seq, nil } -// requestENR simulates an ENR request. +// RequestENR simulates an ENR request. func (t *pingRecorder) RequestENR(n *enode.Node) (*enode.Node, error) { t.mu.Lock() defer t.mu.Unlock() diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index a3e1549075b3..67cd2c004cf6 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -334,7 +334,7 @@ func (t *UDPv4) findnode(toid enode.ID, toaddr *net.UDPAddr, target v4wire.Pubke return nodes, err } -// RequestENR sends enrRequest to the given node and waits for a response. +// RequestENR sends ENRRequest to the given node and waits for a response. func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { addr := &net.UDPAddr{IP: n.IP(), Port: n.UDP()} t.ensureBond(n.ID(), addr) @@ -525,7 +525,7 @@ func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. + // Shut down the loop for permanent errors. if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index aa8dd5c14647..1edc26e9b6c8 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -530,7 +530,7 @@ func TestUDPv4_EIP868(t *testing.T) { t.Fatalf("invalid record: %v", err) } if !reflect.DeepEqual(n, wantNode) { - t.Fatalf("wrong node in enrResponse: %v", n) + t.Fatalf("wrong node in ENRResponse: %v", n) } }) } diff --git a/p2p/discover/v4wire/v4wire.go b/p2p/discover/v4wire/v4wire.go index d6bf3dc4600a..b07a6e341c31 100644 --- a/p2p/discover/v4wire/v4wire.go +++ b/p2p/discover/v4wire/v4wire.go @@ -86,16 +86,16 @@ type ( Rest []rlp.RawValue `rlp:"tail"` } - // enrRequest queries for the remote node's record. + // ENRRequest queries for the remote node's record. ENRRequest struct { Expiration uint64 // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` } - // enrResponse is the reply to enrRequest. + // ENRResponse is the reply to ENRRequest. ENRResponse struct { - ReplyTok []byte // Hash of the enrRequest packet. + ReplyTok []byte // Hash of the ENRRequest packet. Record enr.Record // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 22fab7243501..071ed65adc7f 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -347,7 +347,7 @@ func (t *UDPv5) ping(n *enode.Node) (uint64, error) { } } -// requestENR requests n's record. +// RequestENR requests n's record. func (t *UDPv5) RequestENR(n *enode.Node) (*enode.Node, error) { nodes, err := t.findnode(n, []uint{0}) if err != nil { @@ -407,6 +407,9 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s if err := netutil.CheckRelayIP(c.node.IP(), node.IP()); err != nil { return nil, err } + if t.netrestrict != nil && !t.netrestrict.Contains(node.IP()) { + return nil, errors.New("not contained in netrestrict list") + } if c.node.UDP() <= 1024 { return nil, errLowPort } @@ -622,7 +625,7 @@ func (t *UDPv5) readLoop() { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. + // Shut down the loop for permanent errors. if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 8dc64de6dfd6..45f2f0883bad 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -300,7 +300,7 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error return head, nil } -// encodeHandshakeMessage encodes the handshake message packet header. +// encodeHandshakeHeader encodes the handshake message packet header. func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Whoareyou) (Header, *session, error) { // Ensure calling code sets challenge.node. if challenge.Node == nil { @@ -337,7 +337,7 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who return head, session, err } -// encodeAuthHeader creates the auth header on a request packet following WHOAREYOU. +// makeHandshakeAuth creates the auth header on a request packet following WHOAREYOU. func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoareyou) (*handshakeAuthData, *session, error) { auth := new(handshakeAuthData) auth.h.SrcID = c.localnode.ID() @@ -379,7 +379,7 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey return auth, sec, err } -// encodeMessage encodes an encrypted message packet. +// encodeMessageHeader encodes an encrypted message packet. func (c *Codec) encodeMessageHeader(toID enode.ID, s *session) (Header, error) { head := c.makeHeader(toID, flagMessage, 0) @@ -632,7 +632,7 @@ func (h *StaticHeader) checkValid(packetLen int) error { return nil } -// headerMask returns a cipher for 'masking' / 'unmasking' packet headers. +// mask returns a cipher for 'masking' / 'unmasking' packet headers. func (h *Header) mask(destID enode.ID) cipher.Stream { block, err := aes.NewCipher(destID[:16]) if err != nil { diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go index 5bfa27b43378..d4e0eb8b5aa3 100644 --- a/p2p/msgrate/msgrate.go +++ b/p2p/msgrate/msgrate.go @@ -111,7 +111,7 @@ const tuningImpact = 0.25 // local link is saturated. In that case, the live measurements will force us // to reduce request sizes until the throughput gets stable. // -// Lastly, message rate measurements allows us to detect if a peer is unsuaully +// Lastly, message rate measurements allows us to detect if a peer is unusually // slow compared to other peers, in which case we can decide to keep it around // or free up the slot so someone closer. // @@ -127,7 +127,7 @@ type Tracker struct { // in their sizes. // // Callers of course are free to use the item counter as a byte counter if - // or when their protocol of choise if capped by bytes instead of items. + // or when their protocol of choice if capped by bytes instead of items. // (eg. eth.getHeaders vs snap.getAccountRange). capacity map[uint64]float64 @@ -157,7 +157,7 @@ func NewTracker(caps map[uint64]float64, rtt time.Duration) *Tracker { } // Capacity calculates the number of items the peer is estimated to be able to -// retrieve within the alloted time slot. The method will round up any division +// retrieve within the allotted time slot. The method will round up any division // errors and will add an additional overestimation ratio on top. The reason for // overshooting the capacity is because certain message types might not increase // the load proportionally to the requested items, so fetching a bit more might diff --git a/p2p/netutil/error_test.go b/p2p/netutil/error_test.go index 645e48f83741..84d5c2c20621 100644 --- a/p2p/netutil/error_test.go +++ b/p2p/netutil/error_test.go @@ -66,7 +66,6 @@ func TestIsPacketTooBig(t *testing.T) { for i := range buf { if buf[i] != byte(i) { t.Fatalf("error in pattern") - break } } } diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 35ccdfb06882..7bfa8aab6d10 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -501,7 +501,6 @@ func startExecNodeStack() (*node.Node, error) { // Add the snapshot API. stack.RegisterAPIs([]rpc.API{{ Namespace: "simulation", - Version: "1.0", Service: SnapshotAPI{services}, }}) diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go index a344a8d5cbbe..b221a0597fc4 100644 --- a/p2p/simulations/http.go +++ b/p2p/simulations/http.go @@ -367,7 +367,6 @@ func (s *Server) StopMocker(w http.ResponseWriter, req *http.Request) { // GetMockerList returns a list of available mockers func (s *Server) GetMockers(w http.ResponseWriter, req *http.Request) { - list := GetMockerList() s.JSON(w, http.StatusOK, list) } diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index f5172f3f23db..5775977a41f0 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -489,7 +489,6 @@ func (t *expectEvents) expect(events ...*Event) { } switch expected.Type { - case EventTypeNode: if event.Node == nil { t.Fatal("expected event.Node to be set") @@ -514,7 +513,6 @@ func (t *expectEvents) expect(events ...*Event) { if event.Conn.Up != expected.Conn.Up { t.Fatalf("expected conn event %d to have up=%t, got up=%t", i, expected.Conn.Up, event.Conn.Up) } - } i++ diff --git a/p2p/simulations/mocker.go b/p2p/simulations/mocker.go index 1477b02772b7..47193d83ccb8 100644 --- a/p2p/simulations/mocker.go +++ b/p2p/simulations/mocker.go @@ -157,7 +157,6 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) { } wg.Wait() } - } // connect nodeCount number of nodes in a ring diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 962910dd25bf..d6c5aca73c5c 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -235,7 +235,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub } peer := event.Peer switch event.Type { - case p2p.PeerEventTypeAdd: net.DidConnect(id, peer) @@ -247,7 +246,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub case p2p.PeerEventTypeMsgRecv: net.DidReceive(peer, id, event.Protocol, *event.MsgCode) - } case err := <-sub.Err(): @@ -927,7 +925,6 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn if !haveSvc { cleanedServices = append(cleanedServices, svc) } - } snap.Nodes[i].Node.Config.Lifecycles = cleanedServices } @@ -1021,7 +1018,6 @@ func (net *Network) Load(snap *Snapshot) error { // Start connecting. for _, conn := range snap.Conns { - if !net.GetNode(conn.One).Up() || !net.GetNode(conn.Other).Up() { //in this case, at least one of the nodes of a connection is not up, //so it would result in the snapshot `Load` to fail diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go index fa6936d273c5..ab8cf19462e7 100644 --- a/p2p/simulations/network_test.go +++ b/p2p/simulations/network_test.go @@ -36,7 +36,6 @@ import ( // Tests that a created snapshot with a minimal service only contains the expected connections // and that a network when loaded with this snapshot only contains those same connections func TestSnapshot(t *testing.T) { - // PART I // create snapshot from ring network @@ -204,7 +203,6 @@ OuterTwo: t.Fatal(ctx.Err()) case ev := <-evC: if ev.Type == EventTypeConn && !ev.Control { - // fail on any disconnect if !ev.Conn.Up { t.Fatalf("unexpected disconnect: %v -> %v", ev.Conn.One, ev.Conn.Other) @@ -693,7 +691,6 @@ func BenchmarkMinimalService(b *testing.B) { } func benchmarkMinimalServiceTmp(b *testing.B) { - // stop timer to discard setup time pollution args := strings.Split(b.Name(), "/") nodeCount, err := strconv.ParseInt(args[2], 10, 16) diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 69a49087e2c4..6a733b9ba51e 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -121,7 +121,7 @@ func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint6 } // clean is called automatically when a preset time passes without a response -// being dleivered for the first network request. +// being delivered for the first network request. func (t *Tracker) clean() { t.lock.Lock() defer t.lock.Unlock() diff --git a/params/bootnodes.go b/params/bootnodes.go index 2ad230268bc6..b80997774536 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -116,6 +116,8 @@ func KnownDNSNetwork(genesis common.Hash, protocol string) string { net = "rinkeby" case GoerliGenesisHash: net = "goerli" + case SepoliaGenesisHash: + net = "sepolia" default: return "" } diff --git a/params/config.go b/params/config.go index 87a2baa44464..ef264da941f2 100644 --- a/params/config.go +++ b/params/config.go @@ -57,26 +57,29 @@ var CheckpointOracles = map[common.Hash]*CheckpointOracleConfig{ } var ( + MainnetTerminalTotalDifficulty, _ = new(big.Int).SetString("58_750_000_000_000_000_000_000", 0) + // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1_150_000), - DAOForkBlock: big.NewInt(1_920_000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2_463_000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2_675_000), - EIP158Block: big.NewInt(2_675_000), - ByzantiumBlock: big.NewInt(4_370_000), - ConstantinopleBlock: big.NewInt(7_280_000), - PetersburgBlock: big.NewInt(7_280_000), - IstanbulBlock: big.NewInt(9_069_000), - MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), - LondonBlock: big.NewInt(12_965_000), - ArrowGlacierBlock: big.NewInt(13_773_000), - GrayGlacierBlock: big.NewInt(15_050_000), - Ethash: new(EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2_463_000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), + LondonBlock: big.NewInt(12_965_000), + ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), + TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 + Ethash: new(EthashConfig), } // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. @@ -102,23 +105,24 @@ var ( // RopstenChainConfig contains the chain parameters to run a node on the Ropsten test network. RopstenChainConfig = &ChainConfig{ - ChainID: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1_700_000), - ConstantinopleBlock: big.NewInt(4_230_000), - PetersburgBlock: big.NewInt(4_939_394), - IstanbulBlock: big.NewInt(6_485_846), - MuirGlacierBlock: big.NewInt(7_117_117), - BerlinBlock: big.NewInt(9_812_189), - LondonBlock: big.NewInt(10_499_401), - TerminalTotalDifficulty: new(big.Int).SetUint64(50000000000000000), - Ethash: new(EthashConfig), + ChainID: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1_700_000), + ConstantinopleBlock: big.NewInt(4_230_000), + PetersburgBlock: big.NewInt(4_939_394), + IstanbulBlock: big.NewInt(6_485_846), + MuirGlacierBlock: big.NewInt(7_117_117), + BerlinBlock: big.NewInt(9_812_189), + LondonBlock: big.NewInt(10_499_401), + TerminalTotalDifficulty: new(big.Int).SetUint64(50_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // RopstenTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. @@ -186,22 +190,24 @@ var ( // SepoliaChainConfig contains the chain parameters to run a node on the Sepolia test network. SepoliaChainConfig = &ChainConfig{ - ChainID: big.NewInt(11155111), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - TerminalTotalDifficulty: big.NewInt(17_000_000_000_000_000), - Ethash: new(EthashConfig), + ChainID: big.NewInt(11155111), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(17_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + MergeNetsplitBlock: big.NewInt(1735371), + Ethash: new(EthashConfig), } // SepoliaTrustedCheckpoint contains the light client trusted checkpoint for the Sepolia test network. @@ -258,21 +264,23 @@ var ( // GoerliChainConfig contains the chain parameters to run a node on the Görli test network. GoerliChainConfig = &ChainConfig{ - ChainID: big.NewInt(5), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(1_561_651), - MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(4_460_644), - LondonBlock: big.NewInt(5_062_605), - ArrowGlacierBlock: nil, + ChainID: big.NewInt(5), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(1_561_651), + MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(4_460_644), + LondonBlock: big.NewInt(5_062_605), + ArrowGlacierBlock: nil, + TerminalTotalDifficulty: big.NewInt(10_790_000), + TerminalTotalDifficultyPassed: true, Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -305,16 +313,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, false, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, nil, nil, false, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, false, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int), false) ) @@ -409,11 +417,18 @@ type ChainConfig struct { MergeNetsplitBlock *big.Int `json:"mergeNetsplitBlock,omitempty"` // Virtual fork after The Merge to use as a network splitter PangyoBlock *big.Int `json:"pangyoBlock,omitempty"` // Pangyo switch block (nil = no fork, 0 = already on pangyo) ApplepieBlock *big.Int `json:"applepieBlock,omitempty"` // Applepie switch block (nil = no fork, 0 = already on applepie) + ShanghaiBlock *big.Int `json:"shanghaiBlock,omitempty"` // Shanghai switch block (nil = no fork, 0 = already on shanghai) + CancunBlock *big.Int `json:"cancunBlock,omitempty"` // Cancun switch block (nil = no fork, 0 = already on cancun) // TerminalTotalDifficulty is the amount of total difficulty reached by // the network that triggers the consensus upgrade. TerminalTotalDifficulty *big.Int `json:"terminalTotalDifficulty,omitempty"` + // TerminalTotalDifficultyPassed is a flag specifying that the network already + // passed the terminal total difficulty. Its purpose is to disable legacy sync + // even without having seen the TTD locally (safer long term). + TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` + // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -456,12 +471,16 @@ func (c *ChainConfig) String() string { } else { banner += "Consensus: Ethash (Stake base Proof of Authority for wemix)\n" } + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Ethash (proof-of-work)\n" } else { banner += "Consensus: Beacon (proof-of-stake), merged from Ethash (proof-of-work)\n" } case c.Clique != nil: if c.TerminalTotalDifficulty == nil { banner += "Consensus: Clique (proof-of-authority)\n" + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Clique (proof-of-authority)\n" } else { banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" } @@ -498,20 +517,26 @@ func (c *ChainConfig) String() string { } banner += fmt.Sprintf(" - Pangyo: %-8v\n", c.PangyoBlock) banner += fmt.Sprintf(" - Applepie: %-8v\n", c.ApplepieBlock) + if c.ShanghaiBlock != nil { + banner += fmt.Sprintf(" - Shanghai: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", c.ShanghaiBlock) + } + if c.CancunBlock != nil { + banner += fmt.Sprintf(" - Cancun: %-8v\n", c.CancunBlock) + } banner += "\n" // Add a special section for the merge as it's non-obvious if c.TerminalTotalDifficulty == nil { - banner += "Merge not configured!\n" - banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)" + banner += "The Merge is not yet available for this network!\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md" } else { banner += "Merge configured:\n" - banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)\n" - banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) - banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" + banner += fmt.Sprintf(" - Network known to be merged: %v\n", c.TerminalTotalDifficultyPassed) + banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) + banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) } return banner - } // IsHomestead returns whether num is either equal to the homestead block or greater. @@ -605,6 +630,16 @@ func (c *ChainConfig) IsTerminalPoWBlock(parentTotalDiff *big.Int, totalDiff *bi return parentTotalDiff.Cmp(c.TerminalTotalDifficulty) < 0 && totalDiff.Cmp(c.TerminalTotalDifficulty) >= 0 } +// IsShanghai returns whether num is either equal to the Shanghai fork block or greater. +func (c *ChainConfig) IsShanghai(num *big.Int) bool { + return isForked(c.ShanghaiBlock, num) +} + +// IsCancun returns whether num is either equal to the Cancun fork block or greater. +func (c *ChainConfig) IsCancun(num *big.Int) bool { + return isForked(c.CancunBlock, num) +} + // CheckCompatible checks whether scheduled fork transitions have been imported // with a mismatching chain configuration. func (c *ChainConfig) CheckCompatible(newcfg *ChainConfig, height uint64) *ConfigCompatError { @@ -653,6 +688,8 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "arrowGlacierBlock", block: c.ArrowGlacierBlock, optional: true}, {name: "grayGlacierBlock", block: c.GrayGlacierBlock, optional: true}, {name: "mergeNetsplitBlock", block: c.MergeNetsplitBlock, optional: true}, + {name: "shanghaiBlock", block: c.ShanghaiBlock, optional: true}, + {name: "cancunBlock", block: c.CancunBlock, optional: true}, } { if lastFork.name != "" { // Next one must be higher number @@ -731,6 +768,12 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock, head) { return newCompatError("Merge netsplit fork block", c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock) } + if isForkIncompatible(c.ShanghaiBlock, newcfg.ShanghaiBlock, head) { + return newCompatError("Shanghai fork block", c.ShanghaiBlock, newcfg.ShanghaiBlock) + } + if isForkIncompatible(c.CancunBlock, newcfg.CancunBlock, head) { + return newCompatError("Cancun fork block", c.CancunBlock, newcfg.CancunBlock) + } return nil } @@ -801,6 +844,7 @@ type Rules struct { IsBerlin, IsLondon bool IsMerge bool IsPangyo, IsApplepie bool + IsShanghai, isCancun bool } // Rules ensures c's ChainID is not nil. @@ -824,5 +868,7 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool) Rules { IsMerge: isMerge, IsPangyo: c.IsPangyo(num), IsApplepie: c.IsApplepie(num), + IsShanghai: c.IsShanghai(num), + isCancun: c.IsCancun(num), } } diff --git a/rlp/decode_test.go b/rlp/decode_test.go index e0d33dc43ee9..00722f847bbb 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -1043,7 +1043,6 @@ func TestInvalidOptionalField(t *testing.T) { t.Errorf("wrong error for %T: %v", test.v, err.Error()) } } - } func ExampleDecode() { diff --git a/rlp/iterator.go b/rlp/iterator.go index 353ef09fbdf2..6be574572e61 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -36,7 +36,6 @@ func NewListIterator(data RawValue) (*listIterator, error) { data: data[t : t+c], } return it, nil - } // Next forwards the iterator one step, returns true if it was not at end yet diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go index 17d7e64e0842..25d4393cc656 100644 --- a/rlp/rlpgen/main.go +++ b/rlp/rlpgen/main.go @@ -106,7 +106,7 @@ func (cfg *Config) process() (code []byte, err error) { // Find the type and generate. typ, err := lookupStructType(pkg.Scope(), cfg.Type) if err != nil { - return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err) + return nil, fmt.Errorf("can't find %s in %s: %v", cfg.Type, pkg, err) } code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder) if err != nil { diff --git a/rpc/http.go b/rpc/http.go index 9f4464957349..858d80858652 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -87,6 +87,14 @@ type HTTPTimeouts struct { // ReadHeaderTimeout. It is valid to use them both. ReadTimeout time.Duration + // ReadHeaderTimeout is the amount of time allowed to read + // request headers. The connection's read deadline is reset + // after reading the headers and the Handler can decide what + // is considered too slow for the body. If ReadHeaderTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. + ReadHeaderTimeout time.Duration + // WriteTimeout is the maximum duration before timing out // writes of the response. It is reset whenever a new // request's header is read. Like ReadTimeout, it does not @@ -103,9 +111,10 @@ type HTTPTimeouts struct { // DefaultHTTPTimeouts represents the default timeout values used if further // configuration is not provided. var DefaultHTTPTimeouts = HTTPTimeouts{ - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 120 * time.Second, + ReadTimeout: 30 * time.Second, + ReadHeaderTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, } // DialHTTPWithClient creates a new RPC client that connects to an RPC server over HTTP diff --git a/rpc/server.go b/rpc/server.go index babc5688e264..bf1f71a28e26 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -160,7 +160,7 @@ type PeerInfo struct { // Address of client. This will usually contain the IP address and port. RemoteAddr string - // Addditional information for HTTP and WebSocket connections. + // Additional information for HTTP and WebSocket connections. HTTP struct { // Protocol version, i.e. "HTTP/1.1". This is not set for WebSocket. Version string diff --git a/rpc/types.go b/rpc/types.go index a0d742f49142..e3d1a4896821 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -31,7 +31,7 @@ import ( // API describes the set of methods offered over the RPC interface type API struct { Namespace string // namespace under which the rpc methods of Service are exposed - Version string // api version for DApp's + Version string // deprecated - this field is no longer used, but retained for compatibility Service interface{} // receiver instance which holds the methods Public bool // deprecated - this field is no longer used, but retained for compatibility Authenticated bool // whether the api should only be available behind authentication. @@ -61,6 +61,7 @@ type jsonWriter interface { type BlockNumber int64 const ( + SafeBlockNumber = BlockNumber(-4) FinalizedBlockNumber = BlockNumber(-3) PendingBlockNumber = BlockNumber(-2) LatestBlockNumber = BlockNumber(-1) @@ -92,6 +93,9 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { case "finalized": *bn = FinalizedBlockNumber return nil + case "safe": + *bn = SafeBlockNumber + return nil } blckNum, err := hexutil.DecodeUint64(input) @@ -118,6 +122,8 @@ func (bn BlockNumber) MarshalText() ([]byte, error) { return []byte("pending"), nil case FinalizedBlockNumber: return []byte("finalized"), nil + case SafeBlockNumber: + return []byte("safe"), nil default: return hexutil.Uint64(bn).MarshalText() } @@ -168,6 +174,10 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { bn := FinalizedBlockNumber bnh.BlockNumber = &bn return nil + case "safe": + bn := SafeBlockNumber + bnh.BlockNumber = &bn + return nil default: if len(input) == 66 { hash := common.Hash{} diff --git a/signer/core/api.go b/signer/core/api.go index 867965327c30..5949c3b4276b 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -319,7 +319,6 @@ func (api *SignerAPI) openTrezor(url accounts.URL) { log.Warn("failed to open wallet", "wallet", url, "err", err) return } - } // startUSBListener starts a listener for USB events, for hardware wallet interaction @@ -612,7 +611,6 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args apitypes.SendTxA api.UI.OnApprovedTx(response) // ...and to the external caller return &response, nil - } func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common.MixedcaseAddress, gnosisTx GnosisSafeTx, methodSelector *string) (*GnosisSafeTx, error) { diff --git a/signer/core/api_test.go b/signer/core/api_test.go index 333dff26117c..9bb55bddca31 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -55,7 +55,6 @@ func (ui *headlessUi) RegisterUIServer(api *core.UIServerAPI) {} func (ui *headlessUi) OnApprovedTx(tx ethapi.SignTransactionResult) {} func (ui *headlessUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { - switch <-ui.approveCh { case "Y": return core.SignTxResponse{request.Transaction, true}, nil @@ -125,7 +124,6 @@ func setup(t *testing.T) (*core.SignerAPI, *headlessUi) { am := core.StartClefAccountManager(tmpDirName(t), true, true, "") api := core.NewSignerAPI(am, 1337, true, ui, db, true, &storage.NoStorage{}) return api, ui - } func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { ui.approveCh <- "Y" @@ -139,7 +137,6 @@ func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { } func failCreateAccountWithPassword(ui *headlessUi, api *core.SignerAPI, password string, t *testing.T) { - ui.approveCh <- "Y" // We will be asked three times to provide a suitable password ui.inputCh <- password @@ -169,7 +166,6 @@ func failCreateAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { func list(ui *headlessUi, api *core.SignerAPI, t *testing.T) ([]common.Address, error) { ui.approveCh <- "A" return api.List(context.Background()) - } func TestNewAcc(t *testing.T) { @@ -321,5 +317,4 @@ func TestSignTx(t *testing.T) { if bytes.Equal(res.Raw, res2.Raw) { t.Error("Expected tx to be modified by UI") } - } diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index fea321a1a0bf..47ac2232b741 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -251,6 +251,25 @@ type TypedDataDomain struct { Salt string `json:"salt"` } +// TypedDataAndHash is a helper function that calculates a hash for typed data conforming to EIP-712. +// This hash can then be safely used to calculate a signature. +// +// See https://eips.ethereum.org/EIPS/eip-712 for the full specification. +// +// This gives context to the signed typed data and prevents signing of transactions. +func TypedDataAndHash(typedData TypedData) ([]byte, string, error) { + domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map()) + if err != nil { + return nil, "", err + } + typedDataHash, err := typedData.HashStruct(typedData.PrimaryType, typedData.Message) + if err != nil { + return nil, "", err + } + rawData := fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash)) + return crypto.Keccak256([]byte(rawData)), rawData, nil +} + // HashStruct generates a keccak256 hash of the encoding of the provided data func (typedData *TypedData) HashStruct(primaryType string, data TypedDataMessage) (hexutil.Bytes, error) { encodedData, err := typedData.EncodeData(primaryType, data, 1) @@ -526,7 +545,6 @@ func (typedData *TypedData) EncodePrimitiveValue(encType string, encValue interf return math.U256Bytes(b), nil } return nil, fmt.Errorf("unrecognized type '%s'", encType) - } // dataMismatchError generates an error for a mismatch between @@ -653,13 +671,12 @@ func formatPrimitiveValue(encType string, encValue interface{}) (string, error) } if strings.HasPrefix(encType, "bytes") { return fmt.Sprintf("%s", encValue), nil - } if strings.HasPrefix(encType, "uint") || strings.HasPrefix(encType, "int") { if b, err := parseInteger(encType, encValue); err != nil { return "", err } else { - return fmt.Sprintf("%d (0x%x)", b, b), nil + return fmt.Sprintf("%d (%#x)", b, b), nil } } return "", fmt.Errorf("unhandled type %v", encType) @@ -784,6 +801,8 @@ func isPrimitiveTypeValid(primitiveType string) bool { primitiveType == "int32[]" || primitiveType == "int64" || primitiveType == "int64[]" || + primitiveType == "int96" || + primitiveType == "int96[]" || primitiveType == "int128" || primitiveType == "int128[]" || primitiveType == "int256" || @@ -800,6 +819,8 @@ func isPrimitiveTypeValid(primitiveType string) bool { primitiveType == "uint32[]" || primitiveType == "uint64" || primitiveType == "uint64[]" || + primitiveType == "uint96" || + primitiveType == "uint96[]" || primitiveType == "uint128" || primitiveType == "uint128[]" || primitiveType == "uint256" || diff --git a/signer/core/auditlog.go b/signer/core/auditlog.go index 663d6d131735..a0b292bf714c 100644 --- a/signer/core/auditlog.go +++ b/signer/core/auditlog.go @@ -110,7 +110,6 @@ func (l *AuditLogger) Version(ctx context.Context) (string, error) { data, err := l.api.Version(ctx) l.log.Info("Version", "type", "response", "data", data, "error", err) return data, err - } func NewAuditLogger(path string, api ExternalAPI) (*AuditLogger, error) { diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 05c60906cc0c..187eb1390af7 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -59,7 +59,6 @@ func (ui *CommandlineUI) readString() string { } func (ui *CommandlineUI) OnInputRequired(info UserInputRequest) (UserInputResponse, error) { - fmt.Printf("## %s\n\n%s\n", info.Title, info.Prompt) defer fmt.Println("-----------------------") if info.IsPassword { @@ -147,7 +146,6 @@ func (ui *CommandlineUI) ApproveTx(request *SignTxRequest) (SignTxResponse, erro fmt.Printf(" * %s : %s\n", m.Typ, m.Message) } fmt.Println() - } fmt.Printf("\n") showMetadata(request.Meta) @@ -209,7 +207,6 @@ func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, err // ApproveNewAccount prompt the user for confirmation to create new Account, and reveal to caller func (ui *CommandlineUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResponse, error) { - ui.mu.Lock() defer ui.mu.Unlock() @@ -245,7 +242,6 @@ func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) { } func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) { - fmt.Printf("------- Signer info -------\n") for k, v := range info.Info { fmt.Printf("* %v : %v\n", k, v) diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index 48559bd98cf7..c0da22e62662 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -129,7 +129,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType { Name: "Full message for signing", Typ: "hexdata", - Value: fmt.Sprintf("0x%x", msg), + Value: fmt.Sprintf("%#x", msg), }, } req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Messages: messages, Hash: sighash} @@ -161,7 +161,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType { Name: "Clique header", Typ: "clique", - Value: fmt.Sprintf("clique header %d [0x%x]", header.Number, header.Hash()), + Value: fmt.Sprintf("clique header %d [%#x]", header.Number, header.Hash()), }, } // Clique uses V on the form 0 or 1 @@ -233,23 +233,17 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd // - the signature preimage (hash) func (api *SignerAPI) signTypedData(ctx context.Context, addr common.MixedcaseAddress, typedData apitypes.TypedData, validationMessages *apitypes.ValidationMessages) (hexutil.Bytes, hexutil.Bytes, error) { - domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map()) + sighash, rawData, err := apitypes.TypedDataAndHash(typedData) if err != nil { return nil, nil, err } - typedDataHash, err := typedData.HashStruct(typedData.PrimaryType, typedData.Message) - if err != nil { - return nil, nil, err - } - rawData := []byte(fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash))) - sighash := crypto.Keccak256(rawData) messages, err := typedData.Format() if err != nil { return nil, nil, err } req := &SignDataRequest{ ContentType: apitypes.DataTyped.Mime, - Rawdata: rawData, + Rawdata: []byte(rawData), Messages: messages, Hash: sighash, Address: addr} diff --git a/signer/core/validation_test.go b/signer/core/validation_test.go index 7105691d29c0..6adaa21afd4e 100644 --- a/signer/core/validation_test.go +++ b/signer/core/validation_test.go @@ -38,7 +38,6 @@ func TestPasswordValidation(t *testing.T) { if err == nil && test.shouldFail { t.Errorf("password '%v' should fail validation", test.pw) } else if err != nil && !test.shouldFail { - t.Errorf("password '%v' shound not fail validation, but did: %v", test.pw, err) } } diff --git a/signer/fourbyte/validation_test.go b/signer/fourbyte/validation_test.go index 2e6d9f2d9bb7..1b0ab507a864 100644 --- a/signer/fourbyte/validation_test.go +++ b/signer/fourbyte/validation_test.go @@ -53,7 +53,6 @@ func dummyTxArgs(t txtestcase) *apitypes.SendTxArgs { if t.i != "" { a := hexutil.Bytes(common.FromHex(t.i)) input = &a - } return &apitypes.SendTxArgs{ From: *from, diff --git a/signer/rules/rules.go b/signer/rules/rules.go index 6852d86f3ec7..5ed4514e0227 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -59,6 +59,7 @@ func NewRuleEvaluator(next core.UIClientAPI, jsbackend storage.Storage) (*rulese return c, nil } func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) { + r.next.RegisterUIServer(api) // TODO, make it possible to query from js } @@ -67,7 +68,6 @@ func (r *rulesetUI) Init(javascriptRules string) error { return nil } func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (goja.Value, error) { - // Instantiate a fresh vm engine every time vm := goja.New() diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index c9f99a444369..c35da8ecc188 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -44,7 +44,7 @@ Three things can happen: 3. Anything else; other return values [*], method not implemented or exception occurred during processing. This means that the operation will continue to manual processing, via the regular UI method chosen by the user. -[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not +[*] Note: Future version of the ruleset may use more complex json-based return values, making it possible to not only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all accounts in a list-request. The points above will continue to hold for non-json based responses ("Approve"/"Reject"). @@ -152,7 +152,6 @@ func TestListRequest(t *testing.T) { } func TestSignTxRequest(t *testing.T) { - js := ` function ApproveTx(r){ console.log("transaction.from", r.transaction.from); @@ -245,7 +244,6 @@ func (d *dummyUI) OnSignerStartup(info core.StartupInfo) { // TestForwarding tests that the rule-engine correctly dispatches requests to the next caller func TestForwarding(t *testing.T) { - js := "" ui := &dummyUI{make([]string, 0)} jsBackend := storage.NewEphemeralStorage() @@ -268,11 +266,8 @@ func TestForwarding(t *testing.T) { expCalls := 6 if len(ui.calls) != expCalls { - t.Errorf("Expected %d forwarded calls, got %d: %s", expCalls, len(ui.calls), strings.Join(ui.calls, ",")) - } - } func TestMissingFunc(t *testing.T) { @@ -296,10 +291,8 @@ func TestMissingFunc(t *testing.T) { t.Errorf("Expected missing method to cause non-approval") } t.Logf("Err %v", err) - } func TestStorage(t *testing.T) { - js := ` function testStorage(){ storage.put("mykey", "myvalue") @@ -348,7 +341,6 @@ func TestStorage(t *testing.T) { t.Errorf("Unexpected data, expected '%v', got '%v'", exp, retval) } t.Logf("Err %v", err) - } const ExampleTxWindow = ` @@ -442,7 +434,7 @@ func dummyTx(value hexutil.Big) *core.SignTxRequest { Gas: gas, }, Callinfo: []apitypes.ValidationInfo{ - {Typ: "Warning", Message: "All your base are bellong to us"}, + {Typ: "Warning", Message: "All your base are belong to us"}, }, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, } @@ -548,7 +540,6 @@ func (d *dontCallMe) OnApprovedTx(tx ethapi.SignTransactionResult) { // if it does, that would be bad since developers may rely on that to store data, // instead of using the disk-based data storage func TestContextIsCleared(t *testing.T) { - js := ` function ApproveTx(){ if (typeof foobar == 'undefined') { @@ -580,7 +571,6 @@ func TestContextIsCleared(t *testing.T) { } func TestSignData(t *testing.T) { - js := `function ApproveListing(){ return "Approve" } diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index f09bfa7d4f06..928d643dd618 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -143,7 +143,7 @@ func (s *AESEncryptedStorage) writeEncryptedStorage(creds map[string]storedCrede // encrypt encrypts plaintext with the given key, with additional data // The 'additionalData' is used to place the (plaintext) KV-store key into the V, -// to prevent the possibility to alter a K, or swap two entries in the KV store with eachother. +// to prevent the possibility to alter a K, or swap two entries in the KV store with each other. func encrypt(key []byte, plaintext []byte, additionalData []byte) ([]byte, []byte, error) { block, err := aes.NewCipher(key) if err != nil { diff --git a/signer/storage/aes_gcm_storage_test.go b/signer/storage/aes_gcm_storage_test.go index a2a95d9deedf..e1fea59280a8 100644 --- a/signer/storage/aes_gcm_storage_test.go +++ b/signer/storage/aes_gcm_storage_test.go @@ -51,7 +51,6 @@ func TestEncryption(t *testing.T) { } func TestFileStorage(t *testing.T) { - a := map[string]storedCredential{ "secret": { Iv: common.Hex2Bytes("cdb30036279601aeee60f16b"), diff --git a/tests/difficulty_test_util.go b/tests/difficulty_test_util.go index bda5a9611be8..62b978f9ef2b 100644 --- a/tests/difficulty_test_util.go +++ b/tests/difficulty_test_util.go @@ -65,5 +65,4 @@ func (test *DifficultyTest) Run(config *params.ChainConfig) error { test.CurrentTimestamp, test.CurrentBlockNumber, actual, exp) } return nil - } diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 5a65152aa873..c8106d813e7b 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -163,7 +163,6 @@ func (f *fuzzer) fuzz() int { // Modify something in the proof db // add stuff to proof db // drop stuff from proof db - } if f.exhausted { break diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index c8e7edb0636a..81829b4f50c5 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -140,7 +140,6 @@ func Debug(data []byte) int { } func (f *fuzzer) fuzz() int { - // This spongeDb is used to check the sequence of disk-db-writes var ( spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} @@ -176,10 +175,13 @@ func (f *fuzzer) fuzz() int { return 0 } // Flush trie -> database - rootA, _, err := trieA.Commit(nil) + rootA, nodes, err := trieA.Commit(false) if err != nil { panic(err) } + if nodes != nil { + dbA.Update(trie.NewWithNodeSet(nodes)) + } // Flush memdb -> disk (sponge) dbA.Commit(rootA, false, nil) @@ -187,7 +189,7 @@ func (f *fuzzer) fuzz() int { sort.Sort(vals) for _, kv := range vals { if f.debugging { - fmt.Printf("{\"0x%x\" , \"0x%x\"} // stacktrie.Update\n", kv.k, kv.v) + fmt.Printf("{\"%#x\" , \"%#x\"} // stacktrie.Update\n", kv.k, kv.v) } trieB.Update(kv.k, kv.v) } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 6a41657e52c6..db55c3a7d976 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -51,9 +51,8 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opProve opMax // boundary value, not an actual op @@ -84,11 +83,9 @@ func (ds *dataSource) Ended() bool { } func Generate(input []byte) randTest { - var allKeys [][]byte r := newDataSource(input) genKey := func() []byte { - if len(allKeys) < 2 || r.readByte() < 0x0f { // new key key := make([]byte, r.readByte()%50) @@ -103,7 +100,6 @@ func Generate(input []byte) randTest { var steps randTest for i := 0; !r.Ended(); i++ { - step := randTestStep{op: int(r.readByte()) % opMax} switch step.op { case opUpdate: @@ -143,7 +139,6 @@ func Fuzz(input []byte) int { } func runRandTest(rt randTest) error { - triedb := trie.NewDatabase(memorydb.New()) tr := trie.NewEmpty(triedb) @@ -161,17 +156,20 @@ func runRandTest(rt randTest) error { v := tr.Get(step.key) want := values[string(step.key)] if string(v) != want { - rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) + rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + hash, nodes, err := tr.Commit(false) if err != nil { return err } + if nodes != nil { + if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } newtr, err := trie.New(common.Hash{}, hash, triedb) if err != nil { return err diff --git a/tests/state_test.go b/tests/state_test.go index 93d8a1210626..d33ebc4b00db 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -193,7 +193,7 @@ func runBenchmark(b *testing.B, t *StateTest) { return } vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false) var baseFee *big.Int @@ -249,7 +249,6 @@ func runBenchmark(b *testing.B, t *StateTest) { } statedb.RevertToSnapshot(snapshot) } - }) } } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index f6d8e15001d8..38cdbc4d6504 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -184,7 +184,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) var baseFee *big.Int @@ -220,7 +220,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee - if t.json.Env.Random != nil { + context.Random = nil + if config.IsLondon(new(big.Int)) && t.json.Env.Random != nil { rnd := common.BigToHash(t.json.Env.Random) context.Random = &rnd context.Difficulty = big.NewInt(0) diff --git a/trie/committer.go b/trie/committer.go index 9b7ecbf5fcce..d9f0ecf3dea4 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -17,71 +17,48 @@ package trie import ( - "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" ) -// leafChanSize is the size of the leafCh. It's a pretty arbitrary number, to allow -// some parallelism but not incur too much memory overhead. -const leafChanSize = 200 - -// leaf represents a trie leaf value +// leaf represents a trie leaf node type leaf struct { - size int // size of the rlp data (estimate) - hash common.Hash // hash of rlp data - node node // the node to commit + blob []byte // raw blob of leaf + parent common.Hash // the hash of parent node } -// committer is a type used for the trie Commit operation. A committer has some -// internal preallocated temp space, and also a callback that is invoked when -// leaves are committed. The leafs are passed through the `leafCh`, to allow -// some level of parallelism. -// By 'some level' of parallelism, it's still the case that all leaves will be -// processed sequentially - onleaf will never be called in parallel or out of order. +// committer is the tool used for the trie Commit operation. The committer will +// capture all dirty nodes during the commit process and keep them cached in +// insertion order. type committer struct { - onleaf LeafCallback - leafCh chan *leaf -} - -// committers live in a global sync.Pool -var committerPool = sync.Pool{ - New: func() interface{} { - return &committer{} - }, + nodes *NodeSet + collectLeaf bool } // newCommitter creates a new committer or picks one from the pool. -func newCommitter() *committer { - return committerPool.Get().(*committer) -} - -func returnCommitterToPool(h *committer) { - h.onleaf = nil - h.leafCh = nil - committerPool.Put(h) +func newCommitter(owner common.Hash, collectLeaf bool) *committer { + return &committer{ + nodes: NewNodeSet(owner), + collectLeaf: collectLeaf, + } } // Commit collapses a node down into a hash node and inserts it into the database -func (c *committer) Commit(n node, db *Database) (hashNode, int, error) { - if db == nil { - return nil, 0, errors.New("no db provided") - } - h, committed, err := c.commit(n, db) +func (c *committer) Commit(n node) (hashNode, *NodeSet, error) { + h, err := c.commit(nil, n) if err != nil { - return nil, 0, err + return nil, nil, err } - return h.(hashNode), committed, nil + return h.(hashNode), c.nodes, nil } // commit collapses a node down into a hash node and inserts it into the database -func (c *committer) commit(n node, db *Database) (node, int, error) { +func (c *committer) commit(path []byte, n node) (node, error) { // if this path is clean, use available cached data hash, dirty := n.cache() if hash != nil && !dirty { - return hash, 0, nil + return hash, nil } // Commit children, then parent, and remove the dirty flag. switch cn := n.(type) { @@ -91,36 +68,35 @@ func (c *committer) commit(n node, db *Database) (node, int, error) { // If the child is fullNode, recursively commit, // otherwise it can only be hashNode or valueNode. - var childCommitted int if _, ok := cn.Val.(*fullNode); ok { - childV, committed, err := c.commit(cn.Val, db) + childV, err := c.commit(append(path, cn.Key...), cn.Val) if err != nil { - return nil, 0, err + return nil, err } - collapsed.Val, childCommitted = childV, committed + collapsed.Val = childV } // The key needs to be copied, since we're delivering it to database collapsed.Key = hexToCompact(cn.Key) - hashedNode := c.store(collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil } - return collapsed, childCommitted, nil + return collapsed, nil case *fullNode: - hashedKids, childCommitted, err := c.commitChildren(cn, db) + hashedKids, err := c.commitChildren(path, cn) if err != nil { - return nil, 0, err + return nil, err } collapsed := cn.copy() collapsed.Children = hashedKids - hashedNode := c.store(collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil } - return collapsed, childCommitted, nil + return collapsed, nil case hashNode: - return cn, 0, nil + return cn, nil default: // nil, valuenode shouldn't be committed panic(fmt.Sprintf("%T: invalid node: %v", n, n)) @@ -128,11 +104,8 @@ func (c *committer) commit(n node, db *Database) (node, int, error) { } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, error) { - var ( - committed int - children [17]node - ) +func (c *committer) commitChildren(path []byte, n *fullNode) ([17]node, error) { + var children [17]node for i := 0; i < 16; i++ { child := n.Children[i] if child == nil { @@ -148,87 +121,63 @@ func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, er // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's // possible the type is not hashNode. - hashed, childCommitted, err := c.commit(child, db) + hashed, err := c.commit(append(path, byte(i)), child) if err != nil { - return children, 0, err + return children, err } children[i] = hashed - committed += childCommitted } // For the 17th child, it's possible the type is valuenode. if n.Children[16] != nil { children[16] = n.Children[16] } - return children, committed, nil + return children, nil } // store hashes the node n and if we have a storage layer specified, it writes // the key/value pair to it and tracks any node->child references as well as any // node->external trie references. -func (c *committer) store(n node, db *Database) node { +func (c *committer) store(path []byte, n node) node { // Larger nodes are replaced by their hash and stored in the database. - var ( - hash, _ = n.cache() - size int - ) + var hash, _ = n.cache() + + // This was not generated - must be a small node stored in the parent. + // In theory, we should check if the node is leaf here (embedded node + // usually is leaf node). But small value(less than 32bytes) is not + // our target(leaves in account trie only). if hash == nil { - // This was not generated - must be a small node stored in the parent. - // In theory, we should apply the leafCall here if it's not nil(embedded - // node usually contains value). But small value(less than 32bytes) is - // not our target. return n - } else { - // We have the hash already, estimate the RLP encoding-size of the node. - // The size is used for mem tracking, does not need to be exact - size = estimateSize(n) } - // If we're using channel-based leaf-reporting, send to channel. - // The leaf channel will be active only when there an active leaf-callback - if c.leafCh != nil { - c.leafCh <- &leaf{ - size: size, - hash: common.BytesToHash(hash), - node: n, + // We have the hash already, estimate the RLP encoding-size of the node. + // The size is used for mem tracking, does not need to be exact + var ( + size = estimateSize(n) + nhash = common.BytesToHash(hash) + mnode = &memoryNode{ + hash: nhash, + node: simplifyNode(n), + size: uint16(size), } - } else if db != nil { - // No leaf-callback used, but there's still a database. Do serial - // insertion - db.insert(common.BytesToHash(hash), size, n) - } - return hash -} - -// commitLoop does the actual insert + leaf callback for nodes. -func (c *committer) commitLoop(db *Database) { - for item := range c.leafCh { - var ( - hash = item.hash - size = item.size - n = item.node - ) - // We are pooling the trie nodes into an intermediate memory cache - db.insert(hash, size, n) - - if c.onleaf != nil { - switch n := n.(type) { - case *shortNode: - if child, ok := n.Val.(valueNode); ok { - c.onleaf(nil, nil, child, hash) - } - case *fullNode: - // For children in range [0, 15], it's impossible - // to contain valueNode. Only check the 17th child. - if n.Children[16] != nil { - c.onleaf(nil, nil, n.Children[16].(valueNode), hash) - } + ) + // Collect the dirty node to nodeset for return. + c.nodes.add(string(path), mnode) + + // Collect the corresponding leaf node if it's required. We don't check + // full node since it's impossible to store value in fullNode. The key + // length of leaves should be exactly same. + if c.collectLeaf { + if sn, ok := n.(*shortNode); ok { + if val, ok := sn.Val.(valueNode); ok { + c.nodes.addLeaf(&leaf{blob: val, parent: nhash}) } } } + return hash } // estimateSize estimates the size of an rlp-encoded node, without actually // rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie -// with 1000 leafs, the only errors above 1% are on small shortnodes, where this +// with 1000 leaves, the only errors above 1% are on small shortnodes, where this // method overestimates by 2 or 3 bytes (e.g. 37 instead of 35) func estimateSize(n node) int { switch n := n.(type) { diff --git a/trie/database.go b/trie/database.go index 2df2e859d7b4..8c154ba96df6 100644 --- a/trie/database.go +++ b/trie/database.go @@ -28,6 +28,7 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -74,8 +75,6 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie - gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit gcsize common.StorageSize // Data storage garbage collected since last commit @@ -84,9 +83,9 @@ type Database struct { flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit - dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) - childrenSize common.StorageSize // Storage size of the external children tracking - preimagesSize common.StorageSize // Storage size of the preimages cache + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) + childrenSize common.StorageSize // Storage size of the external children tracking + preimages *preimageStore // The store for caching preimages lock sync.RWMutex } @@ -164,7 +163,10 @@ func (n *cachedNode) rlp() []byte { // or by regenerating it from the rlp encoded blob. func (n *cachedNode) obj(hash common.Hash) node { if node, ok := n.node.(rawNode); ok { - return mustDecodeNode(hash[:], node) + // The raw-blob format nodes are loaded from either from + // clean cache or the database, they are all in their own + // copy and safe to use unsafe decoder. + return mustDecodeNodeUnsafe(hash[:], node) } return expandNode(hash[:], n.node) } @@ -287,15 +289,17 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } + var preimage *preimageStore + if config != nil && config.Preimages { + preimage = newPreimageStore(diskdb) + } db := &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - } - if config == nil || config.Preimages { // TODO(karalabe): Flip to default off in the future - db.preimages = make(map[common.Hash][]byte) + preimages: preimage, } return db } @@ -305,14 +309,10 @@ func (db *Database) DiskDB() ethdb.KeyValueStore { return db.diskdb } -// insert inserts a collapsed trie node into the memory database. -// The blob size must be specified to allow proper size tracking. +// insert inserts a simplified trie node into the memory database. // All nodes inserted by this function will be reference tracked // and in theory should only used for **trie nodes** insertion. func (db *Database) insert(hash common.Hash, size int, node node) { - db.lock.Lock() - defer db.lock.Unlock() - // If the node's already cached, skip if _, ok := db.dirties[hash]; ok { return @@ -321,7 +321,7 @@ func (db *Database) insert(hash common.Hash, size int, node node) { // Create the cached entry for this node entry := &cachedNode{ - node: simplifyNode(node), + node: node, size: uint16(size), flushPrev: db.newest, } @@ -341,24 +341,6 @@ func (db *Database) insert(hash common.Hash, size int, node node) { db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } -// insertPreimage writes a new trie node pre-image to the memory database if it's -// yet unknown. The method will NOT make a copy of the slice, -// only use if the preimage will NOT be changed later on. -// -// Note, this method assumes that the database's lock is held! -func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return - } - // Track the preimage if a yet unknown one - if _, ok := db.preimages[hash]; ok { - return - } - db.preimages[hash] = preimage - db.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) -} - // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. func (db *Database) node(hash common.Hash) node { @@ -367,7 +349,10 @@ func (db *Database) node(hash common.Hash) node { if enc := db.cleans.Get(nil, hash[:]); enc != nil { memcacheCleanHitMeter.Mark(1) memcacheCleanReadMeter.Mark(int64(len(enc))) - return mustDecodeNode(hash[:], enc) + + // The returned value from cache is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } } // Retrieve the node from the dirty cache if available @@ -392,7 +377,9 @@ func (db *Database) node(hash common.Hash) node { memcacheCleanMissMeter.Mark(1) memcacheCleanWriteMeter.Mark(int64(len(enc))) } - return mustDecodeNode(hash[:], enc) + // The returned value from database is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } // Node retrieves an encoded cached trie node from memory. If it cannot be found @@ -435,24 +422,6 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { return nil, errors.New("not found") } -// preimage retrieves a cached trie node pre-image from memory. If it cannot be -// found cached, the method queries the persistent database for the content. -func (db *Database) preimage(hash common.Hash) []byte { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return nil - } - // Retrieve the node from cache if available - db.lock.RLock() - preimage := db.preimages[hash] - db.lock.RUnlock() - - if preimage != nil { - return preimage - } - return rawdb.ReadPreimage(db.diskdb, hash) -} - // Nodes retrieves the hashes of all the nodes cached within the memory database. // This method is extremely expensive and should only be used to validate internal // states in test code. @@ -597,19 +566,8 @@ func (db *Database) Cap(limit common.StorageSize) error { // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. - flushPreimages := db.preimagesSize > 4*1024*1024 - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to write preimages whilst disabled") - } else { - rawdb.WritePreimages(batch, db.preimages) - if batch.ValueSize() > ethdb.IdealBatchSize { - if err := batch.Write(); err != nil { - return err - } - batch.Reset() - } - } + if db.preimages != nil { + db.preimages.commit(false) } // Keep committing nodes from the flush-list until we're below allowance oldest := db.oldest @@ -644,13 +602,6 @@ func (db *Database) Cap(limit common.StorageSize) error { db.lock.Lock() defer db.lock.Unlock() - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to reset preimage cache whilst disabled") - } else { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } - } for db.oldest != oldest { node := db.dirties[db.oldest] delete(db.dirties, db.oldest) @@ -694,13 +645,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H // Move all of the accumulated preimages into a write batch if db.preimages != nil { - rawdb.WritePreimages(batch, db.preimages) - // Since we're going to replay trie node writes into the clean cache, flush out - // any batched pre-images before continuing. - if err := batch.Write(); err != nil { - return err - } - batch.Reset() + db.preimages.commit(true) } // Move the trie itself into the batch, flushing if enough data is accumulated nodes, storage := len(db.dirties), db.dirtiesSize @@ -723,9 +668,6 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H batch.Reset() // Reset the storage counters and bumped metrics - if db.preimages != nil { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } memcacheCommitTimeTimer.Update(time.Since(start)) memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) @@ -826,6 +768,41 @@ func (c *cleaner) Delete(key []byte) error { panic("not implemented") } +// Update inserts the dirty nodes in provided nodeset into database and +// link the account trie with multiple storage tries if necessary. +func (db *Database) Update(nodes *MergedNodeSet) error { + db.lock.Lock() + defer db.lock.Unlock() + + // Insert dirty nodes into the database. In the same tree, it must be + // ensured that children are inserted first, then parent so that children + // can be linked with their parent correctly. The order of writing between + // different tries(account trie, storage tries) is not required. + for owner, subset := range nodes.sets { + for _, path := range subset.paths { + n, ok := subset.nodes[path] + if !ok { + return fmt.Errorf("missing node %x %v", owner, path) + } + db.insert(n.hash, int(n.size), n.node) + } + } + // Link up the account trie and storage trie if the node points + // to an account trie leaf. + if set, present := nodes.sets[common.Hash{}]; present { + for _, n := range set.leaves { + var account types.StateAccount + if err := rlp.DecodeBytes(n.blob, &account); err != nil { + return err + } + if account.Root != emptyRoot { + db.reference(account.Root, n.parent) + } + } + } + return nil +} + // Size returns the current storage size of the memory cache in front of the // persistent database layer. func (db *Database) Size() (common.StorageSize, common.StorageSize) { @@ -837,7 +814,11 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { // counted. var metadataSize = common.StorageSize((len(db.dirties) - 1) * cachedNodeSize) var metarootRefs = common.StorageSize(len(db.dirties[common.Hash{}].children) * (common.HashLength + 2)) - return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, db.preimagesSize + var preimageSize common.StorageSize + if db.preimages != nil { + preimageSize = db.preimages.size() + } + return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, preimageSize } // saveCache saves clean state cache to given directory path @@ -879,3 +860,16 @@ func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, st } } } + +// CommitPreimages flushes the dangling preimages to disk. It is meant to be +// called when closing the blockchain object, so that preimages are persisted +// to the database. +func (db *Database) CommitPreimages() error { + db.lock.Lock() + defer db.lock.Unlock() + + if db.preimages == nil { + return nil + } + return db.preimages.commit(true) +} diff --git a/trie/hasher.go b/trie/hasher.go index 251709941a26..e594d6d6b2ae 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -30,7 +30,7 @@ type hasher struct { sha crypto.KeccakState tmp []byte encbuf rlp.EncoderBuffer - parallel bool // Whether to use paralallel threads when hashing + parallel bool // Whether to use parallel threads when hashing } // hasherPool holds pureHashers @@ -191,7 +191,7 @@ func (h *hasher) hashData(data []byte) hashNode { } // proofHash is used to construct trie proofs, and returns the 'collapsed' -// node (for later RLP encoding) aswell as the hashed node -- unless the +// node (for later RLP encoding) as well as the hashed node -- unless the // node is smaller than 32 bytes, in which case it will be returned as is. // This method does not do anything on value- or hash-nodes. func (h *hasher) proofHash(original node) (collapsed, hashed node) { diff --git a/trie/iterator.go b/trie/iterator.go index e0006ee05e3b..1e76625c6213 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -375,8 +375,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { } } } - resolved, err := it.trie.resolveHash(hash, path) - return resolved, err + return it.trie.resolveHash(hash, path) } func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index e3e6d0e3a8fa..e9d822a9a4f2 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -31,7 +31,7 @@ import ( ) func TestEmptyIterator(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) iter := trie.NodeIterator(nil) seen := make(map[string]struct{}) @@ -44,7 +44,8 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -59,8 +60,13 @@ func TestIterator(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + root, nodes, err := trie.Commit(false) + if err != nil { + t.Fatalf("Failed to commit trie %v", err) + } + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) found := make(map[string]string) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { @@ -80,7 +86,7 @@ type kv struct { } func TestIteratorLargeData(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -173,7 +179,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for _, val := range testdata1 { trie.Update([]byte(val.k), []byte(val.v)) } @@ -214,17 +220,23 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(common.Hash{}, rootA, dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(common.Hash{}, rootB, dbb) found := make(map[string]string) di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) @@ -250,17 +262,23 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(common.Hash{}, rootA, dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(common.Hash{}, rootB, dbb) di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) it := NewIterator(di) @@ -316,7 +334,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } - tr.Commit(nil) + _, nodes, _ := tr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(tr.Hash(), true, nil) } @@ -407,7 +426,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { for _, val := range testdata1 { ctr.Update([]byte(val.k), []byte(val.v)) } - root, _, _ := ctr.Commit(nil) + root, nodes, _ := ctr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } @@ -509,11 +529,11 @@ func (l *loggingDb) Close() error { } // makeLargeTestTrie create a sample test trie -func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { +func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} triedb := NewDatabase(logDb) - trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { @@ -525,7 +545,8 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { val = crypto.Keccak256(val) trie.Update(key, val) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) // Return the generated trie return triedb, trie, logDb } @@ -564,7 +585,8 @@ func TestIteratorNodeBlob(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) triedb.Cap(0) found := make(map[common.Hash][]byte) diff --git a/trie/node.go b/trie/node.go index bf3f024bb8a7..6ce6551ded8c 100644 --- a/trie/node.go +++ b/trie/node.go @@ -99,6 +99,7 @@ func (n valueNode) fstring(ind string) string { return fmt.Sprintf("%x ", []byte(n)) } +// mustDecodeNode is a wrapper of decodeNode and panic if any error is encountered. func mustDecodeNode(hash, buf []byte) node { n, err := decodeNode(hash, buf) if err != nil { @@ -107,8 +108,29 @@ func mustDecodeNode(hash, buf []byte) node { return n } -// decodeNode parses the RLP encoding of a trie node. +// mustDecodeNodeUnsafe is a wrapper of decodeNodeUnsafe and panic if any error is +// encountered. +func mustDecodeNodeUnsafe(hash, buf []byte) node { + n, err := decodeNodeUnsafe(hash, buf) + if err != nil { + panic(fmt.Sprintf("node %x: %v", hash, err)) + } + return n +} + +// decodeNode parses the RLP encoding of a trie node. It will deep-copy the passed +// byte slice for decoding, so it's safe to modify the byte slice afterwards. The- +// decode performance of this function is not optimal, but it is suitable for most +// scenarios with low performance requirements and hard to determine whether the +// byte slice be modified or not. func decodeNode(hash, buf []byte) (node, error) { + return decodeNodeUnsafe(hash, common.CopyBytes(buf)) +} + +// decodeNodeUnsafe parses the RLP encoding of a trie node. The passed byte slice +// will be directly referenced by node without bytes deep copy, so the input MUST +// not be changed after. +func decodeNodeUnsafe(hash, buf []byte) (node, error) { if len(buf) == 0 { return nil, io.ErrUnexpectedEOF } @@ -141,7 +163,7 @@ func decodeShort(hash, elems []byte) (node, error) { if err != nil { return nil, fmt.Errorf("invalid value node: %v", err) } - return &shortNode{key, append(valueNode{}, val...), flag}, nil + return &shortNode{key, valueNode(val), flag}, nil } r, _, err := decodeRef(rest) if err != nil { @@ -164,7 +186,7 @@ func decodeFull(hash, elems []byte) (*fullNode, error) { return n, err } if len(val) > 0 { - n.Children[16] = append(valueNode{}, val...) + n.Children[16] = valueNode(val) } return n, nil } @@ -190,7 +212,7 @@ func decodeRef(buf []byte) (node, []byte, error) { // empty node return nil, rest, nil case kind == rlp.String && len(val) == 32: - return append(hashNode{}, val...), rest, nil + return hashNode(val), rest, nil default: return nil, nil, fmt.Errorf("invalid RLP string size %d (want 0 or 32)", len(val)) } diff --git a/trie/node_test.go b/trie/node_test.go index ac1d8fbef3e6..9b8b33748fa7 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -20,6 +20,7 @@ import ( "bytes" "testing" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -92,3 +93,123 @@ func TestDecodeFullNode(t *testing.T) { t.Fatalf("decode full node err: %v", err) } } + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeShortNode +// BenchmarkEncodeShortNode-8 16878850 70.81 ns/op 48 B/op 1 allocs/op +func BenchmarkEncodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeFullNode +// BenchmarkEncodeFullNode-8 4323273 284.4 ns/op 576 B/op 1 allocs/op +func BenchmarkEncodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNode +// BenchmarkDecodeShortNode-8 7925638 151.0 ns/op 157 B/op 4 allocs/op +func BenchmarkDecodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNodeUnsafe +// BenchmarkDecodeShortNodeUnsafe-8 9027476 128.6 ns/op 109 B/op 3 allocs/op +func BenchmarkDecodeShortNodeUnsafe(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNode +// BenchmarkDecodeFullNode-8 1597462 761.9 ns/op 1280 B/op 18 allocs/op +func BenchmarkDecodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNodeUnsafe +// BenchmarkDecodeFullNodeUnsafe-8 1789070 687.1 ns/op 704 B/op 17 allocs/op +func BenchmarkDecodeFullNodeUnsafe(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} diff --git a/trie/nodeset.go b/trie/nodeset.go new file mode 100644 index 000000000000..08b9b35ebc87 --- /dev/null +++ b/trie/nodeset.go @@ -0,0 +1,94 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" +) + +// memoryNode is all the information we know about a single cached trie node +// in the memory. +type memoryNode struct { + hash common.Hash // Node hash, computed by hashing rlp value + size uint16 // Byte size of the useful cached data + node node // Cached collapsed trie node, or raw rlp data +} + +// NodeSet contains all dirty nodes collected during the commit operation. +// Each node is keyed by path. It's not thread-safe to use. +type NodeSet struct { + owner common.Hash // the identifier of the trie + paths []string // the path of dirty nodes, sort by insertion order + nodes map[string]*memoryNode // the map of dirty nodes, keyed by node path + leaves []*leaf // the list of dirty leaves +} + +// NewNodeSet initializes an empty node set to be used for tracking dirty nodes +// from a specific account or storage trie. The owner is zero for the account +// trie and the owning account address hash for storage tries. +func NewNodeSet(owner common.Hash) *NodeSet { + return &NodeSet{ + owner: owner, + nodes: make(map[string]*memoryNode), + } +} + +// add caches node with provided path and node object. +func (set *NodeSet) add(path string, node *memoryNode) { + set.paths = append(set.paths, path) + set.nodes[path] = node +} + +// addLeaf caches the provided leaf node. +func (set *NodeSet) addLeaf(node *leaf) { + set.leaves = append(set.leaves, node) +} + +// Len returns the number of dirty nodes contained in the set. +func (set *NodeSet) Len() int { + return len(set.nodes) +} + +// MergedNodeSet represents a merged dirty node set for a group of tries. +type MergedNodeSet struct { + sets map[common.Hash]*NodeSet +} + +// NewMergedNodeSet initializes an empty merged set. +func NewMergedNodeSet() *MergedNodeSet { + return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)} +} + +// NewWithNodeSet constructs a merged nodeset with the provided single set. +func NewWithNodeSet(set *NodeSet) *MergedNodeSet { + merged := NewMergedNodeSet() + merged.Merge(set) + return merged +} + +// Merge merges the provided dirty nodes of a trie into the set. The assumption +// is held that no duplicated set belonging to the same trie will be merged twice. +func (set *MergedNodeSet) Merge(other *NodeSet) error { + _, present := set.sets[other.owner] + if present { + return fmt.Errorf("duplicate trie for owner %#x", other.owner) + } + set.sets[other.owner] = other + return nil +} diff --git a/trie/preimages.go b/trie/preimages.go new file mode 100644 index 000000000000..66f34117c1e8 --- /dev/null +++ b/trie/preimages.go @@ -0,0 +1,95 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +// preimageStore is the store for caching preimages of node key. +type preimageStore struct { + lock sync.RWMutex + disk ethdb.KeyValueStore + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + preimagesSize common.StorageSize // Storage size of the preimages cache +} + +// newPreimageStore initializes the store for caching preimages. +func newPreimageStore(disk ethdb.KeyValueStore) *preimageStore { + return &preimageStore{ + disk: disk, + preimages: make(map[common.Hash][]byte), + } +} + +// insertPreimage writes a new trie node pre-image to the memory database if it's +// yet unknown. The method will NOT make a copy of the slice, only use if the +// preimage will NOT be changed later on. +func (store *preimageStore) insertPreimage(preimages map[common.Hash][]byte) { + store.lock.Lock() + defer store.lock.Unlock() + + for hash, preimage := range preimages { + if _, ok := store.preimages[hash]; ok { + continue + } + store.preimages[hash] = preimage + store.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) + } +} + +// preimage retrieves a cached trie node pre-image from memory. If it cannot be +// found cached, the method queries the persistent database for the content. +func (store *preimageStore) preimage(hash common.Hash) []byte { + store.lock.RLock() + preimage := store.preimages[hash] + store.lock.RUnlock() + + if preimage != nil { + return preimage + } + return rawdb.ReadPreimage(store.disk, hash) +} + +// commit flushes the cached preimages into the disk. +func (store *preimageStore) commit(force bool) error { + store.lock.Lock() + defer store.lock.Unlock() + + if store.preimagesSize <= 4*1024*1024 && !force { + return nil + } + batch := store.disk.NewBatch() + rawdb.WritePreimages(batch, store.preimages) + if err := batch.Write(); err != nil { + return err + } + store.preimages, store.preimagesSize = make(map[common.Hash][]byte), 0 + return nil +} + +// size returns the current storage size of accumulated preimages. +func (store *preimageStore) size() common.StorageSize { + store.lock.RLock() + defer store.lock.RUnlock() + + return store.preimagesSize +} diff --git a/trie/proof.go b/trie/proof.go index 5f6cce92302d..3c1f95b4c67f 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -35,9 +36,12 @@ import ( // with the node that proves the absence of the key. func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { // Collect all nodes on the path to key. + var ( + prefix []byte + nodes []node + tn = t.root + ) key = keybytesToHex(key) - var nodes []node - tn := t.root for len(key) > 0 && tn != nil { switch n := tn.(type) { case *shortNode: @@ -46,16 +50,18 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e tn = nil } else { tn = n.Val + prefix = append(prefix, n.Key...) key = key[len(n.Key):] } nodes = append(nodes, n) case *fullNode: tn = n.Children[key[0]] + prefix = append(prefix, key[0]) key = key[1:] nodes = append(nodes, n) case hashNode: var err error - tn, err = t.resolveHash(n, nil) + tn, err = t.resolveHash(n, prefix) if err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) return err @@ -94,7 +100,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *StateTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { return t.trie.Prove(key, fromLevel, proofDb) } @@ -553,7 +559,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key } // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. - tr := newWithRootNode(root) + tr := &Trie{root: root, db: NewDatabase(rawdb.NewMemoryDatabase())} if empty { tr.root = nil } diff --git a/trie/proof_test.go b/trie/proof_test.go index 651078c378bf..fdae877818cf 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -213,7 +213,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { proof := memorydb.New() // Short circuit if the decreased key is same with the previous key - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) if start != 0 && bytes.Equal(first, entries[start-1].k) { continue } @@ -222,7 +222,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { continue } // Short circuit if the increased key is same with the next key - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) if end != len(entries) && bytes.Equal(last, entries[end].k) { continue } @@ -282,7 +282,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 1 start, end := 100, 200 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -305,7 +305,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 2 start, end = 100, 200 - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -351,7 +351,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with left non-existent edge proof start = 1000 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -366,7 +366,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with right non-existent edge proof start = 1000 - last := increseKey(common.CopyBytes(entries[start].k)) + last := increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -381,7 +381,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with two non-existent edge proofs start = 1000 - first, last = decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[start].k)) + first, last = decreaseKey(common.CopyBytes(entries[start].k)), increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -657,9 +657,9 @@ func TestSameSideProofs(t *testing.T) { sort.Sort(entries) pos := 1000 - first := decreseKey(common.CopyBytes(entries[pos].k)) - first = decreseKey(first) - last := decreseKey(common.CopyBytes(entries[pos].k)) + first := decreaseKey(common.CopyBytes(entries[pos].k)) + first = decreaseKey(first) + last := decreaseKey(common.CopyBytes(entries[pos].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -673,9 +673,9 @@ func TestSameSideProofs(t *testing.T) { t.Fatalf("Expected error, got nil") } - first = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(last) + first = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(last) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -781,7 +781,7 @@ func TestEmptyRangeProof(t *testing.T) { } for _, c := range cases { proof := memorydb.New() - first := increseKey(common.CopyBytes(entries[c.pos].k)) + first := increaseKey(common.CopyBytes(entries[c.pos].k)) if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } @@ -920,7 +920,7 @@ func mutateByte(b []byte) { } } -func increseKey(key []byte) []byte { +func increaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]++ if key[i] != 0x0 { @@ -930,7 +930,7 @@ func increseKey(key []byte) []byte { return key } -func decreseKey(key []byte) []byte { +func decreaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]-- if key[i] != 0xff { diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 6a5cc89c9ffd..3d468f56ee0a 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -25,24 +25,35 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// SecureTrie wraps a trie with key hashing. In a secure trie, all +// SecureTrie is the old name of StateTrie. +// Deprecated: use StateTrie. +type SecureTrie = StateTrie + +// NewSecure creates a new StateTrie. +// Deprecated: use NewStateTrie. +func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { + return NewStateTrie(owner, root, db) +} + +// StateTrie wraps a trie with key hashing. In a secure trie, all // access operations hash the key using keccak256. This prevents // calling code from creating long chains of nodes that // increase the access time. // -// Contrary to a regular trie, a SecureTrie can only be created with +// Contrary to a regular trie, a StateTrie can only be created with // New and must have an attached database. The database also stores // the preimage of each key. // -// SecureTrie is not safe for concurrent use. -type SecureTrie struct { +// StateTrie is not safe for concurrent use. +type StateTrie struct { trie Trie + preimages *preimageStore hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte - secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch + secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch } -// NewSecure creates a trie with an existing root node from a backing database +// NewStateTrie creates a trie with an existing root node from a backing database // and optional intermediate in-memory node pool. // // If root is the zero hash or the sha3 hash of an empty string, the @@ -53,7 +64,7 @@ type SecureTrie struct { // Loaded nodes are kept around until their 'cache generation' expires. // A new cache generation is created by each call to Commit. // cachelimit sets the number of past cache generations to keep. -func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { +func NewStateTrie(owner common.Hash, root common.Hash, db *Database) (*StateTrie, error) { if db == nil { panic("trie.NewSecure called without a database") } @@ -61,12 +72,12 @@ func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, if err != nil { return nil, err } - return &SecureTrie{trie: *trie}, nil + return &StateTrie{trie: *trie, preimages: db.preimages}, nil } // Get returns the value for key stored in the trie. // The value bytes must not be modified by the caller. -func (t *SecureTrie) Get(key []byte) []byte { +func (t *StateTrie) Get(key []byte) []byte { res, err := t.TryGet(key) if err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) @@ -77,19 +88,50 @@ func (t *SecureTrie) Get(key []byte) []byte { // TryGet returns the value for key stored in the trie. // The value bytes must not be modified by the caller. // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryGet(key []byte) ([]byte, error) { +func (t *StateTrie) TryGet(key []byte) ([]byte, error) { return t.trie.TryGet(t.hashKey(key)) } +func (t *StateTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + var ret types.StateAccount + res, err := t.trie.TryGet(t.hashKey(key)) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + return &ret, err + } + if res == nil { + return nil, nil + } + err = rlp.DecodeBytes(res, &ret) + return &ret, err +} + +// TryGetAccountWithPreHashedKey does the same thing as TryGetAccount, however +// it expects a key that is already hashed. This constitutes an abstraction leak, +// since the client code needs to know the key format. +func (t *StateTrie) TryGetAccountWithPreHashedKey(key []byte) (*types.StateAccount, error) { + var ret types.StateAccount + res, err := t.trie.TryGet(key) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + return &ret, err + } + if res == nil { + return nil, nil + } + err = rlp.DecodeBytes(res, &ret) + return &ret, err +} + // TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not // possible to use keybyte-encoding as the path might contain odd nibbles. -func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) { +func (t *StateTrie) TryGetNode(path []byte) ([]byte, int, error) { return t.trie.TryGetNode(path) } // TryUpdateAccount account will abstract the write of an account to the // secure trie. -func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { +func (t *StateTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { hk := t.hashKey(key) data, err := rlp.EncodeToBytes(acc) if err != nil { @@ -108,7 +150,7 @@ func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error // // The value bytes must not be modified by the caller while they are // stored in the trie. -func (t *SecureTrie) Update(key, value []byte) { +func (t *StateTrie) Update(key, value []byte) { if err := t.TryUpdate(key, value); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } @@ -122,7 +164,7 @@ func (t *SecureTrie) Update(key, value []byte) { // stored in the trie. // // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryUpdate(key, value []byte) error { +func (t *StateTrie) TryUpdate(key, value []byte) error { hk := t.hashKey(key) err := t.trie.TryUpdate(hk, value) if err != nil { @@ -133,7 +175,7 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error { } // Delete removes any existing value for key from the trie. -func (t *SecureTrie) Delete(key []byte) { +func (t *StateTrie) Delete(key []byte) { if err := t.TryDelete(key); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } @@ -141,7 +183,14 @@ func (t *SecureTrie) Delete(key []byte) { // TryDelete removes any existing value for key from the trie. // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryDelete(key []byte) error { +func (t *StateTrie) TryDelete(key []byte) error { + hk := t.hashKey(key) + delete(t.getSecKeyCache(), string(hk)) + return t.trie.TryDelete(hk) +} + +// TryDeleteACcount abstracts an account deletion from the trie. +func (t *StateTrie) TryDeleteAccount(key []byte) error { hk := t.hashKey(key) delete(t.getSecKeyCache(), string(hk)) return t.trie.TryDelete(hk) @@ -149,58 +198,64 @@ func (t *SecureTrie) TryDelete(key []byte) error { // GetKey returns the sha3 preimage of a hashed key that was // previously used to store a value. -func (t *SecureTrie) GetKey(shaKey []byte) []byte { +func (t *StateTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - return t.trie.db.preimage(common.BytesToHash(shaKey)) + if t.preimages == nil { + return nil + } + return t.preimages.preimage(common.BytesToHash(shaKey)) } -// Commit writes all nodes and the secure hash pre-images to the trie's database. -// Nodes are stored with their sha3 hash as the key. -// -// Committing flushes nodes from memory. Subsequent Get calls will load nodes -// from the database. -func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { +// Commit collects all dirty nodes in the trie and replace them with the +// corresponding node hash. All collected nodes(including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean(nothing to commit). +// All cached preimages will be also flushed if preimages recording is enabled. +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock - t.trie.db.lock.Lock() + if t.preimages != nil { + preimages := make(map[common.Hash][]byte) for hk, key := range t.secKeyCache { - t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) + preimages[common.BytesToHash([]byte(hk))] = key } - t.trie.db.lock.Unlock() + t.preimages.insertPreimage(preimages) } t.secKeyCache = make(map[string][]byte) } // Commit the trie to its intermediate node database - return t.trie.Commit(onleaf) + return t.trie.Commit(collectLeaf) } -// Hash returns the root hash of SecureTrie. It does not write to the +// Hash returns the root hash of StateTrie. It does not write to the // database and can be used even if the trie doesn't have one. -func (t *SecureTrie) Hash() common.Hash { +func (t *StateTrie) Hash() common.Hash { return t.trie.Hash() } -// Copy returns a copy of SecureTrie. -func (t *SecureTrie) Copy() *SecureTrie { - return &SecureTrie{ +// Copy returns a copy of StateTrie. +func (t *StateTrie) Copy() *StateTrie { + return &StateTrie{ trie: *t.trie.Copy(), + preimages: t.preimages, secKeyCache: t.secKeyCache, } } // NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration // starts at the key after the given start key. -func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { +func (t *StateTrie) NodeIterator(start []byte) NodeIterator { return t.trie.NodeIterator(start) } // hashKey returns the hash of key as an ephemeral buffer. // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. -func (t *SecureTrie) hashKey(key []byte) []byte { +func (t *StateTrie) hashKey(key []byte) []byte { h := newHasher(false) h.sha.Reset() h.sha.Write(key) @@ -212,7 +267,7 @@ func (t *SecureTrie) hashKey(key []byte) []byte { // getSecKeyCache returns the current secure key cache, creating a new one if // ownership changed (i.e. the current secure trie is a copy of another owning // the actual cache). -func (t *SecureTrie) getSecKeyCache() map[string][]byte { +func (t *StateTrie) getSecKeyCache() map[string][]byte { if t != t.secKeyCacheOwner { t.secKeyCacheOwner = t t.secKeyCache = make(map[string][]byte) diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index beea5845ad0d..862c3a3ec43d 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "runtime" "sync" "testing" @@ -27,16 +28,16 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) +func newEmptySecure() *StateTrie { + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) return trie } -// makeTestSecureTrie creates a large enough secure trie for testing. -func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { +// makeTestStateTrie creates a large enough secure trie for testing. +func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -57,9 +58,15 @@ func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewSecure(common.Hash{}, root, triedb) return triedb, trie, content } @@ -105,16 +112,16 @@ func TestSecureGetKey(t *testing.T) { } } -func TestSecureTrieConcurrency(t *testing.T) { +func TestStateTrieConcurrency(t *testing.T) { // Create an initial trie and copy if for concurrent access - _, trie, _ := makeTestSecureTrie() + _, trie, _ := makeTestStateTrie() threads := runtime.NumCPU() - tries := make([]*SecureTrie, threads) + tries := make([]*StateTrie, threads) for i := 0; i < threads; i++ { tries[i] = trie.Copy() } - // Start a batch of goroutines interactng with the trie + // Start a batch of goroutines interacting with the trie pend := new(sync.WaitGroup) pend.Add(threads) for i := 0; i < threads; i++ { @@ -135,7 +142,7 @@ func TestSecureTrieConcurrency(t *testing.T) { tries[index].Update(key, val) } } - tries[index].Commit(nil) + tries[index].Commit(false) }(i) } // Wait for all threads to finish diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index f24c749716e5..069e4981d71a 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -345,7 +345,6 @@ func TestStacktrieNotModifyValues(t *testing.T) { if !bytes.Equal(have, want) { t.Fatalf("item %d, have %#x want %#x", i, have, want) } - } } diff --git a/trie/sync.go b/trie/sync.go index db51dd4b036a..303fcbfa22e2 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" ) // ErrNotRequested is returned by the trie sync when it's requested to process a @@ -39,19 +40,6 @@ var ErrAlreadyProcessed = errors.New("already processed") // memory if the node was configured with a significant number of peers. const maxFetchesPerDepth = 16384 -// request represents a scheduled or already in-flight state retrieval request. -type request struct { - path []byte // Merkle path leading to this node for prioritization - hash common.Hash // Hash of the node data content to retrieve - data []byte // Data content of the node, cached until all subtrees complete - code bool // Whether this is a code entry - - parents []*request // Parent state nodes referencing this entry (notify all upon completion) - deps int // Number of dependencies before allowed to commit this node - - callback LeafCallback // Callback to invoke if a leaf node it reached on this branch -} - // SyncPath is a path tuple identifying a particular trie node either in a single // trie (account) or a layered trie (account -> storage). // @@ -85,30 +73,57 @@ func NewSyncPath(path []byte) SyncPath { return SyncPath{hexToKeybytes(path[:64]), hexToCompact(path[64:])} } -// SyncResult is a response with requested data along with it's hash. -type SyncResult struct { - Hash common.Hash // Hash of the originally unknown trie node - Data []byte // Data content of the retrieved node +// nodeRequest represents a scheduled or already in-flight trie node retrieval request. +type nodeRequest struct { + hash common.Hash // Hash of the trie node to retrieve + path []byte // Merkle path leading to this node for prioritization + data []byte // Data content of the node, cached until all subtrees complete + + parent *nodeRequest // Parent state node referencing this entry + deps int // Number of dependencies before allowed to commit this node + callback LeafCallback // Callback to invoke if a leaf node it reached on this branch +} + +// codeRequest represents a scheduled or already in-flight bytecode retrieval request. +type codeRequest struct { + hash common.Hash // Hash of the contract bytecode to retrieve + path []byte // Merkle path leading to this node for prioritization + data []byte // Data content of the node, cached until all subtrees complete + parents []*nodeRequest // Parent state nodes referencing this entry (notify all upon completion) +} + +// NodeSyncResult is a response with requested trie node along with its node path. +type NodeSyncResult struct { + Path string // Path of the originally unknown trie node + Data []byte // Data content of the retrieved trie node +} + +// CodeSyncResult is a response with requested bytecode along with its hash. +type CodeSyncResult struct { + Hash common.Hash // Hash the originally unknown bytecode + Data []byte // Data content of the retrieved bytecode } // syncMemBatch is an in-memory buffer of successfully downloaded but not yet // persisted data items. type syncMemBatch struct { - nodes map[common.Hash][]byte // In-memory membatch of recently completed nodes - codes map[common.Hash][]byte // In-memory membatch of recently completed codes + nodes map[string][]byte // In-memory membatch of recently completed nodes + hashes map[string]common.Hash // Hashes of recently completed nodes + codes map[common.Hash][]byte // In-memory membatch of recently completed codes } // newSyncMemBatch allocates a new memory-buffer for not-yet persisted trie nodes. func newSyncMemBatch() *syncMemBatch { return &syncMemBatch{ - nodes: make(map[common.Hash][]byte), - codes: make(map[common.Hash][]byte), + nodes: make(map[string][]byte), + hashes: make(map[string]common.Hash), + codes: make(map[common.Hash][]byte), } } -// hasNode reports the trie node with specific hash is already cached. -func (batch *syncMemBatch) hasNode(hash common.Hash) bool { - _, ok := batch.nodes[hash] +// hasNode reports the trie node with specific path is already cached. +func (batch *syncMemBatch) hasNode(path []byte) bool { + _, ok := batch.nodes[string(path)] return ok } @@ -122,12 +137,12 @@ func (batch *syncMemBatch) hasCode(hash common.Hash) bool { // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. type Sync struct { - database ethdb.KeyValueReader // Persistent database to check for existing entries - membatch *syncMemBatch // Memory buffer to avoid frequent database writes - nodeReqs map[common.Hash]*request // Pending requests pertaining to a trie node hash - codeReqs map[common.Hash]*request // Pending requests pertaining to a code hash - queue *prque.Prque // Priority queue with the pending requests - fetches map[int]int // Number of active fetches per trie node depth + database ethdb.KeyValueReader // Persistent database to check for existing entries + membatch *syncMemBatch // Memory buffer to avoid frequent database writes + nodeReqs map[string]*nodeRequest // Pending requests pertaining to a trie node path + codeReqs map[common.Hash]*codeRequest // Pending requests pertaining to a code hash + queue *prque.Prque // Priority queue with the pending requests + fetches map[int]int // Number of active fetches per trie node depth } // NewSync creates a new trie data download scheduler. @@ -135,51 +150,51 @@ func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallb ts := &Sync{ database: database, membatch: newSyncMemBatch(), - nodeReqs: make(map[common.Hash]*request), - codeReqs: make(map[common.Hash]*request), + nodeReqs: make(map[string]*nodeRequest), + codeReqs: make(map[common.Hash]*codeRequest), queue: prque.New(nil), fetches: make(map[int]int), } - ts.AddSubTrie(root, nil, common.Hash{}, callback) + ts.AddSubTrie(root, nil, common.Hash{}, nil, callback) return ts } -// AddSubTrie registers a new trie to the sync code, rooted at the designated parent. -func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, callback LeafCallback) { +// AddSubTrie registers a new trie to the sync code, rooted at the designated +// parent for completion tracking. The given path is a unique node path in +// hex format and contain all the parent path if it's layered trie node. +func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, parentPath []byte, callback LeafCallback) { // Short circuit if the trie is empty or already known if root == emptyRoot { return } - if s.membatch.hasNode(root) { + if s.membatch.hasNode(path) { return } - // If database says this is a duplicate, then at least the trie node is - // present, and we hold the assumption that it's NOT legacy contract code. if rawdb.HasTrieNode(s.database, root) { return } // Assemble the new sub-trie sync request - req := &request{ - path: path, + req := &nodeRequest{ hash: root, + path: path, callback: callback, } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.nodeReqs[parent] + ancestor := s.nodeReqs[string(parentPath)] if ancestor == nil { panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent)) } ancestor.deps++ - req.parents = append(req.parents, ancestor) + req.parent = ancestor } - s.schedule(req) + s.scheduleNodeRequest(req) } // AddCodeEntry schedules the direct retrieval of a contract code that should not // be interpreted as a trie node, but rather accepted and stored into the database // as is. -func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash) { +func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash, parentPath []byte) { // Short circuit if the entry is empty or already known if hash == emptyState { return @@ -196,30 +211,29 @@ func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash) { return } // Assemble the new sub-trie sync request - req := &request{ + req := &codeRequest{ path: path, hash: hash, - code: true, } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.nodeReqs[parent] // the parent of codereq can ONLY be nodereq + ancestor := s.nodeReqs[string(parentPath)] // the parent of codereq can ONLY be nodereq if ancestor == nil { panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent)) } ancestor.deps++ req.parents = append(req.parents, ancestor) } - s.schedule(req) + s.scheduleCodeRequest(req) } // Missing retrieves the known missing nodes from the trie for retrieval. To aid // both eth/6x style fast sync and snap/1x style state sync, the paths of trie // nodes are returned too, as well as separate hash list for codes. -func (s *Sync) Missing(max int) (nodes []common.Hash, paths []SyncPath, codes []common.Hash) { +func (s *Sync) Missing(max int) ([]string, []common.Hash, []common.Hash) { var ( + nodePaths []string nodeHashes []common.Hash - nodePaths []SyncPath codeHashes []common.Hash ) for !s.queue.Empty() && (max == 0 || len(nodeHashes)+len(codeHashes) < max) { @@ -235,62 +249,76 @@ func (s *Sync) Missing(max int) (nodes []common.Hash, paths []SyncPath, codes [] s.queue.Pop() s.fetches[depth]++ - hash := item.(common.Hash) - if req, ok := s.nodeReqs[hash]; ok { - nodeHashes = append(nodeHashes, hash) - nodePaths = append(nodePaths, NewSyncPath(req.path)) - } else { - codeHashes = append(codeHashes, hash) + switch item := item.(type) { + case common.Hash: + codeHashes = append(codeHashes, item) + case string: + req, ok := s.nodeReqs[item] + if !ok { + log.Error("Missing node request", "path", item) + continue // System very wrong, shouldn't happen + } + nodePaths = append(nodePaths, item) + nodeHashes = append(nodeHashes, req.hash) } } - return nodeHashes, nodePaths, codeHashes + return nodePaths, nodeHashes, codeHashes } -// Process injects the received data for requested item. Note it can +// ProcessCode injects the received data for requested item. Note it can // happpen that the single response commits two pending requests(e.g. // there are two requests one for code and one for node but the hash // is same). In this case the second response for the same hash will // be treated as "non-requested" item or "already-processed" item but // there is no downside. -func (s *Sync) Process(result SyncResult) error { - // If the item was not requested either for code or node, bail out - if s.nodeReqs[result.Hash] == nil && s.codeReqs[result.Hash] == nil { +func (s *Sync) ProcessCode(result CodeSyncResult) error { + // If the code was not requested or it's already processed, bail out + req := s.codeReqs[result.Hash] + if req == nil { return ErrNotRequested } - // There is an pending code request for this data, commit directly - var filled bool - if req := s.codeReqs[result.Hash]; req != nil && req.data == nil { - filled = true - req.data = result.Data - s.commit(req) + if req.data != nil { + return ErrAlreadyProcessed } - // There is an pending node request for this data, fill it. - if req := s.nodeReqs[result.Hash]; req != nil && req.data == nil { - filled = true - // Decode the node data content and update the request - node, err := decodeNode(result.Hash[:], result.Data) - if err != nil { - return err - } - req.data = result.Data + req.data = result.Data + return s.commitCodeRequest(req) +} - // Create and schedule a request for all the children nodes - requests, err := s.children(req, node) - if err != nil { - return err - } - if len(requests) == 0 && req.deps == 0 { - s.commit(req) - } else { - req.deps += len(requests) - for _, child := range requests { - s.schedule(child) - } - } +// ProcessNode injects the received data for requested item. Note it can +// happen that the single response commits two pending requests(e.g. +// there are two requests one for code and one for node but the hash +// is same). In this case the second response for the same hash will +// be treated as "non-requested" item or "already-processed" item but +// there is no downside. +func (s *Sync) ProcessNode(result NodeSyncResult) error { + // If the trie node was not requested or it's already processed, bail out + req := s.nodeReqs[result.Path] + if req == nil { + return ErrNotRequested } - if !filled { + if req.data != nil { return ErrAlreadyProcessed } + // Decode the node data content and update the request + node, err := decodeNode(req.hash.Bytes(), result.Data) + if err != nil { + return err + } + req.data = result.Data + + // Create and schedule a request for all the children nodes + requests, err := s.children(req, node) + if err != nil { + return err + } + if len(requests) == 0 && req.deps == 0 { + s.commitNodeRequest(req) + } else { + req.deps += len(requests) + for _, child := range requests { + s.scheduleNodeRequest(child) + } + } return nil } @@ -298,11 +326,11 @@ func (s *Sync) Process(result SyncResult) error { // storage, returning any occurred error. func (s *Sync) Commit(dbw ethdb.Batch) error { // Dump the membatch into a database dbw - for key, value := range s.membatch.nodes { - rawdb.WriteTrieNode(dbw, key, value) + for path, value := range s.membatch.nodes { + rawdb.WriteTrieNode(dbw, s.membatch.hashes[path], value) } - for key, value := range s.membatch.codes { - rawdb.WriteCode(dbw, key, value) + for hash, value := range s.membatch.codes { + rawdb.WriteCode(dbw, hash, value) } // Drop the membatch data and return s.membatch = newSyncMemBatch() @@ -317,23 +345,31 @@ func (s *Sync) Pending() int { // schedule inserts a new state retrieval request into the fetch queue. If there // is already a pending request for this node, the new request will be discarded // and only a parent reference added to the old one. -func (s *Sync) schedule(req *request) { - var reqset = s.nodeReqs - if req.code { - reqset = s.codeReqs +func (s *Sync) scheduleNodeRequest(req *nodeRequest) { + s.nodeReqs[string(req.path)] = req + + // Schedule the request for future retrieval. This queue is shared + // by both node requests and code requests. + prio := int64(len(req.path)) << 56 // depth >= 128 will never happen, storage leaves will be included in their parents + for i := 0; i < 14 && i < len(req.path); i++ { + prio |= int64(15-req.path[i]) << (52 - i*4) // 15-nibble => lexicographic order } + s.queue.Push(string(req.path), prio) +} + +// schedule inserts a new state retrieval request into the fetch queue. If there +// is already a pending request for this node, the new request will be discarded +// and only a parent reference added to the old one. +func (s *Sync) scheduleCodeRequest(req *codeRequest) { // If we're already requesting this node, add a new reference and stop - if old, ok := reqset[req.hash]; ok { + if old, ok := s.codeReqs[req.hash]; ok { old.parents = append(old.parents, req.parents...) return } - reqset[req.hash] = req + s.codeReqs[req.hash] = req // Schedule the request for future retrieval. This queue is shared - // by both node requests and code requests. It can happen that there - // is a trie node and code has same hash. In this case two elements - // with same hash and same or different depth will be pushed. But it's - // ok the worst case is the second response will be treated as duplicated. + // by both node requests and code requests. prio := int64(len(req.path)) << 56 // depth >= 128 will never happen, storage leaves will be included in their parents for i := 0; i < 14 && i < len(req.path); i++ { prio |= int64(15-req.path[i]) << (52 - i*4) // 15-nibble => lexicographic order @@ -343,7 +379,7 @@ func (s *Sync) schedule(req *request) { // children retrieves all the missing children of a state trie entry for future // retrieval scheduling. -func (s *Sync) children(req *request, object node) ([]*request, error) { +func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // Gather all the children of the node, irrelevant whether known or not type child struct { path []byte @@ -374,7 +410,7 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { panic(fmt.Sprintf("unknown node: %+v", node)) } // Iterate over the children, and request all unknown ones - requests := make([]*request, 0, len(children)) + requests := make([]*nodeRequest, 0, len(children)) for _, child := range children { // Notify any external watcher of a new key/value node if req.callback != nil { @@ -386,7 +422,7 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { paths = append(paths, hexToKeybytes(child.path[:2*common.HashLength])) paths = append(paths, hexToKeybytes(child.path[2*common.HashLength:])) } - if err := req.callback(paths, child.path, node, req.hash); err != nil { + if err := req.callback(paths, child.path, node, req.hash, req.path); err != nil { return nil, err } } @@ -394,20 +430,20 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // If the child references another node, resolve or schedule if node, ok := (child.node).(hashNode); ok { // Try to resolve the node from the local database - hash := common.BytesToHash(node) - if s.membatch.hasNode(hash) { + if s.membatch.hasNode(child.path) { continue } // If database says duplicate, then at least the trie node is present // and we hold the assumption that it's NOT legacy contract code. - if rawdb.HasTrieNode(s.database, hash) { + chash := common.BytesToHash(node) + if rawdb.HasTrieNode(s.database, chash) { continue } // Locally unknown node, schedule for retrieval - requests = append(requests, &request{ + requests = append(requests, &nodeRequest{ path: child.path, - hash: hash, - parents: []*request{req}, + hash: chash, + parent: req, callback: req.callback, }) } @@ -418,22 +454,40 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // commit finalizes a retrieval request and stores it into the membatch. If any // of the referencing parent requests complete due to this commit, they are also // committed themselves. -func (s *Sync) commit(req *request) (err error) { +func (s *Sync) commitNodeRequest(req *nodeRequest) error { // Write the node content to the membatch - if req.code { - s.membatch.codes[req.hash] = req.data - delete(s.codeReqs, req.hash) - s.fetches[len(req.path)]-- - } else { - s.membatch.nodes[req.hash] = req.data - delete(s.nodeReqs, req.hash) - s.fetches[len(req.path)]-- + s.membatch.nodes[string(req.path)] = req.data + s.membatch.hashes[string(req.path)] = req.hash + + delete(s.nodeReqs, string(req.path)) + s.fetches[len(req.path)]-- + + // Check parent for completion + if req.parent != nil { + req.parent.deps-- + if req.parent.deps == 0 { + if err := s.commitNodeRequest(req.parent); err != nil { + return err + } + } } + return nil +} + +// commit finalizes a retrieval request and stores it into the membatch. If any +// of the referencing parent requests complete due to this commit, they are also +// committed themselves. +func (s *Sync) commitCodeRequest(req *codeRequest) error { + // Write the node content to the membatch + s.membatch.codes[req.hash] = req.data + delete(s.codeReqs, req.hash) + s.fetches[len(req.path)]-- + // Check all parents for completion for _, parent := range req.parents { parent.deps-- if parent.deps == 0 { - if err := s.commit(parent); err != nil { + if err := s.commitNodeRequest(parent); err != nil { return err } } diff --git a/trie/sync_test.go b/trie/sync_test.go index 87b17dcee1b1..68dcd7be7b0b 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "testing" "github.com/ethereum/go-ethereum/common" @@ -26,10 +27,10 @@ import ( ) // makeTestTrie create a sample test trie to test node-wise reconstruction. -func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { +func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -50,9 +51,15 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewSecure(common.Hash{}, root, triedb) return triedb, trie, content } @@ -60,7 +67,7 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { // content map. func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { // Check root availability and trie contents - trie, err := NewSecure(common.Hash{}, common.BytesToHash(root), db) + trie, err := NewStateTrie(common.Hash{}, common.BytesToHash(root), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } @@ -77,7 +84,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db *Database, root common.Hash) error { // Create and iterate a trie rooted in a subnode - trie, err := NewSecure(common.Hash{}, root, db) + trie, err := NewStateTrie(common.Hash{}, root, db) if err != nil { return nil // Consider a non existent state consistent } @@ -87,6 +94,13 @@ func checkTrieConsistency(db *Database, root common.Hash) error { return it.Error() } +// trieElement represents the element in the state trie(bytecode or trie node). +type trieElement struct { + path string + hash common.Hash + syncPath SyncPath +} + // Tests that an empty trie is not scheduled for syncing. func TestEmptySync(t *testing.T) { dbA := NewDatabase(memorydb.New()) @@ -96,8 +110,8 @@ func TestEmptySync(t *testing.T) { for i, trie := range []*Trie{emptyA, emptyB} { sync := NewSync(trie.Hash(), memorydb.New(), nil) - if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { - t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, nodes, paths, codes) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes) } } } @@ -118,35 +132,38 @@ func testIterativeSync(t *testing.T, count int, bypath bool) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, paths, codes := sched.Missing(count) - var ( - hashQueue []common.Hash - pathQueue []SyncPath - ) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(count) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) } - for len(hashQueue)+len(pathQueue) > 0 { - results := make([]SyncResult, len(hashQueue)+len(pathQueue)) - for i, hash := range hashQueue { - data, err := srcDb.Node(hash) - if err != nil { - t.Fatalf("failed to retrieve node data for hash %x: %v", hash, err) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + if !bypath { + for i, element := range elements { + data, err := srcDb.Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err) + } + results[i] = NodeSyncResult{element.path, data} } - results[i] = SyncResult{hash, data} - } - for i, path := range pathQueue { - data, _, err := srcTrie.TryGetNode(path[0]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) + } else { + for i, element := range elements { + data, _, err := srcTrie.TryGetNode(element.syncPath[len(element.syncPath)-1]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", element.path, err) + } + results[i] = NodeSyncResult{element.path, data} } - results[len(hashQueue)+i] = SyncResult{crypto.Keccak256Hash(data), data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -156,12 +173,14 @@ func testIterativeSync(t *testing.T, count int, bypath bool) { } batch.Write() - nodes, paths, codes = sched.Missing(count) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, _ = sched.Missing(count) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) } } // Cross check that the two tries are in sync @@ -179,21 +198,29 @@ func TestIterativeDelayedSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, _, codes := sched.Missing(10000) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(10000) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } + for len(elements) > 0 { // Sync only half of the scheduled nodes - results := make([]SyncResult, len(queue)/2+1) - for i, hash := range queue[:len(results)] { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, len(elements)/2+1) + for i, element := range elements[:len(results)] { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -203,8 +230,15 @@ func TestIterativeDelayedSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(10000) - queue = append(append(queue[len(results):], nodes...), codes...) + paths, nodes, _ = sched.Missing(10000) + elements = elements[len(results):] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) @@ -225,24 +259,30 @@ func testIterativeRandomSync(t *testing.T, count int) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(count) + queue := make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + } } for len(queue) > 0 { // Fetch all the queued nodes in a random order - results := make([]SyncResult, 0, len(queue)) - for hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, 0, len(queue)) + for path, element := range queue { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, NodeSyncResult{path, data}) } // Feed the retrieved results back and queue new tasks for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -252,10 +292,14 @@ func testIterativeRandomSync(t *testing.T, count int) { } batch.Write() - queue = make(map[common.Hash]struct{}) - nodes, _, codes = sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + paths, nodes, _ = sched.Missing(count) + queue = make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } } // Cross check that the two tries are in sync @@ -273,20 +317,26 @@ func TestIterativeRandomDelayedSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(10000) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(10000) + queue := make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } for len(queue) > 0 { // Sync only half of the scheduled nodes, even those in random order - results := make([]SyncResult, 0, len(queue)/2+1) - for hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, 0, len(queue)/2+1) + for path, element := range queue { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, NodeSyncResult{path, data}) if len(results) >= cap(results) { break @@ -294,7 +344,7 @@ func TestIterativeRandomDelayedSync(t *testing.T) { } // Feed the retrieved results back and queue new tasks for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -304,11 +354,15 @@ func TestIterativeRandomDelayedSync(t *testing.T) { } batch.Write() for _, result := range results { - delete(queue, result.Hash) - } - nodes, _, codes = sched.Missing(10000) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + delete(queue, result.Path) + } + paths, nodes, _ = sched.Missing(10000) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } } // Cross check that the two tries are in sync @@ -326,26 +380,35 @@ func TestDuplicateAvoidanceSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, _, codes := sched.Missing(0) - queue := append(append([]common.Hash{}, nodes...), codes...) + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(0) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } requested := make(map[common.Hash]struct{}) - for len(queue) > 0 { - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - if _, ok := requested[hash]; ok { - t.Errorf("hash %x already requested once", hash) + if _, ok := requested[element.hash]; ok { + t.Errorf("hash %x already requested once", element.hash) } - requested[hash] = struct{}{} + requested[element.hash] = struct{}{} - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -355,8 +418,15 @@ func TestDuplicateAvoidanceSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(0) - queue = append(append(queue[:0], nodes...), codes...) + paths, nodes, _ = sched.Missing(0) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) @@ -377,23 +447,34 @@ func TestIncompleteSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - var added []common.Hash - - nodes, _, codes := sched.Missing(1) - queue := append(append([]common.Hash{}, nodes...), codes...) - for len(queue) > 0 { + // The code requests are ignored here since there is no code + // at the testing trie. + var ( + added []common.Hash + elements []trieElement + root = srcTrie.Hash() + ) + paths, nodes, _ := sched.Missing(1) + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } + for len(elements) > 0 { // Fetch a batch of trie nodes - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } // Process each of the trie nodes for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -402,27 +483,36 @@ func TestIncompleteSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() + for _, result := range results { - added = append(added, result.Hash) + hash := crypto.Keccak256Hash(result.Data) + if hash != root { + added = append(added, hash) + } // Check that all known sub-tries in the synced trie are complete - if err := checkTrieConsistency(triedb, result.Hash); err != nil { + if err := checkTrieConsistency(triedb, hash); err != nil { t.Fatalf("trie inconsistent: %v", err) } } // Fetch the next batch to retrieve - nodes, _, codes = sched.Missing(1) - queue = append(append(queue[:0], nodes...), codes...) + paths, nodes, _ = sched.Missing(1) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Sanity check that removing any node from the database is detected - for _, node := range added[1:] { - key := node.Bytes() - value, _ := diskdb.Get(key) - - diskdb.Delete(key) - if err := checkTrieConsistency(triedb, added[0]); err == nil { - t.Fatalf("trie inconsistency not caught, missing: %x", key) + for _, hash := range added { + value, _ := diskdb.Get(hash.Bytes()) + diskdb.Delete(hash.Bytes()) + if err := checkTrieConsistency(triedb, root); err == nil { + t.Fatalf("trie inconsistency not caught, missing: %x", hash) } - diskdb.Put(key, value) + diskdb.Put(hash.Bytes(), value) } } @@ -437,21 +527,33 @@ func TestSyncOrdering(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, paths, _ := sched.Missing(1) - queue := append([]common.Hash{}, nodes...) - reqs := append([]SyncPath{}, paths...) + // The code requests are ignored here since there is no code + // at the testing trie. + var ( + reqs []SyncPath + elements []trieElement + ) + paths, nodes, _ := sched.Missing(1) + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + reqs = append(reqs, NewSyncPath([]byte(paths[i]))) + } - for len(queue) > 0 { - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -461,9 +563,16 @@ func TestSyncOrdering(t *testing.T) { } batch.Write() - nodes, paths, _ = sched.Missing(1) - queue = append(queue[:0], nodes...) - reqs = append(reqs, paths...) + paths, nodes, _ = sched.Missing(1) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + reqs = append(reqs, NewSyncPath([]byte(paths[i]))) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) diff --git a/trie/trie.go b/trie/trie.go index 0c81cb2c3901..9274d88380cc 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -21,14 +21,10 @@ import ( "bytes" "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -42,36 +38,41 @@ var ( // LeafCallback is a callback type invoked when a trie operation reaches a leaf // node. // -// The paths is a path tuple identifying a particular trie node either in a single -// trie (account) or a layered trie (account -> storage). Each path in the tuple +// The keys is a path tuple identifying a particular trie node either in a single +// trie (account) or a layered trie (account -> storage). Each key in the tuple // is in the raw format(32 bytes). // -// The hexpath is a composite hexary path identifying the trie node. All the key +// The path is a composite hexary path identifying the trie node. All the key // bytes are converted to the hexary nibbles and composited with the parent path // if the trie node is in a layered trie. // // It's used by state sync and commit to allow handling external references // between account and storage tries. And also it's used in the state healing // for extracting the raw states(leaf nodes) with corresponding paths. -type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error +type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error -// Trie is a Merkle Patricia Trie. -// The zero value is an empty trie with no database. -// Use New to create a trie that sits on top of a database. +// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on +// top of a database. Whenever trie performs a commit operation, the generated +// nodes will be gathered and returned in a set. Once the trie is committed, +// it's not usable anymore. Callers have to re-create the trie with new root +// based on the updated trie database. // // Trie is not safe for concurrent use. type Trie struct { - db *Database root node owner common.Hash // Keep track of the number leaves which have been inserted since the last // hashing operation. This number will not directly map to the number of - // actually unhashed nodes + // actually unhashed nodes. unhashed int - // tracer is the state diff tracer can be used to track newly added/deleted - // trie node. It will be reset after each commit operation. + // db is the handler trie can retrieve nodes from. It's + // only for reading purpose and not available for writing. + db *Database + + // tracer is the tool to track the trie changes. + // It will be reset after each commit operation. tracer *tracer } @@ -83,10 +84,10 @@ func (t *Trie) newFlag() nodeFlag { // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - db: t.db, root: t.root, owner: t.owner, unhashed: t.unhashed, + db: t.db, tracer: t.tracer.copy(), } } @@ -99,33 +100,9 @@ func (t *Trie) Copy() *Trie { // New will panic if db is nil and returns a MissingNodeError if root does // not exist in the database. Accessing the trie loads nodes from db on demand. func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { - return newTrie(owner, root, db) -} - -// NewEmpty is a shortcut to create empty tree. It's mostly used in tests. -func NewEmpty(db *Database) *Trie { - tr, _ := newTrie(common.Hash{}, common.Hash{}, db) - return tr -} - -// newWithRootNode initializes the trie with the given root node. -// It's only used by range prover. -func newWithRootNode(root node) *Trie { - return &Trie{ - root: root, - //tracer: newTracer(), - db: NewDatabase(rawdb.NewMemoryDatabase()), - } -} - -// newTrie is the internal function used to construct the trie with given parameters. -func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { - if db == nil { - panic("trie.New called without a database") - } trie := &Trie{ - db: db, owner: owner, + db: db, //tracer: newTracer(), } if root != (common.Hash{}) && root != emptyRoot { @@ -138,6 +115,12 @@ func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { return trie, nil } +// NewEmpty is a shortcut to create empty tree. It's mostly used in tests. +func NewEmpty(db *Database) *Trie { + tr, _ := New(common.Hash{}, common.Hash{}, db) + return tr +} + // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at // the key after the given start key. func (t *Trie) NodeIterator(start []byte) NodeIterator { @@ -290,14 +273,6 @@ func (t *Trie) Update(key, value []byte) { } } -func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { - data, err := rlp.EncodeToBytes(acc) - if err != nil { - return fmt.Errorf("can't encode object at %x: %w", key[:], err) - } - return t.TryUpdate(key, data) -} - // TryUpdate associates key with value in the trie. Subsequent calls to // Get will return value. If value has length zero, any existing value // is deleted from the trie and calls to Get will return nil. @@ -307,6 +282,12 @@ func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + return t.tryUpdate(key, value) +} + +// tryUpdate expects an RLP-encoded value and performs the core function +// for TryUpdate and TryUpdateAccount. +func (t *Trie) tryUpdate(key, value []byte) error { t.unhashed++ k := keybytesToHex(key) if len(value) != 0 { @@ -512,7 +493,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // shortNode{..., shortNode{...}}. Since the entry // might not be loaded yet, resolve it just for this // check. - cnode, err := t.resolve(n.Children[pos], prefix) + cnode, err := t.resolve(n.Children[pos], append(prefix, byte(pos))) if err != nil { return false, nil, err } @@ -572,6 +553,8 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) { return n, nil } +// resolveHash loads node from the underlying database with the provided +// node hash and path prefix. func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { hash := common.BytesToHash(n) if node := t.db.node(hash); node != nil { @@ -580,6 +563,8 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} } +// resolveHash loads rlp-encoded node blob from the underlying database +// with the provided node hash and path prefix. func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { hash := common.BytesToHash(n) blob, _ := t.db.Node(hash) @@ -597,56 +582,37 @@ func (t *Trie) Hash() common.Hash { return common.BytesToHash(hash.(hashNode)) } -// Commit writes all nodes to the trie's memory database, tracking the internal -// and external (for account tries) references. -func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) { - if t.db == nil { - panic("commit called on trie with nil database") - } +// Commit collects all dirty nodes in the trie and replace them with the +// corresponding node hash. All collected nodes(including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean(nothing to commit). +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { defer t.tracer.reset() if t.root == nil { - return emptyRoot, 0, nil + return emptyRoot, nil, nil } // Derive the hash for all dirty nodes first. We hold the assumption // in the following procedure that all nodes are hashed. rootHash := t.Hash() - h := newCommitter() - defer returnCommitterToPool(h) - // Do a quick check if we really need to commit, before we spin - // up goroutines. This can happen e.g. if we load a trie for reading storage - // values, but don't write to it. + // Do a quick check if we really need to commit. This can happen e.g. + // if we load a trie for reading storage values, but don't write to it. if hashedNode, dirty := t.root.cache(); !dirty { // Replace the root node with the origin hash in order to // ensure all resolved nodes are dropped after the commit. t.root = hashedNode - return rootHash, 0, nil - } - var wg sync.WaitGroup - if onleaf != nil { - h.onleaf = onleaf - h.leafCh = make(chan *leaf, leafChanSize) - wg.Add(1) - go func() { - defer wg.Done() - h.commitLoop(t.db) - }() - } - newRoot, committed, err := h.Commit(t.root, t.db) - if onleaf != nil { - // The leafch is created in newCommitter if there was an onleaf callback - // provided. The commitLoop only _reads_ from it, and the commit - // operation was the sole writer. Therefore, it's safe to close this - // channel here. - close(h.leafCh) - wg.Wait() + return rootHash, nil, nil } + h := newCommitter(t.owner, collectLeaf) + newRoot, nodes, err := h.Commit(t.root) if err != nil { - return common.Hash{}, 0, err + return common.Hash{}, nil, err } t.root = newRoot - return rootHash, committed, nil + return rootHash, nodes, nil } // hashRoot calculates the root hash of the given trie @@ -667,10 +633,6 @@ func (t *Trie) Reset() { t.root = nil t.owner = common.Hash{} t.unhashed = 0 + //t.db = nil t.tracer.reset() } - -// Owner returns the associated trie owner. -func (t *Trie) Owner() common.Hash { - return t.owner -} diff --git a/trie/trie_test.go b/trie/trie_test.go index c813257520a3..6ce9c8463ae7 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -24,7 +24,6 @@ import ( "hash" "math/big" "math/rand" - "os" "reflect" "testing" "testing/quick" @@ -35,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" @@ -46,12 +44,6 @@ func init() { spew.Config.DisableMethods = false } -// Used for testing -func newEmpty() *Trie { - trie := NewEmpty(NewDatabase(memorydb.New())) - return trie -} - func TestEmptyTrie(t *testing.T) { trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) res := trie.Hash() @@ -91,7 +83,8 @@ func testMissingNode(t *testing.T, memonly bool) { trie := NewEmpty(triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } @@ -157,7 +150,7 @@ func testMissingNode(t *testing.T, memonly bool) { } func TestInsert(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -169,11 +162,11 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = newEmpty() + trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, _, err := trie.Commit(nil) + root, _, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -183,7 +176,8 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") updateString(trie, "dogglesworth", "cat") @@ -193,21 +187,21 @@ func TestGet(t *testing.T) { if !bytes.Equal(res, []byte("puppy")) { t.Errorf("expected puppy got %x", res) } - unknown := getString(trie, "unknown") if unknown != nil { t.Errorf("expected nil got %x", unknown) } - if i == 1 { return } - trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) } } func TestDelete(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -234,7 +228,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -258,7 +252,8 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -271,13 +266,14 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - exp, _, err := trie.Commit(nil) + exp, nodes, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } + triedb.Update(NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. - trie2, err := New(common.Hash{}, exp, trie.db) + trie2, err := New(common.Hash{}, exp, triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -286,7 +282,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, _, err := trie2.Commit(nil) + hash, nodes, err := trie2.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -294,6 +290,14 @@ func TestReplication(t *testing.T) { t.Errorf("root failure. expected %x got %x", exp, hash) } + // recreate the trie after commit + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } + trie2, err = New(common.Hash{}, hash, triedb) + if err != nil { + t.Fatalf("can't recreate trie at %x: %v", exp, err) + } // perform some insertions on the new trie. vals2 := []struct{ k, v string }{ {"do", "verb"}, @@ -315,7 +319,7 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update([]byte("key1"), []byte{99, 99, 99, 99}) trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() @@ -352,7 +356,6 @@ func TestRandomCases(t *testing.T) { {op: 1, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("")}, // step 25 } runRandTest(rt) - } // randTest performs random trie operations. @@ -370,9 +373,8 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opNodeDiff opMax // boundary value, not an actual op @@ -432,19 +434,19 @@ func runRandTest(rt randTest) bool { v := tr.Get(step.key) want := values[string(step.key)] if string(v) != want { - rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) + rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) - origTrie = tr.Copy() case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + hash, nodes, err := tr.Commit(false) if err != nil { rt[i].err = err return false } + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } newtr, err := New(common.Hash{}, hash, triedb) if err != nil { rt[i].err = err @@ -534,44 +536,31 @@ func TestRandom(t *testing.T) { } } -func BenchmarkGet(b *testing.B) { benchGet(b, false) } -func BenchmarkGetDB(b *testing.B) { benchGet(b, true) } +func BenchmarkGet(b *testing.B) { benchGet(b) } func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) } func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 -func benchGet(b *testing.B, commit bool) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - if commit { - tmpdb := tempDB(b) - trie = NewEmpty(tmpdb) - } +func benchGet(b *testing.B) { + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { binary.LittleEndian.PutUint64(k, uint64(i)) trie.Update(k, k) } binary.LittleEndian.PutUint64(k, benchElemCount/2) - if commit { - trie.Commit(nil) - } b.ResetTimer() for i := 0; i < b.N; i++ { trie.Get(k) } b.StopTimer() - - if commit { - ldb := trie.db.diskdb.(*leveldb.Database) - ldb.Close() - os.RemoveAll(ldb.Path()) - } } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -601,7 +590,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) i := 0 for ; i < len(addresses)/2; i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -622,22 +611,17 @@ func BenchmarkHash(b *testing.B) { // insert into the trie before measuring the hashing. func BenchmarkCommitAfterHash(b *testing.B) { b.Run("no-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, nil) + benchmarkCommitAfterHash(b, false) }) - var a types.StateAccount - onleaf := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { - rlp.DecodeBytes(leaf, &a) - return nil - } b.Run("with-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, onleaf) + benchmarkCommitAfterHash(b, true) }) } -func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { +func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -645,13 +629,13 @@ func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { trie.Hash() b.ResetTimer() b.ReportAllocs() - trie.Commit(onleaf) + trie.Commit(collectLeaf) } func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -664,7 +648,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr := NewEmpty(trie.db) + checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -677,19 +661,19 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() - trie.Commit(nil) + trie.Commit(false) root := trie.Hash() exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6") if exp != root { t.Errorf("got %x, exp %x", root, exp) } - root, _, _ = trie.Commit(nil) + root, _, _ = trie.Commit(false) if exp != root { t.Errorf("got %x, exp %x", root, exp) } @@ -798,7 +782,8 @@ func TestCommitSequence(t *testing.T) { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -850,7 +835,8 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { trie.Update(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -876,7 +862,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) // Fill the trie with elements - for i := 1; i < count; i++ { + for i := 0; i < count; i++ { // For the stack trie, we need to do inserts in proper order key := make([]byte, 32) binary.BigEndian.PutUint64(key, uint64(i)) @@ -892,8 +878,9 @@ func TestCommitSequenceStackTrie(t *testing.T) { stTrie.TryUpdate(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -937,8 +924,9 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie.TryUpdate(key, []byte{0x1}) stTrie.TryUpdate(key, []byte{0x1}) // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -1000,7 +988,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1051,14 +1039,14 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() b.StartTimer() - trie.Commit(nil) + trie.Commit(false) b.StopTimer() } @@ -1103,26 +1091,19 @@ func BenchmarkDerefRootFixedSize(b *testing.B) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } h := trie.Hash() - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) b.StartTimer() - trie.db.Dereference(h) + triedb.Dereference(h) b.StopTimer() } -func tempDB(tb testing.TB) *Database { - dir := tb.TempDir() - diskdb, err := leveldb.New(dir, 256, 0, "", false) - if err != nil { - panic(fmt.Sprintf("can't create temporary database: %v", err)) - } - return NewDatabase(diskdb) -} - func getString(trie *Trie, k string) []byte { return trie.Get([]byte(k)) } diff --git a/trie/util_test.go b/trie/util_test.go index 589eca62423a..252dc09e0804 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -19,12 +19,14 @@ package trie import ( "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" ) // Tests if the trie diffs are tracked correctly. func TestTrieTracer(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted @@ -65,8 +67,11 @@ func TestTrieTracer(t *testing.T) { t.Fatalf("Unexpected deleted node tracked %d", len(deleted)) } - // Commit the changes - trie.Commit(nil) + // Commit the changes and re-create with new root + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) + trie.tracer = newTracer() // Delete all the elements, check deletion set for _, val := range vals { diff --git a/wemix/etcdutil.go b/wemix/etcdutil.go index 080ae9e28ec8..8f86fefe4749 100644 --- a/wemix/etcdutil.go +++ b/wemix/etcdutil.go @@ -895,7 +895,6 @@ again: } } } - } return err } @@ -990,7 +989,6 @@ again: ).Commit() if err == nil && !txresp.Succeeded { - var ( tokenFound, workFound bool = false, false foundToken []byte diff --git a/wemix/rewards_test.go b/wemix/rewards_test.go index 0737abd492c1..af3b43b3ba39 100644 --- a/wemix/rewards_test.go +++ b/wemix/rewards_test.go @@ -12,7 +12,6 @@ import ( // TestDistributeRewards tests the DistributeRewards function func TestDistributeRewards(t *testing.T) { - hexToAddressPtr := func(addr string) *common.Address { address := common.HexToAddress(addr) return &address diff --git a/wemix/scripts/gwemix.sh b/wemix/scripts/gwemix.sh index 8b7c5b64ae60..4152a7c13be3 100755 --- a/wemix/scripts/gwemix.sh +++ b/wemix/scripts/gwemix.sh @@ -147,6 +147,8 @@ function start () [ "$PORT" = "" ] || RPCOPT="${RPCOPT} --http.port ${PORT}" RPCOPT="${RPCOPT} --ws --ws.addr 0.0.0.0" [ "$PORT" = "" ] || RPCOPT="${RPCOPT} --ws.port $((${PORT}+10))" + RPCOPT="${RPCOPT} --authrpc.addr 0.0.0.0" + [ "$PORT" = "" ] || RPCOPT="${RPCOPT} --authrpc.port $((${PORT}+11))" [ "$NONCE_LIMIT" = "" ] || NONCE_LIMIT="--noncelimit $NONCE_LIMIT" [ "$BOOT_NODES" = "" ] || BOOT_NODES="--bootnodes $BOOT_NODES" [ "$TESTNET" = "1" ] && TESTNET=--wemix-testnet