diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9d61c4b359c7..1cdbfe866863 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -225,6 +225,9 @@ var ( utils.BlockMinBuildTime, utils.BlockMinBuildTxs, utils.BlockTrailTime, + utils.PublicRequestsCacheLocation, + utils.MaxPublicRequests, + utils.BootnodeCount, } ) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index c1d2abc09be6..1aa1f77cce9d 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -235,6 +235,9 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.BlockMinBuildTime, utils.BlockMinBuildTxs, utils.BlockTrailTime, + utils.PublicRequestsCacheLocation, + utils.MaxPublicRequests, + utils.BootnodeCount, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c1ec692a8efc..96e8cd654c53 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -23,6 +23,7 @@ import ( "io" "math" "math/big" + "math/rand" "os" "path/filepath" godebug "runtime/debug" @@ -916,6 +917,21 @@ var ( Usage: "Time to leave for block data transfer in ms", Value: params.BlockTrailTime, } + PublicRequestsCacheLocation = cli.StringFlag{ + Name: "wemix.publicrequests.cache", + Usage: "Public requests cache location", + Value: params.PublicRequestsCacheLocation, + } + MaxPublicRequests = cli.Int64Flag{ + Name: "wemix.publicrequests.max", + Usage: "Max # of concurrent public requests", + Value: params.MaxPublicRequests, + } + BootnodeCount = cli.IntFlag{ + Name: "wemix.bootnodecount", + Usage: "Default bootnode peer count", + Value: params.BootnodeCount, + } ) var ( @@ -1011,15 +1027,40 @@ func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) { } } +// setRandomBootstrapNodes setting a random list of bootstrap nodes using the command line +func setRandomBootstrapNodes(ctx *cli.Context, bootnodes []string) []string { + rand.Seed(time.Now().UnixNano()) + bootnodeslen := len(bootnodes) + + // check command line + if ctx.GlobalIsSet(BootnodeCount.Name) { + setcount := ctx.GlobalInt(BootnodeCount.Name) + if setcount > 0 && setcount <= bootnodeslen { + params.BootnodeCount = setcount + } + } + // select random bootnodes + selectcount := params.BootnodeCount + urls := make([]string, selectcount) + tempnode := make([]string, bootnodeslen) + copy(tempnode, bootnodes) + for i := 0; i < selectcount; i++ { + index := rand.Intn(len(tempnode)) + urls = append(urls, tempnode[index]) + tempnode = append(tempnode[:index], tempnode[index+1:]...) + } + return urls +} + // setBootstrapNodes creates a list of bootstrap nodes from the command line // flags, reverting to pre-configured ones if none have been specified. func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { - urls := params.WemixMainnetBootnodes + urls := setRandomBootstrapNodes(ctx, params.WemixMainnetBootnodes) switch { case ctx.GlobalIsSet(BootnodesFlag.Name): urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) case ctx.GlobalBool(WemixTestnetFlag.Name): - urls = params.WemixTestnetBootnodes + urls = setRandomBootstrapNodes(ctx, params.WemixTestnetBootnodes) case ctx.GlobalBool(RopstenFlag.Name): urls = params.RopstenBootnodes case ctx.GlobalBool(SepoliaFlag.Name): @@ -1982,6 +2023,12 @@ func SetWemixConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(BlockTrailTime.Name) { params.BlockTrailTime = ctx.GlobalInt64(BlockTrailTime.Name) } + if ctx.GlobalIsSet(PublicRequestsCacheLocation.Name) { + params.PublicRequestsCacheLocation = ctx.GlobalString(PublicRequestsCacheLocation.Name) + } + if ctx.GlobalIsSet(MaxPublicRequests.Name) { + params.MaxPublicRequests = ctx.GlobalInt64(MaxPublicRequests.Name) + } if params.ConsensusMethod == params.ConsensusInvalid { params.ConsensusMethod = params.ConsensusPoW diff --git a/eth/api.go b/eth/api.go index 7ec802b15852..4ebd7231e162 100644 --- a/eth/api.go +++ b/eth/api.go @@ -412,7 +412,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, } else { blockRlp = fmt.Sprintf("0x%x", rlpBytes) } - if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.APIBackend.ChainConfig()); err != nil { + if blockJSON, err = ethapi.RPCMarshalBlock(ctx, block, true, true, api.eth.APIBackend.ChainConfig()); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} } results = append(results, &BadBlockArgs{ diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 481ceaf5323d..8263fef4b60f 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -34,7 +34,7 @@ import ( const ( // maxKnownTxs is the maximum transactions hashes to keep in the known list // before starting to randomly evict them. - maxKnownTxs = 2000000 + maxKnownTxs = 100000 // maxKnownBlocks is the maximum block hashes to keep in the known list // before starting to randomly evict them. diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index c351f4fdfb46..08d85442cc50 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -20,9 +20,12 @@ import ( "context" "errors" "fmt" + "io" "math/big" "os" + "runtime" "strings" + "sync" "time" "github.com/davecgh/go-spew/spew" @@ -42,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/vrf" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" @@ -50,6 +54,29 @@ import ( "github.com/tyler-smith/go-bip39" ) +var apiRequestsCache ethdb.Database +var apiRequestsThrottle chan struct{} +var apiRequestsTokens chan struct{} + +func apiRequestsEnter() { + if len(apiRequestsThrottle) >= int(params.MaxPublicRequests) { + pc, _, _, _ := runtime.Caller(1) + var name string + parts := strings.Split(runtime.FuncForPC(pc).Name(), ".") + if len(parts) > 0 { + name = parts[len(parts)-1] + } else { + name = runtime.FuncForPC(pc).Name() + } + log.Warn("Too many API requests", "func", name, "count", len(apiRequestsThrottle)) + } + apiRequestsThrottle <- struct{}{} +} + +func apiRequestsLeave() { + <-apiRequestsThrottle +} + // PublicEthereumAPI provides an API to access Ethereum related information. // It offers only methods that operate on public data that is freely available to anyone. type PublicEthereumAPI struct { @@ -681,6 +708,20 @@ type PublicBlockChainAPI struct { // NewPublicBlockChainAPI creates a new Ethereum blockchain API. func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI { + if len(params.PublicRequestsCacheLocation) > 0 { + var err error + apiRequestsCache, err = apiCacheOpen(params.PublicRequestsCacheLocation) + if err != nil { + panic(err) + } + } + apiRequestsThrottle = make(chan struct{}, params.MaxPublicRequests) + tokens := runtime.NumCPU() * 8 / 10 + if tokens < 4 { + tokens = 4 + } + apiRequestsTokens = make(chan struct{}, tokens) + return &PublicBlockChainAPI{b} } @@ -701,6 +742,21 @@ func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 { // GetBlockReceipts returns all the transaction receipts for the given block hash. func (s *PublicBlockChainAPI) GetReceiptsByHash(ctx context.Context, blockHash common.Hash) ([]map[string]interface{}, error) { + apiRequestsEnter() + defer apiRequestsLeave() + + select { + case <-ctx.Done(): + return nil, io.EOF + default: + } + + if apiRequestsCache != nil { + if fields, err := apiCacheGetReceipts(apiRequestsCache, blockHash.Bytes()); err == nil { + log.Debug("API Cache", "found receipts", blockHash) + return fields, nil + } + } block, err1 := s.b.BlockByHash(ctx, blockHash) if block == nil && err1 == nil { @@ -723,6 +779,11 @@ func (s *PublicBlockChainAPI) GetReceiptsByHash(ctx context.Context, blockHash c fieldsList := make([]map[string]interface{}, 0, len(receipts)) for index, receipt := range receipts { + select { + case <-ctx.Done(): + return nil, io.EOF + default: + } bigblock := new(big.Int).SetUint64(block.NumberU64()) signer := types.MakeSigner(s.b.ChainConfig(), bigblock) @@ -770,6 +831,9 @@ func (s *PublicBlockChainAPI) GetReceiptsByHash(ctx context.Context, blockHash c fieldsList = append(fieldsList, fields) } + if apiRequestsCache != nil { + apiCachePutReceipts(apiRequestsCache, blockHash.Bytes(), fieldsList) + } return fieldsList, nil } @@ -884,6 +948,9 @@ func (s *PublicBlockChainAPI) GetHeaderByHash(ctx context.Context, hash common.H // - When fullTx is true all transactions in the block are returned, otherwise // only the transaction hash is returned. func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { + apiRequestsEnter() + defer apiRequestsLeave() + block, err := s.b.BlockByNumber(ctx, number) if block != nil && err == nil { response, err := s.rpcMarshalBlock(ctx, block, true, fullTx) @@ -901,6 +968,9 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.B // GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full // detail, otherwise only the transaction hash is returned. func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) { + apiRequestsEnter() + defer apiRequestsLeave() + block, err := s.b.BlockByHash(ctx, hash) if block != nil { return s.rpcMarshalBlock(ctx, block, true, fullTx) @@ -910,6 +980,9 @@ func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Ha // GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { + apiRequestsEnter() + defer apiRequestsLeave() + block, err := s.b.BlockByNumber(ctx, blockNr) if block != nil { uncles := block.Uncles() @@ -925,6 +998,9 @@ func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, // GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) { + apiRequestsEnter() + defer apiRequestsLeave() + block, err := s.b.BlockByHash(ctx, blockHash) if block != nil { uncles := block.Uncles() @@ -940,6 +1016,9 @@ func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, b // GetUncleCountByBlockNumber returns number of uncles in the block for the given block number func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { + apiRequestsEnter() + defer apiRequestsLeave() + if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { n := hexutil.Uint(len(block.Uncles())) return &n @@ -949,6 +1028,9 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, bl // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { + apiRequestsEnter() + defer apiRequestsLeave() + if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { n := hexutil.Uint(len(block.Uncles())) return &n @@ -1299,7 +1381,20 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { // RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are // returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain // transaction hashes. -func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *params.ChainConfig) (map[string]interface{}, error) { +func RPCMarshalBlock(ctx context.Context, block *types.Block, inclTx bool, fullTx bool, config *params.ChainConfig) (map[string]interface{}, error) { + select { + case <-ctx.Done(): + return nil, io.EOF + default: + } + + if fullTx && apiRequestsCache != nil { + if fields, err := apiCacheGetBlock(apiRequestsCache, block.Hash().Bytes()); err == nil { + log.Debug("API Cache", "found block", block.Number()) + return fields, nil + } + } + fields := RPCMarshalHeader(block.Header()) fields["size"] = hexutil.Uint64(block.Size()) @@ -1314,11 +1409,33 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *param } txs := block.Transactions() transactions := make([]interface{}, len(txs)) + var wg sync.WaitGroup var err error for i, tx := range txs { - if transactions[i], err = formatTx(tx); err != nil { - return nil, err - } + wg.Add(1) + go func(ii int, itx *types.Transaction) { + apiRequestsTokens <- struct{}{} + defer func() { + wg.Done() + <-apiRequestsTokens + }() + + select { + case <-ctx.Done(): + err = io.EOF + return + default: + } + var err2 error + transactions[ii], err2 = formatTx(itx) + if err2 != nil { + err = err2 + } + }(i, tx) + } + wg.Wait() + if err != nil { + return nil, err } fields["transactions"] = transactions } @@ -1329,6 +1446,10 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *param } fields["uncles"] = uncleHashes + if fullTx && apiRequestsCache != nil { + apiCachePutBlock(apiRequestsCache, block.Hash().Bytes(), fields) + } + return fields, nil } @@ -1343,7 +1464,7 @@ func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *type // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields, err := RPCMarshalBlock(b, inclTx, fullTx, s.b.ChainConfig()) + fields, err := RPCMarshalBlock(ctx, b, inclTx, fullTx, s.b.ChainConfig()) if err != nil { return nil, err } diff --git a/internal/ethapi/api_cache.go b/internal/ethapi/api_cache.go new file mode 100644 index 000000000000..fd950a999923 --- /dev/null +++ b/internal/ethapi/api_cache.go @@ -0,0 +1,126 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "io" + + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +var marshaledBlockPrefix = []byte(".blck.") +var marshaledReceiptsPrefix = []byte(".rcpt.") + +func apiCacheOpen(fn string) (ethdb.Database, error) { + return rawdb.NewDB(fn, 0, 0, "", false) +} + +func apiCacheClose(db ethdb.Database) { + db.Close() +} + +func apiCacheGet(db ethdb.Database, prefix, hash []byte) ([]byte, error) { + key := append(prefix, hash...) + data, err := db.Get(key) + if err != nil { + return nil, err + } + + gzReader, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + defer gzReader.Close() + + decompressedData, err := io.ReadAll(gzReader) + if err != nil { + return nil, err + } + return decompressedData, nil +} + +func apiCacheHas(db ethdb.Database, prefix, hash []byte) (bool, error) { + key := append(prefix, hash...) + return db.Has(key) +} + +func apiCachePut(db ethdb.Database, prefix, hash, data []byte) error { + var buf bytes.Buffer + gzWriter := gzip.NewWriter(&buf) + if _, err := gzWriter.Write(data); err != nil { + return err + } + if err := gzWriter.Close(); err != nil { + return err + } + + key := append(prefix, hash...) + return db.Put(key, buf.Bytes()) +} + +func apiCacheDelete(db ethdb.Database, prefix, hash []byte) error { + key := append(prefix, hash...) + return db.Delete(key) +} + +func apiCacheGetBlock(db ethdb.Database, hash []byte) (map[string]interface{}, error) { + data, err := apiCacheGet(db, marshaledBlockPrefix, hash) + if err != nil { + return nil, err + } + + var fields map[string]interface{} + if err = json.Unmarshal(data, &fields); err != nil { + return nil, err + } + return fields, nil +} + +func apiCachePutBlock(db ethdb.Database, hash []byte, fields map[string]interface{}) error { + data, err := json.Marshal(fields) + if err != nil { + return err + } + return apiCachePut(db, marshaledBlockPrefix, hash, data) +} + +func apiCacheGetReceipts(db ethdb.Database, hash []byte) ([]map[string]interface{}, error) { + data, err := apiCacheGet(db, marshaledReceiptsPrefix, hash) + if err != nil { + return nil, err + } + + var fields []map[string]interface{} + if err = json.Unmarshal(data, &fields); err != nil { + return nil, err + } + return fields, nil +} + +func apiCachePutReceipts(db ethdb.Database, hash []byte, fields []map[string]interface{}) error { + data, err := json.Marshal(fields) + if err != nil { + return err + } + return apiCachePut(db, marshaledReceiptsPrefix, hash, data) +} + +// EoF diff --git a/params/protocol_params.go b/params/protocol_params.go index b64e82ba54d8..86623d79d990 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -202,4 +202,8 @@ var ( BlockMinBuildTime int64 = 300 // Minimum block generation time in ms BlockMinBuildTxs int64 = 2500 // Minimum txs in a block with pending txs BlockTrailTime int64 = 300 // Time to leave for block data transfer transfer in ms + + PublicRequestsCacheLocation string = "" // Cache DB location + MaxPublicRequests int64 = 100 // Max # of concurrent public requests + BootnodeCount int = 3 // Default bootnode peer count. ) diff --git a/params/version.go b/params/version.go index fe5e96fcb700..65bce5efb03b 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 0 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release + VersionPatch = 7 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string )