diff --git a/app/cmd/cli/accounts.go b/app/cmd/cli/accounts.go index 03184924..9056ea77 100644 --- a/app/cmd/cli/accounts.go +++ b/app/cmd/cli/accounts.go @@ -116,7 +116,11 @@ var getNodesLean = &cobra.Command{ fmt.Println("Lean pocket is not enabled") return } - keys, _ := app.ReadValidatorPrivateKeyFileLean(datadir) + keys, err := app.ReadValidatorPrivateKeyFileLean(datadir) + if err != nil { + fmt.Println("Failed to read lean valida") + return + } for _, value := range keys { fmt.Printf("Lean Address: %s\n", strings.ToLower(value.PublicKey().Address().String())) } @@ -139,27 +143,32 @@ var getValidator = &cobra.Command{ }, } -var setValidatorsLean = &cobra.Command{ - Use: `set-validators-lean `, - Short: "Sets the main validator accounts for tendermint; NOTE: keyfile should be a json string array of private keys", - Long: `Sets the main validator accounts that will be used across all Tendermint functions; NOTE: keyfile should be a json string array of private keys`, - Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - app.InitConfig(datadir, tmNode, persistentPeers, seeds, remoteCLIURL) - - if !app.GlobalConfig.PocketConfig.LeanPocket { - fmt.Println("Lean pocket is not enabled") - return - } - - keys, err := app.ReadValidatorPrivateKeyFileLean(args[0]) - if err != nil { - fmt.Println("Failed to set validators ", err) - os.Exit(1) - } - app.SetValidatorsFilesLean(keys) - }, -} +//var setValidatorsLean = &cobra.Command{ +// Use: `set-validators-lean `, +// Short: "Sets the main validator accounts for tendermint; NOTE: keyfile should be a json string array of private keys", +// Long: `Sets the main validator accounts that will be used across all Tendermint functions; NOTE: keyfile should be a json string array of private keys`, +// Args: cobra.MinimumNArgs(1), +// Run: func(cmd *cobra.Command, args []string) { +// app.InitConfig(datadir, tmNode, persistentPeers, seeds, remoteCLIURL) +// +// if !app.GlobalConfig.PocketConfig.LeanPocket { +// fmt.Println("Lean pocket is not enabled") +// return +// } +// +// //keys, err := app.ReadValidatorPrivateKeyFileLean(args[0]) +// //if err != nil { +// // fmt.Println("Failed to read validators json file ", err) +// // os.Exit(1) +// //} +// //// err = app.SetValidatorsFilesLean(keys) +// //if err != nil { +// // fmt.Println("Failed to set validators", err) +// // os.Exit(1) +// // return +// //} +// }, +//} var setValidator = &cobra.Command{ Use: "set-validator
", diff --git a/app/cmd/cli/root.go b/app/cmd/cli/root.go index 4f649880..90338408 100644 --- a/app/cmd/cli/root.go +++ b/app/cmd/cli/root.go @@ -62,8 +62,7 @@ func init() { startCmd.Flags().BoolVar(&profileApp, "profileApp", false, "expose cpu & memory profiling") startCmd.Flags().BoolVar(&useCache, "useCache", false, "use cache") startCmd.Flags().BoolVar(&useLean, "useLean", false, "enable usage of lean pocket through cli") - startCmd.Flags().StringVar(&validatorsPathLean, "validators-path-lean", "", "path of your lean pocket validators private keys to hot reload") - startCmd.Flags().StringVar(&peerNodeLean, "set-peer-node-lean", "", "change the node lean file to be one of the specified validator by address") + //startCmd.Flags().StringVar(&peerNodeLean, "set-peer-node-lean", "", "change the node lean file to be one of the specified validator by address") rootCmd.AddCommand(startCmd) rootCmd.AddCommand(resetCmd) rootCmd.AddCommand(version) diff --git a/app/config.go b/app/config.go index dd5a6bd9..b8738315 100644 --- a/app/config.go +++ b/app/config.go @@ -43,6 +43,7 @@ import ( "io/ioutil" log2 "log" "os" + "path" fp "path/filepath" "strings" "sync" @@ -92,7 +93,24 @@ func InitApp(datadir, tmNode, persistentPeers, seeds, remoteCLIURL string, keyba // init cache InitPocketCoreConfig(chains, logger) - InitKeyfiles() + // prestart hook, so users don't have to create their own set-validator prestart script + if GlobalConfig.PocketConfig.LeanPocket { + userProvidedKeyPath := GlobalConfig.PocketConfig.GetLeanPocketUserKeyFilePath() + // add a read to nodes.json -> convert with setValidators() + keys, err := ReadValidatorPrivateKeyFileLean(userProvidedKeyPath) + if err != nil { + logger.Error("Can't read user provided validator keys, did you create keys in", userProvidedKeyPath, err) + os.Exit(1) + } + // check if peering address is set + err = SetValidatorsFilesLean(keys) + if err != nil { + logger.Error("Failed to set validators for user provided file, try pocket accounts set-validators", userProvidedKeyPath, err) + os.Exit(1) + } + } + + InitKeyfiles(logger) // get hosted blockchains @@ -101,6 +119,7 @@ func InitApp(datadir, tmNode, persistentPeers, seeds, remoteCLIURL string, keyba // log the config and chains logger.Debug(fmt.Sprintf("Pocket Config: \n%v", GlobalConfig)) // init the tendermint node + return InitTendermint(keybase, chains, logger) } @@ -346,12 +365,13 @@ func InitTendermint(keybase bool, chains *types.HostedBlockchains, logger log.Lo return tmNode } -func InitKeyfiles() { +func InitKeyfiles(logger log.Logger) { if GlobalConfig.PocketConfig.LeanPocket { - err := InitNodesLean() + err := InitNodesLean(logger) if err != nil { - log2.Fatal(err) + logger.Error("Failed to init lean nodes", err) + os.Exit(1) } return } @@ -385,36 +405,82 @@ func InitKeyfiles() { } } -func InitNodesLean() error { - datadir := GlobalConfig.PocketConfig.DataDir - filePathLean := datadir + FS + GlobalConfig.TendermintConfig.PrivValidatorKey +func watchFile(filePath string) error { + initialStat, err := os.Stat(filePath) + if err != nil { + return err + } + for { + stat, err := os.Stat(filePath) + if err != nil { + return err + } - if _, err := os.Stat(filePathLean); err != nil { + if stat.Size() != initialStat.Size() || stat.ModTime() != initialStat.ModTime() { + break + } + + time.Sleep(1 * time.Second) + } + return nil +} + +func InitNodesLean(logger log.Logger) error { + pvkName := path.Join(GlobalConfig.PocketConfig.DataDir, GlobalConfig.TendermintConfig.PrivValidatorKey) + if _, err := os.Stat(pvkName); err != nil { if os.IsNotExist(err) { return errors.New("pocket accounts set-validators must be ran first") } - return errors.New("Failed to retrieve information on " + filePathLean) + return errors.New("Failed to retrieve information on " + pvkName) } - leanNodesTm, err := loadFilePVKeysFromFileLean(filePathLean) + leanNodesTm, err := loadFilePVKeysFromFileLean(pvkName) if err != nil { return err } + if len(leanNodesTm) == 0 { + return errors.New("failed to load lean validators, length of zero") + } + // logic here to remove / stop lean nodes that are added + types.GlobalNodeLeanRWLock.Lock() + defer types.GlobalNodeLeanRWLock.Unlock() + addedNodesSet := map[crypto.PrivateKey]bool{} for _, node := range leanNodesTm { key, err := crypto.PrivKeyToPrivateKey(node.PrivKey) if err != nil { return errors.New("failed to convert tendermint private key over to pocket private key") } - types.InitNodeWithCacheLean(key) + addedNodesSet[key] = true + types.InitNodeWithCacheLean(key, logger) } - if len(types.GlobalNodesLean) <= 1 { - return errors.New("lean pocket should be enabled with atleast two nodes") + for k, v := range types.GlobalNodesLean { + if v == nil { + continue + } + _, shouldKeep := addedNodesSet[v.PrivateKey] + if !shouldKeep { + + go func() { + types.GlobalNodeLeanRWLock.Lock() + defer types.GlobalNodeLeanRWLock.Unlock() + node := types.GlobalNodesLean[k] + logger.Info("Detected node " + k + " has been deleted, flushing session/evidence DB first.") + node.SessionCache.FlushToDB() + node.EvidenceCache.FlushToDB() + node.EvidenceCache.DB.Close() + logger.Info(k + " evidence DB successfully closed") + + + delete(types.GlobalNodesLean, k) + }() + } } + // set the "main (legacy)" validator to first lean node types.InitPVKeyFile(leanNodesTm[0]) return nil @@ -515,7 +581,7 @@ func loadFilePVKeysFromFileLean(path string) ([]privval.FilePVKey, error) { return pvKey, nil } -func privValKeysLean(res []crypto.PrivateKey) { +func privValKeysLean(res []crypto.PrivateKey) error { var pvKL []privval.FilePVKey for _, pk := range res { pvKL = append(pvKL, privval.FilePVKey{ @@ -526,16 +592,17 @@ func privValKeysLean(res []crypto.PrivateKey) { } pvkBz, err := cdc.MarshalJSONIndent(pvKL, "", " ") if err != nil { - log2.Fatal(err) + return err } - pvFile, err := os.OpenFile(GlobalConfig.PocketConfig.DataDir+FS+GlobalConfig.TendermintConfig.PrivValidatorKey, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + pvFile, err := os.OpenFile(GlobalConfig.PocketConfig.DataDir+FS+GlobalConfig.TendermintConfig.PrivValidatorKey, os.O_RDWR|os.O_CREATE, 0666) if err != nil { - log2.Fatal(err) + return err } _, err = pvFile.Write(pvkBz) if err != nil { - log2.Fatal(err) + return err } + return nil } func privValKey(res crypto.PrivateKey) { @@ -577,19 +644,39 @@ func nodeKey(res crypto.PrivateKey) { } } -func privValStateLean(size int) { +func nodeKeyLean(res crypto.PrivateKey) error { + nodeKey := p2p.NodeKey{ + PrivKey: res.PrivKey(), + } + pvkBz, err := cdc.MarshalJSONIndent(nodeKey, "", " ") + if err != nil { + return err + } + pvFile, err := os.OpenFile(GlobalConfig.PocketConfig.DataDir+FS+GlobalConfig.TendermintConfig.NodeKey, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + return err + } + _, err = pvFile.Write(pvkBz) + if err != nil { + return err + } + return nil +} + +func privValStateLean(size int) error { pvkBz, err := cdc.MarshalJSONIndent(make([]privval.FilePVLastSignState, size), "", " ") if err != nil { - log2.Fatal(err) + return err } pvFile, err := os.OpenFile(GlobalConfig.PocketConfig.DataDir+FS+GlobalConfig.TendermintConfig.PrivValidatorState, os.O_RDWR|os.O_CREATE, 0666) if err != nil { - log2.Fatal(err) + return err } _, err = pvFile.Write(pvkBz) if err != nil { - log2.Fatal(err) + return err } + return nil } func privValState() { @@ -905,11 +992,29 @@ func ReadValidatorPrivateKeyFileLean(filePath string) ([]crypto.PrivateKey, erro return pks, nil } -func SetValidatorsFilesLean(keys []crypto.PrivateKey) { +func SetValidatorsFilesLean(keys []crypto.PrivateKey) error { + return SetValidatorsFilesWithPeerLean(keys, keys[0].PublicKey().Address().String()) +} + +func SetValidatorsFilesWithPeerLean(keys []crypto.PrivateKey, address string) error { resetFilePVLean(GlobalConfig.PocketConfig.DataDir+FS+GlobalConfig.TendermintConfig.PrivValidatorKey, GlobalConfig.PocketConfig.DataDir+FS+GlobalConfig.TendermintConfig.PrivValidatorState, log.NewNopLogger()) - privValKeysLean(keys) - privValStateLean(len(keys)) - nodeKey(keys[0]) + + err := privValKeysLean(keys) + if err != nil { + return err + } + err = privValStateLean(len(keys)) + if err != nil { + return err + } + for _, k := range keys { + if strings.EqualFold(k.PublicKey().Address().String(), address) { + err := nodeKeyLean(k) + return err + } + } + log2.Println("Could not find " + address + " setting default peering to address: " + keys[0].PublicKey().Address().String()) + return nodeKeyLean(keys[0]) } func SetValidator(address sdk.Address, passphrase string) { @@ -979,11 +1084,12 @@ func resetFilePV(privValKeyFile, privValStateFile string, logger log.Logger) { "stateFile", privValStateFile) } -func resetFilePVLean(privValKeyFile, privValStateFile string, logger log.Logger) { - if _, err := os.Stat(privValKeyFile); err == nil { - _ = os.Truncate(privValKeyFile, 0) - _ = os.Truncate(privValStateFile, 0) - _ = os.Truncate(GlobalConfig.PocketConfig.DataDir + FS + GlobalConfig.TendermintConfig.NodeKey, 0) +func resetFilePVLean(privValKeyFile, privValStateFile string, logger log.Logger){ + _, err := os.Stat(privValKeyFile) + if err == nil { + _ = os.Remove(privValKeyFile) + _ = os.Remove(privValStateFile) + _ = os.Remove(GlobalConfig.PocketConfig.DataDir + FS + GlobalConfig.TendermintConfig.NodeKey) } logger.Info("Reset private validator file", "keyFile", privValKeyFile, "stateFile", privValStateFile) diff --git a/app/tendermint.go b/app/tendermint.go index ac40b592..03ce759a 100644 --- a/app/tendermint.go +++ b/app/tendermint.go @@ -1,6 +1,7 @@ package app import ( + "errors" "github.com/pokt-network/pocket-core/codec" sdk "github.com/pokt-network/pocket-core/types" cfg "github.com/tendermint/tendermint/config" @@ -33,27 +34,59 @@ func loadFilePVWithConfig(c config) *pvm.FilePVLite { } } +func ReloadValidatorKeys(c config, tmNode *node.Node) error { + + keys, err := ReadValidatorPrivateKeyFileLean(GlobalConfig.PocketConfig.GetLeanPocketUserKeyFilePath()) + if err != nil { + return err + } + + if len(keys) == 0 { + return errors.New("user key file contained zero validator keys") + } + + err = SetValidatorsFilesLean(keys) + if err != nil { + return err + } -func loadValidatorsLean(c config, tmNode *node.Node) error { - // add a read to nodes.json -> convert with setValidators() validators := loadFilePVWithConfig(c) - err := InitNodesLean() // reinitialize lean nodes tmNode.ConsensusState().SetPrivValidators(validators) // set new lean nodes + + err = InitNodesLean(c.Logger) // initialize lean nodes if err != nil { return err } + return nil } -// hotReloadValidatorsLean - spins off a goroutine that reads from validator files TODO: add load file with error +// hotReloadValidatorsLean - spins off a goroutine that reads from validator files func hotReloadValidatorsLean(c config, tmNode *node.Node) { - go func() { - for { - loadValidatorsLean(c, tmNode) - // init light node, but add removal code for validators that aren't part of it. - time.Sleep(time.Minute * 1) + userKeysPath := GlobalConfig.PocketConfig.GetLeanPocketUserKeyFilePath() + stat, err := os.Stat(userKeysPath) + if err != nil { + c.Logger.Error("Cannot find user provided key file to hot reload") + return + } + for { + time.Sleep(time.Second * 5) + c.Logger.Info("Checking for hot reload") + newStat, err := os.Stat(userKeysPath) + if err != nil { + continue } - }() + if newStat.Size() != stat.Size() || stat.ModTime() != newStat.ModTime() { + c.Logger.Info("Detected change in files, hot reloading validators") + err := ReloadValidatorKeys(c, tmNode) + if err != nil { + c.Logger.Error("Failed to hot reload validators") + continue + } + c.Logger.Info("Successfully hot reloaded validators") + stat = newStat + } + } } func NewClient(c config, creator AppCreator) (*node.Node, *PocketCoreApp, error) { @@ -97,8 +130,6 @@ func NewClient(c config, creator AppCreator) (*node.Node, *PocketCoreApp, error) c.Logger.With("module", "node"), ) - - if err != nil { return nil, nil, err } diff --git a/types/config.go b/types/config.go index 62aaf577..91d16853 100644 --- a/types/config.go +++ b/types/config.go @@ -3,6 +3,7 @@ package types import ( "github.com/tendermint/tendermint/config" db "github.com/tendermint/tm-db" + "path" "sync" "time" ) @@ -17,34 +18,39 @@ type SDKConfig struct { } type PocketConfig struct { - DataDir string `json:"data_dir"` - GenesisName string `json:"genesis_file"` - ChainsName string `json:"chains_name"` - EvidenceDBName string `json:"evidence_db_name"` - TendermintURI string `json:"tendermint_uri"` - KeybaseName string `json:"keybase_name"` - RPCPort string `json:"rpc_port"` - ClientBlockSyncAllowance int `json:"client_block_sync_allowance"` - MaxEvidenceCacheEntires int `json:"max_evidence_cache_entries"` - MaxSessionCacheEntries int `json:"max_session_cache_entries"` - JSONSortRelayResponses bool `json:"json_sort_relay_responses"` - RemoteCLIURL string `json:"remote_cli_url"` - UserAgent string `json:"user_agent"` - ValidatorCacheSize int64 `json:"validator_cache_size"` - ApplicationCacheSize int64 `json:"application_cache_size"` - RPCTimeout int64 `json:"rpc_timeout"` - PrometheusAddr string `json:"pocket_prometheus_port"` - PrometheusMaxOpenfiles int `json:"prometheus_max_open_files"` - MaxClaimAgeForProofRetry int `json:"max_claim_age_for_proof_retry"` - ProofPrevalidation bool `json:"proof_prevalidation"` - CtxCacheSize int `json:"ctx_cache_size"` - ABCILogging bool `json:"abci_logging"` - RelayErrors bool `json:"show_relay_errors"` - DisableTxEvents bool `json:"disable_tx_events"` - Cache bool `json:"-"` - IavlCacheSize int64 `json:"iavl_cache_size"` - ChainsHotReload bool `json:"chains_hot_reload"` - LeanPocket bool `json:"lean_pocket"` + DataDir string `json:"data_dir"` + GenesisName string `json:"genesis_file"` + ChainsName string `json:"chains_name"` + EvidenceDBName string `json:"evidence_db_name"` + TendermintURI string `json:"tendermint_uri"` + KeybaseName string `json:"keybase_name"` + RPCPort string `json:"rpc_port"` + ClientBlockSyncAllowance int `json:"client_block_sync_allowance"` + MaxEvidenceCacheEntires int `json:"max_evidence_cache_entries"` + MaxSessionCacheEntries int `json:"max_session_cache_entries"` + JSONSortRelayResponses bool `json:"json_sort_relay_responses"` + RemoteCLIURL string `json:"remote_cli_url"` + UserAgent string `json:"user_agent"` + ValidatorCacheSize int64 `json:"validator_cache_size"` + ApplicationCacheSize int64 `json:"application_cache_size"` + RPCTimeout int64 `json:"rpc_timeout"` + PrometheusAddr string `json:"pocket_prometheus_port"` + PrometheusMaxOpenfiles int `json:"prometheus_max_open_files"` + MaxClaimAgeForProofRetry int `json:"max_claim_age_for_proof_retry"` + ProofPrevalidation bool `json:"proof_prevalidation"` + CtxCacheSize int `json:"ctx_cache_size"` + ABCILogging bool `json:"abci_logging"` + RelayErrors bool `json:"show_relay_errors"` + DisableTxEvents bool `json:"disable_tx_events"` + Cache bool `json:"-"` + IavlCacheSize int64 `json:"iavl_cache_size"` + ChainsHotReload bool `json:"chains_hot_reload"` + LeanPocket bool `json:"lean_pocket"` + LeanPocketUserKeyFileName string `json:"lean_pocket_user_key_file"` +} + +func (c PocketConfig) GetLeanPocketUserKeyFilePath() string { + return path.Join(c.DataDir, c.LeanPocketUserKeyFileName) } type Config struct { @@ -102,6 +108,7 @@ const ( DefaultIavlCacheSize = 5000000 DefaultChainHotReload = false DefaultLeanPocket = false + DefaultLeanPocketUserKeyFileName = "lean_nodes_keys.json" ) func DefaultConfig(dataDir string) Config { @@ -109,7 +116,7 @@ func DefaultConfig(dataDir string) Config { TendermintConfig: *config.DefaultConfig(), PocketConfig: PocketConfig{ DataDir: dataDir, - GenesisName: DefaultGenesisName, + GenesisName: DefaultGenesisName, ChainsName: DefaultChainsName, EvidenceDBName: DefaultEvidenceDBName, TendermintURI: DefaultTMURI, @@ -135,6 +142,7 @@ func DefaultConfig(dataDir string) Config { IavlCacheSize: DefaultIavlCacheSize, ChainsHotReload: DefaultChainHotReload, LeanPocket: DefaultLeanPocket, + LeanPocketUserKeyFileName: DefaultLeanPocketUserKeyFileName, }, } c.TendermintConfig.LevelDBOptions = config.DefaultLevelDBOpts() diff --git a/x/pocketcore/keeper/service_test.go b/x/pocketcore/keeper/service_test.go index b665845d..73a51df9 100644 --- a/x/pocketcore/keeper/service_test.go +++ b/x/pocketcore/keeper/service_test.go @@ -2,6 +2,7 @@ package keeper import ( "encoding/hex" + "github.com/tendermint/tendermint/libs/log" "testing" sdk "github.com/pokt-network/pocket-core/types" @@ -115,7 +116,7 @@ func TestKeeper_HandleRelayLean(t *testing.T) { npk := kp.PublicKey nodePubKey := npk.RawString() - types.InitNodeWithCacheLean(pk) + types.InitNodeWithCacheLean(pk, log.NewNopLogger()) p := types.Payload{ Data: "{\"jsonrpc\":\"2.0\",\"method\":\"web3_clientVersion\",\"params\":[],\"id\":67}", diff --git a/x/pocketcore/types/cache_test.go b/x/pocketcore/types/cache_test.go index 50c41e66..5b792ec1 100644 --- a/x/pocketcore/types/cache_test.go +++ b/x/pocketcore/types/cache_test.go @@ -21,7 +21,7 @@ func InitCacheTest() { // init needed maps for cache servicerPk := GetRandomPrivateKey() - InitNodeWithCacheLean(servicerPk) + InitNodeWithCacheLean(servicerPk, logger) } diff --git a/x/pocketcore/types/lean_node.go b/x/pocketcore/types/lean_node.go index 63b7a932..dd89a95f 100644 --- a/x/pocketcore/types/lean_node.go +++ b/x/pocketcore/types/lean_node.go @@ -4,10 +4,12 @@ import ( "fmt" "github.com/pokt-network/pocket-core/crypto" sdk "github.com/pokt-network/pocket-core/types" + "github.com/tendermint/tendermint/libs/log" "sync" ) -var GlobalNodesLean map[string]*LeanNode +var GlobalNodesLean = map[string]*LeanNode{} +var GlobalNodeLeanRWLock = sync.RWMutex{} type LeanNode struct { PrivateKey crypto.PrivateKey @@ -16,26 +18,22 @@ type LeanNode struct { EvidenceSealedMap *sync.Map } -func InitNodeWithCacheLean(pk crypto.PrivateKey) { - - if GlobalNodesLean == nil { - GlobalNodesLean = make(map[string]*LeanNode) - } - - leanNode := LeanNode{PrivateKey: pk, EvidenceCache: new(CacheStorage), SessionCache: new(CacheStorage), EvidenceSealedMap: &sync.Map{}} +func InitNodeWithCacheLean(pk crypto.PrivateKey, logger log.Logger) { key := sdk.GetAddress(pk.PublicKey()).String() _, exists := GlobalNodesLean[key] if exists { - fmt.Println(key + " already added as a lean node") return } - + logger.Info("Adding " + key + " as lean node") + leanNode := LeanNode{PrivateKey: pk, EvidenceCache: new(CacheStorage), SessionCache: new(CacheStorage), EvidenceSealedMap: &sync.Map{}} leanNode.EvidenceCache.Init(GlobalPocketConfig.DataDir, GlobalPocketConfig.EvidenceDBName+"_"+key, GlobalTenderMintConfig.LevelDBOptions, GlobalPocketConfig.MaxEvidenceCacheEntires, false) leanNode.SessionCache.Init(GlobalPocketConfig.DataDir, "", GlobalTenderMintConfig.LevelDBOptions, GlobalPocketConfig.MaxSessionCacheEntries, true) GlobalNodesLean[key] = &leanNode } func GetNodeLean(address *sdk.Address) (*LeanNode, error) { + GlobalNodeLeanRWLock.RLock() + defer GlobalNodeLeanRWLock.RUnlock() node, ok := GlobalNodesLean[address.String()] if !ok { return nil, fmt.Errorf("failed to find private key for %s", address.String())