From 5eaba03db11e0b1f5331f21a1a5b5d4670b0f051 Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 14 Apr 2023 00:19:09 +0800 Subject: [PATCH 1/8] atomic boolean in monitor --- monitor/monitor.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/monitor/monitor.go b/monitor/monitor.go index caef23f5..50321f44 100644 --- a/monitor/monitor.go +++ b/monitor/monitor.go @@ -58,7 +58,7 @@ type Monitor struct { //dl *backend.TorrentManager exitCh chan any - terminated atomic.Int32 + terminated atomic.Bool lastNumber uint64 startNumber uint64 scope uint64 @@ -117,7 +117,7 @@ func New(flag *params.Config, cache, compress, listen bool, fs *backend.ChainDB, //start: mclock.Now(), } m.currentNumber.Store(0) - m.terminated.Store(0) + m.terminated.Store(false) m.blockCache, _ = lru.New(delay) m.sizeCache, _ = lru.New(batch) m.listen = listen @@ -287,7 +287,7 @@ func (m *Monitor) buildConnection(ipcpath string, rpcuri string) (*rpc.Client, e return cl, nil } - if m.terminated.Load() == 1 { + if m.terminated.Load() { log.Info("Connection builder break") return nil, errors.New("ipc connection terminated") } @@ -495,10 +495,10 @@ func (m *Monitor) Stop() { m.lock.Lock() defer m.lock.Unlock() //m.closeOnce.Do(func() { - if m.terminated.Load() == 1 { + if m.terminated.Load() { return } - m.terminated.Store(1) + m.terminated.Store(true) m.exit() log.Info("Monitor is waiting to be closed") @@ -715,7 +715,7 @@ func (m *Monitor) syncLastBlock() uint64 { } //start := mclock.Now() for i := minNumber; i <= maxNumber; { // i++ { - if m.terminated.Load() == 1 { + if m.terminated.Load() { log.Warn("Fs scan terminated", "number", i) maxNumber = i - 1 break From 8562c8d3c064e31b5871c00b24ed8d8e5ed94403 Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 14 Apr 2023 01:12:33 +0800 Subject: [PATCH 2/8] peer channel quit --- peer.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/peer.go b/peer.go index 152ddb41..46946c49 100644 --- a/peer.go +++ b/peer.go @@ -220,14 +220,20 @@ func (peer *Peer) handshake() error { Files: uint64(peer.host.Congress()), Leafs: uint64(len(peer.host.chain().Blocks())), } - errc <- p2p.SendItems(peer.ws, params.StatusCode, params.ProtocolVersion, &info) + select { + case errc <- p2p.SendItems(peer.ws, params.StatusCode, params.ProtocolVersion, &info): + case <-peer.quit: + } log.Debug("Nas send items OK", "status", params.StatusCode, "version", params.ProtocolVersion, "len", len(errc)) }() // Fetch the remote status packet and verify protocol match peer.wg.Add(1) go func() { defer peer.wg.Done() - errc <- peer.readStatus() + select { + case errc <- peer.readStatus(): + case <-peer.quit: + } }() timeout := time.NewTimer(params.HandshakeTimeout) @@ -241,6 +247,8 @@ func (peer *Peer) handshake() error { case <-timeout.C: log.Info("Handshake timeout") return fmt.Errorf("peer [%x] timeout", peer.ID()) + case <-peer.quit: + return nil } } From 5e8947e1eacdf2c5fd6a29f0a514f536a8e37651 Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 14 Apr 2023 02:20:50 +0800 Subject: [PATCH 3/8] tracker healthy check --- wormhole/wormhole.go | 105 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 22 deletions(-) diff --git a/wormhole/wormhole.go b/wormhole/wormhole.go index 4406b102..361571d7 100644 --- a/wormhole/wormhole.go +++ b/wormhole/wormhole.go @@ -17,10 +17,14 @@ package wormhole import ( + "errors" + "github.com/CortexFoundation/CortexTheseus/common" + "github.com/CortexFoundation/CortexTheseus/common/mclock" "github.com/CortexFoundation/CortexTheseus/log" "github.com/CortexFoundation/torrentfs/params" "net" "net/url" + "time" //"sync" resty "github.com/go-resty/resty/v2" @@ -28,7 +32,6 @@ import ( mapset "github.com/deckarep/golang-set/v2" "strings" - "time" ) var ( @@ -54,36 +57,61 @@ func BestTrackers() (ret []string) { log.Warn("Global tracker lost", "err", err) continue } + client.SetTimeout(time.Second * 3) + retCh := make(chan string) + //var wg sync.WaitGroup str := strings.Split(resp.String(), "\n\n") + start := mclock.Now() for _, s := range str { - if len(ret) < CAP { - log.Debug("Global best trackers", "url", s) - switch { - case strings.HasPrefix(s, "http"), strings.HasPrefix(s, "https"): - if _, err := client.R().Post(s); err != nil { - log.Warn("tracker failed", "err", err) - } else { - ret = append(ret, s) - } - case strings.HasPrefix(s, "udp"): - if u, err := url.Parse(s); err == nil { - if host, port, err := net.SplitHostPort(u.Host); err == nil { - if err := ping(host, port); err == nil { - ret = append(ret, s) - } else { - log.Warn("UDP ping err", "s", s, "err", err) - } + //if len(ret) < CAP { + // wg.Add(1) + go func(ss string) { + // defer wg.Done() + if err := HealthCheck(ss); err == nil { + //ret = append(ret, s) + retCh <- ss + } else { + retCh <- "" + } + }(s) + /*switch { + case strings.HasPrefix(s, "http"), strings.HasPrefix(s, "https"): + if _, err := client.R().Post(s); err != nil { + log.Warn("tracker failed", "err", err) + } else { + ret = append(ret, s) + } + case strings.HasPrefix(s, "udp"): + if u, err := url.Parse(s); err == nil { + if host, port, err := net.SplitHostPort(u.Host); err == nil { + if err := ping(host, port); err == nil { + ret = append(ret, s) + } else { + log.Warn("UDP ping err", "s", s, "err", err) } } - default: - log.Warn("Other protocols trackers", "s", s) } - } else { - break + default: + log.Warn("Other protocols trackers", "s", s) + }*/ + //} else { + // break + //} + } + + for i := 0; i < len(str); i++ { + select { + case x := <-retCh: + if len(x) > 0 { + log.Info("Healthy tracker", "url", x, "latency", common.PrettyDuration(time.Duration(mclock.Now())-time.Duration(start))) + ret = append(ret, x) + } } } + //wg.Wait() + if len(ret) > 0 { return } @@ -92,6 +120,39 @@ func BestTrackers() (ret []string) { return } +func HealthCheck(s string) error { + log.Debug("Global best trackers", "url", s) + switch { + case strings.HasPrefix(s, "http"), strings.HasPrefix(s, "https"): + if _, err := client.R().Post(s); err != nil { + log.Warn("tracker failed", "err", err) + return err + } else { + //ret = append(ret, s) + return nil + } + case strings.HasPrefix(s, "udp"): + if u, err := url.Parse(s); err == nil { + if host, port, err := net.SplitHostPort(u.Host); err == nil { + if err := ping(host, port); err == nil { + //ret = append(ret, s) + return nil + } else { + log.Warn("UDP ping err", "s", s, "err", err) + return err + } + } + } else { + return err + } + default: + log.Warn("Other protocols trackers", "s", s) + return errors.New("invalid url protocol") + } + + return errors.New("unhealthy url") +} + func ColaList() mapset.Set[string] { m := mapset.NewSet[string]() for _, url := range params.ColaUrl { From 0c9eb6657ca94db9d97c0634f48af4ceae2813eb Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 14 Apr 2023 02:28:22 +0800 Subject: [PATCH 4/8] tracker timeout tolerate --- wormhole/wormhole.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wormhole/wormhole.go b/wormhole/wormhole.go index 361571d7..0a9eba66 100644 --- a/wormhole/wormhole.go +++ b/wormhole/wormhole.go @@ -57,7 +57,7 @@ func BestTrackers() (ret []string) { log.Warn("Global tracker lost", "err", err) continue } - client.SetTimeout(time.Second * 3) + client.SetTimeout(time.Second * 2) retCh := make(chan string) //var wg sync.WaitGroup From adca668baf292443c6a6ce8f11a2b87c56861918 Mon Sep 17 00:00:00 2001 From: ucwong Date: Sat, 15 Apr 2023 00:58:27 +0800 Subject: [PATCH 5/8] reset worm timeout --- wormhole/wormhole.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/wormhole/wormhole.go b/wormhole/wormhole.go index 0a9eba66..2ecbf51f 100644 --- a/wormhole/wormhole.go +++ b/wormhole/wormhole.go @@ -50,19 +50,23 @@ func Tunnel(hash string) error { } func BestTrackers() (ret []string) { + defer client.SetTimeout(time.Second * 10) + for _, ur := range params.BestTrackerUrl { resp, err := client.R().Get(ur) - if err != nil { + if err != nil || resp == nil || len(resp.String()) == 0 { log.Warn("Global tracker lost", "err", err) continue } client.SetTimeout(time.Second * 2) - retCh := make(chan string) //var wg sync.WaitGroup - str := strings.Split(resp.String(), "\n\n") - start := mclock.Now() + var ( + str = strings.Split(resp.String(), "\n\n") + retCh = make(chan string, len(str)) + start = mclock.Now() + ) for _, s := range str { //if len(ret) < CAP { // wg.Add(1) From 9f14d925d630cfff9dd76bc8a962299ae1db5b65 Mon Sep 17 00:00:00 2001 From: ucwong Date: Sat, 15 Apr 2023 15:02:14 +0800 Subject: [PATCH 6/8] close dht default --- params/default.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/default.go b/params/default.go index e4baae8a..95be0a60 100644 --- a/params/default.go +++ b/params/default.go @@ -51,7 +51,7 @@ var DefaultConfig = Config{ BoostNodes: TorrentBoostNodes, Mode: "default", DisableUTP: true, - DisableDHT: false, + DisableDHT: true, DisableTCP: false, DisableIPv6: false, MaxSeedingNum: LimitSeeding / 2, From 51867955ee80106d12b1098c82499238419e5379 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 20 Apr 2023 22:06:21 +0800 Subject: [PATCH 7/8] cola09 --- go.mod | 4 ++-- go.sum | 8 ++++---- params/version.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 9089a9a5..954dfd41 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/ucwong/filecache v1.0.5-0.20230402204106-c6ff7d7930ef github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb - github.com/ucwong/golang-kv v1.0.16-0.20230412200348-60af39b46284 + github.com/ucwong/golang-kv v1.0.16-0.20230420022227-55629d4173fc github.com/ucwong/shard v0.0.0-20230406003402-a003024d1f4f github.com/urfave/cli/v2 v2.25.1 go.etcd.io/bbolt v1.3.7 @@ -61,7 +61,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble v0.0.0-20230412195903-4e468412c552 // indirect + github.com/cockroachdb/pebble v0.0.0-20230420011906-6002e39ce756 // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect diff --git a/go.sum b/go.sum index ee64ffac..02cce40f 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,8 @@ github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZO github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230412195903-4e468412c552 h1:n4wDCFjAFEp9aWLJPKVfviVlFrx8GWYjO7UTeT0IJpQ= -github.com/cockroachdb/pebble v0.0.0-20230412195903-4e468412c552/go.mod h1:9lRMC4XN3/BLPtIp6kAKwIaHu369NOf2rMucPzipz50= +github.com/cockroachdb/pebble v0.0.0-20230420011906-6002e39ce756 h1:NgUWWgwtcPD4JjclHKE6GtJGJmZm0mWHfGjomCA4CsY= +github.com/cockroachdb/pebble v0.0.0-20230420011906-6002e39ce756/go.mod h1:9lRMC4XN3/BLPtIp6kAKwIaHu369NOf2rMucPzipz50= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -575,8 +575,8 @@ github.com/ucwong/filecache v1.0.5-0.20230402204106-c6ff7d7930ef h1:bbArCHS5UrTT github.com/ucwong/filecache v1.0.5-0.20230402204106-c6ff7d7930ef/go.mod h1:ddwX+NCjMZPdpzcGh1fcEbNTUTCtKgt2hC2rqvmLKgA= github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb h1:dVZH3AH9f7zB3VBmsjn25B7lfcAyMP4QxdFYTrfj7tg= github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb/go.mod h1:3yswsBsVuwsOjDvFfC5Na9XSEf4HC7mj3W3g6jvSY/s= -github.com/ucwong/golang-kv v1.0.16-0.20230412200348-60af39b46284 h1:GbObPT3x5fgGJo8owcQmJTT6Ir5G/K3AkYXzbqmmeaM= -github.com/ucwong/golang-kv v1.0.16-0.20230412200348-60af39b46284/go.mod h1:k6qOzS8yK+ZSVuN560N9em5kf1wqvr/b961FE8E+Kd4= +github.com/ucwong/golang-kv v1.0.16-0.20230420022227-55629d4173fc h1:WO23vpzFOYv6M2iDC+hreH8PMpgs1CdEToQWDfoL1fs= +github.com/ucwong/golang-kv v1.0.16-0.20230420022227-55629d4173fc/go.mod h1:30cMOyY29NBwtYV2oblXs/5YwA5QpLi3xcoWZZ5mWtw= github.com/ucwong/shard v0.0.0-20230406003402-a003024d1f4f h1:q9Y7MB/zxaHiR3Bxp8/qp9iRju/2cCFIE2RM1o79NWo= github.com/ucwong/shard v0.0.0-20230406003402-a003024d1f4f/go.mod h1:Y/x75znJgnXLUzsTENLnGpiUPVEeizYyg6qwpx9M3T0= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/params/version.go b/params/version.go index 0afadb2c..01f5f1cc 100644 --- a/params/version.go +++ b/params/version.go @@ -17,5 +17,5 @@ package params const ( - ClientVersion = "-COLA08-" + ClientVersion = "-COLA09-" ) From 064b741940a36a498658da9d978f38c15c978b5d Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 20 Apr 2023 22:10:32 +0800 Subject: [PATCH 8/8] cola09-1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 954dfd41..bdc5a6b6 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/ucwong/filecache v1.0.5-0.20230402204106-c6ff7d7930ef github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb - github.com/ucwong/golang-kv v1.0.16-0.20230420022227-55629d4173fc + github.com/ucwong/golang-kv v1.0.16-0.20230420140948-f85ce47d4928 github.com/ucwong/shard v0.0.0-20230406003402-a003024d1f4f github.com/urfave/cli/v2 v2.25.1 go.etcd.io/bbolt v1.3.7 diff --git a/go.sum b/go.sum index 02cce40f..3b385625 100644 --- a/go.sum +++ b/go.sum @@ -575,8 +575,8 @@ github.com/ucwong/filecache v1.0.5-0.20230402204106-c6ff7d7930ef h1:bbArCHS5UrTT github.com/ucwong/filecache v1.0.5-0.20230402204106-c6ff7d7930ef/go.mod h1:ddwX+NCjMZPdpzcGh1fcEbNTUTCtKgt2hC2rqvmLKgA= github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb h1:dVZH3AH9f7zB3VBmsjn25B7lfcAyMP4QxdFYTrfj7tg= github.com/ucwong/go-ttlmap v1.0.2-0.20221020173635-331e7ddde2bb/go.mod h1:3yswsBsVuwsOjDvFfC5Na9XSEf4HC7mj3W3g6jvSY/s= -github.com/ucwong/golang-kv v1.0.16-0.20230420022227-55629d4173fc h1:WO23vpzFOYv6M2iDC+hreH8PMpgs1CdEToQWDfoL1fs= -github.com/ucwong/golang-kv v1.0.16-0.20230420022227-55629d4173fc/go.mod h1:30cMOyY29NBwtYV2oblXs/5YwA5QpLi3xcoWZZ5mWtw= +github.com/ucwong/golang-kv v1.0.16-0.20230420140948-f85ce47d4928 h1:CbZ1HSx9YG6dHRbMCWS30d9evLzTWK2/hm3fZTNaU3I= +github.com/ucwong/golang-kv v1.0.16-0.20230420140948-f85ce47d4928/go.mod h1:pFPRDx8PPYFli7tBrpt5+ltAzRbWccwUnba5y5wrn/M= github.com/ucwong/shard v0.0.0-20230406003402-a003024d1f4f h1:q9Y7MB/zxaHiR3Bxp8/qp9iRju/2cCFIE2RM1o79NWo= github.com/ucwong/shard v0.0.0-20230406003402-a003024d1f4f/go.mod h1:Y/x75znJgnXLUzsTENLnGpiUPVEeizYyg6qwpx9M3T0= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=