From 2717d47671a23b51534ac412a13165922c5489d1 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 13 Jun 2024 08:26:57 -0500 Subject: [PATCH 1/5] Track last receive time from websocket --- internal/metrics/metrics.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index ccb4dcf..ad3e7c2 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -59,6 +59,7 @@ type Metrics struct { metricsPort uint16 client *rpc.Client network *string + lastReceive time.Time // httpClient is another instance of the rpc.Client in HTTP mode // This is used rarely, to request data in response to a websocket event that is too large to fit on a single @@ -327,6 +328,8 @@ func (m *Metrics) OpenWebsocket() error { service.SetupPollingMetrics() } + m.lastReceive = time.Now() + return nil } @@ -354,6 +357,8 @@ func (m *Metrics) websocketReceive(resp *types.WebsocketResponse, err error) { return } + m.lastReceive = time.Now() + log.Printf("recv: %s %s\n", resp.Origin, resp.Command) log.Debugf("origin: %s command: %s destination: %s data: %s\n", resp.Origin, resp.Command, resp.Destination, string(resp.Data)) From 6fe0ccf62360f7af30c139f267c02912c8f09b5b Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 13 Jun 2024 08:32:58 -0500 Subject: [PATCH 2/5] Pull new client into its own function so we can recreate it if necessary --- internal/metrics/metrics.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index ad3e7c2..b0abe45 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -90,10 +90,7 @@ func NewMetrics(port uint16, logLevel log.Level) (*Metrics, error) { log.SetLevel(logLevel) - metrics.client, err = rpc.NewClient(rpc.ConnectionModeWebsocket, rpc.WithAutoConfig(), rpc.WithBaseURL(&url.URL{ - Scheme: "wss", - Host: viper.GetString("hostname"), - })) + err = metrics.setNewClient() if err != nil { return nil, err } @@ -138,6 +135,19 @@ func NewMetrics(port uint16, logLevel log.Level) (*Metrics, error) { return metrics, nil } +func (m *Metrics) setNewClient() error { + m.client = nil + client, err := rpc.NewClient(rpc.ConnectionModeWebsocket, rpc.WithAutoConfig(), rpc.WithBaseURL(&url.URL{ + Scheme: "wss", + Host: viper.GetString("hostname"), + })) + if err != nil { + return err + } + m.client = client + return nil +} + func (m *Metrics) createDBClient() error { var err error From 7c7420f89668f3c6308369614fe1c4fdbab6a597 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 13 Jun 2024 08:57:01 -0500 Subject: [PATCH 3/5] If we haven't received a websocket event in 5 minutes, recreate the websocket client --- internal/metrics/metrics.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index b0abe45..077ba1d 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -339,6 +339,32 @@ func (m *Metrics) OpenWebsocket() error { } m.lastReceive = time.Now() + go func() { + for { + // If we don't get any events for 5 minutes, we'll reset the connection + time.Sleep(10 * time.Second) + + if m.lastReceive.Before(time.Now().Add(-5*time.Minute)) { + log.Info("Websocket connection seems down. Recreating...") + m.disconnectHandler() + err := m.setNewClient() + if err != nil { + log.Errorf("Error creating new client: %s", err.Error()) + continue + } + + err = m.OpenWebsocket() + if err != nil { + log.Errorf("Error opening websocket on new client: %s", err.Error()) + continue + } + + // Got the new connection open, so stop the loop on the old connection + // since we called this function again and a new loop was created + break + } + } + }() return nil } From 1a534b4de7f513c3d0180ba5aae1277496bf139c Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 13 Jun 2024 08:57:39 -0500 Subject: [PATCH 4/5] fmt/lint/test/tidy --- go.mod | 2 +- go.sum | 4 ++-- internal/metrics/metrics.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 64d0a46..1cd943e 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.1 // indirect + github.com/gorilla/websocket v1.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/go.sum b/go.sum index eac8a63..937f224 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.2 h1:qoW6V1GT3aZxybsbC6oLnailWnB+qTMVwMreOso9XUw= +github.com/gorilla/websocket v1.5.2/go.mod h1:0n9H61RBAcf5/38py2MCYbxzPIY9rOkpvvMT24Rqs30= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 077ba1d..6406cd2 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -344,7 +344,7 @@ func (m *Metrics) OpenWebsocket() error { // If we don't get any events for 5 minutes, we'll reset the connection time.Sleep(10 * time.Second) - if m.lastReceive.Before(time.Now().Add(-5*time.Minute)) { + if m.lastReceive.Before(time.Now().Add(-5 * time.Minute)) { log.Info("Websocket connection seems down. Recreating...") m.disconnectHandler() err := m.setNewClient() From 60530cde1acbe06f848959dceae0177dc9a32e83 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 13 Jun 2024 09:55:35 -0500 Subject: [PATCH 5/5] Update dependencies --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 1cd943e..5807039 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/chia-network/chia-exporter go 1.21 require ( - github.com/chia-network/go-chia-libs v0.8.1 + github.com/chia-network/go-chia-libs v0.8.2 github.com/chia-network/go-modules v0.0.4 github.com/go-sql-driver/mysql v1.8.1 github.com/oschwald/maxminddb-golang v1.13.0 @@ -28,9 +28,9 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.53.0 // indirect - github.com/prometheus/procfs v0.15.0 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/prometheus/common v0.54.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/samber/mo v1.11.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -39,11 +39,11 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 937f224..a10b3a3 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chia-network/go-chia-libs v0.8.1 h1:yj9HHjrDPwLzVUABAvi2hm7wPhB+By5qfaNRfsSbMlg= -github.com/chia-network/go-chia-libs v0.8.1/go.mod h1:XY6eMATROQBVP94RppwvvdiPJ0l3QoSZYdntUBSXlLE= +github.com/chia-network/go-chia-libs v0.8.2 h1:ExVs0LW/BZQNGCrDc1n7ixPmyka9MRNaLRWX/MNQ6MQ= +github.com/chia-network/go-chia-libs v0.8.2/go.mod h1:qICIVdfTQg15wXb9soXMvZbXB2tYop7udSv+aZGBbwc= github.com/chia-network/go-modules v0.0.4 h1:XlCcuT4j1krLvsFT1Y49Un5xORwcTc8jjE4SHih7OTI= github.com/chia-network/go-modules v0.0.4/go.mod h1:JP8mG/9ieE76VcGIbzD5G3/4YDmvNhRryiQwp8GQr1U= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -53,15 +53,15 @@ github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQ github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= -github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= -github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= +github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= +github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/samber/mo v1.11.0 h1:ZOiSkrGGpNhVv/1dxP02risztdMTIwE8KSW9OG4k5bY= @@ -94,18 +94,18 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=