From 1b82d33433a6040c22a9d6432aec9551dbd4b300 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 31 May 2024 17:03:56 -0400 Subject: [PATCH 001/182] Create a new config format so we can expand listener configuration for proxy protocol. --- cmd/outline-ss-server/config.go | 86 ++++++++++++++++ .../config_example.deprecated.yml | 15 +++ cmd/outline-ss-server/config_example.yml | 36 ++++--- cmd/outline-ss-server/config_test.go | 79 +++++++++++++++ cmd/outline-ss-server/main.go | 98 ++++++++++++------- net/address.go | 63 ++++++++++++ net/address_test.go | 82 ++++++++++++++++ 7 files changed, 408 insertions(+), 51 deletions(-) create mode 100644 cmd/outline-ss-server/config.go create mode 100644 cmd/outline-ss-server/config_example.deprecated.yml create mode 100644 cmd/outline-ss-server/config_test.go create mode 100644 net/address.go create mode 100644 net/address_test.go diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go new file mode 100644 index 00000000..1f97e182 --- /dev/null +++ b/cmd/outline-ss-server/config.go @@ -0,0 +1,86 @@ +// Copyright 2024 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v2" +) + +type Service struct { + Listeners []Listener + Keys []Key +} + +type Listener struct { + Type string + Address string +} + +type Key struct { + ID string + Cipher string + Secret string +} + +type Config struct { + Services []Service + + // Deprecated: Keys exists for historical compatibility. This is ignored if top-level `services` is specified. + Keys []struct { + ID string + Port int + Cipher string + Secret string + } +} + +// Reads a config from a filename and parses it as a [Config]. +func ReadConfig(filename string) (*Config, error) { + config := Config{} + configData, err := os.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to read config: %w", err) + } + err = yaml.Unmarshal(configData, &config) + if err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + if config.Services == nil { + // This is a deprecated config format. We need to transform it to to the new format. + ports := make(map[int][]Key) + for _, keyConfig := range config.Keys { + ports[keyConfig.Port] = append(ports[keyConfig.Port], Key{ + ID: keyConfig.ID, + Cipher: keyConfig.Cipher, + Secret: keyConfig.Secret, + }) + } + for port, keys := range ports { + s := Service{ + Listeners: []Listener{ + Listener{Type: "direct", Address: fmt.Sprintf("tcp://[::]:%d", port)}, + Listener{Type: "direct", Address: fmt.Sprintf("udp://[::]:%d", port)}, + }, + Keys: keys, + } + config.Services = append(config.Services, s) + } + } + config.Keys = nil + return &config, nil +} diff --git a/cmd/outline-ss-server/config_example.deprecated.yml b/cmd/outline-ss-server/config_example.deprecated.yml new file mode 100644 index 00000000..8895b86d --- /dev/null +++ b/cmd/outline-ss-server/config_example.deprecated.yml @@ -0,0 +1,15 @@ +keys: + - id: user-0 + port: 9000 + cipher: chacha20-ietf-poly1305 + secret: Secret0 + + - id: user-1 + port: 9000 + cipher: chacha20-ietf-poly1305 + secret: Secret1 + + - id: user-2 + port: 9001 + cipher: chacha20-ietf-poly1305 + secret: Secret2 diff --git a/cmd/outline-ss-server/config_example.yml b/cmd/outline-ss-server/config_example.yml index 8895b86d..66009c10 100644 --- a/cmd/outline-ss-server/config_example.yml +++ b/cmd/outline-ss-server/config_example.yml @@ -1,15 +1,23 @@ -keys: - - id: user-0 - port: 9000 - cipher: chacha20-ietf-poly1305 - secret: Secret0 +services: + - listeners: + - type: direct + address: "tcp://[::]:9000" + - type: direct + address: "udp://[::]:9000" + keys: + - id: user-0 + cipher: chacha20-ietf-poly1305 + secret: Secret0 + - id: user-1 + cipher: chacha20-ietf-poly1305 + secret: Secret1 - - id: user-1 - port: 9000 - cipher: chacha20-ietf-poly1305 - secret: Secret1 - - - id: user-2 - port: 9001 - cipher: chacha20-ietf-poly1305 - secret: Secret2 + - listeners: + - type: direct + address: "tcp://[::]:9001" + - type: direct + address: "udp://[::]:9001" + keys: + - id: user-2 + cipher: chacha20-ietf-poly1305 + secret: Secret2 diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go new file mode 100644 index 00000000..0d46489c --- /dev/null +++ b/cmd/outline-ss-server/config_test.go @@ -0,0 +1,79 @@ +// Copyright 2024 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadConfig(t *testing.T) { + config, _ := ReadConfig("./config_example.yml") + + expected := Config{ + Services: []Service{ + Service{ + Listeners: []Listener{ + Listener{Type: "direct", Address: "tcp://[::]:9000"}, + Listener{Type: "direct", Address: "udp://[::]:9000"}, + }, + Keys: []Key{ + Key{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + Key{"user-1", "chacha20-ietf-poly1305", "Secret1"}, + }, + }, + Service{ + Listeners: []Listener{ + Listener{Type: "direct", Address: "tcp://[::]:9001"}, + Listener{Type: "direct", Address: "udp://[::]:9001"}, + }, + Keys: []Key{ + Key{"user-2", "chacha20-ietf-poly1305", "Secret2"}, + }, + }, + }, + } + require.Equal(t, expected, *config) +} + +func TestReadConfigParsesDeprecatedFormat(t *testing.T) { + config, _ := ReadConfig("./config_example.deprecated.yml") + + expected := Config{ + Services: []Service{ + Service{ + Listeners: []Listener{ + Listener{Type: "direct", Address: "tcp://[::]:9000"}, + Listener{Type: "direct", Address: "udp://[::]:9000"}, + }, + Keys: []Key{ + Key{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + Key{"user-1", "chacha20-ietf-poly1305", "Secret1"}, + }, + }, + Service{ + Listeners: []Listener{ + Listener{Type: "direct", Address: "tcp://[::]:9001"}, + Listener{Type: "direct", Address: "udp://[::]:9001"}, + }, + Keys: []Key{ + Key{"user-2", "chacha20-ietf-poly1305", "Secret2"}, + }, + }, + }, + } + require.Equal(t, expected, *config) +} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 47b686a9..165e5217 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -28,13 +28,14 @@ import ( "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" + "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + onet "github.com/Jigsaw-Code/outline-ss-server/net" "github.com/Jigsaw-Code/outline-ss-server/service" "github.com/op/go-logging" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/term" - "gopkg.in/yaml.v2" ) var logger *logging.Logger @@ -48,6 +49,8 @@ const tcpReadTimeout time.Duration = 59 * time.Second // A UDP NAT timeout of at least 5 minutes is recommended in RFC 4787 Section 4.3. const defaultNatTimeout time.Duration = 5 * time.Minute +var directListenerType = "direct" + func init() { var prefix = "%{level:.1s}%{time:2006-01-02T15:04:05.000Z07:00} %{pid} %{shortfile}]" if term.IsTerminal(int(os.Stderr.Fd())) { @@ -125,26 +128,67 @@ func (s *SSServer) removePort(portNum int) error { } func (s *SSServer) loadConfig(filename string) error { - config, err := readConfig(filename) + config, err := ReadConfig(filename) if err != nil { return fmt.Errorf("failed to load config (%v): %w", filename, err) } portChanges := make(map[int]int) portCiphers := make(map[int]*list.List) // Values are *List of *CipherEntry. - for _, keyConfig := range config.Keys { - portChanges[keyConfig.Port] = 1 - cipherList, ok := portCiphers[keyConfig.Port] - if !ok { - cipherList = list.New() - portCiphers[keyConfig.Port] = cipherList + for _, serviceConfig := range config.Services { + if serviceConfig.Listeners == nil || serviceConfig.Keys == nil { + return fmt.Errorf("must specify at least 1 listener and 1 key per service") + } + addrs := []net.Addr{} + for _, listener := range serviceConfig.Listeners { + switch t := listener.Type; t { + // TODO: Support more listener types. + case directListenerType: + addr, err := onet.ResolveAddr(listener.Address) + if err != nil { + return fmt.Errorf("failed to resolve direct address: %v: %w", listener.Address, err) + } + addrs = append(addrs, addr) + port, err := onet.GetPort(addr) + if err != nil { + return err + } + portChanges[int(port)] = 1 + default: + return fmt.Errorf("unsupported listener type: %s", t) + } } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) - if err != nil { - return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + + type key struct { + c string + s string + } + existingCipher := make(map[key]bool) + for _, keyConfig := range serviceConfig.Keys { + for _, addr := range addrs { + port, err := onet.GetPort(addr) + if err != nil { + return err + } + cipherList, ok := portCiphers[port] + if !ok { + cipherList = list.New() + portCiphers[port] = cipherList + } + _, ok = existingCipher[key{keyConfig.Cipher, keyConfig.Secret}] + if ok { + logger.Debugf("encryption key already exists for port=%v, ID=`%v`. Skipping.", port, keyConfig.ID) + continue + } + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + } + entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + cipherList.PushBack(&entry) + existingCipher[key{keyConfig.Cipher, keyConfig.Secret}] = true + } } - entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - cipherList.PushBack(&entry) } for port := range s.ports { portChanges[port] = portChanges[port] - 1 @@ -160,11 +204,13 @@ func (s *SSServer) loadConfig(filename string) error { } } } + numServices := 0 for portNum, cipherList := range portCiphers { s.ports[portNum].cipherList.Update(cipherList) + numServices += cipherList.Len() } - logger.Infof("Loaded %v access keys over %v ports", len(config.Keys), len(s.ports)) - s.m.SetNumAccessKeys(len(config.Keys), len(portCiphers)) + logger.Infof("Loaded %v access keys over %v ports", numServices, len(s.ports)) + s.m.SetNumAccessKeys(numServices, len(s.ports)) return nil } @@ -203,28 +249,6 @@ func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, return server, nil } -type Config struct { - Keys []struct { - ID string - Port int - Cipher string - Secret string - } -} - -func readConfig(filename string) (*Config, error) { - config := Config{} - configData, err := os.ReadFile(filename) - if err != nil { - return nil, fmt.Errorf("failed to read config: %w", err) - } - err = yaml.Unmarshal(configData, &config) - if err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - return &config, nil -} - func main() { var flags struct { ConfigFile string diff --git a/net/address.go b/net/address.go new file mode 100644 index 00000000..a74e4f13 --- /dev/null +++ b/net/address.go @@ -0,0 +1,63 @@ +// Copyright 2024 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net + +import ( + "fmt" + "net" + "net/url" +) + +// Resolves a URL-style listen address specification as a [net.Addr] +// +// Examples: +// +// udp6://127.0.0.1:8000 +// unix:///tmp/foo.sock +// tcp://127.0.0.1:9002 +func ResolveAddr(addr string) (net.Addr, error) { + u, err := url.Parse(addr) + if err != nil { + return nil, err + } + switch u.Scheme { + case "tcp", "tcp4", "tcp6": + return net.ResolveTCPAddr(u.Scheme, u.Host) + case "udp", "udp4", "udp6": + return net.ResolveUDPAddr(u.Scheme, u.Host) + case "unix", "unixgram", "unixpacket": + var path string + if u.Opaque != "" { + path = u.Opaque + } else { + path = u.Path + } + return net.ResolveUnixAddr(u.Scheme, path) + default: + return nil, net.UnknownNetworkError(u.Scheme) + } +} + +// Returns the port from a given address. +func GetPort(addr net.Addr) (port int, err error) { + switch t := addr.(type) { + case *net.TCPAddr: + return t.Port, nil + case *net.UDPAddr: + return t.Port, nil + default: + return -1, fmt.Errorf("failed to get port from address: %v", addr) + } +} diff --git a/net/address_test.go b/net/address_test.go new file mode 100644 index 00000000..f681f056 --- /dev/null +++ b/net/address_test.go @@ -0,0 +1,82 @@ +// Copyright 2024 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net + +import ( + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +type fakeAddr string + +func (a fakeAddr) String() string { return string(a) } +func (a fakeAddr) Network() string { return "" } + +func TestResolveAddrReturnsTCPAddr(t *testing.T) { + addr, err := ResolveAddr("tcp://0.0.0.0:9000") + + require.NoError(t, err) + if _, ok := addr.(*net.TCPAddr); !ok { + t.Errorf("expected a *net.TCPAddr; it is a %T", addr) + } +} + +func TestResolveAddrReturnsUDPAddr(t *testing.T) { + addr, err := ResolveAddr("udp://[::]:9001") + + require.NoError(t, err) + if _, ok := addr.(*net.UDPAddr); !ok { + t.Errorf("expected a *net.UDPAddr; it is a %T", addr) + } +} + +func TestResolveAddrReturnsUnixAddr(t *testing.T) { + addr, err := ResolveAddr("unix:///path/to/stream_socket") + + require.NoError(t, err) + if _, ok := addr.(*net.UnixAddr); !ok { + t.Errorf("expected a *net.UnixAddr; it is a %T", addr) + } +} + +func TestResolveAddrReturnsErrorForUnknownScheme(t *testing.T) { + addr, err := ResolveAddr("foobar") + + require.Nil(t, addr) + require.Error(t, err) +} + +func TestGetPortFromTCPAddr(t *testing.T) { + port, err := GetPort(&net.TCPAddr{IP: net.ParseIP("1.2.3.4"), Port: 1234}) + + require.NoError(t, err) + require.Equal(t, 1234, port) +} + +func TestGetPortFromUDPPAddr(t *testing.T) { + port, err := GetPort(&net.UDPAddr{IP: net.ParseIP("1.2.3.4"), Port: 5678}) + + require.NoError(t, err) + require.Equal(t, 5678, port) +} + +func TestGetPortReturnsErrorForUnsupportedAddressType(t *testing.T) { + port, err := GetPort(&net.UnixAddr{Name: "/path/to/foo", Net: "unix"}) + + require.Equal(t, -1, port) + require.Error(t, err) +} From a4c200718dad9cdf6e62161789584e0293c14035 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 31 May 2024 17:04:48 -0400 Subject: [PATCH 002/182] Remove unused `fakeAddr`. --- net/address_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/net/address_test.go b/net/address_test.go index f681f056..d7fe1d01 100644 --- a/net/address_test.go +++ b/net/address_test.go @@ -21,11 +21,6 @@ import ( "github.com/stretchr/testify/require" ) -type fakeAddr string - -func (a fakeAddr) String() string { return string(a) } -func (a fakeAddr) Network() string { return "" } - func TestResolveAddrReturnsTCPAddr(t *testing.T) { addr, err := ResolveAddr("tcp://0.0.0.0:9000") From 72b27d7327bf50af84d86e03da78711623e78e4f Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 3 Jun 2024 12:33:15 -0400 Subject: [PATCH 003/182] Split `startPort` up between TCP and UDP. --- cmd/outline-ss-server/main.go | 39 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 165e5217..f5b247da 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -75,25 +75,16 @@ type SSServer struct { ports map[int]*ssPort } -func (s *SSServer) startPort(portNum int) error { +func (s *SSServer) startTCP(portNum int, cipherList service.CipherList) (*net.TCPListener, error) { listener, err := net.ListenTCP("tcp", &net.TCPAddr{Port: portNum}) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks TCP service failed to start on port %v: %w", portNum, err) + return nil, fmt.Errorf("Shadowsocks TCP service failed to start on port %v: %w", portNum, err) } logger.Infof("Shadowsocks TCP service listening on %v", listener.Addr().String()) - packetConn, err := net.ListenUDP("udp", &net.UDPAddr{Port: portNum}) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks UDP service failed to start on port %v: %w", portNum, err) - } - logger.Infof("Shadowsocks UDP service listening on %v", packetConn.LocalAddr().String()) - port := &ssPort{tcpListener: listener, packetConn: packetConn, cipherList: service.NewCipherList()} - authFunc := service.NewShadowsocksStreamAuthenticator(port.cipherList, &s.replayCache, s.m) + authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &s.replayCache, s.m) // TODO: Register initial data metrics at zero. tcpHandler := service.NewTCPHandler(portNum, authFunc, s.m, tcpReadTimeout) - packetHandler := service.NewPacketHandler(s.natTimeout, port.cipherList, s.m) - s.ports[portNum] = port accept := func() (transport.StreamConn, error) { conn, err := listener.AcceptTCP() if err == nil { @@ -102,8 +93,19 @@ func (s *SSServer) startPort(portNum int) error { return conn, err } go service.StreamServe(accept, tcpHandler.Handle) - go packetHandler.Handle(port.packetConn) - return nil + return listener, nil +} + +func (s *SSServer) startUDP(portNum int, cipherList service.CipherList) (*net.UDPConn, error) { + packetConn, err := net.ListenUDP("udp", &net.UDPAddr{Port: portNum}) + if err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return nil, fmt.Errorf("Shadowsocks UDP service failed to start on port %v: %w", portNum, err) + } + logger.Infof("Shadowsocks UDP service listening on %v", packetConn.LocalAddr().String()) + packetHandler := service.NewPacketHandler(s.natTimeout, cipherList, s.m) + go packetHandler.Handle(packetConn) + return packetConn, nil } func (s *SSServer) removePort(portNum int) error { @@ -199,9 +201,16 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("failed to remove port %v: %w", portNum, err) } } else if count == +1 { - if err := s.startPort(portNum); err != nil { + cipherList := service.NewCipherList() + tcpListener, err := s.startTCP(portNum, cipherList) + if err != nil { + return err + } + packetConn, err := s.startUDP(portNum, cipherList) + if err != nil { return err } + s.ports[portNum] = &ssPort{tcpListener, packetConn, cipherList} } } numServices := 0 From fddfc5720b8c5becfbdeaef0dd14fb7d8a9575a8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 3 Jun 2024 17:29:31 -0400 Subject: [PATCH 004/182] Use listeners to configure TCP and/or UDP services as needed. --- cmd/outline-ss-server/main.go | 169 +++++++++++++++++----------------- 1 file changed, 83 insertions(+), 86 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index f5b247da..206ff05d 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -18,6 +18,7 @@ import ( "container/list" "flag" "fmt" + "io" "net" "net/http" "os" @@ -62,29 +63,28 @@ func init() { logger = logging.MustGetLogger("") } -type ssPort struct { - tcpListener *net.TCPListener - packetConn net.PacketConn - cipherList service.CipherList +type ssListener struct { + io.Closer + cipherList service.CipherList } type SSServer struct { natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache - ports map[int]*ssPort + listeners map[string]*ssListener } -func (s *SSServer) startTCP(portNum int, cipherList service.CipherList) (*net.TCPListener, error) { - listener, err := net.ListenTCP("tcp", &net.TCPAddr{Port: portNum}) +func (s *SSServer) startDirectTCP(addr *net.TCPAddr, cipherList service.CipherList) (*net.TCPListener, error) { + listener, err := net.ListenTCP("tcp", addr) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. - return nil, fmt.Errorf("Shadowsocks TCP service failed to start on port %v: %w", portNum, err) + return nil, fmt.Errorf("Shadowsocks TCP service failed to start on address %v: %w", addr.String(), err) } logger.Infof("Shadowsocks TCP service listening on %v", listener.Addr().String()) authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &s.replayCache, s.m) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(portNum, authFunc, s.m, tcpReadTimeout) + tcpHandler := service.NewTCPHandler(addr.Port, authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { conn, err := listener.AcceptTCP() if err == nil { @@ -96,11 +96,11 @@ func (s *SSServer) startTCP(portNum int, cipherList service.CipherList) (*net.TC return listener, nil } -func (s *SSServer) startUDP(portNum int, cipherList service.CipherList) (*net.UDPConn, error) { - packetConn, err := net.ListenUDP("udp", &net.UDPAddr{Port: portNum}) +func (s *SSServer) startDirectUDP(addr *net.UDPAddr, cipherList service.CipherList) (*net.UDPConn, error) { + packetConn, err := net.ListenUDP("udp", addr) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. - return nil, fmt.Errorf("Shadowsocks UDP service failed to start on port %v: %w", portNum, err) + return nil, fmt.Errorf("Shadowsocks UDP service failed to start on address %v: %w", addr.String(), err) } logger.Infof("Shadowsocks UDP service listening on %v", packetConn.LocalAddr().String()) packetHandler := service.NewPacketHandler(s.natTimeout, cipherList, s.m) @@ -108,24 +108,29 @@ func (s *SSServer) startUDP(portNum int, cipherList service.CipherList) (*net.UD return packetConn, nil } -func (s *SSServer) removePort(portNum int) error { - port, ok := s.ports[portNum] - if !ok { - return fmt.Errorf("port %v doesn't exist", portNum) +func (s *SSServer) start(addr net.Addr, cipherList service.CipherList) (io.Closer, error) { + switch t := addr.(type) { + case *net.TCPAddr: + return s.startDirectTCP(t, cipherList) + case *net.UDPAddr: + return s.startDirectUDP(t, cipherList) + default: + return nil, fmt.Errorf("unable to start address: %s", t) } - tcpErr := port.tcpListener.Close() - udpErr := port.packetConn.Close() - delete(s.ports, portNum) - if tcpErr != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks TCP service on port %v failed to stop: %w", portNum, tcpErr) +} + +func (s *SSServer) remove(addr string) error { + listener, ok := s.listeners[addr] + if !ok { + return fmt.Errorf("address %v doesn't exist", addr) } - logger.Infof("Shadowsocks TCP service on port %v stopped", portNum) - if udpErr != nil { + err := listener.Close() + delete(s.listeners, addr) + if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks UDP service on port %v failed to stop: %w", portNum, udpErr) + return fmt.Errorf("Shadowsocks service on address %v failed to stop: %w", addr, err) } - logger.Infof("Shadowsocks UDP service on port %v stopped", portNum) + logger.Infof("Shadowsocks service on address %v stopped", addr) return nil } @@ -135,98 +140,90 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("failed to load config (%v): %w", filename, err) } - portChanges := make(map[int]int) - portCiphers := make(map[int]*list.List) // Values are *List of *CipherEntry. + uniqueCiphers := 0 + addrChanges := make(map[string]int) + type addrWithCiphers struct { + address net.Addr + ciphers *list.List // Values are *List of *CipherEntry. + } + addrs := make(map[string]*addrWithCiphers) for _, serviceConfig := range config.Services { if serviceConfig.Listeners == nil || serviceConfig.Keys == nil { return fmt.Errorf("must specify at least 1 listener and 1 key per service") } - addrs := []net.Addr{} + + ciphers := list.New() + type cipherKey struct { + cipher string + secret string + } + existingCiphers := make(map[cipherKey]bool) + for _, keyConfig := range serviceConfig.Keys { + key := cipherKey{keyConfig.Cipher, keyConfig.Secret} + _, ok := existingCiphers[key] + if ok { + logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) + continue + } + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + } + entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + ciphers.PushBack(&entry) + existingCiphers[key] = true + } + uniqueCiphers += ciphers.Len() + for _, listener := range serviceConfig.Listeners { switch t := listener.Type; t { // TODO: Support more listener types. case directListenerType: + //var addr net.Addr addr, err := onet.ResolveAddr(listener.Address) if err != nil { return fmt.Errorf("failed to resolve direct address: %v: %w", listener.Address, err) } - addrs = append(addrs, addr) - port, err := onet.GetPort(addr) - if err != nil { - return err - } - portChanges[int(port)] = 1 + addrChanges[listener.Address] = 1 + addrs[listener.Address] = &addrWithCiphers{addr, ciphers} default: return fmt.Errorf("unsupported listener type: %s", t) } } - - type key struct { - c string - s string - } - existingCipher := make(map[key]bool) - for _, keyConfig := range serviceConfig.Keys { - for _, addr := range addrs { - port, err := onet.GetPort(addr) - if err != nil { - return err - } - cipherList, ok := portCiphers[port] - if !ok { - cipherList = list.New() - portCiphers[port] = cipherList - } - _, ok = existingCipher[key{keyConfig.Cipher, keyConfig.Secret}] - if ok { - logger.Debugf("encryption key already exists for port=%v, ID=`%v`. Skipping.", port, keyConfig.ID) - continue - } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) - if err != nil { - return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) - } - entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - cipherList.PushBack(&entry) - existingCipher[key{keyConfig.Cipher, keyConfig.Secret}] = true - } - } } - for port := range s.ports { - portChanges[port] = portChanges[port] - 1 + for listener := range s.listeners { + addrChanges[listener] = addrChanges[listener] - 1 } - for portNum, count := range portChanges { + for addr, count := range addrChanges { if count == -1 { - if err := s.removePort(portNum); err != nil { - return fmt.Errorf("failed to remove port %v: %w", portNum, err) + if err := s.remove(addr); err != nil { + return fmt.Errorf("failed to remove address %v: %w", addr, err) } } else if count == +1 { cipherList := service.NewCipherList() - tcpListener, err := s.startTCP(portNum, cipherList) + listener, err := s.start(addrs[addr].address, cipherList) if err != nil { return err } - packetConn, err := s.startUDP(portNum, cipherList) - if err != nil { - return err - } - s.ports[portNum] = &ssPort{tcpListener, packetConn, cipherList} + s.listeners[addr] = &ssListener{Closer: listener, cipherList: cipherList} } } - numServices := 0 - for portNum, cipherList := range portCiphers { - s.ports[portNum].cipherList.Update(cipherList) - numServices += cipherList.Len() + for addr, addrWithCiphers := range addrs { + listener, ok := s.listeners[addr] + if !ok { + return fmt.Errorf("unable to find listener for address: %v", addr) + } + listener.cipherList.Update(addrWithCiphers.ciphers) } - logger.Infof("Loaded %v access keys over %v ports", numServices, len(s.ports)) - s.m.SetNumAccessKeys(numServices, len(s.ports)) + logger.Infof("Loaded %v access keys over %v listeners", uniqueCiphers, len(s.listeners)) + s.m.SetNumAccessKeys(uniqueCiphers, len(s.listeners)) return nil } // Stop serving on all ports. func (s *SSServer) Stop() error { - for portNum := range s.ports { - if err := s.removePort(portNum); err != nil { + for addr := range s.listeners { + if err := s.remove(addr); err != nil { return err } } @@ -239,7 +236,7 @@ func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, natTimeout: natTimeout, m: sm, replayCache: service.NewReplayCache(replayHistory), - ports: make(map[int]*ssPort), + listeners: make(map[string]*ssListener), } err := server.loadConfig(filename) if err != nil { From c1ee12f672571754d87d8bd8e6bb4b3dd6918844 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 3 Jun 2024 17:37:36 -0400 Subject: [PATCH 005/182] Remove commented out line. --- cmd/outline-ss-server/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 206ff05d..f45c8f4a 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -179,7 +179,6 @@ func (s *SSServer) loadConfig(filename string) error { switch t := listener.Type; t { // TODO: Support more listener types. case directListenerType: - //var addr net.Addr addr, err := onet.ResolveAddr(listener.Address) if err != nil { return fmt.Errorf("failed to resolve direct address: %v: %w", listener.Address, err) From 354301eafdd3dd99f2b37177a79776d6d1a8d9db Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 3 Jun 2024 17:42:19 -0400 Subject: [PATCH 006/182] Use `ElementsMatch` to compare the services irrespective of element ordering. --- cmd/outline-ss-server/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 0d46489c..1718a1fc 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -75,5 +75,5 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { }, }, } - require.Equal(t, expected, *config) + require.ElementsMatch(t, expected.Services, config.Services) } From 751d1643f1eb26f0e40c85caf2a5caeb9d9ac876 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 12 Jun 2024 13:58:55 -0400 Subject: [PATCH 007/182] Do not ignore the `keys` field if `services` is used as well. --- cmd/outline-ss-server/config.go | 54 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 1f97e182..21934e31 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -37,16 +37,17 @@ type Key struct { Secret string } +type LegacyKeyService struct { + Key `yaml:",inline"` + Port int +} + type Config struct { Services []Service - // Deprecated: Keys exists for historical compatibility. This is ignored if top-level `services` is specified. - Keys []struct { - ID string - Port int - Cipher string - Secret string - } + // Deprecated: `keys` exists for backward compatibility. Prefer to configure + // using the newer `services` format. + Keys []LegacyKeyService } // Reads a config from a filename and parses it as a [Config]. @@ -60,27 +61,28 @@ func ReadConfig(filename string) (*Config, error) { if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) } - if config.Services == nil { - // This is a deprecated config format. We need to transform it to to the new format. - ports := make(map[int][]Key) - for _, keyConfig := range config.Keys { - ports[keyConfig.Port] = append(ports[keyConfig.Port], Key{ - ID: keyConfig.ID, - Cipher: keyConfig.Cipher, - Secret: keyConfig.Secret, - }) - } - for port, keys := range ports { - s := Service{ - Listeners: []Listener{ - Listener{Type: "direct", Address: fmt.Sprintf("tcp://[::]:%d", port)}, - Listener{Type: "direct", Address: fmt.Sprintf("udp://[::]:%d", port)}, - }, - Keys: keys, - } - config.Services = append(config.Services, s) + + // Specifying keys in `config.Keys` is a deprecated config format. We need to + // transform it to to the new format. + ports := make(map[int][]Key) + for _, keyConfig := range config.Keys { + ports[keyConfig.Port] = append(ports[keyConfig.Port], Key{ + ID: keyConfig.ID, + Cipher: keyConfig.Cipher, + Secret: keyConfig.Secret, + }) + } + for port, keys := range ports { + s := Service{ + Listeners: []Listener{ + Listener{Type: "direct", Address: fmt.Sprintf("tcp://[::]:%d", port)}, + Listener{Type: "direct", Address: fmt.Sprintf("udp://[::]:%d", port)}, + }, + Keys: keys, } + config.Services = append(config.Services, s) } config.Keys = nil + return &config, nil } From 6297304d5170de3224ccd762f1ced7b6f9a82e63 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 12 Jun 2024 14:05:28 -0400 Subject: [PATCH 008/182] Add some more tests for failure scenarios and empty files. --- cmd/outline-ss-server/config_test.go | 33 ++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 1718a1fc..329c517f 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -15,14 +15,16 @@ package main import ( + "os" "testing" "github.com/stretchr/testify/require" ) func TestReadConfig(t *testing.T) { - config, _ := ReadConfig("./config_example.yml") + config, err := ReadConfig("./config_example.yml") + require.NoError(t, err) expected := Config{ Services: []Service{ Service{ @@ -50,8 +52,9 @@ func TestReadConfig(t *testing.T) { } func TestReadConfigParsesDeprecatedFormat(t *testing.T) { - config, _ := ReadConfig("./config_example.deprecated.yml") + config, err := ReadConfig("./config_example.deprecated.yml") + require.NoError(t, err) expected := Config{ Services: []Service{ Service{ @@ -77,3 +80,29 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { } require.ElementsMatch(t, expected.Services, config.Services) } + +func TestReadConfigFromEmptyFile(t *testing.T) { + file, _ := os.CreateTemp("", "empty.yaml") + + config, err := ReadConfig(file.Name()) + + require.NoError(t, err) + require.ElementsMatch(t, Config{}, config) +} + +func TestReadConfigFromNonExistingFileFails(t *testing.T) { + config, err := ReadConfig("./foo") + + require.Error(t, err) + require.ElementsMatch(t, nil, config) +} + +func TestReadConfigFromIncorrectFormatFails(t *testing.T) { + file, _ := os.CreateTemp("", "empty.yaml") + file.WriteString("foo") + + config, err := ReadConfig(file.Name()) + + require.Error(t, err) + require.ElementsMatch(t, Config{}, config) +} From 0ac0a724be44fb4bbcd3a3d260ae341d6d62d3d9 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 12 Jun 2024 14:12:48 -0400 Subject: [PATCH 009/182] Remove unused `GetPort()`. --- net/address.go | 13 ------------- net/address_test.go | 21 --------------------- 2 files changed, 34 deletions(-) diff --git a/net/address.go b/net/address.go index a74e4f13..7438ee16 100644 --- a/net/address.go +++ b/net/address.go @@ -15,7 +15,6 @@ package net import ( - "fmt" "net" "net/url" ) @@ -49,15 +48,3 @@ func ResolveAddr(addr string) (net.Addr, error) { return nil, net.UnknownNetworkError(u.Scheme) } } - -// Returns the port from a given address. -func GetPort(addr net.Addr) (port int, err error) { - switch t := addr.(type) { - case *net.TCPAddr: - return t.Port, nil - case *net.UDPAddr: - return t.Port, nil - default: - return -1, fmt.Errorf("failed to get port from address: %v", addr) - } -} diff --git a/net/address_test.go b/net/address_test.go index d7fe1d01..f0904781 100644 --- a/net/address_test.go +++ b/net/address_test.go @@ -54,24 +54,3 @@ func TestResolveAddrReturnsErrorForUnknownScheme(t *testing.T) { require.Nil(t, addr) require.Error(t, err) } - -func TestGetPortFromTCPAddr(t *testing.T) { - port, err := GetPort(&net.TCPAddr{IP: net.ParseIP("1.2.3.4"), Port: 1234}) - - require.NoError(t, err) - require.Equal(t, 1234, port) -} - -func TestGetPortFromUDPPAddr(t *testing.T) { - port, err := GetPort(&net.UDPAddr{IP: net.ParseIP("1.2.3.4"), Port: 5678}) - - require.NoError(t, err) - require.Equal(t, 5678, port) -} - -func TestGetPortReturnsErrorForUnsupportedAddressType(t *testing.T) { - port, err := GetPort(&net.UnixAddr{Name: "/path/to/foo", Net: "unix"}) - - require.Equal(t, -1, port) - require.Error(t, err) -} From 794f860fec4facedd4b45b80fde0877e1b7b1cac Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 12 Jun 2024 16:06:24 -0400 Subject: [PATCH 010/182] Move `ResolveAddr` to config.go. --- cmd/outline-ss-server/config.go | 32 ++++++++++++++++ cmd/outline-ss-server/config_test.go | 35 +++++++++++++++++ cmd/outline-ss-server/main.go | 3 +- net/address.go | 50 ------------------------- net/address_test.go | 56 ---------------------------- 5 files changed, 68 insertions(+), 108 deletions(-) delete mode 100644 net/address.go delete mode 100644 net/address_test.go diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 21934e31..e2fcbd87 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -16,6 +16,8 @@ package main import ( "fmt" + "net" + "net/url" "os" "gopkg.in/yaml.v2" @@ -86,3 +88,33 @@ func ReadConfig(filename string) (*Config, error) { return &config, nil } + +// Resolves a URL-style listen address specification as a [net.Addr]. +// +// Examples: +// +// udp6://127.0.0.1:8000 +// unix:///tmp/foo.sock +// tcp://127.0.0.1:9002 +func ResolveAddr(addr string) (net.Addr, error) { + u, err := url.Parse(addr) + if err != nil { + return nil, err + } + switch u.Scheme { + case "tcp", "tcp4", "tcp6": + return net.ResolveTCPAddr(u.Scheme, u.Host) + case "udp", "udp4", "udp6": + return net.ResolveUDPAddr(u.Scheme, u.Host) + case "unix", "unixgram", "unixpacket": + var path string + if u.Opaque != "" { + path = u.Opaque + } else { + path = u.Path + } + return net.ResolveUnixAddr(u.Scheme, path) + default: + return nil, net.UnknownNetworkError(u.Scheme) + } +} diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 329c517f..247d7e68 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -15,6 +15,7 @@ package main import ( + "net" "os" "testing" @@ -106,3 +107,37 @@ func TestReadConfigFromIncorrectFormatFails(t *testing.T) { require.Error(t, err) require.ElementsMatch(t, Config{}, config) } + +func TestResolveAddrReturnsTCPAddr(t *testing.T) { + addr, err := ResolveAddr("tcp://0.0.0.0:9000") + + require.NoError(t, err) + if _, ok := addr.(*net.TCPAddr); !ok { + t.Errorf("expected a *net.TCPAddr; it is a %T", addr) + } +} + +func TestResolveAddrReturnsUDPAddr(t *testing.T) { + addr, err := ResolveAddr("udp://[::]:9001") + + require.NoError(t, err) + if _, ok := addr.(*net.UDPAddr); !ok { + t.Errorf("expected a *net.UDPAddr; it is a %T", addr) + } +} + +func TestResolveAddrReturnsUnixAddr(t *testing.T) { + addr, err := ResolveAddr("unix:///path/to/stream_socket") + + require.NoError(t, err) + if _, ok := addr.(*net.UnixAddr); !ok { + t.Errorf("expected a *net.UnixAddr; it is a %T", addr) + } +} + +func TestResolveAddrReturnsErrorForUnknownScheme(t *testing.T) { + addr, err := ResolveAddr("foobar") + + require.Nil(t, addr) + require.Error(t, err) +} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index f45c8f4a..546bfcb5 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -31,7 +31,6 @@ import ( "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" - onet "github.com/Jigsaw-Code/outline-ss-server/net" "github.com/Jigsaw-Code/outline-ss-server/service" "github.com/op/go-logging" "github.com/prometheus/client_golang/prometheus" @@ -179,7 +178,7 @@ func (s *SSServer) loadConfig(filename string) error { switch t := listener.Type; t { // TODO: Support more listener types. case directListenerType: - addr, err := onet.ResolveAddr(listener.Address) + addr, err := ResolveAddr(listener.Address) if err != nil { return fmt.Errorf("failed to resolve direct address: %v: %w", listener.Address, err) } diff --git a/net/address.go b/net/address.go deleted file mode 100644 index 7438ee16..00000000 --- a/net/address.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2024 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package net - -import ( - "net" - "net/url" -) - -// Resolves a URL-style listen address specification as a [net.Addr] -// -// Examples: -// -// udp6://127.0.0.1:8000 -// unix:///tmp/foo.sock -// tcp://127.0.0.1:9002 -func ResolveAddr(addr string) (net.Addr, error) { - u, err := url.Parse(addr) - if err != nil { - return nil, err - } - switch u.Scheme { - case "tcp", "tcp4", "tcp6": - return net.ResolveTCPAddr(u.Scheme, u.Host) - case "udp", "udp4", "udp6": - return net.ResolveUDPAddr(u.Scheme, u.Host) - case "unix", "unixgram", "unixpacket": - var path string - if u.Opaque != "" { - path = u.Opaque - } else { - path = u.Path - } - return net.ResolveUnixAddr(u.Scheme, path) - default: - return nil, net.UnknownNetworkError(u.Scheme) - } -} diff --git a/net/address_test.go b/net/address_test.go deleted file mode 100644 index f0904781..00000000 --- a/net/address_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2024 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package net - -import ( - "net" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestResolveAddrReturnsTCPAddr(t *testing.T) { - addr, err := ResolveAddr("tcp://0.0.0.0:9000") - - require.NoError(t, err) - if _, ok := addr.(*net.TCPAddr); !ok { - t.Errorf("expected a *net.TCPAddr; it is a %T", addr) - } -} - -func TestResolveAddrReturnsUDPAddr(t *testing.T) { - addr, err := ResolveAddr("udp://[::]:9001") - - require.NoError(t, err) - if _, ok := addr.(*net.UDPAddr); !ok { - t.Errorf("expected a *net.UDPAddr; it is a %T", addr) - } -} - -func TestResolveAddrReturnsUnixAddr(t *testing.T) { - addr, err := ResolveAddr("unix:///path/to/stream_socket") - - require.NoError(t, err) - if _, ok := addr.(*net.UnixAddr); !ok { - t.Errorf("expected a *net.UnixAddr; it is a %T", addr) - } -} - -func TestResolveAddrReturnsErrorForUnknownScheme(t *testing.T) { - addr, err := ResolveAddr("foobar") - - require.Nil(t, addr) - require.Error(t, err) -} From 01b7e8a20b727dffeaccdee11c79b2658ed0a4a1 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 12 Jun 2024 18:24:50 -0400 Subject: [PATCH 011/182] Remove use of `net.Addr` type. --- cmd/outline-ss-server/config.go | 32 ---------- cmd/outline-ss-server/config_test.go | 35 ---------- cmd/outline-ss-server/main.go | 95 ++++++++++++++-------------- service/tcp.go | 8 +-- 4 files changed, 53 insertions(+), 117 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index e2fcbd87..21934e31 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -16,8 +16,6 @@ package main import ( "fmt" - "net" - "net/url" "os" "gopkg.in/yaml.v2" @@ -88,33 +86,3 @@ func ReadConfig(filename string) (*Config, error) { return &config, nil } - -// Resolves a URL-style listen address specification as a [net.Addr]. -// -// Examples: -// -// udp6://127.0.0.1:8000 -// unix:///tmp/foo.sock -// tcp://127.0.0.1:9002 -func ResolveAddr(addr string) (net.Addr, error) { - u, err := url.Parse(addr) - if err != nil { - return nil, err - } - switch u.Scheme { - case "tcp", "tcp4", "tcp6": - return net.ResolveTCPAddr(u.Scheme, u.Host) - case "udp", "udp4", "udp6": - return net.ResolveUDPAddr(u.Scheme, u.Host) - case "unix", "unixgram", "unixpacket": - var path string - if u.Opaque != "" { - path = u.Opaque - } else { - path = u.Path - } - return net.ResolveUnixAddr(u.Scheme, path) - default: - return nil, net.UnknownNetworkError(u.Scheme) - } -} diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 247d7e68..329c517f 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -15,7 +15,6 @@ package main import ( - "net" "os" "testing" @@ -107,37 +106,3 @@ func TestReadConfigFromIncorrectFormatFails(t *testing.T) { require.Error(t, err) require.ElementsMatch(t, Config{}, config) } - -func TestResolveAddrReturnsTCPAddr(t *testing.T) { - addr, err := ResolveAddr("tcp://0.0.0.0:9000") - - require.NoError(t, err) - if _, ok := addr.(*net.TCPAddr); !ok { - t.Errorf("expected a *net.TCPAddr; it is a %T", addr) - } -} - -func TestResolveAddrReturnsUDPAddr(t *testing.T) { - addr, err := ResolveAddr("udp://[::]:9001") - - require.NoError(t, err) - if _, ok := addr.(*net.UDPAddr); !ok { - t.Errorf("expected a *net.UDPAddr; it is a %T", addr) - } -} - -func TestResolveAddrReturnsUnixAddr(t *testing.T) { - addr, err := ResolveAddr("unix:///path/to/stream_socket") - - require.NoError(t, err) - if _, ok := addr.(*net.UnixAddr); !ok { - t.Errorf("expected a *net.UnixAddr; it is a %T", addr) - } -} - -func TestResolveAddrReturnsErrorForUnknownScheme(t *testing.T) { - addr, err := ResolveAddr("foobar") - - require.Nil(t, addr) - require.Error(t, err) -} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 546bfcb5..294934a5 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -21,6 +21,7 @@ import ( "io" "net" "net/http" + "net/url" "os" "os/signal" "strings" @@ -74,48 +75,58 @@ type SSServer struct { listeners map[string]*ssListener } -func (s *SSServer) startDirectTCP(addr *net.TCPAddr, cipherList service.CipherList) (*net.TCPListener, error) { - listener, err := net.ListenTCP("tcp", addr) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return nil, fmt.Errorf("Shadowsocks TCP service failed to start on address %v: %w", addr.String(), err) - } - logger.Infof("Shadowsocks TCP service listening on %v", listener.Addr().String()) - authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &s.replayCache, s.m) - // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(addr.Port, authFunc, s.m, tcpReadTimeout) - accept := func() (transport.StreamConn, error) { - conn, err := listener.AcceptTCP() - if err == nil { - conn.SetKeepAlive(true) +func (s *SSServer) serve(listener io.Closer, cipherList service.CipherList) error { + switch ln := listener.(type) { + case net.Listener: + authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &s.replayCache, s.m) + // TODO: Register initial data metrics at zero. + tcpHandler := service.NewTCPHandler(authFunc, s.m, tcpReadTimeout) + accept := func() (transport.StreamConn, error) { + conn, err := ln.Accept() + if err == nil { + conn.(*net.TCPConn).SetKeepAlive(true) + } + return conn.(transport.StreamConn), err } - return conn, err + go service.StreamServe(accept, tcpHandler.Handle) + case net.PacketConn: + packetHandler := service.NewPacketHandler(s.natTimeout, cipherList, s.m) + go packetHandler.Handle(ln) + default: + return fmt.Errorf("unknown listener type: %v", ln) } - go service.StreamServe(accept, tcpHandler.Handle) - return listener, nil + return nil } -func (s *SSServer) startDirectUDP(addr *net.UDPAddr, cipherList service.CipherList) (*net.UDPConn, error) { - packetConn, err := net.ListenUDP("udp", addr) +func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, error) { + u, err := url.Parse(addr) if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return nil, fmt.Errorf("Shadowsocks UDP service failed to start on address %v: %w", addr.String(), err) + return nil, err } - logger.Infof("Shadowsocks UDP service listening on %v", packetConn.LocalAddr().String()) - packetHandler := service.NewPacketHandler(s.natTimeout, cipherList, s.m) - go packetHandler.Handle(packetConn) - return packetConn, nil -} -func (s *SSServer) start(addr net.Addr, cipherList service.CipherList) (io.Closer, error) { - switch t := addr.(type) { - case *net.TCPAddr: - return s.startDirectTCP(t, cipherList) - case *net.UDPAddr: - return s.startDirectUDP(t, cipherList) + var listener io.Closer + switch u.Scheme { + case "tcp", "tcp4", "tcp6": + // TODO: Validate `u` address. + listener, err = net.Listen(u.Scheme, u.Host) + case "udp", "udp4", "udp6": + // TODO: Validate `u` address. + listener, err = net.ListenPacket(u.Scheme, u.Host) default: - return nil, fmt.Errorf("unable to start address: %s", t) + return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme) } + if err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return nil, fmt.Errorf("Shadowsocks service failed to start on address %v: %w", addr, err) + } + logger.Infof("Shadowsocks service listening on %v", addr) + + err = s.serve(listener, cipherList) + if err != nil { + return nil, fmt.Errorf("failed to serve on listener %w: %w", listener, err) + } + + return listener, nil } func (s *SSServer) remove(addr string) error { @@ -141,11 +152,7 @@ func (s *SSServer) loadConfig(filename string) error { uniqueCiphers := 0 addrChanges := make(map[string]int) - type addrWithCiphers struct { - address net.Addr - ciphers *list.List // Values are *List of *CipherEntry. - } - addrs := make(map[string]*addrWithCiphers) + addrCiphers := make(map[string]*list.List) // Values are *List of *CipherEntry. for _, serviceConfig := range config.Services { if serviceConfig.Listeners == nil || serviceConfig.Keys == nil { return fmt.Errorf("must specify at least 1 listener and 1 key per service") @@ -178,12 +185,8 @@ func (s *SSServer) loadConfig(filename string) error { switch t := listener.Type; t { // TODO: Support more listener types. case directListenerType: - addr, err := ResolveAddr(listener.Address) - if err != nil { - return fmt.Errorf("failed to resolve direct address: %v: %w", listener.Address, err) - } addrChanges[listener.Address] = 1 - addrs[listener.Address] = &addrWithCiphers{addr, ciphers} + addrCiphers[listener.Address] = ciphers default: return fmt.Errorf("unsupported listener type: %s", t) } @@ -199,19 +202,19 @@ func (s *SSServer) loadConfig(filename string) error { } } else if count == +1 { cipherList := service.NewCipherList() - listener, err := s.start(addrs[addr].address, cipherList) + listener, err := s.start(addr, cipherList) if err != nil { return err } s.listeners[addr] = &ssListener{Closer: listener, cipherList: cipherList} } } - for addr, addrWithCiphers := range addrs { + for addr, ciphers := range addrCiphers { listener, ok := s.listeners[addr] if !ok { return fmt.Errorf("unable to find listener for address: %v", addr) } - listener.cipherList.Update(addrWithCiphers.ciphers) + listener.cipherList.Update(ciphers) } logger.Infof("Loaded %v access keys over %v listeners", uniqueCiphers, len(s.listeners)) s.m.SetNumAccessKeys(uniqueCiphers, len(s.listeners)) diff --git a/service/tcp.go b/service/tcp.go index 85ab9990..484a1b90 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -170,9 +170,8 @@ type tcpHandler struct { } // NewTCPService creates a TCPService -func NewTCPHandler(port int, authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { +func NewTCPHandler(authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { return &tcpHandler{ - port: port, m: m, readTimeout: timeout, authenticate: authenticate, @@ -342,7 +341,8 @@ func (h *tcpHandler) handleConnection(ctx context.Context, outerConn transport.S id, innerConn, authErr := h.authenticate(outerConn) if authErr != nil { // Drain to protect against probing attacks. - h.absorbProbe(outerConn, authErr.Status, proxyMetrics) + port := outerConn.LocalAddr().(*net.TCPAddr).Port + h.absorbProbe(outerConn, port, authErr.Status, proxyMetrics) return id, authErr } h.m.AddAuthenticatedTCPConnection(outerConn.RemoteAddr(), id) @@ -370,7 +370,7 @@ func (h *tcpHandler) handleConnection(ctx context.Context, outerConn transport.S // Keep the connection open until we hit the authentication deadline to protect against probing attacks // `proxyMetrics` is a pointer because its value is being mutated by `clientConn`. -func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, status string, proxyMetrics *metrics.ProxyMetrics) { +func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, port int, status string, proxyMetrics *metrics.ProxyMetrics) { // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) From 87a15650e2ca5116e50199b63986f813dc3d8fa5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 12 Jun 2024 18:26:28 -0400 Subject: [PATCH 012/182] Pull listener creation into its own function. --- cmd/outline-ss-server/main.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 294934a5..8d853ffd 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -98,23 +98,26 @@ func (s *SSServer) serve(listener io.Closer, cipherList service.CipherList) erro return nil } -func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, error) { +func newListener(addr string) (io.Closer, error) { u, err := url.Parse(addr) if err != nil { return nil, err } - var listener io.Closer switch u.Scheme { case "tcp", "tcp4", "tcp6": // TODO: Validate `u` address. - listener, err = net.Listen(u.Scheme, u.Host) + return net.Listen(u.Scheme, u.Host) case "udp", "udp4", "udp6": // TODO: Validate `u` address. - listener, err = net.ListenPacket(u.Scheme, u.Host) + return net.ListenPacket(u.Scheme, u.Host) default: return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme) } +} + +func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, error) { + listener, err := newListener(addr) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return nil, fmt.Errorf("Shadowsocks service failed to start on address %v: %w", addr, err) From 51a13a7a52801a61d0bc06eaa2bae83eacf8fa84 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 14 Jun 2024 10:10:36 -0400 Subject: [PATCH 013/182] Move listener validation/creation to `config.go`. --- cmd/outline-ss-server/config.go | 46 +++++++++++++++++++++++++++++++++ cmd/outline-ss-server/main.go | 21 +-------------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 21934e31..8d8cd37e 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -15,7 +15,11 @@ package main import ( + "errors" "fmt" + "io" + "net" + "net/url" "os" "gopkg.in/yaml.v2" @@ -86,3 +90,45 @@ func ReadConfig(filename string) (*Config, error) { return &config, nil } + +// validateListener asserts that a listener URI conforms to the expected format. +func validateListener(u *url.URL) error { + if u.Opaque != "" { + return errors.New("URI cannot have an opaque part") + } + if u.User != nil { + return errors.New("URI cannot have an userdata part") + } + if u.RawQuery != "" || u.ForceQuery { + return errors.New("URI cannot have a query part") + } + if u.Fragment != "" { + return errors.New("URI cannot have a fragement") + } + if u.Path != "" && u.Path != "/" { + return errors.New("URI path not allowed") + } + return nil +} + +func NewListener(addr string) (io.Closer, error) { + u, err := url.Parse(addr) + if err != nil { + return nil, err + } + + switch u.Scheme { + case "tcp", "tcp4", "tcp6": + if err := validateListener(u); err != nil { + return nil, fmt.Errorf("invalid listener `%s`: %v", u, err) + } + return net.Listen(u.Scheme, u.Host) + case "udp", "udp4", "udp6": + if err := validateListener(u); err != nil { + return nil, fmt.Errorf("invalid listener `%s`: %v", u, err) + } + return net.ListenPacket(u.Scheme, u.Host) + default: + return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme) + } +} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 8d853ffd..323bcda7 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -21,7 +21,6 @@ import ( "io" "net" "net/http" - "net/url" "os" "os/signal" "strings" @@ -98,26 +97,8 @@ func (s *SSServer) serve(listener io.Closer, cipherList service.CipherList) erro return nil } -func newListener(addr string) (io.Closer, error) { - u, err := url.Parse(addr) - if err != nil { - return nil, err - } - - switch u.Scheme { - case "tcp", "tcp4", "tcp6": - // TODO: Validate `u` address. - return net.Listen(u.Scheme, u.Host) - case "udp", "udp4", "udp6": - // TODO: Validate `u` address. - return net.ListenPacket(u.Scheme, u.Host) - default: - return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme) - } -} - func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, error) { - listener, err := newListener(addr) + listener, err := NewListener(addr) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return nil, fmt.Errorf("Shadowsocks service failed to start on address %v: %w", addr, err) From f8d7aa5b90bd13bae3c1b2542e684adb6497774f Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 14 Jun 2024 10:30:16 -0400 Subject: [PATCH 014/182] Use a custom type for listener type. --- cmd/outline-ss-server/config.go | 12 ++++++++---- cmd/outline-ss-server/config_test.go | 16 ++++++++-------- cmd/outline-ss-server/main.go | 8 +++----- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 8d8cd37e..112ba58f 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -30,8 +30,12 @@ type Service struct { Keys []Key } +type ListenerType string + +const listenerTypeDirect ListenerType = "direct" + type Listener struct { - Type string + Type ListenerType Address string } @@ -79,8 +83,8 @@ func ReadConfig(filename string) (*Config, error) { for port, keys := range ports { s := Service{ Listeners: []Listener{ - Listener{Type: "direct", Address: fmt.Sprintf("tcp://[::]:%d", port)}, - Listener{Type: "direct", Address: fmt.Sprintf("udp://[::]:%d", port)}, + Listener{Type: listenerTypeDirect, Address: fmt.Sprintf("tcp://[::]:%d", port)}, + Listener{Type: listenerTypeDirect, Address: fmt.Sprintf("udp://[::]:%d", port)}, }, Keys: keys, } @@ -111,7 +115,7 @@ func validateListener(u *url.URL) error { return nil } -func NewListener(addr string) (io.Closer, error) { +func newListener(addr string) (io.Closer, error) { u, err := url.Parse(addr) if err != nil { return nil, err diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 329c517f..88660388 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -29,8 +29,8 @@ func TestReadConfig(t *testing.T) { Services: []Service{ Service{ Listeners: []Listener{ - Listener{Type: "direct", Address: "tcp://[::]:9000"}, - Listener{Type: "direct", Address: "udp://[::]:9000"}, + Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, + Listener{Type: listenerTypeDirect, Address: "udp://[::]:9000"}, }, Keys: []Key{ Key{"user-0", "chacha20-ietf-poly1305", "Secret0"}, @@ -39,8 +39,8 @@ func TestReadConfig(t *testing.T) { }, Service{ Listeners: []Listener{ - Listener{Type: "direct", Address: "tcp://[::]:9001"}, - Listener{Type: "direct", Address: "udp://[::]:9001"}, + Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9001"}, + Listener{Type: listenerTypeDirect, Address: "udp://[::]:9001"}, }, Keys: []Key{ Key{"user-2", "chacha20-ietf-poly1305", "Secret2"}, @@ -59,8 +59,8 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { Services: []Service{ Service{ Listeners: []Listener{ - Listener{Type: "direct", Address: "tcp://[::]:9000"}, - Listener{Type: "direct", Address: "udp://[::]:9000"}, + Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, + Listener{Type: listenerTypeDirect, Address: "udp://[::]:9000"}, }, Keys: []Key{ Key{"user-0", "chacha20-ietf-poly1305", "Secret0"}, @@ -69,8 +69,8 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { }, Service{ Listeners: []Listener{ - Listener{Type: "direct", Address: "tcp://[::]:9001"}, - Listener{Type: "direct", Address: "udp://[::]:9001"}, + Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9001"}, + Listener{Type: listenerTypeDirect, Address: "udp://[::]:9001"}, }, Keys: []Key{ Key{"user-2", "chacha20-ietf-poly1305", "Secret2"}, diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 323bcda7..2af841cc 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -49,8 +49,6 @@ const tcpReadTimeout time.Duration = 59 * time.Second // A UDP NAT timeout of at least 5 minutes is recommended in RFC 4787 Section 4.3. const defaultNatTimeout time.Duration = 5 * time.Minute -var directListenerType = "direct" - func init() { var prefix = "%{level:.1s}%{time:2006-01-02T15:04:05.000Z07:00} %{pid} %{shortfile}]" if term.IsTerminal(int(os.Stderr.Fd())) { @@ -98,7 +96,7 @@ func (s *SSServer) serve(listener io.Closer, cipherList service.CipherList) erro } func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, error) { - listener, err := NewListener(addr) + listener, err := newListener(addr) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return nil, fmt.Errorf("Shadowsocks service failed to start on address %v: %w", addr, err) @@ -107,7 +105,7 @@ func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, err = s.serve(listener, cipherList) if err != nil { - return nil, fmt.Errorf("failed to serve on listener %w: %w", listener, err) + return nil, fmt.Errorf("failed to serve on listener %v: %w", listener, err) } return listener, nil @@ -168,7 +166,7 @@ func (s *SSServer) loadConfig(filename string) error { for _, listener := range serviceConfig.Listeners { switch t := listener.Type; t { // TODO: Support more listener types. - case directListenerType: + case listenerTypeDirect: addrChanges[listener.Address] = 1 addrCiphers[listener.Address] = ciphers default: From 19520364bcd8f06322455deb06ad28e4dd2642dc Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 14 Jun 2024 10:45:38 -0400 Subject: [PATCH 015/182] Fix accept handler. --- cmd/outline-ss-server/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 2af841cc..085b8c0e 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -80,10 +80,12 @@ func (s *SSServer) serve(listener io.Closer, cipherList service.CipherList) erro tcpHandler := service.NewTCPHandler(authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { conn, err := ln.Accept() - if err == nil { - conn.(*net.TCPConn).SetKeepAlive(true) + if err != nil { + return nil, err } - return conn.(transport.StreamConn), err + c := conn.(*net.TCPConn) + c.SetKeepAlive(true) + return c, err } go service.StreamServe(accept, tcpHandler.Handle) case net.PacketConn: From 7212265bdc10c5fac9bb957329ae019f983e6b2a Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 14 Jun 2024 10:56:48 -0400 Subject: [PATCH 016/182] Add doc comment. --- cmd/outline-ss-server/config.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 112ba58f..8ce54066 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -115,6 +115,12 @@ func validateListener(u *url.URL) error { return nil } +// newListener creates a new listener from a URL-style address specification. +// +// Example addresses: +// +// tcp4://127.0.0.1:8000 +// udp://127.0.0.1:9000 func newListener(addr string) (io.Closer, error) { u, err := url.Parse(addr) if err != nil { From 6e2068d2491d3277ce44d5a6965a0312c123e24e Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 14 Jun 2024 11:00:29 -0400 Subject: [PATCH 017/182] Fix tests still supplying the port. --- internal/integration_test/integration_test.go | 8 ++++---- service/tcp_test.go | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index 4ca2f120..43109b7a 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -133,7 +133,7 @@ func TestTCPEcho(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -202,7 +202,7 @@ func TestRestrictedAddresses(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { service.StreamServe(service.WrapStreamListener(proxyListener.AcceptTCP), handler.Handle) @@ -384,7 +384,7 @@ func BenchmarkTCPThroughput(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -448,7 +448,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { diff --git a/service/tcp_test.go b/service/tcp_test.go index 1a70ed67..14069756 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -281,7 +281,7 @@ func TestProbeRandom(t *testing.T) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) @@ -358,7 +358,7 @@ func TestProbeClientBytesBasicTruncated(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -393,7 +393,7 @@ func TestProbeClientBytesBasicModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -429,7 +429,7 @@ func TestProbeClientBytesCoalescedModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -472,7 +472,7 @@ func TestProbeServerBytesModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) @@ -503,7 +503,7 @@ func TestReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -582,7 +582,7 @@ func TestReverseReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -653,7 +653,7 @@ func probeExpectTimeout(t *testing.T, payloadSize int) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { From 7114434738b7369894de50586f8215527156a0ac Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 14 Jun 2024 14:36:02 -0400 Subject: [PATCH 018/182] Move old config parsing to `loadConfig`. --- cmd/outline-ss-server/config.go | 23 --------------------- cmd/outline-ss-server/config_test.go | 31 +++++++++++----------------- cmd/outline-ss-server/main.go | 20 ++++++++++++++++++ 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 8ce54066..41a44d7a 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -69,29 +69,6 @@ func ReadConfig(filename string) (*Config, error) { if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) } - - // Specifying keys in `config.Keys` is a deprecated config format. We need to - // transform it to to the new format. - ports := make(map[int][]Key) - for _, keyConfig := range config.Keys { - ports[keyConfig.Port] = append(ports[keyConfig.Port], Key{ - ID: keyConfig.ID, - Cipher: keyConfig.Cipher, - Secret: keyConfig.Secret, - }) - } - for port, keys := range ports { - s := Service{ - Listeners: []Listener{ - Listener{Type: listenerTypeDirect, Address: fmt.Sprintf("tcp://[::]:%d", port)}, - Listener{Type: listenerTypeDirect, Address: fmt.Sprintf("udp://[::]:%d", port)}, - }, - Keys: keys, - } - config.Services = append(config.Services, s) - } - config.Keys = nil - return &config, nil } diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 88660388..2793bbc4 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -56,29 +56,22 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { require.NoError(t, err) expected := Config{ - Services: []Service{ - Service{ - Listeners: []Listener{ - Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, - Listener{Type: listenerTypeDirect, Address: "udp://[::]:9000"}, - }, - Keys: []Key{ - Key{"user-0", "chacha20-ietf-poly1305", "Secret0"}, - Key{"user-1", "chacha20-ietf-poly1305", "Secret1"}, - }, + Keys: []LegacyKeyService{ + LegacyKeyService{ + Key: Key{ID: "user-0", Cipher: "chacha20-ietf-poly1305", Secret: "Secret0"}, + Port: 9000, }, - Service{ - Listeners: []Listener{ - Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9001"}, - Listener{Type: listenerTypeDirect, Address: "udp://[::]:9001"}, - }, - Keys: []Key{ - Key{"user-2", "chacha20-ietf-poly1305", "Secret2"}, - }, + LegacyKeyService{ + Key: Key{ID: "user-1", Cipher: "chacha20-ietf-poly1305", Secret: "Secret1"}, + Port: 9000, + }, + LegacyKeyService{ + Key: Key{ID: "user-2", Cipher: "chacha20-ietf-poly1305", Secret: "Secret2"}, + Port: 9001, }, }, } - require.ElementsMatch(t, expected.Services, config.Services) + require.Equal(t, expected, *config) } func TestReadConfigFromEmptyFile(t *testing.T) { diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 085b8c0e..50c13922 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -137,6 +137,26 @@ func (s *SSServer) loadConfig(filename string) error { uniqueCiphers := 0 addrChanges := make(map[string]int) addrCiphers := make(map[string]*list.List) // Values are *List of *CipherEntry. + + for _, legacyKeyConfig := range config.Keys { + cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyConfig.Cipher, legacyKeyConfig.Secret) + if err != nil { + return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyConfig.ID, err) + } + entry := service.MakeCipherEntry(legacyKeyConfig.ID, cryptoKey, legacyKeyConfig.Secret) + for _, ln := range []string{"tcp", "udp"} { + addr := fmt.Sprintf("%s://[::]:%d", ln, legacyKeyConfig.Port) + addrChanges[addr] = 1 + ciphers, ok := addrCiphers[addr] + if !ok { + ciphers = list.New() + addrCiphers[addr] = ciphers + } + ciphers.PushBack(&entry) + } + uniqueCiphers += 1 + } + for _, serviceConfig := range config.Services { if serviceConfig.Listeners == nil || serviceConfig.Keys == nil { return fmt.Errorf("must specify at least 1 listener and 1 key per service") From 1b2dd42b392bd65b35ab485bf739163df0b1b86e Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 14 Jun 2024 14:43:29 -0400 Subject: [PATCH 019/182] Lowercase `readConfig`. --- cmd/outline-ss-server/config.go | 4 ++-- cmd/outline-ss-server/config_test.go | 10 +++++----- cmd/outline-ss-server/main.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 41a44d7a..4df7de83 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -58,8 +58,8 @@ type Config struct { Keys []LegacyKeyService } -// Reads a config from a filename and parses it as a [Config]. -func ReadConfig(filename string) (*Config, error) { +// readConfig attempts to read a config from a filename and parses it as a [Config]. +func readConfig(filename string) (*Config, error) { config := Config{} configData, err := os.ReadFile(filename) if err != nil { diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 2793bbc4..fce0d8f8 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -22,7 +22,7 @@ import ( ) func TestReadConfig(t *testing.T) { - config, err := ReadConfig("./config_example.yml") + config, err := readConfig("./config_example.yml") require.NoError(t, err) expected := Config{ @@ -52,7 +52,7 @@ func TestReadConfig(t *testing.T) { } func TestReadConfigParsesDeprecatedFormat(t *testing.T) { - config, err := ReadConfig("./config_example.deprecated.yml") + config, err := readConfig("./config_example.deprecated.yml") require.NoError(t, err) expected := Config{ @@ -77,14 +77,14 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { func TestReadConfigFromEmptyFile(t *testing.T) { file, _ := os.CreateTemp("", "empty.yaml") - config, err := ReadConfig(file.Name()) + config, err := readConfig(file.Name()) require.NoError(t, err) require.ElementsMatch(t, Config{}, config) } func TestReadConfigFromNonExistingFileFails(t *testing.T) { - config, err := ReadConfig("./foo") + config, err := readConfig("./foo") require.Error(t, err) require.ElementsMatch(t, nil, config) @@ -94,7 +94,7 @@ func TestReadConfigFromIncorrectFormatFails(t *testing.T) { file, _ := os.CreateTemp("", "empty.yaml") file.WriteString("foo") - config, err := ReadConfig(file.Name()) + config, err := readConfig(file.Name()) require.Error(t, err) require.ElementsMatch(t, Config{}, config) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 50c13922..190657e8 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -129,7 +129,7 @@ func (s *SSServer) remove(addr string) error { } func (s *SSServer) loadConfig(filename string) error { - config, err := ReadConfig(filename) + config, err := readConfig(filename) if err != nil { return fmt.Errorf("failed to load config (%v): %w", filename, err) } From 4ce06f0ee394d1b784a3f6e1a35be3cacd19c3a0 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 13:37:26 -0400 Subject: [PATCH 020/182] Use `Config` suffix for config types. --- cmd/outline-ss-server/config.go | 20 ++++++------ cmd/outline-ss-server/config_test.go | 48 ++++++++++++++-------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 4df7de83..666d45da 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -25,37 +25,37 @@ import ( "gopkg.in/yaml.v2" ) -type Service struct { - Listeners []Listener - Keys []Key +type ServiceConfig struct { + Listeners []ListenerConfig + Keys []KeyConfig } type ListenerType string const listenerTypeDirect ListenerType = "direct" -type Listener struct { +type ListenerConfig struct { Type ListenerType Address string } -type Key struct { +type KeyConfig struct { ID string Cipher string Secret string } -type LegacyKeyService struct { - Key `yaml:",inline"` - Port int +type LegacyKeyServiceConfig struct { + KeyConfig `yaml:",inline"` + Port int } type Config struct { - Services []Service + Services []ServiceConfig // Deprecated: `keys` exists for backward compatibility. Prefer to configure // using the newer `services` format. - Keys []LegacyKeyService + Keys []LegacyKeyServiceConfig } // readConfig attempts to read a config from a filename and parses it as a [Config]. diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index fce0d8f8..af42c9fe 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -26,24 +26,24 @@ func TestReadConfig(t *testing.T) { require.NoError(t, err) expected := Config{ - Services: []Service{ - Service{ - Listeners: []Listener{ - Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, - Listener{Type: listenerTypeDirect, Address: "udp://[::]:9000"}, + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, + ListenerConfig{Type: listenerTypeDirect, Address: "udp://[::]:9000"}, }, - Keys: []Key{ - Key{"user-0", "chacha20-ietf-poly1305", "Secret0"}, - Key{"user-1", "chacha20-ietf-poly1305", "Secret1"}, + Keys: []KeyConfig{ + KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + KeyConfig{"user-1", "chacha20-ietf-poly1305", "Secret1"}, }, }, - Service{ - Listeners: []Listener{ - Listener{Type: listenerTypeDirect, Address: "tcp://[::]:9001"}, - Listener{Type: listenerTypeDirect, Address: "udp://[::]:9001"}, + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9001"}, + ListenerConfig{Type: listenerTypeDirect, Address: "udp://[::]:9001"}, }, - Keys: []Key{ - Key{"user-2", "chacha20-ietf-poly1305", "Secret2"}, + Keys: []KeyConfig{ + KeyConfig{"user-2", "chacha20-ietf-poly1305", "Secret2"}, }, }, }, @@ -56,18 +56,18 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { require.NoError(t, err) expected := Config{ - Keys: []LegacyKeyService{ - LegacyKeyService{ - Key: Key{ID: "user-0", Cipher: "chacha20-ietf-poly1305", Secret: "Secret0"}, - Port: 9000, + Keys: []LegacyKeyServiceConfig{ + LegacyKeyServiceConfig{ + KeyConfig: KeyConfig{ID: "user-0", Cipher: "chacha20-ietf-poly1305", Secret: "Secret0"}, + Port: 9000, }, - LegacyKeyService{ - Key: Key{ID: "user-1", Cipher: "chacha20-ietf-poly1305", Secret: "Secret1"}, - Port: 9000, + LegacyKeyServiceConfig{ + KeyConfig: KeyConfig{ID: "user-1", Cipher: "chacha20-ietf-poly1305", Secret: "Secret1"}, + Port: 9000, }, - LegacyKeyService{ - Key: Key{ID: "user-2", Cipher: "chacha20-ietf-poly1305", Secret: "Secret2"}, - Port: 9001, + LegacyKeyServiceConfig{ + KeyConfig: KeyConfig{ID: "user-2", Cipher: "chacha20-ietf-poly1305", Secret: "Secret2"}, + Port: 9001, }, }, } From 866003227e23e63aca544aaf3527fc15a81e5be4 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 13:39:23 -0400 Subject: [PATCH 021/182] Remove the IP version specifiers from the `newListener` config handling. --- cmd/outline-ss-server/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 666d45da..95c441c9 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -96,7 +96,7 @@ func validateListener(u *url.URL) error { // // Example addresses: // -// tcp4://127.0.0.1:8000 +// tcp://127.0.0.1:8000 // udp://127.0.0.1:9000 func newListener(addr string) (io.Closer, error) { u, err := url.Parse(addr) @@ -105,12 +105,12 @@ func newListener(addr string) (io.Closer, error) { } switch u.Scheme { - case "tcp", "tcp4", "tcp6": + case "tcp": if err := validateListener(u); err != nil { return nil, fmt.Errorf("invalid listener `%s`: %v", u, err) } return net.Listen(u.Scheme, u.Host) - case "udp", "udp4", "udp6": + case "udp": if err := validateListener(u); err != nil { return nil, fmt.Errorf("invalid listener `%s`: %v", u, err) } From 26b9100320c42ea459e336df8b03db6e99b94da5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 13:58:53 -0400 Subject: [PATCH 022/182] refactor: remove use of port in proving metric --- cmd/outline-ss-server/main.go | 2 +- cmd/outline-ss-server/metrics.go | 5 ++--- cmd/outline-ss-server/metrics_test.go | 4 ++-- service/tcp.go | 12 ++++++------ service/tcp_test.go | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 47b686a9..5f4b2821 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -88,7 +88,7 @@ func (s *SSServer) startPort(portNum int) error { port := &ssPort{tcpListener: listener, packetConn: packetConn, cipherList: service.NewCipherList()} authFunc := service.NewShadowsocksStreamAuthenticator(port.cipherList, &s.replayCache, s.m) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(portNum, authFunc, s.m, tcpReadTimeout) + tcpHandler := service.NewTCPHandler(listener.Addr().String(), authFunc, s.m, tcpReadTimeout) packetHandler := service.NewPacketHandler(s.natTimeout, port.cipherList, s.m) s.ports[portNum] = port accept := func() (transport.StreamConn, error) { diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 531c16ba..e95ceeb3 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -18,7 +18,6 @@ import ( "fmt" "net" "net/netip" - "strconv" "sync" "time" @@ -357,8 +356,8 @@ func (m *outlineMetrics) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string } } -func (m *outlineMetrics) AddTCPProbe(status, drainResult string, port int, clientProxyBytes int64) { - m.tcpProbes.WithLabelValues(strconv.Itoa(port), status, drainResult).Observe(float64(clientProxyBytes)) +func (m *outlineMetrics) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { + m.tcpProbes.WithLabelValues(listenerId, status, drainResult).Observe(float64(clientProxyBytes)) } func (m *outlineMetrics) AddTCPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { diff --git a/cmd/outline-ss-server/metrics_test.go b/cmd/outline-ss-server/metrics_test.go index 353520e4..e2605918 100644 --- a/cmd/outline-ss-server/metrics_test.go +++ b/cmd/outline-ss-server/metrics_test.go @@ -68,7 +68,7 @@ func TestMethodsDontPanic(t *testing.T) { ssMetrics.AddUDPPacketFromTarget(ipInfo, "3", "OK", 10, 20) ssMetrics.AddUDPNatEntry(fakeAddr("127.0.0.1:9"), "key-1") ssMetrics.RemoveUDPNatEntry(fakeAddr("127.0.0.1:9"), "key-1") - ssMetrics.AddTCPProbe("ERR_CIPHER", "eof", 443, proxyMetrics.ClientProxy) + ssMetrics.AddTCPProbe("ERR_CIPHER", "eof", "127.0.0.1:443", proxyMetrics.ClientProxy) ssMetrics.AddTCPCipherSearch(true, 10*time.Millisecond) ssMetrics.AddUDPCipherSearch(true, 10*time.Millisecond) } @@ -168,7 +168,7 @@ func BenchmarkProbe(b *testing.B) { data := metrics.ProxyMetrics{} b.ResetTimer() for i := 0; i < b.N; i++ { - ssMetrics.AddTCPProbe(status, drainResult, port, data.ClientProxy) + ssMetrics.AddTCPProbe(status, drainResult, "127.0.0.1:12345", data.ClientProxy) } } diff --git a/service/tcp.go b/service/tcp.go index 85ab9990..2195480b 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -44,7 +44,7 @@ type TCPMetrics interface { AddOpenTCPConnection(clientInfo ipinfo.IPInfo) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, clientAddr net.Addr, accessKey string, status string, data metrics.ProxyMetrics, duration time.Duration) - AddTCPProbe(status, drainResult string, port int, clientProxyBytes int64) + AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) } func remoteIP(conn net.Conn) netip.Addr { @@ -162,7 +162,7 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa } type tcpHandler struct { - port int + listenerId string m TCPMetrics readTimeout time.Duration authenticate StreamAuthenticateFunc @@ -170,9 +170,9 @@ type tcpHandler struct { } // NewTCPService creates a TCPService -func NewTCPHandler(port int, authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { +func NewTCPHandler(listenerId string, authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { return &tcpHandler{ - port: port, + listenerId: listenerId, m: m, readTimeout: timeout, authenticate: authenticate, @@ -375,7 +375,7 @@ func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, status string, proxyM _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) logger.Debugf("Drain error: %v, drain result: %v", drainErr, drainResult) - h.m.AddTCPProbe(status, drainResult, h.port, proxyMetrics.ClientProxy) + h.m.AddTCPProbe(status, drainResult, h.listenerId, proxyMetrics.ClientProxy) } func drainErrToString(drainErr error) string { @@ -404,6 +404,6 @@ func (m *NoOpTCPMetrics) GetIPInfo(net.IP) (ipinfo.IPInfo, error) { func (m *NoOpTCPMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) {} func (m *NoOpTCPMetrics) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { } -func (m *NoOpTCPMetrics) AddTCPProbe(status, drainResult string, port int, clientProxyBytes int64) { +func (m *NoOpTCPMetrics) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { } func (m *NoOpTCPMetrics) AddTCPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) {} diff --git a/service/tcp_test.go b/service/tcp_test.go index 1a70ed67..2f66a7aa 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -239,7 +239,7 @@ func (m *probeTestMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { func (m *probeTestMetrics) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { } -func (m *probeTestMetrics) AddTCPProbe(status, drainResult string, port int, clientProxyBytes int64) { +func (m *probeTestMetrics) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { m.mu.Lock() m.probeData = append(m.probeData, clientProxyBytes) m.probeStatus = append(m.probeStatus, status) From 1b8e9038625a6cf62e8e630b0866bcb20df8c936 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 14:07:36 -0400 Subject: [PATCH 023/182] Fix tests. --- cmd/outline-ss-server/metrics_test.go | 1 - internal/integration_test/integration_test.go | 8 ++++---- service/tcp_test.go | 16 ++++++++-------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/outline-ss-server/metrics_test.go b/cmd/outline-ss-server/metrics_test.go index e2605918..80e81817 100644 --- a/cmd/outline-ss-server/metrics_test.go +++ b/cmd/outline-ss-server/metrics_test.go @@ -164,7 +164,6 @@ func BenchmarkProbe(b *testing.B) { ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry()) status := "ERR_REPLAY" drainResult := "other" - port := 12345 data := metrics.ProxyMetrics{} b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index 4ca2f120..f98319f4 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -133,7 +133,7 @@ func TestTCPEcho(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -202,7 +202,7 @@ func TestRestrictedAddresses(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { service.StreamServe(service.WrapStreamListener(proxyListener.AcceptTCP), handler.Handle) @@ -384,7 +384,7 @@ func BenchmarkTCPThroughput(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -448,7 +448,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { diff --git a/service/tcp_test.go b/service/tcp_test.go index 2f66a7aa..5c3bc9df 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -281,7 +281,7 @@ func TestProbeRandom(t *testing.T) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) @@ -358,7 +358,7 @@ func TestProbeClientBytesBasicTruncated(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -393,7 +393,7 @@ func TestProbeClientBytesBasicModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -429,7 +429,7 @@ func TestProbeClientBytesCoalescedModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -472,7 +472,7 @@ func TestProbeServerBytesModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) @@ -503,7 +503,7 @@ func TestReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -582,7 +582,7 @@ func TestReverseReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -653,7 +653,7 @@ func probeExpectTimeout(t *testing.T, payloadSize int) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().(*net.TCPAddr).Port, authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { From 4216ce3233825deae5f6079b51e0e91a175abbb8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 14:47:50 -0400 Subject: [PATCH 024/182] Add a TODO comment to allow short-form direct listener config. --- cmd/outline-ss-server/config_example.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/outline-ss-server/config_example.yml b/cmd/outline-ss-server/config_example.yml index 66009c10..f7fc2e71 100644 --- a/cmd/outline-ss-server/config_example.yml +++ b/cmd/outline-ss-server/config_example.yml @@ -1,5 +1,7 @@ services: - listeners: + # TODO(sbruens): Allow a string-based listener config, as a convenient short-form + # to create a direct listener, e.g. `- tcp://[::]:9000`. - type: direct address: "tcp://[::]:9000" - type: direct From 35c828db2fba3a5863075e674a3a404772cf7842 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 15:48:18 -0400 Subject: [PATCH 025/182] Make legacy key config name consistent with type. --- cmd/outline-ss-server/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 066299aa..f7b45dc1 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -138,14 +138,14 @@ func (s *SSServer) loadConfig(filename string) error { addrChanges := make(map[string]int) addrCiphers := make(map[string]*list.List) // Values are *List of *CipherEntry. - for _, legacyKeyConfig := range config.Keys { - cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyConfig.Cipher, legacyKeyConfig.Secret) + for _, legacyKeyServiceConfig := range config.Keys { + cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyServiceConfig.Cipher, legacyKeyServiceConfig.Secret) if err != nil { - return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyConfig.ID, err) + return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyServiceConfig.ID, err) } - entry := service.MakeCipherEntry(legacyKeyConfig.ID, cryptoKey, legacyKeyConfig.Secret) + entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) for _, ln := range []string{"tcp", "udp"} { - addr := fmt.Sprintf("%s://[::]:%d", ln, legacyKeyConfig.Port) + addr := fmt.Sprintf("%s://[::]:%d", ln, legacyKeyServiceConfig.Port) addrChanges[addr] = 1 ciphers, ok := addrCiphers[addr] if !ok { From 1322f2d124c5398d65db626baa6596244d5554f5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 15:59:38 -0400 Subject: [PATCH 026/182] Move config validation out of the `loadConfig` function. --- cmd/outline-ss-server/config.go | 17 +++++++++ cmd/outline-ss-server/config_test.go | 54 ++++++++++++++++++++++++++++ cmd/outline-ss-server/main.go | 20 ++++------- 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 95c441c9..73550454 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -58,6 +58,23 @@ type Config struct { Keys []LegacyKeyServiceConfig } +// Validate checks that the config is valid. +func (c *Config) Validate() error { + for _, serviceConfig := range c.Services { + if serviceConfig.Listeners == nil || serviceConfig.Keys == nil { + return errors.New("must specify at least 1 listener and 1 key per service") + } + + for _, listener := range serviceConfig.Listeners { + // TODO: Support more listener types. + if listener.Type != listenerTypeDirect { + return fmt.Errorf("unsupported listener type: %s", listener.Type) + } + } + } + return nil +} + // readConfig attempts to read a config from a filename and parses it as a [Config]. func readConfig(filename string) (*Config, error) { config := Config{} diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index af42c9fe..d8628f65 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -21,6 +21,60 @@ import ( "github.com/stretchr/testify/require" ) +func TestValidateConfigFails(t *testing.T) { + tests := []struct { + name string + cfg *Config + }{ + { + name: "WithoutListeners", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Keys: []KeyConfig{ + KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + }, + }, + }, + }, + }, + { + name: "WithoutKeys", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, + }, + }, + }, + }, + }, + { + name: "WithUnknownListenerType", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: "foo", Address: "tcp://[::]:9000"}, + }, + Keys: []KeyConfig{ + KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.Validate() + require.Error(t, err) + }) + } +} + func TestReadConfig(t *testing.T) { config, err := readConfig("./config_example.yml") diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index f7b45dc1..b457b990 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -133,7 +133,9 @@ func (s *SSServer) loadConfig(filename string) error { if err != nil { return fmt.Errorf("failed to load config (%v): %w", filename, err) } - + if err := config.Validate(); err != nil { + return err + } uniqueCiphers := 0 addrChanges := make(map[string]int) addrCiphers := make(map[string]*list.List) // Values are *List of *CipherEntry. @@ -158,10 +160,6 @@ func (s *SSServer) loadConfig(filename string) error { } for _, serviceConfig := range config.Services { - if serviceConfig.Listeners == nil || serviceConfig.Keys == nil { - return fmt.Errorf("must specify at least 1 listener and 1 key per service") - } - ciphers := list.New() type cipherKey struct { cipher string @@ -186,14 +184,8 @@ func (s *SSServer) loadConfig(filename string) error { uniqueCiphers += ciphers.Len() for _, listener := range serviceConfig.Listeners { - switch t := listener.Type; t { - // TODO: Support more listener types. - case listenerTypeDirect: - addrChanges[listener.Address] = 1 - addrCiphers[listener.Address] = ciphers - default: - return fmt.Errorf("unsupported listener type: %s", t) - } + addrChanges[listener.Address] = 1 + addrCiphers[listener.Address] = ciphers } } for listener := range s.listeners { @@ -245,7 +237,7 @@ func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, } err := server.loadConfig(filename) if err != nil { - return nil, fmt.Errorf("failed configure server: %w", err) + return nil, fmt.Errorf("failed to configure server: %w", err) } sigHup := make(chan os.Signal, 1) signal.Notify(sigHup, syscall.SIGHUP) From adc11f2b6b3711ea333cec1f65b757e675c8b3be Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 21 Jun 2024 16:12:03 -0400 Subject: [PATCH 027/182] Remove unused port from bad merge. --- service/tcp.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/service/tcp.go b/service/tcp.go index 297ed478..2195480b 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -342,8 +342,7 @@ func (h *tcpHandler) handleConnection(ctx context.Context, outerConn transport.S id, innerConn, authErr := h.authenticate(outerConn) if authErr != nil { // Drain to protect against probing attacks. - port := outerConn.LocalAddr().(*net.TCPAddr).Port - h.absorbProbe(outerConn, port, authErr.Status, proxyMetrics) + h.absorbProbe(outerConn, authErr.Status, proxyMetrics) return id, authErr } h.m.AddAuthenticatedTCPConnection(outerConn.RemoteAddr(), id) @@ -371,7 +370,7 @@ func (h *tcpHandler) handleConnection(ctx context.Context, outerConn transport.S // Keep the connection open until we hit the authentication deadline to protect against probing attacks // `proxyMetrics` is a pointer because its value is being mutated by `clientConn`. -func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, port int, status string, proxyMetrics *metrics.ProxyMetrics) { +func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, status string, proxyMetrics *metrics.ProxyMetrics) { // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) From 3084dfd2be13d8256607644121b193f5a50fa9a2 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 24 Jun 2024 10:57:45 -0400 Subject: [PATCH 028/182] Add comment describing keys. --- cmd/outline-ss-server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index b457b990..ce377e79 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -69,7 +69,7 @@ type SSServer struct { natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache - listeners map[string]*ssListener + listeners map[string]*ssListener // Keys are addresses, e.g. `tcp://[::]:9000` } func (s *SSServer) serve(addr string, listener io.Closer, cipherList service.CipherList) error { From 7e5aae52b16d149057a11014b474d6a803ad1513 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 24 Jun 2024 12:02:21 -0400 Subject: [PATCH 029/182] Move validation of listeners to config's `Validate()` function. --- cmd/outline-ss-server/config.go | 29 +++++++++++++++++----------- cmd/outline-ss-server/config_test.go | 15 ++++++++++++++ cmd/outline-ss-server/main.go | 2 +- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 73550454..ea96d0e8 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -65,10 +65,23 @@ func (c *Config) Validate() error { return errors.New("must specify at least 1 listener and 1 key per service") } - for _, listener := range serviceConfig.Listeners { + for _, listenerConfig := range serviceConfig.Listeners { // TODO: Support more listener types. - if listener.Type != listenerTypeDirect { - return fmt.Errorf("unsupported listener type: %s", listener.Type) + if listenerConfig.Type != listenerTypeDirect { + return fmt.Errorf("unsupported listener type: %s", listenerConfig.Type) + } + + u, err := url.Parse(listenerConfig.Address) + if err != nil { + return err + } + switch u.Scheme { + case "tcp", "udp": + if err := validateListenerAddress(u); err != nil { + return fmt.Errorf("invalid listener address `%s`: %v", u, err) + } + default: + return fmt.Errorf("unsupported protocol: %s", u.Scheme) } } } @@ -89,8 +102,8 @@ func readConfig(filename string) (*Config, error) { return &config, nil } -// validateListener asserts that a listener URI conforms to the expected format. -func validateListener(u *url.URL) error { +// validateListenerAddress asserts that a listener URI conforms to the expected format. +func validateListenerAddress(u *url.URL) error { if u.Opaque != "" { return errors.New("URI cannot have an opaque part") } @@ -123,14 +136,8 @@ func newListener(addr string) (io.Closer, error) { switch u.Scheme { case "tcp": - if err := validateListener(u); err != nil { - return nil, fmt.Errorf("invalid listener `%s`: %v", u, err) - } return net.Listen(u.Scheme, u.Host) case "udp": - if err := validateListener(u); err != nil { - return nil, fmt.Errorf("invalid listener `%s`: %v", u, err) - } return net.ListenPacket(u.Scheme, u.Host) default: return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme) diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index d8628f65..72c11083 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -65,6 +65,21 @@ func TestValidateConfigFails(t *testing.T) { }, }, }, + { + name: "WithInvalidListenerAddress", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9000/path"}, + }, + Keys: []KeyConfig{ + KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + }, + }, + }, + }, + }, } for _, tc := range tests { diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index ce377e79..5e38804d 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -134,7 +134,7 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("failed to load config (%v): %w", filename, err) } if err := config.Validate(); err != nil { - return err + return fmt.Errorf("failed to validate config: %w", err) } uniqueCiphers := 0 addrChanges := make(map[string]int) From b136c79de18d163848c75d39a0da5d90b98b4689 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 25 Jun 2024 12:03:36 -0400 Subject: [PATCH 030/182] Introduce a `NetworkAdd` to centralize parsing and creation of listeners. --- cmd/outline-ss-server/config.go | 58 +------------- cmd/outline-ss-server/config_example.yml | 10 +-- cmd/outline-ss-server/config_test.go | 14 ++-- cmd/outline-ss-server/listeners.go | 99 ++++++++++++++++++++++++ cmd/outline-ss-server/main.go | 10 ++- 5 files changed, 122 insertions(+), 69 deletions(-) create mode 100644 cmd/outline-ss-server/listeners.go diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index ea96d0e8..b970c120 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -17,9 +17,6 @@ package main import ( "errors" "fmt" - "io" - "net" - "net/url" "os" "gopkg.in/yaml.v2" @@ -71,17 +68,12 @@ func (c *Config) Validate() error { return fmt.Errorf("unsupported listener type: %s", listenerConfig.Type) } - u, err := url.Parse(listenerConfig.Address) + network, _, _, err := SplitNetworkAddr(listenerConfig.Address) if err != nil { - return err + return fmt.Errorf("invalid listener address `%s`: %v", listenerConfig.Address, err) } - switch u.Scheme { - case "tcp", "udp": - if err := validateListenerAddress(u); err != nil { - return fmt.Errorf("invalid listener address `%s`: %v", u, err) - } - default: - return fmt.Errorf("unsupported protocol: %s", u.Scheme) + if network != "tcp" && network != "udp" { + return fmt.Errorf("unsupported network: %s", network) } } } @@ -101,45 +93,3 @@ func readConfig(filename string) (*Config, error) { } return &config, nil } - -// validateListenerAddress asserts that a listener URI conforms to the expected format. -func validateListenerAddress(u *url.URL) error { - if u.Opaque != "" { - return errors.New("URI cannot have an opaque part") - } - if u.User != nil { - return errors.New("URI cannot have an userdata part") - } - if u.RawQuery != "" || u.ForceQuery { - return errors.New("URI cannot have a query part") - } - if u.Fragment != "" { - return errors.New("URI cannot have a fragement") - } - if u.Path != "" && u.Path != "/" { - return errors.New("URI path not allowed") - } - return nil -} - -// newListener creates a new listener from a URL-style address specification. -// -// Example addresses: -// -// tcp://127.0.0.1:8000 -// udp://127.0.0.1:9000 -func newListener(addr string) (io.Closer, error) { - u, err := url.Parse(addr) - if err != nil { - return nil, err - } - - switch u.Scheme { - case "tcp": - return net.Listen(u.Scheme, u.Host) - case "udp": - return net.ListenPacket(u.Scheme, u.Host) - default: - return nil, fmt.Errorf("unsupported protocol: %s", u.Scheme) - } -} diff --git a/cmd/outline-ss-server/config_example.yml b/cmd/outline-ss-server/config_example.yml index f7fc2e71..bbfd265f 100644 --- a/cmd/outline-ss-server/config_example.yml +++ b/cmd/outline-ss-server/config_example.yml @@ -1,11 +1,11 @@ services: - listeners: # TODO(sbruens): Allow a string-based listener config, as a convenient short-form - # to create a direct listener, e.g. `- tcp://[::]:9000`. + # to create a direct listener, e.g. `- tcp/[::]:9000`. - type: direct - address: "tcp://[::]:9000" + address: "tcp/[::]:9000" - type: direct - address: "udp://[::]:9000" + address: "udp/[::]:9000" keys: - id: user-0 cipher: chacha20-ietf-poly1305 @@ -16,9 +16,9 @@ services: - listeners: - type: direct - address: "tcp://[::]:9001" + address: "tcp/[::]:9001" - type: direct - address: "udp://[::]:9001" + address: "udp/[::]:9001" keys: - id: user-2 cipher: chacha20-ietf-poly1305 diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 72c11083..3e76fa57 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -44,7 +44,7 @@ func TestValidateConfigFails(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, + ListenerConfig{Type: listenerTypeDirect, Address: "tcp/[::]:9000"}, }, }, }, @@ -56,7 +56,7 @@ func TestValidateConfigFails(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: "foo", Address: "tcp://[::]:9000"}, + ListenerConfig{Type: "foo", Address: "tcp/[::]:9000"}, }, Keys: []KeyConfig{ KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, @@ -71,7 +71,7 @@ func TestValidateConfigFails(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9000/path"}, + ListenerConfig{Type: listenerTypeDirect, Address: "tcp//[::]:9000"}, }, Keys: []KeyConfig{ KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, @@ -98,8 +98,8 @@ func TestReadConfig(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9000"}, - ListenerConfig{Type: listenerTypeDirect, Address: "udp://[::]:9000"}, + ListenerConfig{Type: listenerTypeDirect, Address: "tcp/[::]:9000"}, + ListenerConfig{Type: listenerTypeDirect, Address: "udp/[::]:9000"}, }, Keys: []KeyConfig{ KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, @@ -108,8 +108,8 @@ func TestReadConfig(t *testing.T) { }, ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp://[::]:9001"}, - ListenerConfig{Type: listenerTypeDirect, Address: "udp://[::]:9001"}, + ListenerConfig{Type: listenerTypeDirect, Address: "tcp/[::]:9001"}, + ListenerConfig{Type: listenerTypeDirect, Address: "udp/[::]:9001"}, }, Keys: []KeyConfig{ KeyConfig{"user-2", "chacha20-ietf-poly1305", "Secret2"}, diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go new file mode 100644 index 00000000..09f24a10 --- /dev/null +++ b/cmd/outline-ss-server/listeners.go @@ -0,0 +1,99 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" +) + +type NetworkAddr struct { + network string + Host string + Port uint +} + +// String returns a human-readable representation of the [NetworkAddr]. +func (na *NetworkAddr) Network() string { + return na.network +} + +// String returns a human-readable representation of the [NetworkAddr]. +func (na *NetworkAddr) String() string { + return na.JoinHostPort() +} + +// JoinHostPort is a convenience wrapper around [net.JoinHostPort]. +func (na *NetworkAddr) JoinHostPort() string { + return net.JoinHostPort(na.Host, strconv.Itoa(int(na.Port))) +} + +// Listen creates a new listener for the [NetworkAddr]. +func (na *NetworkAddr) Listen() (io.Closer, error) { + address := na.JoinHostPort() + + switch na.network { + + case "tcp": + return net.Listen(na.network, address) + case "udp": + return net.ListenPacket(na.network, address) + default: + return nil, fmt.Errorf("unsupported network: %s", na.network) + } +} + +// ParseNetworkAddr parses an address into a [NetworkAddr]. The input +// string is expected to be of the form "network/host:port" where any part is +// optional. +// +// Examples: +// +// tcp/127.0.0.1:8000 +// udp/127.0.0.1:9000 +func ParseNetworkAddr(addr string) (NetworkAddr, error) { + var host, port string + network, host, port, err := SplitNetworkAddr(addr) + if err != nil { + return NetworkAddr{}, err + } + if network == "" { + return NetworkAddr{}, errors.New("missing network") + } + p, err := strconv.ParseUint(port, 10, 16) + if err != nil { + return NetworkAddr{}, fmt.Errorf("invalid port: %v", err) + } + return NetworkAddr{ + network: network, + Host: host, + Port: uint(p), + }, nil +} + +// SplitNetworkAddr splits a into its network, host, and port components. +func SplitNetworkAddr(a string) (network, host, port string, err error) { + beforeSlash, afterSlash, slashFound := strings.Cut(a, "/") + if slashFound { + network = strings.ToLower(strings.TrimSpace(beforeSlash)) + a = afterSlash + } + host, port, err = net.SplitHostPort(a) + return +} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 5e38804d..f6b0f8cf 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -69,7 +69,7 @@ type SSServer struct { natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache - listeners map[string]*ssListener // Keys are addresses, e.g. `tcp://[::]:9000` + listeners map[string]*ssListener // Keys are addresses, e.g. `tcp/[::]:9000` } func (s *SSServer) serve(addr string, listener io.Closer, cipherList service.CipherList) error { @@ -98,7 +98,11 @@ func (s *SSServer) serve(addr string, listener io.Closer, cipherList service.Cip } func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, error) { - listener, err := newListener(addr) + listenAddr, err := ParseNetworkAddr(addr) + if err != nil { + return nil, fmt.Errorf("error parsing listener address `%s`: %v", addr, err) + } + listener, err := listenAddr.Listen() if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return nil, fmt.Errorf("Shadowsocks service failed to start on address %v: %w", addr, err) @@ -147,7 +151,7 @@ func (s *SSServer) loadConfig(filename string) error { } entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) for _, ln := range []string{"tcp", "udp"} { - addr := fmt.Sprintf("%s://[::]:%d", ln, legacyKeyServiceConfig.Port) + addr := fmt.Sprintf("%s/[::]:%d", ln, legacyKeyServiceConfig.Port) addrChanges[addr] = 1 ciphers, ok := addrCiphers[addr] if !ok { From 4bf9c272dd792e9638d37ebd6718f70b1d3b9bbf Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 24 Jun 2024 15:17:57 -0400 Subject: [PATCH 031/182] Use `net.ListenConfig` to listen. --- cmd/outline-ss-server/listeners.go | 7 ++++--- cmd/outline-ss-server/main.go | 12 ++++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index 09f24a10..a5c9c8b0 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -15,6 +15,7 @@ package main import ( + "context" "errors" "fmt" "io" @@ -45,15 +46,15 @@ func (na *NetworkAddr) JoinHostPort() string { } // Listen creates a new listener for the [NetworkAddr]. -func (na *NetworkAddr) Listen() (io.Closer, error) { +func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (io.Closer, error) { address := na.JoinHostPort() switch na.network { case "tcp": - return net.Listen(na.network, address) + return config.Listen(ctx, na.network, address) case "udp": - return net.ListenPacket(na.network, address) + return config.ListenPacket(ctx, na.network, address) default: return nil, fmt.Errorf("unsupported network: %s", na.network) } diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index f6b0f8cf..ca019810 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -16,6 +16,7 @@ package main import ( "container/list" + "context" "flag" "fmt" "io" @@ -79,13 +80,8 @@ func (s *SSServer) serve(addr string, listener io.Closer, cipherList service.Cip // TODO: Register initial data metrics at zero. tcpHandler := service.NewTCPHandler(addr, authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { - conn, err := ln.Accept() - if err != nil { - return nil, err - } - c := conn.(*net.TCPConn) - c.SetKeepAlive(true) - return c, err + c, err := ln.Accept() + return c.(transport.StreamConn), err } go service.StreamServe(accept, tcpHandler.Handle) case net.PacketConn: @@ -102,7 +98,7 @@ func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, if err != nil { return nil, fmt.Errorf("error parsing listener address `%s`: %v", addr, err) } - listener, err := listenAddr.Listen() + listener, err := listenAddr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return nil, fmt.Errorf("Shadowsocks service failed to start on address %v: %w", addr, err) From b7bb65bf605a4a9e24c51d7174a5f7a318b0d96e Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 25 Jun 2024 15:14:11 -0400 Subject: [PATCH 032/182] Simplify how we create new listeners. This does not yet deal with reused sockets. --- cmd/outline-ss-server/main.go | 128 ++++++++++++++-------------------- 1 file changed, 52 insertions(+), 76 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index ca019810..6d48a628 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -61,24 +61,19 @@ func init() { logger = logging.MustGetLogger("") } -type ssListener struct { - io.Closer - cipherList service.CipherList -} - type SSServer struct { natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache - listeners map[string]*ssListener // Keys are addresses, e.g. `tcp/[::]:9000` + listeners []io.Closer } -func (s *SSServer) serve(addr string, listener io.Closer, cipherList service.CipherList) error { +func (s *SSServer) serve(addr NetworkAddr, listener io.Closer, cipherList service.CipherList) error { switch ln := listener.(type) { case net.Listener: authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &s.replayCache, s.m) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(addr, authFunc, s.m, tcpReadTimeout) + tcpHandler := service.NewTCPHandler(addr.String(), authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { c, err := ln.Accept() return c.(transport.StreamConn), err @@ -93,41 +88,6 @@ func (s *SSServer) serve(addr string, listener io.Closer, cipherList service.Cip return nil } -func (s *SSServer) start(addr string, cipherList service.CipherList) (io.Closer, error) { - listenAddr, err := ParseNetworkAddr(addr) - if err != nil { - return nil, fmt.Errorf("error parsing listener address `%s`: %v", addr, err) - } - listener, err := listenAddr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return nil, fmt.Errorf("Shadowsocks service failed to start on address %v: %w", addr, err) - } - logger.Infof("Shadowsocks service listening on %v", addr) - - err = s.serve(addr, listener, cipherList) - if err != nil { - return nil, fmt.Errorf("failed to serve on listener %v: %w", listener, err) - } - - return listener, nil -} - -func (s *SSServer) remove(addr string) error { - listener, ok := s.listeners[addr] - if !ok { - return fmt.Errorf("address %v doesn't exist", addr) - } - err := listener.Close() - delete(s.listeners, addr) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks service on address %v failed to stop: %w", addr, err) - } - logger.Infof("Shadowsocks service on address %v stopped", addr) - return nil -} - func (s *SSServer) loadConfig(filename string) error { config, err := readConfig(filename) if err != nil { @@ -137,8 +97,9 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("failed to validate config: %w", err) } uniqueCiphers := 0 - addrChanges := make(map[string]int) - addrCiphers := make(map[string]*list.List) // Values are *List of *CipherEntry. + // TODO: Clone existing listeners so we can close them after starting the new ones. + addrs := make([]NetworkAddr, 0) + addrCiphers := make(map[NetworkAddr]*list.List) // Values are *List of *CipherEntry. for _, legacyKeyServiceConfig := range config.Keys { cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyServiceConfig.Cipher, legacyKeyServiceConfig.Secret) @@ -146,9 +107,13 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyServiceConfig.ID, err) } entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) - for _, ln := range []string{"tcp", "udp"} { - addr := fmt.Sprintf("%s/[::]:%d", ln, legacyKeyServiceConfig.Port) - addrChanges[addr] = 1 + for _, network := range []string{"tcp", "udp"} { + addr := NetworkAddr{ + network: network, + Host: "::", + Port: uint(legacyKeyServiceConfig.Port), + } + addrs = append(addrs, addr) ciphers, ok := addrCiphers[addr] if !ok { ciphers = list.New() @@ -168,8 +133,7 @@ func (s *SSServer) loadConfig(filename string) error { existingCiphers := make(map[cipherKey]bool) for _, keyConfig := range serviceConfig.Keys { key := cipherKey{keyConfig.Cipher, keyConfig.Secret} - _, ok := existingCiphers[key] - if ok { + if _, exists := existingCiphers[key]; exists { logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) continue } @@ -184,45 +148,57 @@ func (s *SSServer) loadConfig(filename string) error { uniqueCiphers += ciphers.Len() for _, listener := range serviceConfig.Listeners { - addrChanges[listener.Address] = 1 - addrCiphers[listener.Address] = ciphers - } - } - for listener := range s.listeners { - addrChanges[listener] = addrChanges[listener] - 1 - } - for addr, count := range addrChanges { - if count == -1 { - if err := s.remove(addr); err != nil { - return fmt.Errorf("failed to remove address %v: %w", addr, err) - } - } else if count == +1 { - cipherList := service.NewCipherList() - listener, err := s.start(addr, cipherList) + addr, err := ParseNetworkAddr(listener.Address) if err != nil { - return err + return fmt.Errorf("error parsing listener address `%s`: %v", listener.Address, err) } - s.listeners[addr] = &ssListener{Closer: listener, cipherList: cipherList} + addrs = append(addrs, addr) + addrCiphers[addr] = ciphers } } - for addr, ciphers := range addrCiphers { - listener, ok := s.listeners[addr] + + for _, addr := range addrs { + cipherList := service.NewCipherList() + ciphers, ok := addrCiphers[addr] if !ok { - return fmt.Errorf("unable to find listener for address: %v", addr) + return fmt.Errorf("unable to find ciphers for address: %v", addr) + } + cipherList.Update(ciphers) + + listener, err := addr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) + if err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", addr.Network(), addr.String(), err) + } + logger.Infof("Shadowsocks %s service listening on %s", addr.Network(), addr.String()) + s.listeners = append(s.listeners, listener) + + if err = s.serve(addr, listener, cipherList); err != nil { + return fmt.Errorf("failed to serve on listener %v: %w", listener, err) } - listener.cipherList.Update(ciphers) } logger.Infof("Loaded %v access keys over %v listeners", uniqueCiphers, len(s.listeners)) s.m.SetNumAccessKeys(uniqueCiphers, len(s.listeners)) return nil } -// Stop serving on all ports. +// Stop serving on all listeners. func (s *SSServer) Stop() error { - for addr := range s.listeners { - if err := s.remove(addr); err != nil { - return err + for _, listener := range s.listeners { + var addr net.Addr + switch ln := listener.(type) { + case net.Listener: + addr = ln.Addr() + case net.PacketConn: + addr = ln.LocalAddr() + default: + return fmt.Errorf("unknown listener type: %v", ln) + } + if err := listener.Close(); err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks service on address %s %s failed to stop: %w", addr.Network(), addr.String(), err) } + logger.Infof("Shadowsocks service on address %s %s stopped", addr.Network(), addr.String()) } return nil } @@ -233,7 +209,7 @@ func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, natTimeout: natTimeout, m: sm, replayCache: service.NewReplayCache(replayHistory), - listeners: make(map[string]*ssListener), + listeners: nil, } err := server.loadConfig(filename) if err != nil { From af3ca3155f8335159c7e11dbe167928675a01fcb Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 28 Jun 2024 16:47:33 -0400 Subject: [PATCH 033/182] Do not use `io.Closer`. --- cmd/outline-ss-server/listeners.go | 9 ++++++-- cmd/outline-ss-server/main.go | 36 ++++++++++++++++++------------ service/tcp.go | 2 +- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index a5c9c8b0..3457b794 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -18,7 +18,6 @@ import ( "context" "errors" "fmt" - "io" "net" "strconv" "strings" @@ -45,8 +44,14 @@ func (na *NetworkAddr) JoinHostPort() string { return net.JoinHostPort(na.Host, strconv.Itoa(int(na.Port))) } +// Key returns a representative string useful to retrieve this entity from a +// map. This is used to uniquely identify reusable listeners. +func (na *NetworkAddr) Key() string { + return na.network + "/" + na.JoinHostPort() +} + // Listen creates a new listener for the [NetworkAddr]. -func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (io.Closer, error) { +func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (Listener, error) { address := na.JoinHostPort() switch na.network { diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 6d48a628..cd96b7ea 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -19,7 +19,6 @@ import ( "context" "flag" "fmt" - "io" "net" "net/http" "os" @@ -65,15 +64,22 @@ type SSServer struct { natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache - listeners []io.Closer + listeners []Listener } -func (s *SSServer) serve(addr NetworkAddr, listener io.Closer, cipherList service.CipherList) error { +// The implementations of listeners for different network types are not +// interchangeable. The type of listener depends on the network type. +// TODO(sbruens): Create a custom `Listener` type so we can share serving logic, +// dispatching to the handlers based on connection type instead of on the +// listener type. +type Listener = any + +func (s *SSServer) serve(addr NetworkAddr, listener Listener, cipherList service.CipherList) error { switch ln := listener.(type) { case net.Listener: authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &s.replayCache, s.m) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(addr.String(), authFunc, s.m, tcpReadTimeout) + tcpHandler := service.NewTCPHandler(addr.Key(), authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { c, err := ln.Accept() return c.(transport.StreamConn), err @@ -161,7 +167,7 @@ func (s *SSServer) loadConfig(filename string) error { cipherList := service.NewCipherList() ciphers, ok := addrCiphers[addr] if !ok { - return fmt.Errorf("unable to find ciphers for address: %v", addr) + return fmt.Errorf("unable to find ciphers for address: %v", addr.Key()) } cipherList.Update(ciphers) @@ -174,7 +180,7 @@ func (s *SSServer) loadConfig(filename string) error { s.listeners = append(s.listeners, listener) if err = s.serve(addr, listener, cipherList); err != nil { - return fmt.Errorf("failed to serve on listener %v: %w", listener, err) + return fmt.Errorf("failed to serve on %s listener on address %s: %w", addr.Network(), addr.String(), err) } } logger.Infof("Loaded %v access keys over %v listeners", uniqueCiphers, len(s.listeners)) @@ -185,20 +191,22 @@ func (s *SSServer) loadConfig(filename string) error { // Stop serving on all listeners. func (s *SSServer) Stop() error { for _, listener := range s.listeners { - var addr net.Addr switch ln := listener.(type) { case net.Listener: - addr = ln.Addr() + err := ln.Close() + if err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) + } case net.PacketConn: - addr = ln.LocalAddr() + err := ln.Close() + if err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) + } default: return fmt.Errorf("unknown listener type: %v", ln) } - if err := listener.Close(); err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks service on address %s %s failed to stop: %w", addr.Network(), addr.String(), err) - } - logger.Infof("Shadowsocks service on address %s %s stopped", addr.Network(), addr.String()) } return nil } diff --git a/service/tcp.go b/service/tcp.go index 2195480b..ced85a54 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -236,7 +236,7 @@ func StreamServe(accept StreamListener, handle StreamHandler) { if errors.Is(err, net.ErrClosed) { break } - logger.Warningf("AcceptTCP failed: %v. Continuing to listen.", err) + logger.Warningf("Accept failed: %v. Continuing to listen.", err) continue } From fc725937c549d9341fae774edac7aa55ae0e54f7 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 1 Jul 2024 15:08:33 -0400 Subject: [PATCH 034/182] Use an inline error check. --- cmd/outline-ss-server/main.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index cd96b7ea..097d3653 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -193,14 +193,12 @@ func (s *SSServer) Stop() error { for _, listener := range s.listeners { switch ln := listener.(type) { case net.Listener: - err := ln.Close() - if err != nil { + if err := ln.Close(); err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) } case net.PacketConn: - err := ln.Close() - if err != nil { + if err := ln.Close(); err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) } From b24a3390d5e7e5c3544c32ec365b93c6f9744f9f Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 1 Jul 2024 14:49:14 -0400 Subject: [PATCH 035/182] Use shared listeners and packet connections. This allows us to reload a config while the existing one is still running. They share the same underlying listener, which is actually closed when the last user closes it. --- cmd/outline-ss-server/listeners.go | 195 ++++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 5 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index 3457b794..d819587e 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -21,8 +21,133 @@ import ( "net" "strconv" "strings" + "sync" + "sync/atomic" + "time" ) +var ( + listeners = make(map[string]*globalListener) + listenersMu sync.Mutex +) + +type sharedListener struct { + net.Listener + key string + closed atomic.Int32 + usage *atomic.Int32 + deadline *bool + deadlineMu *sync.Mutex +} + +// Accept accepts connections until Close() is called. +func (sl *sharedListener) Accept() (net.Conn, error) { + if sl.closed.Load() == 1 { + return nil, &net.OpError{ + Op: "accept", + Net: sl.Listener.Addr().Network(), + Addr: sl.Listener.Addr(), + Err: fmt.Errorf("listener closed"), + } + } + + conn, err := sl.Listener.Accept() + if err == nil { + return conn, nil + } + + sl.deadlineMu.Lock() + if *sl.deadline { + switch ln := sl.Listener.(type) { + case *net.TCPListener: + ln.SetDeadline(time.Time{}) + } + *sl.deadline = false + } + sl.deadlineMu.Unlock() + + if sl.closed.Load() == 1 { + // In `Close()` we set a deadline in the past to force currently-blocked + // listeners to close without having to close the underlying socket. To + // avoid callers from retrying, we avoid returning timeout errors and + // instead make sure we return a fake "closed" error. + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + return nil, &net.OpError{ + Op: "accept", + Net: sl.Listener.Addr().Network(), + Addr: sl.Listener.Addr(), + Err: fmt.Errorf("listener closed"), + } + } + } + + return nil, err +} + +// Close stops accepting new connections without closing the underlying socket. +// Only when the last user closes it, we actually close it. +func (sl *sharedListener) Close() error { + if sl.closed.CompareAndSwap(0, 1) { + // NOTE: In order to cancel current calls to Accept(), we set a deadline in + // the past, as we cannot actually close the listener. + sl.deadlineMu.Lock() + if !*sl.deadline { + switch ln := sl.Listener.(type) { + case *net.TCPListener: + ln.SetDeadline(time.Now().Add(-1 * time.Minute)) + } + *sl.deadline = true + } + sl.deadlineMu.Unlock() + + // See if we need to actually close the underlying listener. + if sl.usage.Add(-1) == 0 { + listenersMu.Lock() + delete(listeners, sl.key) + listenersMu.Unlock() + err := sl.Listener.Close() + if err != nil { + return err + } + } + + } + + return nil +} + +type sharedPacketConn struct { + net.PacketConn + key string + closed atomic.Int32 + usage *atomic.Int32 +} + +func (spc *sharedPacketConn) Close() error { + if spc.closed.CompareAndSwap(0, 1) { + // See if we need to actually close the underlying listener. + if spc.usage.Add(-1) == 0 { + listenersMu.Lock() + delete(listeners, spc.key) + listenersMu.Unlock() + err := spc.PacketConn.Close() + if err != nil { + return err + } + } + } + + return nil +} + +type globalListener struct { + ln net.Listener + pc net.PacketConn + usage atomic.Int32 + deadline bool + deadlineMu sync.Mutex +} + type NetworkAddr struct { network string Host string @@ -51,17 +176,77 @@ func (na *NetworkAddr) Key() string { } // Listen creates a new listener for the [NetworkAddr]. -func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (Listener, error) { - address := na.JoinHostPort() - +// +// Listeners can overlap one another, because during config changes the new +// config is started before the old config is destroyed. This is done by using +// reusable listener wrappers, which do not actually close the underlying socket +// until all uses of the shared listener have been closed. +func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (any, error) { switch na.network { case "tcp": - return config.Listen(ctx, na.network, address) + listenersMu.Lock() + defer listenersMu.Unlock() + + if lnGlobal, ok := listeners[na.Key()]; ok { + lnGlobal.usage.Add(1) + return &sharedListener{ + usage: &lnGlobal.usage, + deadline: &lnGlobal.deadline, + deadlineMu: &lnGlobal.deadlineMu, + key: na.Key(), + Listener: lnGlobal.ln, + }, nil + } + + ln, err := config.Listen(ctx, na.network, na.JoinHostPort()) + if err != nil { + return nil, err + } + + lnGlobal := &globalListener{ln: ln} + lnGlobal.usage.Store(1) + listeners[na.Key()] = lnGlobal + + return &sharedListener{ + usage: &lnGlobal.usage, + deadline: &lnGlobal.deadline, + deadlineMu: &lnGlobal.deadlineMu, + key: na.Key(), + Listener: ln, + }, nil + case "udp": - return config.ListenPacket(ctx, na.network, address) + listenersMu.Lock() + defer listenersMu.Unlock() + + if lnGlobal, ok := listeners[na.Key()]; ok { + lnGlobal.usage.Add(1) + return &sharedPacketConn{ + usage: &lnGlobal.usage, + key: na.Key(), + PacketConn: lnGlobal.pc, + }, nil + } + + pc, err := config.ListenPacket(ctx, na.network, na.JoinHostPort()) + if err != nil { + return nil, err + } + + lnGlobal := &globalListener{pc: pc} + lnGlobal.usage.Store(1) + listeners[na.Key()] = lnGlobal + + return &sharedPacketConn{ + usage: &lnGlobal.usage, + key: na.Key(), + PacketConn: pc, + }, nil + default: return nil, fmt.Errorf("unsupported network: %s", na.network) + } } From 3bc76bc5a11d72d01e506c925684b4ddcc89afee Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 1 Jul 2024 16:23:19 -0400 Subject: [PATCH 036/182] Close existing listeners once the new ones are serving. --- cmd/outline-ss-server/listeners.go | 4 ++-- cmd/outline-ss-server/main.go | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index d819587e..fc83b5d7 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -47,7 +47,7 @@ func (sl *sharedListener) Accept() (net.Conn, error) { Op: "accept", Net: sl.Listener.Addr().Network(), Addr: sl.Listener.Addr(), - Err: fmt.Errorf("listener closed"), + Err: net.ErrClosed, } } @@ -76,7 +76,7 @@ func (sl *sharedListener) Accept() (net.Conn, error) { Op: "accept", Net: sl.Listener.Addr().Network(), Addr: sl.Listener.Addr(), - Err: fmt.Errorf("listener closed"), + Err: net.ErrClosed, } } } diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 097d3653..2d3ac56a 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -82,7 +82,10 @@ func (s *SSServer) serve(addr NetworkAddr, listener Listener, cipherList service tcpHandler := service.NewTCPHandler(addr.Key(), authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { c, err := ln.Accept() - return c.(transport.StreamConn), err + if err == nil { + return c.(transport.StreamConn), err + } + return nil, err } go service.StreamServe(accept, tcpHandler.Handle) case net.PacketConn: @@ -102,8 +105,8 @@ func (s *SSServer) loadConfig(filename string) error { if err := config.Validate(); err != nil { return fmt.Errorf("failed to validate config: %w", err) } + uniqueCiphers := 0 - // TODO: Clone existing listeners so we can close them after starting the new ones. addrs := make([]NetworkAddr, 0) addrCiphers := make(map[NetworkAddr]*list.List) // Values are *List of *CipherEntry. @@ -163,6 +166,8 @@ func (s *SSServer) loadConfig(filename string) error { } } + // Create new listeners based on the configured network addresses. + newListeners := make([]Listener, 0) for _, addr := range addrs { cipherList := service.NewCipherList() ciphers, ok := addrCiphers[addr] @@ -177,18 +182,25 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", addr.Network(), addr.String(), err) } logger.Infof("Shadowsocks %s service listening on %s", addr.Network(), addr.String()) - s.listeners = append(s.listeners, listener) + newListeners = append(newListeners, listener) if err = s.serve(addr, listener, cipherList); err != nil { return fmt.Errorf("failed to serve on %s listener on address %s: %w", addr.Network(), addr.String(), err) } } + + // Take down the old listeners now that the new ones are serving. + if err := s.Stop(); err != nil { + logger.Warningf("Failed to stop old listeners: %w", err) + } + s.listeners = newListeners + logger.Infof("Loaded %v access keys over %v listeners", uniqueCiphers, len(s.listeners)) s.m.SetNumAccessKeys(uniqueCiphers, len(s.listeners)) return nil } -// Stop serving on all listeners. +// Stop serving on all existing listeners. func (s *SSServer) Stop() error { for _, listener := range s.listeners { switch ln := listener.(type) { From f71b13d6bbcfaa680714b2a4924801c69c955b77 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 1 Jul 2024 16:56:46 -0400 Subject: [PATCH 037/182] Elevate failure to stop listeners to `ERROR` level. --- cmd/outline-ss-server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 2d3ac56a..9b3d4d65 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -191,7 +191,7 @@ func (s *SSServer) loadConfig(filename string) error { // Take down the old listeners now that the new ones are serving. if err := s.Stop(); err != nil { - logger.Warningf("Failed to stop old listeners: %w", err) + logger.Errorf("Failed to stop old listeners: %w", err) } s.listeners = newListeners From 32cc180b2c50bc3755f4d83c6b11c6dfa5de324f Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 2 Jul 2024 10:30:56 -0400 Subject: [PATCH 038/182] Be more lenient in config validation to allow empty listeners or keys. --- cmd/outline-ss-server/config.go | 5 ----- cmd/outline-ss-server/config_test.go | 30 ++++++---------------------- cmd/outline-ss-server/main.go | 1 + 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index b970c120..d101bf26 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -15,7 +15,6 @@ package main import ( - "errors" "fmt" "os" @@ -58,10 +57,6 @@ type Config struct { // Validate checks that the config is valid. func (c *Config) Validate() error { for _, serviceConfig := range c.Services { - if serviceConfig.Listeners == nil || serviceConfig.Keys == nil { - return errors.New("must specify at least 1 listener and 1 key per service") - } - for _, listenerConfig := range serviceConfig.Listeners { // TODO: Support more listener types. if listenerConfig.Type != listenerTypeDirect { diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 3e76fa57..463de266 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -27,54 +27,36 @@ func TestValidateConfigFails(t *testing.T) { cfg *Config }{ { - name: "WithoutListeners", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Keys: []KeyConfig{ - KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, - }, - }, - }, - }, - }, - { - name: "WithoutKeys", + name: "WithUnknownListenerType", cfg: &Config{ Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp/[::]:9000"}, + ListenerConfig{Type: "foo", Address: "tcp/[::]:9000"}, }, }, }, }, }, { - name: "WithUnknownListenerType", + name: "WithInvalidListenerAddress", cfg: &Config{ Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: "foo", Address: "tcp/[::]:9000"}, - }, - Keys: []KeyConfig{ - KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + ListenerConfig{Type: listenerTypeDirect, Address: "tcp//[::]:9000"}, }, }, }, }, }, { - name: "WithInvalidListenerAddress", + name: "WithUnsupportedNetworkType", cfg: &Config{ Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp//[::]:9000"}, - }, - Keys: []KeyConfig{ - KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + ListenerConfig{Type: listenerTypeDirect, Address: "foo/[::]:9000"}, }, }, }, diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index b9e70497..9b3d4d65 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -27,6 +27,7 @@ import ( "syscall" "time" + "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" From 640f80f0089e26f364f9eab8b9d97068c4d44931 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 3 Jul 2024 17:16:14 -0400 Subject: [PATCH 039/182] Ensure the address is an IP address. --- cmd/outline-ss-server/config.go | 6 +++++- cmd/outline-ss-server/config_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index d101bf26..2341f4a5 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -16,6 +16,7 @@ package main import ( "fmt" + "net" "os" "gopkg.in/yaml.v2" @@ -63,13 +64,16 @@ func (c *Config) Validate() error { return fmt.Errorf("unsupported listener type: %s", listenerConfig.Type) } - network, _, _, err := SplitNetworkAddr(listenerConfig.Address) + network, host, _, err := SplitNetworkAddr(listenerConfig.Address) if err != nil { return fmt.Errorf("invalid listener address `%s`: %v", listenerConfig.Address, err) } if network != "tcp" && network != "udp" { return fmt.Errorf("unsupported network: %s", network) } + if ip := net.ParseIP(host); ip == nil { + return fmt.Errorf("address must be IP, found: %s", host) + } } } return nil diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 463de266..86d20afb 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -62,6 +62,18 @@ func TestValidateConfigFails(t *testing.T) { }, }, }, + { + name: "WithHostnameAddress", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeDirect, Address: "tcp/example.com:9000"}, + }, + }, + }, + }, + }, } for _, tc := range tests { From 22638c7370442990491d9ecb639714674befb250 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 3 Jul 2024 17:18:29 -0400 Subject: [PATCH 040/182] Use `yaml.v3`. --- cmd/outline-ss-server/config.go | 2 +- go.mod | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 2341f4a5..d678679e 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -19,7 +19,7 @@ import ( "net" "os" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type ServiceConfig struct { diff --git a/go.mod b/go.mod index 04a9ddab..5c1419d2 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/stretchr/testify v1.8.4 golang.org/x/crypto v0.17.0 golang.org/x/term v0.16.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -263,7 +263,7 @@ require ( gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect gopkg.in/src-d/go-git.v4 v4.13.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.90.0 // indirect mvdan.cc/sh/v3 v3.7.0 // indirect sigs.k8s.io/kind v0.17.0 // indirect From 2631b872ccd9d3e1d36390e76ca8f0725c0a80f4 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 12:41:14 -0400 Subject: [PATCH 041/182] Move file reading back to `main.go`. --- cmd/outline-ss-server/config.go | 10 ++-------- cmd/outline-ss-server/config_test.go | 20 +++++++++----------- cmd/outline-ss-server/main.go | 6 +++++- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index d678679e..5929f202 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -17,7 +17,6 @@ package main import ( "fmt" "net" - "os" "gopkg.in/yaml.v3" ) @@ -80,14 +79,9 @@ func (c *Config) Validate() error { } // readConfig attempts to read a config from a filename and parses it as a [Config]. -func readConfig(filename string) (*Config, error) { +func readConfig(configData []byte) (*Config, error) { config := Config{} - configData, err := os.ReadFile(filename) - if err != nil { - return nil, fmt.Errorf("failed to read config: %w", err) - } - err = yaml.Unmarshal(configData, &config) - if err != nil { + if err := yaml.Unmarshal(configData, &config); err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) } return &config, nil diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 86d20afb..3a8b6cf8 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -85,7 +85,7 @@ func TestValidateConfigFails(t *testing.T) { } func TestReadConfig(t *testing.T) { - config, err := readConfig("./config_example.yml") + config, err := readConfigFile("./config_example.yml") require.NoError(t, err) expected := Config{ @@ -115,7 +115,7 @@ func TestReadConfig(t *testing.T) { } func TestReadConfigParsesDeprecatedFormat(t *testing.T) { - config, err := readConfig("./config_example.deprecated.yml") + config, err := readConfigFile("./config_example.deprecated.yml") require.NoError(t, err) expected := Config{ @@ -140,25 +140,23 @@ func TestReadConfigParsesDeprecatedFormat(t *testing.T) { func TestReadConfigFromEmptyFile(t *testing.T) { file, _ := os.CreateTemp("", "empty.yaml") - config, err := readConfig(file.Name()) + config, err := readConfigFile(file.Name()) require.NoError(t, err) require.ElementsMatch(t, Config{}, config) } -func TestReadConfigFromNonExistingFileFails(t *testing.T) { - config, err := readConfig("./foo") - - require.Error(t, err) - require.ElementsMatch(t, nil, config) -} - func TestReadConfigFromIncorrectFormatFails(t *testing.T) { file, _ := os.CreateTemp("", "empty.yaml") file.WriteString("foo") - config, err := readConfig(file.Name()) + config, err := readConfigFile(file.Name()) require.Error(t, err) require.ElementsMatch(t, Config{}, config) } + +func readConfigFile(filename string) (*Config, error) { + configData, _ := os.ReadFile(filename) + return readConfig(configData) +} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 9b3d4d65..1deb3454 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -98,7 +98,11 @@ func (s *SSServer) serve(addr NetworkAddr, listener Listener, cipherList service } func (s *SSServer) loadConfig(filename string) error { - config, err := readConfig(filename) + configData, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read config file %s: %w", filename, err) + } + config, err := readConfig(configData) if err != nil { return fmt.Errorf("failed to load config (%v): %w", filename, err) } From d76efd2f7a6a5a23e4cc465657853d99888c04aa Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 13:25:20 -0400 Subject: [PATCH 042/182] Do not embed the `net.Listener` type. --- cmd/outline-ss-server/listeners.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index fc83b5d7..fefa6920 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -32,7 +32,7 @@ var ( ) type sharedListener struct { - net.Listener + listener net.Listener key string closed atomic.Int32 usage *atomic.Int32 @@ -45,20 +45,20 @@ func (sl *sharedListener) Accept() (net.Conn, error) { if sl.closed.Load() == 1 { return nil, &net.OpError{ Op: "accept", - Net: sl.Listener.Addr().Network(), - Addr: sl.Listener.Addr(), + Net: sl.listener.Addr().Network(), + Addr: sl.listener.Addr(), Err: net.ErrClosed, } } - conn, err := sl.Listener.Accept() + conn, err := sl.listener.Accept() if err == nil { return conn, nil } sl.deadlineMu.Lock() if *sl.deadline { - switch ln := sl.Listener.(type) { + switch ln := sl.listener.(type) { case *net.TCPListener: ln.SetDeadline(time.Time{}) } @@ -74,8 +74,8 @@ func (sl *sharedListener) Accept() (net.Conn, error) { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { return nil, &net.OpError{ Op: "accept", - Net: sl.Listener.Addr().Network(), - Addr: sl.Listener.Addr(), + Net: sl.listener.Addr().Network(), + Addr: sl.listener.Addr(), Err: net.ErrClosed, } } @@ -92,7 +92,7 @@ func (sl *sharedListener) Close() error { // the past, as we cannot actually close the listener. sl.deadlineMu.Lock() if !*sl.deadline { - switch ln := sl.Listener.(type) { + switch ln := sl.listener.(type) { case *net.TCPListener: ln.SetDeadline(time.Now().Add(-1 * time.Minute)) } @@ -105,7 +105,7 @@ func (sl *sharedListener) Close() error { listenersMu.Lock() delete(listeners, sl.key) listenersMu.Unlock() - err := sl.Listener.Close() + err := sl.listener.Close() if err != nil { return err } @@ -116,6 +116,10 @@ func (sl *sharedListener) Close() error { return nil } +func (sl *sharedListener) Addr() net.Addr { + return sl.listener.Addr() +} + type sharedPacketConn struct { net.PacketConn key string @@ -195,7 +199,7 @@ func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (any deadline: &lnGlobal.deadline, deadlineMu: &lnGlobal.deadlineMu, key: na.Key(), - Listener: lnGlobal.ln, + listener: lnGlobal.ln, }, nil } @@ -213,7 +217,7 @@ func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (any deadline: &lnGlobal.deadline, deadlineMu: &lnGlobal.deadlineMu, key: na.Key(), - Listener: ln, + listener: ln, }, nil case "udp": From b8c5ab8db267099e8e5f74d0f212c87394854f38 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 14:47:52 -0400 Subject: [PATCH 043/182] Use a `Service` object to abstract away some of the complex logic of managing listeners. --- cmd/outline-ss-server/main.go | 266 ++++++++++++++++++++-------------- 1 file changed, 161 insertions(+), 105 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 1deb3454..6093bb68 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -60,13 +60,6 @@ func init() { logger = logging.MustGetLogger("") } -type SSServer struct { - natTimeout time.Duration - m *outlineMetrics - replayCache service.ReplayCache - listeners []Listener -} - // The implementations of listeners for different network types are not // interchangeable. The type of listener depends on the network type. // TODO(sbruens): Create a custom `Listener` type so we can share serving logic, @@ -74,10 +67,18 @@ type SSServer struct { // listener type. type Listener = any -func (s *SSServer) serve(addr NetworkAddr, listener Listener, cipherList service.CipherList) error { +type Service struct { + natTimeout time.Duration + m *outlineMetrics + replayCache *service.ReplayCache + Listeners []Listener + Ciphers *list.List // Values are *List of *CipherEntry. +} + +func (s *Service) Serve(addr NetworkAddr, listener Listener, cipherList service.CipherList) error { switch ln := listener.(type) { case net.Listener: - authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &s.replayCache, s.m) + authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, s.replayCache, s.m) // TODO: Register initial data metrics at zero. tcpHandler := service.NewTCPHandler(addr.Key(), authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { @@ -97,131 +98,187 @@ func (s *SSServer) serve(addr NetworkAddr, listener Listener, cipherList service return nil } -func (s *SSServer) loadConfig(filename string) error { - configData, err := os.ReadFile(filename) - if err != nil { - return fmt.Errorf("failed to read config file %s: %w", filename, err) +func (s *Service) Stop() error { + for _, listener := range s.Listeners { + switch ln := listener.(type) { + case net.Listener: + if err := ln.Close(); err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) + } + case net.PacketConn: + if err := ln.Close(); err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) + } + default: + return fmt.Errorf("unknown listener type: %v", ln) + } } - config, err := readConfig(configData) + return nil +} + +// AddListener adds a new listener to the service. +func (s *Service) AddListener(addr NetworkAddr) error { + // Create new listeners based on the configured network addresses. + cipherList := service.NewCipherList() + cipherList.Update(s.Ciphers) + + listener, err := addr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) if err != nil { - return fmt.Errorf("failed to load config (%v): %w", filename, err) + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", addr.Network(), addr.String(), err) } - if err := config.Validate(); err != nil { - return fmt.Errorf("failed to validate config: %w", err) + s.Listeners = append(s.Listeners, listener) + logger.Infof("Shadowsocks %s service listening on %s", addr.Network(), addr.String()) + if err = s.Serve(addr, listener, cipherList); err != nil { + return fmt.Errorf("failed to serve on %s listener on address %s: %w", addr.Network(), addr.String(), err) } + return nil +} - uniqueCiphers := 0 - addrs := make([]NetworkAddr, 0) - addrCiphers := make(map[NetworkAddr]*list.List) // Values are *List of *CipherEntry. +// NewService creates a new Service. +func NewService(config ServiceConfig, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { + s := Service{ + natTimeout: natTimeout, + m: m, + replayCache: replayCache, + Ciphers: list.New(), + } - for _, legacyKeyServiceConfig := range config.Keys { - cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyServiceConfig.Cipher, legacyKeyServiceConfig.Secret) - if err != nil { - return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyServiceConfig.ID, err) + type cipherKey struct { + cipher string + secret string + } + existingCiphers := make(map[cipherKey]bool) + for _, keyConfig := range config.Keys { + key := cipherKey{keyConfig.Cipher, keyConfig.Secret} + if _, exists := existingCiphers[key]; exists { + logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) + continue } - entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) - for _, network := range []string{"tcp", "udp"} { - addr := NetworkAddr{ - network: network, - Host: "::", - Port: uint(legacyKeyServiceConfig.Port), - } - addrs = append(addrs, addr) - ciphers, ok := addrCiphers[addr] - if !ok { - ciphers = list.New() - addrCiphers[addr] = ciphers - } - ciphers.PushBack(&entry) + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) } - uniqueCiphers += 1 + entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + s.Ciphers.PushBack(&entry) + existingCiphers[key] = true } - for _, serviceConfig := range config.Services { - ciphers := list.New() - type cipherKey struct { - cipher string - secret string + for _, listener := range config.Listeners { + addr, err := ParseNetworkAddr(listener.Address) + if err != nil { + return nil, fmt.Errorf("error parsing listener address `%s`: %v", listener.Address, err) } - existingCiphers := make(map[cipherKey]bool) - for _, keyConfig := range serviceConfig.Keys { - key := cipherKey{keyConfig.Cipher, keyConfig.Secret} - if _, exists := existingCiphers[key]; exists { - logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) - continue - } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) - if err != nil { - return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) - } - entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - ciphers.PushBack(&entry) - existingCiphers[key] = true + if err := s.AddListener(addr); err != nil { + return nil, err } - uniqueCiphers += ciphers.Len() + } - for _, listener := range serviceConfig.Listeners { - addr, err := ParseNetworkAddr(listener.Address) - if err != nil { - return fmt.Errorf("error parsing listener address `%s`: %v", listener.Address, err) - } - addrs = append(addrs, addr) - addrCiphers[addr] = ciphers - } + return &s, nil +} + +func NewLegacyKeyService(config LegacyKeyServiceConfig, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { + s := Service{ + natTimeout: natTimeout, + m: m, + replayCache: replayCache, + Ciphers: list.New(), } - // Create new listeners based on the configured network addresses. - newListeners := make([]Listener, 0) - for _, addr := range addrs { - cipherList := service.NewCipherList() - ciphers, ok := addrCiphers[addr] - if !ok { - return fmt.Errorf("unable to find ciphers for address: %v", addr.Key()) + cryptoKey, err := shadowsocks.NewEncryptionKey(config.Cipher, config.Secret) + if err != nil { + return nil, fmt.Errorf("failed to create encyption key for key %v: %w", config.ID, err) + } + entry := service.MakeCipherEntry(config.ID, cryptoKey, config.Secret) + s.Ciphers.PushBack(&entry) + + for _, network := range []string{"tcp", "udp"} { + addr := NetworkAddr{ + network: network, + Host: "::", + Port: uint(config.Port), } - cipherList.Update(ciphers) + if err := s.AddListener(addr); err != nil { + return nil, err + } + } - listener, err := addr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) + return &s, nil +} + +type SSServer struct { + natTimeout time.Duration + m *outlineMetrics + replayCache service.ReplayCache + services []Service +} + +func (s *SSServer) loadConfig(filename string) error { + configData, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read config file %s: %w", filename, err) + } + config, err := readConfig(configData) + if err != nil { + return fmt.Errorf("failed to load config (%v): %w", filename, err) + } + if err := config.Validate(); err != nil { + return fmt.Errorf("failed to validate config: %w", err) + } + + // We hot swap the services by having them both live at the same time. This + // means we create services for the new config first, and then take down the + // services from the old config. + newServices := make([]Service, 0) + for _, legacyKeyServiceConfig := range config.Keys { + service, err := NewLegacyKeyService(legacyKeyServiceConfig, s.natTimeout, s.m, &s.replayCache) if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", addr.Network(), addr.String(), err) + return fmt.Errorf("Failed to create new service: %v", err) } - logger.Infof("Shadowsocks %s service listening on %s", addr.Network(), addr.String()) - newListeners = append(newListeners, listener) - - if err = s.serve(addr, listener, cipherList); err != nil { - return fmt.Errorf("failed to serve on %s listener on address %s: %w", addr.Network(), addr.String(), err) + newServices = append(newServices, *service) + } + for _, serviceConfig := range config.Services { + service, err := NewService(serviceConfig, s.natTimeout, s.m, &s.replayCache) + if err != nil { + return fmt.Errorf("Failed to create new service: %v", err) } + newServices = append(newServices, *service) } - // Take down the old listeners now that the new ones are serving. + // Take down the old services now that the new ones are created and serving. if err := s.Stop(); err != nil { - logger.Errorf("Failed to stop old listeners: %w", err) + logger.Errorf("Failed to stop old services: %w", err) } - s.listeners = newListeners - - logger.Infof("Loaded %v access keys over %v listeners", uniqueCiphers, len(s.listeners)) - s.m.SetNumAccessKeys(uniqueCiphers, len(s.listeners)) + s.services = newServices + + // Gather some basic stats for logging. + var ( + listenerCount int + cipherCount int + ) + for _, service := range s.services { + listenerCount += len(service.Listeners) + cipherCount += service.Ciphers.Len() + } + logger.Infof("Loaded %d services with %d access keys over %d listeners", len(s.services), cipherCount, listenerCount) + s.m.SetNumAccessKeys(cipherCount, len(s.services)) return nil } -// Stop serving on all existing listeners. +// Stop serving on all existing services. func (s *SSServer) Stop() error { - for _, listener := range s.listeners { - switch ln := listener.(type) { - case net.Listener: - if err := ln.Close(); err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) - } - case net.PacketConn: - if err := ln.Close(); err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) - } - default: - return fmt.Errorf("unknown listener type: %v", ln) + if len(s.services) == 0 { + return nil + } + + for _, service := range s.services { + if err := service.Stop(); err != nil { + return err } } + logger.Infof("Stopped %d old services", len(s.services)) return nil } @@ -231,7 +288,6 @@ func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, natTimeout: natTimeout, m: sm, replayCache: service.NewReplayCache(replayHistory), - listeners: nil, } err := server.loadConfig(filename) if err != nil { From 5ac0f46d99359a36ed7baf1e709cce39625d0542 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 16:17:44 -0400 Subject: [PATCH 044/182] Fix how we deal with legacy services. --- cmd/outline-ss-server/main.go | 201 ++++++------------------------- cmd/outline-ss-server/metrics.go | 14 +-- cmd/outline-ss-server/service.go | 167 +++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 171 deletions(-) create mode 100644 cmd/outline-ss-server/service.go diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 6093bb68..2b1f7842 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -16,10 +16,8 @@ package main import ( "container/list" - "context" "flag" "fmt" - "net" "net/http" "os" "os/signal" @@ -27,9 +25,7 @@ import ( "syscall" "time" - "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" - "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" "github.com/op/go-logging" @@ -60,159 +56,11 @@ func init() { logger = logging.MustGetLogger("") } -// The implementations of listeners for different network types are not -// interchangeable. The type of listener depends on the network type. -// TODO(sbruens): Create a custom `Listener` type so we can share serving logic, -// dispatching to the handlers based on connection type instead of on the -// listener type. -type Listener = any - -type Service struct { - natTimeout time.Duration - m *outlineMetrics - replayCache *service.ReplayCache - Listeners []Listener - Ciphers *list.List // Values are *List of *CipherEntry. -} - -func (s *Service) Serve(addr NetworkAddr, listener Listener, cipherList service.CipherList) error { - switch ln := listener.(type) { - case net.Listener: - authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, s.replayCache, s.m) - // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(addr.Key(), authFunc, s.m, tcpReadTimeout) - accept := func() (transport.StreamConn, error) { - c, err := ln.Accept() - if err == nil { - return c.(transport.StreamConn), err - } - return nil, err - } - go service.StreamServe(accept, tcpHandler.Handle) - case net.PacketConn: - packetHandler := service.NewPacketHandler(s.natTimeout, cipherList, s.m) - go packetHandler.Handle(ln) - default: - return fmt.Errorf("unknown listener type: %v", ln) - } - return nil -} - -func (s *Service) Stop() error { - for _, listener := range s.Listeners { - switch ln := listener.(type) { - case net.Listener: - if err := ln.Close(); err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) - } - case net.PacketConn: - if err := ln.Close(); err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) - } - default: - return fmt.Errorf("unknown listener type: %v", ln) - } - } - return nil -} - -// AddListener adds a new listener to the service. -func (s *Service) AddListener(addr NetworkAddr) error { - // Create new listeners based on the configured network addresses. - cipherList := service.NewCipherList() - cipherList.Update(s.Ciphers) - - listener, err := addr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", addr.Network(), addr.String(), err) - } - s.Listeners = append(s.Listeners, listener) - logger.Infof("Shadowsocks %s service listening on %s", addr.Network(), addr.String()) - if err = s.Serve(addr, listener, cipherList); err != nil { - return fmt.Errorf("failed to serve on %s listener on address %s: %w", addr.Network(), addr.String(), err) - } - return nil -} - -// NewService creates a new Service. -func NewService(config ServiceConfig, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { - s := Service{ - natTimeout: natTimeout, - m: m, - replayCache: replayCache, - Ciphers: list.New(), - } - - type cipherKey struct { - cipher string - secret string - } - existingCiphers := make(map[cipherKey]bool) - for _, keyConfig := range config.Keys { - key := cipherKey{keyConfig.Cipher, keyConfig.Secret} - if _, exists := existingCiphers[key]; exists { - logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) - continue - } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) - if err != nil { - return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) - } - entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - s.Ciphers.PushBack(&entry) - existingCiphers[key] = true - } - - for _, listener := range config.Listeners { - addr, err := ParseNetworkAddr(listener.Address) - if err != nil { - return nil, fmt.Errorf("error parsing listener address `%s`: %v", listener.Address, err) - } - if err := s.AddListener(addr); err != nil { - return nil, err - } - } - - return &s, nil -} - -func NewLegacyKeyService(config LegacyKeyServiceConfig, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { - s := Service{ - natTimeout: natTimeout, - m: m, - replayCache: replayCache, - Ciphers: list.New(), - } - - cryptoKey, err := shadowsocks.NewEncryptionKey(config.Cipher, config.Secret) - if err != nil { - return nil, fmt.Errorf("failed to create encyption key for key %v: %w", config.ID, err) - } - entry := service.MakeCipherEntry(config.ID, cryptoKey, config.Secret) - s.Ciphers.PushBack(&entry) - - for _, network := range []string{"tcp", "udp"} { - addr := NetworkAddr{ - network: network, - Host: "::", - Port: uint(config.Port), - } - if err := s.AddListener(addr); err != nil { - return nil, err - } - } - - return &s, nil -} - type SSServer struct { natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache - services []Service + services []*Service } func (s *SSServer) loadConfig(filename string) error { @@ -231,21 +79,47 @@ func (s *SSServer) loadConfig(filename string) error { // We hot swap the services by having them both live at the same time. This // means we create services for the new config first, and then take down the // services from the old config. - newServices := make([]Service, 0) + newServices := make([]*Service, 0) + + legacyPortService := make(map[int]*Service) // Values are *List of *CipherEntry. for _, legacyKeyServiceConfig := range config.Keys { - service, err := NewLegacyKeyService(legacyKeyServiceConfig, s.natTimeout, s.m, &s.replayCache) + legacyService, ok := legacyPortService[legacyKeyServiceConfig.Port] + if !ok { + legacyService = &Service{ + natTimeout: s.natTimeout, + m: s.m, + replayCache: &s.replayCache, + ciphers: list.New(), + } + for _, network := range []string{"tcp", "udp"} { + addr := NetworkAddr{ + network: network, + Host: "::", + Port: uint(legacyKeyServiceConfig.Port), + } + if err := legacyService.AddListener(addr); err != nil { + return err + } + } + newServices = append(newServices, legacyService) + legacyPortService[legacyKeyServiceConfig.Port] = legacyService + } + cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyServiceConfig.Cipher, legacyKeyServiceConfig.Secret) if err != nil { - return fmt.Errorf("Failed to create new service: %v", err) + return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyServiceConfig.ID, err) } - newServices = append(newServices, *service) + entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) + legacyService.AddCipher(&entry) } + for _, serviceConfig := range config.Services { service, err := NewService(serviceConfig, s.natTimeout, s.m, &s.replayCache) if err != nil { return fmt.Errorf("Failed to create new service: %v", err) } - newServices = append(newServices, *service) + newServices = append(newServices, service) } + logger.Infof("Loaded %d new services", len(newServices)) // Take down the old services now that the new ones are created and serving. if err := s.Stop(); err != nil { @@ -253,17 +127,16 @@ func (s *SSServer) loadConfig(filename string) error { } s.services = newServices - // Gather some basic stats for logging. var ( listenerCount int cipherCount int ) for _, service := range s.services { - listenerCount += len(service.Listeners) - cipherCount += service.Ciphers.Len() + listenerCount += service.NumListeners() + cipherCount += service.NumCiphers() } - logger.Infof("Loaded %d services with %d access keys over %d listeners", len(s.services), cipherCount, listenerCount) - s.m.SetNumAccessKeys(cipherCount, len(s.services)) + logger.Infof("%d services active: %d access keys over %d listeners", len(s.services), cipherCount, listenerCount) + s.m.SetNumAccessKeys(cipherCount, listenerCount) return nil } @@ -272,13 +145,13 @@ func (s *SSServer) Stop() error { if len(s.services) == 0 { return nil } - for _, service := range s.services { if err := service.Stop(); err != nil { return err } } logger.Infof("Stopped %d old services", len(s.services)) + s.services = nil return nil } diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index e95ceeb3..600cea16 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -38,7 +38,7 @@ type outlineMetrics struct { buildInfo *prometheus.GaugeVec accessKeys prometheus.Gauge - ports prometheus.Gauge + listeners prometheus.Gauge dataBytes *prometheus.CounterVec dataBytesPerLocation *prometheus.CounterVec timeToCipherMs *prometheus.HistogramVec @@ -183,10 +183,10 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus Name: "keys", Help: "Count of access keys", }), - ports: prometheus.NewGauge(prometheus.GaugeOpts{ + listeners: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, - Name: "ports", - Help: "Count of open Shadowsocks ports", + Name: "listeners", + Help: "Count of open Shadowsocks listeners", }), tcpProbes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, @@ -265,7 +265,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus m.tunnelTimeCollector = newTunnelTimeCollector(ip2info, registerer) // TODO: Is it possible to pass where to register the collectors? - registerer.MustRegister(m.buildInfo, m.accessKeys, m.ports, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, + registerer.MustRegister(m.buildInfo, m.accessKeys, m.listeners, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, m.dataBytes, m.dataBytesPerLocation, m.timeToCipherMs, m.udpPacketsFromClientPerLocation, m.udpAddedNatEntries, m.udpRemovedNatEntries, m.tunnelTimeCollector) return m @@ -275,9 +275,9 @@ func (m *outlineMetrics) SetBuildInfo(version string) { m.buildInfo.WithLabelValues(version).Set(1) } -func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) { +func (m *outlineMetrics) SetNumAccessKeys(numKeys int, listeners int) { m.accessKeys.Set(float64(numKeys)) - m.ports.Set(float64(ports)) + m.listeners.Set(float64(listeners)) } func (m *outlineMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { diff --git a/cmd/outline-ss-server/service.go b/cmd/outline-ss-server/service.go new file mode 100644 index 00000000..dc91e5a5 --- /dev/null +++ b/cmd/outline-ss-server/service.go @@ -0,0 +1,167 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "container/list" + "context" + "fmt" + "net" + "time" + + "github.com/Jigsaw-Code/outline-sdk/transport" + "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" + "github.com/Jigsaw-Code/outline-ss-server/service" +) + +// The implementations of listeners for different network types are not +// interchangeable. The type of listener depends on the network type. +// TODO(sbruens): Create a custom `Listener` type so we can share serving logic, +// dispatching to the handlers based on connection type instead of on the +// listener type. +type Listener = any + +type Service struct { + natTimeout time.Duration + m *outlineMetrics + replayCache *service.ReplayCache + listeners []Listener + ciphers *list.List // Values are *List of *service.CipherEntry. +} + +func (s *Service) Serve(addr NetworkAddr, listener Listener, cipherList service.CipherList) error { + switch ln := listener.(type) { + case net.Listener: + authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, s.replayCache, s.m) + // TODO: Register initial data metrics at zero. + tcpHandler := service.NewTCPHandler(addr.Key(), authFunc, s.m, tcpReadTimeout) + accept := func() (transport.StreamConn, error) { + c, err := ln.Accept() + if err == nil { + return c.(transport.StreamConn), err + } + return nil, err + } + go service.StreamServe(accept, tcpHandler.Handle) + case net.PacketConn: + packetHandler := service.NewPacketHandler(s.natTimeout, cipherList, s.m) + go packetHandler.Handle(ln) + default: + return fmt.Errorf("unknown listener type: %v", ln) + } + return nil +} + +func (s *Service) Stop() error { + for _, listener := range s.listeners { + switch ln := listener.(type) { + case net.Listener: + if err := ln.Close(); err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) + } + case net.PacketConn: + if err := ln.Close(); err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) + } + default: + return fmt.Errorf("unknown listener type: %v", ln) + } + } + return nil +} + +// AddListener adds a new listener to the service. +func (s *Service) AddListener(addr NetworkAddr) error { + // Create new listeners based on the configured network addresses. + cipherList := service.NewCipherList() + cipherList.Update(s.ciphers) + + listener, err := addr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) + if err != nil { + //lint:ignore ST1005 Shadowsocks is capitalized. + return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", addr.Network(), addr.String(), err) + } + s.listeners = append(s.listeners, listener) + logger.Infof("Shadowsocks %s service listening on %s", addr.Network(), addr.String()) + if err = s.Serve(addr, listener, cipherList); err != nil { + return fmt.Errorf("failed to serve on %s listener on address %s: %w", addr.Network(), addr.String(), err) + } + return nil +} + +func (s *Service) NumListeners() int { + return len(s.listeners) +} + +func (s *Service) AddCipher(entry *service.CipherEntry) { + s.ciphers.PushBack(entry) +} + +func (s *Service) NumCiphers() int { + return s.ciphers.Len() +} + +// func NewService(natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) Service { +// return &Service{ +// natTimeout: natTimeout, +// m: m, +// replayCache: replayCache, +// ciphers: list.New(), +// } +// } + +// NewService creates a new Service based on a config +func NewService(config ServiceConfig, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { + s := Service{ + natTimeout: natTimeout, + m: m, + replayCache: replayCache, + ciphers: list.New(), + } + + type cipherKey struct { + cipher string + secret string + } + existingCiphers := make(map[cipherKey]bool) + for _, keyConfig := range config.Keys { + key := cipherKey{keyConfig.Cipher, keyConfig.Secret} + if _, exists := existingCiphers[key]; exists { + logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) + continue + } + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + } + entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + s.AddCipher(&entry) + existingCiphers[key] = true + } + + for _, listener := range config.Listeners { + addr, err := ParseNetworkAddr(listener.Address) + if err != nil { + return nil, fmt.Errorf("error parsing listener address `%s`: %v", listener.Address, err) + } + if err := s.AddListener(addr); err != nil { + return nil, err + } + } + + return &s, nil +} From 1f097be05a9ed4343f753a73306bab3c4b6e8aa6 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 16:21:22 -0400 Subject: [PATCH 045/182] Remove commented out lines. --- cmd/outline-ss-server/service.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cmd/outline-ss-server/service.go b/cmd/outline-ss-server/service.go index dc91e5a5..5635de3e 100644 --- a/cmd/outline-ss-server/service.go +++ b/cmd/outline-ss-server/service.go @@ -115,15 +115,6 @@ func (s *Service) NumCiphers() int { return s.ciphers.Len() } -// func NewService(natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) Service { -// return &Service{ -// natTimeout: natTimeout, -// m: m, -// replayCache: replayCache, -// ciphers: list.New(), -// } -// } - // NewService creates a new Service based on a config func NewService(config ServiceConfig, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { s := Service{ From 80b25b16ac2645477b51cd759113a970421ba9e4 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 16:55:28 -0400 Subject: [PATCH 046/182] Use `tcp` and `udp` types for direct listeners. --- cmd/outline-ss-server/config.go | 10 +-- cmd/outline-ss-server/config_example.yml | 16 ++-- cmd/outline-ss-server/config_test.go | 26 ++----- cmd/outline-ss-server/listeners.go | 99 ++++-------------------- cmd/outline-ss-server/main.go | 10 +-- cmd/outline-ss-server/service.go | 24 +++--- 6 files changed, 50 insertions(+), 135 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index 5929f202..e8a8a43f 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -28,7 +28,8 @@ type ServiceConfig struct { type ListenerType string -const listenerTypeDirect ListenerType = "direct" +const listenerTypeTCP ListenerType = "tcp" +const listenerTypeUDP ListenerType = "udp" type ListenerConfig struct { Type ListenerType @@ -59,17 +60,14 @@ func (c *Config) Validate() error { for _, serviceConfig := range c.Services { for _, listenerConfig := range serviceConfig.Listeners { // TODO: Support more listener types. - if listenerConfig.Type != listenerTypeDirect { + if listenerConfig.Type != listenerTypeTCP && listenerConfig.Type != listenerTypeUDP { return fmt.Errorf("unsupported listener type: %s", listenerConfig.Type) } - network, host, _, err := SplitNetworkAddr(listenerConfig.Address) + host, _, err := net.SplitHostPort(listenerConfig.Address) if err != nil { return fmt.Errorf("invalid listener address `%s`: %v", listenerConfig.Address, err) } - if network != "tcp" && network != "udp" { - return fmt.Errorf("unsupported network: %s", network) - } if ip := net.ParseIP(host); ip == nil { return fmt.Errorf("address must be IP, found: %s", host) } diff --git a/cmd/outline-ss-server/config_example.yml b/cmd/outline-ss-server/config_example.yml index bbfd265f..7af360b2 100644 --- a/cmd/outline-ss-server/config_example.yml +++ b/cmd/outline-ss-server/config_example.yml @@ -2,10 +2,10 @@ services: - listeners: # TODO(sbruens): Allow a string-based listener config, as a convenient short-form # to create a direct listener, e.g. `- tcp/[::]:9000`. - - type: direct - address: "tcp/[::]:9000" - - type: direct - address: "udp/[::]:9000" + - type: tcp + address: "[::]:9000" + - type: udp + address: "[::]:9000" keys: - id: user-0 cipher: chacha20-ietf-poly1305 @@ -15,10 +15,10 @@ services: secret: Secret1 - listeners: - - type: direct - address: "tcp/[::]:9001" - - type: direct - address: "udp/[::]:9001" + - type: tcp + address: "[::]:9001" + - type: udp + address: "[::]:9001" keys: - id: user-2 cipher: chacha20-ietf-poly1305 diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 3a8b6cf8..25895111 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -32,7 +32,7 @@ func TestValidateConfigFails(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: "foo", Address: "tcp/[::]:9000"}, + ListenerConfig{Type: "foo", Address: "[::]:9000"}, }, }, }, @@ -44,19 +44,7 @@ func TestValidateConfigFails(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp//[::]:9000"}, - }, - }, - }, - }, - }, - { - name: "WithUnsupportedNetworkType", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "foo/[::]:9000"}, + ListenerConfig{Type: listenerTypeTCP, Address: "tcp/[::]:9000"}, }, }, }, @@ -68,7 +56,7 @@ func TestValidateConfigFails(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp/example.com:9000"}, + ListenerConfig{Type: listenerTypeTCP, Address: "example.com:9000"}, }, }, }, @@ -92,8 +80,8 @@ func TestReadConfig(t *testing.T) { Services: []ServiceConfig{ ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp/[::]:9000"}, - ListenerConfig{Type: listenerTypeDirect, Address: "udp/[::]:9000"}, + ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, + ListenerConfig{Type: listenerTypeUDP, Address: "[::]:9000"}, }, Keys: []KeyConfig{ KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, @@ -102,8 +90,8 @@ func TestReadConfig(t *testing.T) { }, ServiceConfig{ Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeDirect, Address: "tcp/[::]:9001"}, - ListenerConfig{Type: listenerTypeDirect, Address: "udp/[::]:9001"}, + ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9001"}, + ListenerConfig{Type: listenerTypeUDP, Address: "[::]:9001"}, }, Keys: []KeyConfig{ KeyConfig{"user-2", "chacha20-ietf-poly1305", "Secret2"}, diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index fefa6920..2f8f7c59 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -16,11 +16,8 @@ package main import ( "context" - "errors" "fmt" "net" - "strconv" - "strings" "sync" "sync/atomic" "time" @@ -152,71 +149,46 @@ type globalListener struct { deadlineMu sync.Mutex } -type NetworkAddr struct { - network string - Host string - Port uint -} - -// String returns a human-readable representation of the [NetworkAddr]. -func (na *NetworkAddr) Network() string { - return na.network -} - -// String returns a human-readable representation of the [NetworkAddr]. -func (na *NetworkAddr) String() string { - return na.JoinHostPort() -} - -// JoinHostPort is a convenience wrapper around [net.JoinHostPort]. -func (na *NetworkAddr) JoinHostPort() string { - return net.JoinHostPort(na.Host, strconv.Itoa(int(na.Port))) -} - -// Key returns a representative string useful to retrieve this entity from a -// map. This is used to uniquely identify reusable listeners. -func (na *NetworkAddr) Key() string { - return na.network + "/" + na.JoinHostPort() -} - -// Listen creates a new listener for the [NetworkAddr]. +// Listen creates a new listener for a given network and address. // // Listeners can overlap one another, because during config changes the new // config is started before the old config is destroyed. This is done by using // reusable listener wrappers, which do not actually close the underlying socket // until all uses of the shared listener have been closed. -func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (any, error) { - switch na.network { +func Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (any, error) { + lnKey := network + "/" + addr + + switch network { case "tcp": listenersMu.Lock() defer listenersMu.Unlock() - if lnGlobal, ok := listeners[na.Key()]; ok { + if lnGlobal, ok := listeners[lnKey]; ok { lnGlobal.usage.Add(1) return &sharedListener{ usage: &lnGlobal.usage, deadline: &lnGlobal.deadline, deadlineMu: &lnGlobal.deadlineMu, - key: na.Key(), + key: lnKey, listener: lnGlobal.ln, }, nil } - ln, err := config.Listen(ctx, na.network, na.JoinHostPort()) + ln, err := config.Listen(ctx, network, addr) if err != nil { return nil, err } lnGlobal := &globalListener{ln: ln} lnGlobal.usage.Store(1) - listeners[na.Key()] = lnGlobal + listeners[lnKey] = lnGlobal return &sharedListener{ usage: &lnGlobal.usage, deadline: &lnGlobal.deadline, deadlineMu: &lnGlobal.deadlineMu, - key: na.Key(), + key: lnKey, listener: ln, }, nil @@ -224,71 +196,32 @@ func (na *NetworkAddr) Listen(ctx context.Context, config net.ListenConfig) (any listenersMu.Lock() defer listenersMu.Unlock() - if lnGlobal, ok := listeners[na.Key()]; ok { + if lnGlobal, ok := listeners[lnKey]; ok { lnGlobal.usage.Add(1) return &sharedPacketConn{ usage: &lnGlobal.usage, - key: na.Key(), + key: lnKey, PacketConn: lnGlobal.pc, }, nil } - pc, err := config.ListenPacket(ctx, na.network, na.JoinHostPort()) + pc, err := config.ListenPacket(ctx, network, addr) if err != nil { return nil, err } lnGlobal := &globalListener{pc: pc} lnGlobal.usage.Store(1) - listeners[na.Key()] = lnGlobal + listeners[lnKey] = lnGlobal return &sharedPacketConn{ usage: &lnGlobal.usage, - key: na.Key(), + key: lnKey, PacketConn: pc, }, nil default: - return nil, fmt.Errorf("unsupported network: %s", na.network) - - } -} - -// ParseNetworkAddr parses an address into a [NetworkAddr]. The input -// string is expected to be of the form "network/host:port" where any part is -// optional. -// -// Examples: -// -// tcp/127.0.0.1:8000 -// udp/127.0.0.1:9000 -func ParseNetworkAddr(addr string) (NetworkAddr, error) { - var host, port string - network, host, port, err := SplitNetworkAddr(addr) - if err != nil { - return NetworkAddr{}, err - } - if network == "" { - return NetworkAddr{}, errors.New("missing network") - } - p, err := strconv.ParseUint(port, 10, 16) - if err != nil { - return NetworkAddr{}, fmt.Errorf("invalid port: %v", err) - } - return NetworkAddr{ - network: network, - Host: host, - Port: uint(p), - }, nil -} + return nil, fmt.Errorf("unsupported network: %s", network) -// SplitNetworkAddr splits a into its network, host, and port components. -func SplitNetworkAddr(a string) (network, host, port string, err error) { - beforeSlash, afterSlash, slashFound := strings.Cut(a, "/") - if slashFound { - network = strings.ToLower(strings.TrimSpace(beforeSlash)) - a = afterSlash } - host, port, err = net.SplitHostPort(a) - return } diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 2b1f7842..fbc40e89 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -18,9 +18,11 @@ import ( "container/list" "flag" "fmt" + "net" "net/http" "os" "os/signal" + "strconv" "strings" "syscall" "time" @@ -92,12 +94,8 @@ func (s *SSServer) loadConfig(filename string) error { ciphers: list.New(), } for _, network := range []string{"tcp", "udp"} { - addr := NetworkAddr{ - network: network, - Host: "::", - Port: uint(legacyKeyServiceConfig.Port), - } - if err := legacyService.AddListener(addr); err != nil { + addr := net.JoinHostPort("::", strconv.Itoa(legacyKeyServiceConfig.Port)) + if err := legacyService.AddListener(network, addr); err != nil { return err } } diff --git a/cmd/outline-ss-server/service.go b/cmd/outline-ss-server/service.go index 5635de3e..9dc98657 100644 --- a/cmd/outline-ss-server/service.go +++ b/cmd/outline-ss-server/service.go @@ -41,12 +41,12 @@ type Service struct { ciphers *list.List // Values are *List of *service.CipherEntry. } -func (s *Service) Serve(addr NetworkAddr, listener Listener, cipherList service.CipherList) error { +func (s *Service) Serve(lnKey string, listener Listener, cipherList service.CipherList) error { switch ln := listener.(type) { case net.Listener: authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, s.replayCache, s.m) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(addr.Key(), authFunc, s.m, tcpReadTimeout) + tcpHandler := service.NewTCPHandler(lnKey, authFunc, s.m, tcpReadTimeout) accept := func() (transport.StreamConn, error) { c, err := ln.Accept() if err == nil { @@ -85,20 +85,21 @@ func (s *Service) Stop() error { } // AddListener adds a new listener to the service. -func (s *Service) AddListener(addr NetworkAddr) error { +func (s *Service) AddListener(network string, addr string) error { // Create new listeners based on the configured network addresses. cipherList := service.NewCipherList() cipherList.Update(s.ciphers) - listener, err := addr.Listen(context.TODO(), net.ListenConfig{KeepAlive: 0}) + listener, err := Listen(context.TODO(), network, addr, net.ListenConfig{KeepAlive: 0}) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", addr.Network(), addr.String(), err) + return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", network, addr, err) } s.listeners = append(s.listeners, listener) - logger.Infof("Shadowsocks %s service listening on %s", addr.Network(), addr.String()) - if err = s.Serve(addr, listener, cipherList); err != nil { - return fmt.Errorf("failed to serve on %s listener on address %s: %w", addr.Network(), addr.String(), err) + logger.Infof("Shadowsocks %s service listening on %s", network, addr) + lnKey := network + "/" + addr + if err = s.Serve(lnKey, listener, cipherList); err != nil { + return fmt.Errorf("failed to serve on %s listener on address %s: %w", network, addr, err) } return nil } @@ -145,11 +146,8 @@ func NewService(config ServiceConfig, natTimeout time.Duration, m *outlineMetric } for _, listener := range config.Listeners { - addr, err := ParseNetworkAddr(listener.Address) - if err != nil { - return nil, fmt.Errorf("error parsing listener address `%s`: %v", listener.Address, err) - } - if err := s.AddListener(addr); err != nil { + network := string(listener.Type) + if err := s.AddListener(network, listener.Address); err != nil { return nil, err } } From 2070d40ff9cb36ec4f054b766057e318f8bddb88 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 17:41:37 -0400 Subject: [PATCH 047/182] Use a `ListenerManager` instead of globals to manage listener state. --- cmd/outline-ss-server/listeners.go | 82 +++++++++++++++++++----------- cmd/outline-ss-server/main.go | 5 +- cmd/outline-ss-server/service.go | 6 ++- 3 files changed, 59 insertions(+), 34 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index 2f8f7c59..c8a5d644 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -23,13 +23,9 @@ import ( "time" ) -var ( - listeners = make(map[string]*globalListener) - listenersMu sync.Mutex -) - type sharedListener struct { listener net.Listener + manager ListenerManager key string closed atomic.Int32 usage *atomic.Int32 @@ -99,9 +95,7 @@ func (sl *sharedListener) Close() error { // See if we need to actually close the underlying listener. if sl.usage.Add(-1) == 0 { - listenersMu.Lock() - delete(listeners, sl.key) - listenersMu.Unlock() + sl.manager.Delete(sl.key) err := sl.listener.Close() if err != nil { return err @@ -119,18 +113,17 @@ func (sl *sharedListener) Addr() net.Addr { type sharedPacketConn struct { net.PacketConn - key string - closed atomic.Int32 - usage *atomic.Int32 + manager ListenerManager + key string + closed atomic.Int32 + usage *atomic.Int32 } func (spc *sharedPacketConn) Close() error { if spc.closed.CompareAndSwap(0, 1) { // See if we need to actually close the underlying listener. if spc.usage.Add(-1) == 0 { - listenersMu.Lock() - delete(listeners, spc.key) - listenersMu.Unlock() + spc.manager.Delete(spc.key) err := spc.PacketConn.Close() if err != nil { return err @@ -149,29 +142,47 @@ type globalListener struct { deadlineMu sync.Mutex } +// ListenerManager holds and manages the state of shared listeners. +type ListenerManager interface { + Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (any, error) + Delete(key string) +} + +type listenerManager struct { + listeners map[string]*globalListener + listenersMu sync.Mutex +} + +func NewListenerManager() ListenerManager { + return &listenerManager{ + listeners: make(map[string]*globalListener), + } +} + // Listen creates a new listener for a given network and address. // // Listeners can overlap one another, because during config changes the new // config is started before the old config is destroyed. This is done by using // reusable listener wrappers, which do not actually close the underlying socket // until all uses of the shared listener have been closed. -func Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (any, error) { +func (m *listenerManager) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (any, error) { lnKey := network + "/" + addr switch network { case "tcp": - listenersMu.Lock() - defer listenersMu.Unlock() + m.listenersMu.Lock() + defer m.listenersMu.Unlock() - if lnGlobal, ok := listeners[lnKey]; ok { + if lnGlobal, ok := m.listeners[lnKey]; ok { lnGlobal.usage.Add(1) return &sharedListener{ + listener: lnGlobal.ln, + manager: m, + key: lnKey, usage: &lnGlobal.usage, deadline: &lnGlobal.deadline, deadlineMu: &lnGlobal.deadlineMu, - key: lnKey, - listener: lnGlobal.ln, }, nil } @@ -182,26 +193,28 @@ func Listen(ctx context.Context, network string, addr string, config net.ListenC lnGlobal := &globalListener{ln: ln} lnGlobal.usage.Store(1) - listeners[lnKey] = lnGlobal + m.listeners[lnKey] = lnGlobal return &sharedListener{ + listener: ln, + manager: m, + key: lnKey, usage: &lnGlobal.usage, deadline: &lnGlobal.deadline, deadlineMu: &lnGlobal.deadlineMu, - key: lnKey, - listener: ln, }, nil case "udp": - listenersMu.Lock() - defer listenersMu.Unlock() + m.listenersMu.Lock() + defer m.listenersMu.Unlock() - if lnGlobal, ok := listeners[lnKey]; ok { + if lnGlobal, ok := m.listeners[lnKey]; ok { lnGlobal.usage.Add(1) return &sharedPacketConn{ - usage: &lnGlobal.usage, - key: lnKey, PacketConn: lnGlobal.pc, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, }, nil } @@ -212,12 +225,13 @@ func Listen(ctx context.Context, network string, addr string, config net.ListenC lnGlobal := &globalListener{pc: pc} lnGlobal.usage.Store(1) - listeners[lnKey] = lnGlobal + m.listeners[lnKey] = lnGlobal return &sharedPacketConn{ - usage: &lnGlobal.usage, - key: lnKey, PacketConn: pc, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, }, nil default: @@ -225,3 +239,9 @@ func Listen(ctx context.Context, network string, addr string, config net.ListenC } } + +func (m *listenerManager) Delete(key string) { + m.listenersMu.Lock() + delete(m.listeners, key) + m.listenersMu.Unlock() +} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index fbc40e89..92bb62b4 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -59,6 +59,7 @@ func init() { } type SSServer struct { + lnManager ListenerManager natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache @@ -88,6 +89,7 @@ func (s *SSServer) loadConfig(filename string) error { legacyService, ok := legacyPortService[legacyKeyServiceConfig.Port] if !ok { legacyService = &Service{ + lnManager: s.lnManager, natTimeout: s.natTimeout, m: s.m, replayCache: &s.replayCache, @@ -111,7 +113,7 @@ func (s *SSServer) loadConfig(filename string) error { } for _, serviceConfig := range config.Services { - service, err := NewService(serviceConfig, s.natTimeout, s.m, &s.replayCache) + service, err := NewService(serviceConfig, s.lnManager, s.natTimeout, s.m, &s.replayCache) if err != nil { return fmt.Errorf("Failed to create new service: %v", err) } @@ -156,6 +158,7 @@ func (s *SSServer) Stop() error { // RunSSServer starts a shadowsocks server running, and returns the server or an error. func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ + lnManager: NewListenerManager(), natTimeout: natTimeout, m: sm, replayCache: service.NewReplayCache(replayHistory), diff --git a/cmd/outline-ss-server/service.go b/cmd/outline-ss-server/service.go index 9dc98657..a8ac9f04 100644 --- a/cmd/outline-ss-server/service.go +++ b/cmd/outline-ss-server/service.go @@ -34,6 +34,7 @@ import ( type Listener = any type Service struct { + lnManager ListenerManager natTimeout time.Duration m *outlineMetrics replayCache *service.ReplayCache @@ -90,7 +91,7 @@ func (s *Service) AddListener(network string, addr string) error { cipherList := service.NewCipherList() cipherList.Update(s.ciphers) - listener, err := Listen(context.TODO(), network, addr, net.ListenConfig{KeepAlive: 0}) + listener, err := s.lnManager.Listen(context.TODO(), network, addr, net.ListenConfig{KeepAlive: 0}) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", network, addr, err) @@ -117,8 +118,9 @@ func (s *Service) NumCiphers() int { } // NewService creates a new Service based on a config -func NewService(config ServiceConfig, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { +func NewService(config ServiceConfig, lnManager ListenerManager, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { s := Service{ + lnManager: lnManager, natTimeout: natTimeout, m: m, replayCache: replayCache, From eacfa0e7fd3e5b8c7b0bc8170bfa018622d5ca13 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 8 Jul 2024 17:50:49 -0400 Subject: [PATCH 048/182] Add validation check that no two services have the same listener. --- cmd/outline-ss-server/config.go | 17 +++++++++++------ cmd/outline-ss-server/config_test.go | 17 +++++++++++++++++ cmd/outline-ss-server/listeners.go | 6 +++++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/cmd/outline-ss-server/config.go b/cmd/outline-ss-server/config.go index e8a8a43f..1a734720 100644 --- a/cmd/outline-ss-server/config.go +++ b/cmd/outline-ss-server/config.go @@ -57,20 +57,25 @@ type Config struct { // Validate checks that the config is valid. func (c *Config) Validate() error { + existingListeners := make(map[string]bool) for _, serviceConfig := range c.Services { - for _, listenerConfig := range serviceConfig.Listeners { + for _, lnConfig := range serviceConfig.Listeners { // TODO: Support more listener types. - if listenerConfig.Type != listenerTypeTCP && listenerConfig.Type != listenerTypeUDP { - return fmt.Errorf("unsupported listener type: %s", listenerConfig.Type) + if lnConfig.Type != listenerTypeTCP && lnConfig.Type != listenerTypeUDP { + return fmt.Errorf("unsupported listener type: %s", lnConfig.Type) } - - host, _, err := net.SplitHostPort(listenerConfig.Address) + host, _, err := net.SplitHostPort(lnConfig.Address) if err != nil { - return fmt.Errorf("invalid listener address `%s`: %v", listenerConfig.Address, err) + return fmt.Errorf("invalid listener address `%s`: %v", lnConfig.Address, err) } if ip := net.ParseIP(host); ip == nil { return fmt.Errorf("address must be IP, found: %s", host) } + key := listenerKey(string(lnConfig.Type), lnConfig.Address) + if _, exists := existingListeners[key]; exists { + return fmt.Errorf("listener of type %s with address %s already exists.", lnConfig.Type, lnConfig.Address) + } + existingListeners[key] = true } } return nil diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go index 25895111..f183ff5a 100644 --- a/cmd/outline-ss-server/config_test.go +++ b/cmd/outline-ss-server/config_test.go @@ -62,6 +62,23 @@ func TestValidateConfigFails(t *testing.T) { }, }, }, + { + name: "WithDuplicateListeners", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, + }, + }, + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, + }, + }, + }, + }, + }, } for _, tc := range tests { diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index c8a5d644..77716acf 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -166,7 +166,7 @@ func NewListenerManager() ListenerManager { // reusable listener wrappers, which do not actually close the underlying socket // until all uses of the shared listener have been closed. func (m *listenerManager) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (any, error) { - lnKey := network + "/" + addr + lnKey := listenerKey(network, addr) switch network { @@ -245,3 +245,7 @@ func (m *listenerManager) Delete(key string) { delete(m.listeners, key) m.listenersMu.Unlock() } + +func listenerKey(network string, addr string) string { + return network + "/" + addr +} From dc1075a1ddae2d0407b91f92fe2c4d03cac9e2e3 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 10 Jul 2024 15:45:35 -0400 Subject: [PATCH 049/182] Use channels to notify shared listeners they need to stop acceoting. --- cmd/outline-ss-server/listeners.go | 118 +++++++++++------------------ 1 file changed, 45 insertions(+), 73 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index 77716acf..d00ac30b 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -20,78 +20,44 @@ import ( "net" "sync" "sync/atomic" - "time" ) +type acceptResponse struct { + conn net.Conn + err error +} + type sharedListener struct { - listener net.Listener - manager ListenerManager - key string - closed atomic.Int32 - usage *atomic.Int32 - deadline *bool - deadlineMu *sync.Mutex + listener net.Listener + manager ListenerManager + key string + closed atomic.Int32 + usage *atomic.Int32 + acceptCh chan acceptResponse + closeCh chan struct{} } // Accept accepts connections until Close() is called. func (sl *sharedListener) Accept() (net.Conn, error) { if sl.closed.Load() == 1 { - return nil, &net.OpError{ - Op: "accept", - Net: sl.listener.Addr().Network(), - Addr: sl.listener.Addr(), - Err: net.ErrClosed, - } + return nil, net.ErrClosed } - - conn, err := sl.listener.Accept() - if err == nil { - return conn, nil - } - - sl.deadlineMu.Lock() - if *sl.deadline { - switch ln := sl.listener.(type) { - case *net.TCPListener: - ln.SetDeadline(time.Time{}) - } - *sl.deadline = false - } - sl.deadlineMu.Unlock() - - if sl.closed.Load() == 1 { - // In `Close()` we set a deadline in the past to force currently-blocked - // listeners to close without having to close the underlying socket. To - // avoid callers from retrying, we avoid returning timeout errors and - // instead make sure we return a fake "closed" error. - if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - return nil, &net.OpError{ - Op: "accept", - Net: sl.listener.Addr().Network(), - Addr: sl.listener.Addr(), - Err: net.ErrClosed, - } + select { + case acceptResponse := <-sl.acceptCh: + if acceptResponse.err != nil { + return nil, acceptResponse.err } + return acceptResponse.conn, nil + case <-sl.closeCh: + return nil, net.ErrClosed } - - return nil, err } // Close stops accepting new connections without closing the underlying socket. // Only when the last user closes it, we actually close it. func (sl *sharedListener) Close() error { if sl.closed.CompareAndSwap(0, 1) { - // NOTE: In order to cancel current calls to Accept(), we set a deadline in - // the past, as we cannot actually close the listener. - sl.deadlineMu.Lock() - if !*sl.deadline { - switch ln := sl.listener.(type) { - case *net.TCPListener: - ln.SetDeadline(time.Now().Add(-1 * time.Minute)) - } - *sl.deadline = true - } - sl.deadlineMu.Unlock() + close(sl.closeCh) // See if we need to actually close the underlying listener. if sl.usage.Add(-1) == 0 { @@ -135,11 +101,10 @@ func (spc *sharedPacketConn) Close() error { } type globalListener struct { - ln net.Listener - pc net.PacketConn - usage atomic.Int32 - deadline bool - deadlineMu sync.Mutex + ln net.Listener + pc net.PacketConn + usage atomic.Int32 + acceptCh chan acceptResponse } // ListenerManager holds and manages the state of shared listeners. @@ -177,12 +142,12 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin if lnGlobal, ok := m.listeners[lnKey]; ok { lnGlobal.usage.Add(1) return &sharedListener{ - listener: lnGlobal.ln, - manager: m, - key: lnKey, - usage: &lnGlobal.usage, - deadline: &lnGlobal.deadline, - deadlineMu: &lnGlobal.deadlineMu, + listener: lnGlobal.ln, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, + acceptCh: lnGlobal.acceptCh, + closeCh: make(chan struct{}), }, nil } @@ -191,17 +156,24 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin return nil, err } - lnGlobal := &globalListener{ln: ln} + lnGlobal := &globalListener{ln: ln, acceptCh: make(chan acceptResponse)} lnGlobal.usage.Store(1) m.listeners[lnKey] = lnGlobal + go func() { + for { + conn, err := lnGlobal.ln.Accept() + lnGlobal.acceptCh <- acceptResponse{conn, err} + } + }() + return &sharedListener{ - listener: ln, - manager: m, - key: lnKey, - usage: &lnGlobal.usage, - deadline: &lnGlobal.deadline, - deadlineMu: &lnGlobal.deadlineMu, + listener: ln, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, + acceptCh: lnGlobal.acceptCh, + closeCh: make(chan struct{}), }, nil case "udp": From 2a343e2efbffbe247458e86fb8e058af79b4ce8b Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 10 Jul 2024 15:51:37 -0400 Subject: [PATCH 050/182] Pass TCP timeout to service. --- cmd/outline-ss-server/main.go | 3 ++- cmd/outline-ss-server/service.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 92bb62b4..111b1c2d 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -90,6 +90,7 @@ func (s *SSServer) loadConfig(filename string) error { if !ok { legacyService = &Service{ lnManager: s.lnManager, + tcpTimeout: tcpReadTimeout, natTimeout: s.natTimeout, m: s.m, replayCache: &s.replayCache, @@ -113,7 +114,7 @@ func (s *SSServer) loadConfig(filename string) error { } for _, serviceConfig := range config.Services { - service, err := NewService(serviceConfig, s.lnManager, s.natTimeout, s.m, &s.replayCache) + service, err := NewService(serviceConfig, s.lnManager, tcpReadTimeout, s.natTimeout, s.m, &s.replayCache) if err != nil { return fmt.Errorf("Failed to create new service: %v", err) } diff --git a/cmd/outline-ss-server/service.go b/cmd/outline-ss-server/service.go index a8ac9f04..d935b0de 100644 --- a/cmd/outline-ss-server/service.go +++ b/cmd/outline-ss-server/service.go @@ -35,6 +35,7 @@ type Listener = any type Service struct { lnManager ListenerManager + tcpTimeout time.Duration natTimeout time.Duration m *outlineMetrics replayCache *service.ReplayCache @@ -47,7 +48,7 @@ func (s *Service) Serve(lnKey string, listener Listener, cipherList service.Ciph case net.Listener: authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, s.replayCache, s.m) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(lnKey, authFunc, s.m, tcpReadTimeout) + tcpHandler := service.NewTCPHandler(lnKey, authFunc, s.m, s.tcpTimeout) accept := func() (transport.StreamConn, error) { c, err := ln.Accept() if err == nil { @@ -118,9 +119,10 @@ func (s *Service) NumCiphers() int { } // NewService creates a new Service based on a config -func NewService(config ServiceConfig, lnManager ListenerManager, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { +func NewService(config ServiceConfig, lnManager ListenerManager, tcpTimeout time.Duration, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { s := Service{ lnManager: lnManager, + tcpTimeout: tcpTimeout, natTimeout: natTimeout, m: m, replayCache: replayCache, From e58b79dcf55ef5a0d567d1c8682d247c8c346c48 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 10 Jul 2024 15:57:23 -0400 Subject: [PATCH 051/182] Move go routine call up. --- cmd/outline-ss-server/listeners.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index d00ac30b..03be496d 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -67,7 +67,6 @@ func (sl *sharedListener) Close() error { return err } } - } return nil @@ -157,15 +156,14 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin } lnGlobal := &globalListener{ln: ln, acceptCh: make(chan acceptResponse)} - lnGlobal.usage.Store(1) - m.listeners[lnKey] = lnGlobal - go func() { for { conn, err := lnGlobal.ln.Accept() lnGlobal.acceptCh <- acceptResponse{conn, err} } }() + lnGlobal.usage.Store(1) + m.listeners[lnKey] = lnGlobal return &sharedListener{ listener: ln, From c7465fbe0496e9a36a6211aa503689acb01e70c2 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 11 Jul 2024 12:37:17 -0400 Subject: [PATCH 052/182] Allow inserting single elements directly into the cipher list. --- service/cipher_list.go | 6 ++++++ service/cipher_list_testing.go | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/service/cipher_list.go b/service/cipher_list.go index 3b6f1957..171b4236 100644 --- a/service/cipher_list.go +++ b/service/cipher_list.go @@ -62,6 +62,8 @@ type CipherList interface { // which is a List of *CipherEntry. Update takes ownership of `contents`, // which must not be read or written after this call. Update(contents *list.List) + // PushBack inserts a new cipher at the back of the list. + PushBack(entry *CipherEntry) *list.Element } type cipherList struct { @@ -116,3 +118,7 @@ func (cl *cipherList) Update(src *list.List) { cl.list = src cl.mu.Unlock() } + +func (cl *cipherList) PushBack(entry *CipherEntry) *list.Element { + return cl.list.PushBack(entry) +} diff --git a/service/cipher_list_testing.go b/service/cipher_list_testing.go index a77427ed..d8532f79 100644 --- a/service/cipher_list_testing.go +++ b/service/cipher_list_testing.go @@ -15,7 +15,6 @@ package service import ( - "container/list" "fmt" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" @@ -24,7 +23,7 @@ import ( // MakeTestCiphers creates a CipherList containing one fresh AEAD cipher // for each secret in `secrets`. func MakeTestCiphers(secrets []string) (CipherList, error) { - l := list.New() + cipherList := NewCipherList() for i := 0; i < len(secrets); i++ { cipherID := fmt.Sprintf("id-%v", i) cipher, err := shadowsocks.NewEncryptionKey(shadowsocks.CHACHA20IETFPOLY1305, secrets[i]) @@ -32,10 +31,8 @@ func MakeTestCiphers(secrets []string) (CipherList, error) { return nil, fmt.Errorf("failed to create cipher %v: %w", i, err) } entry := MakeCipherEntry(cipherID, cipher, secrets[i]) - l.PushBack(&entry) + cipherList.PushBack(&entry) } - cipherList := NewCipherList() - cipherList.Update(l) return cipherList, nil } From 43fa0d61bc26c2b09cae07d9149549734523549b Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 11 Jul 2024 16:14:40 -0400 Subject: [PATCH 053/182] Add the concept of a listener set to track existing listeners and close them all. --- cmd/outline-ss-server/listeners.go | 83 ++++++++- cmd/outline-ss-server/main.go | 154 ++++++++++++----- cmd/outline-ss-server/service.go | 160 ------------------ internal/integration_test/integration_test.go | 6 +- service/cipher_list.go | 5 + service/udp.go | 5 +- service/udp_test.go | 5 +- 7 files changed, 208 insertions(+), 210 deletions(-) delete mode 100644 cmd/outline-ss-server/service.go diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go index 03be496d..f45339b1 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/cmd/outline-ss-server/listeners.go @@ -20,6 +20,9 @@ import ( "net" "sync" "sync/atomic" + + "github.com/Jigsaw-Code/outline-sdk/transport" + "github.com/Jigsaw-Code/outline-ss-server/service" ) type acceptResponse struct { @@ -27,6 +30,10 @@ type acceptResponse struct { err error } +type SharedListener interface { + SetHandler(handler Handler) +} + type sharedListener struct { listener net.Listener manager ListenerManager @@ -37,6 +44,20 @@ type sharedListener struct { closeCh chan struct{} } +func (sl *sharedListener) SetHandler(handler Handler) { + accept := func() (transport.StreamConn, error) { + c, err := sl.Accept() + if err == nil { + return c.(transport.StreamConn), err + } + return nil, err + } + handle := func(ctx context.Context, conn transport.StreamConn) { + handler.Handle(ctx, conn) + } + go service.StreamServe(accept, handle) +} + // Accept accepts connections until Close() is called. func (sl *sharedListener) Accept() (net.Conn, error) { if sl.closed.Load() == 1 { @@ -99,6 +120,10 @@ func (spc *sharedPacketConn) Close() error { return nil } +func (spc *sharedPacketConn) SetHandler(handler Handler) { + go handler.Handle(context.TODO(), spc.PacketConn) +} + type globalListener struct { ln net.Listener pc net.PacketConn @@ -106,9 +131,56 @@ type globalListener struct { acceptCh chan acceptResponse } +type ListenerSet interface { + Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) + Close() error + Len() int +} + +type listenerSet struct { + manager ListenerManager + listeners map[string]*SharedListener +} + +func (ls *listenerSet) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) { + lnKey := listenerKey(network, addr) + if _, exists := ls.listeners[lnKey]; exists { + return nil, fmt.Errorf("listener %s already exists", lnKey) + } + ln, err := ls.manager.Listen(ctx, network, addr, config) + if err != nil { + return nil, err + } + ls.listeners[lnKey] = &ln + return ln, nil +} + +func (ls *listenerSet) Close() error { + for _, listener := range ls.listeners { + switch ln := (*listener).(type) { + case net.Listener: + if err := ln.Close(); err != nil { + return fmt.Errorf("%s listener on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) + } + case net.PacketConn: + if err := ln.Close(); err != nil { + return fmt.Errorf("%s listener on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) + } + default: + return fmt.Errorf("unknown listener type: %v", ln) + } + } + return nil +} + +func (ls *listenerSet) Len() int { + return len(ls.listeners) +} + // ListenerManager holds and manages the state of shared listeners. type ListenerManager interface { - Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (any, error) + NewListenerSet() ListenerSet + Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) Delete(key string) } @@ -123,13 +195,20 @@ func NewListenerManager() ListenerManager { } } +func (m *listenerManager) NewListenerSet() ListenerSet { + return &listenerSet{ + manager: m, + listeners: make(map[string]*SharedListener), + } +} + // Listen creates a new listener for a given network and address. // // Listeners can overlap one another, because during config changes the new // config is started before the old config is destroyed. This is done by using // reusable listener wrappers, which do not actually close the underlying socket // until all uses of the shared listener have been closed. -func (m *listenerManager) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (any, error) { +func (m *listenerManager) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) { lnKey := listenerKey(network, addr) switch network { diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 111b1c2d..10375744 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -15,7 +15,7 @@ package main import ( - "container/list" + "context" "flag" "fmt" "net" @@ -27,6 +27,7 @@ import ( "syscall" "time" + "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" @@ -58,12 +59,80 @@ func init() { logger = logging.MustGetLogger("") } +type Handler interface { + NumCiphers() int + AddCipher(entry *service.CipherEntry) + Handle(ctx context.Context, conn any) +} + +type connHandler struct { + tcpTimeout time.Duration + natTimeout time.Duration + replayCache *service.ReplayCache + m *outlineMetrics + ciphers service.CipherList +} + +// NewHandler creates a new Handler handler based on a service config. +func NewHandler(config ServiceConfig, tcpTimeout time.Duration, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (Handler, error) { + type cipherKey struct { + cipher string + secret string + } + ciphers := service.NewCipherList() + existingCiphers := make(map[cipherKey]bool) + for _, keyConfig := range config.Keys { + key := cipherKey{keyConfig.Cipher, keyConfig.Secret} + if _, exists := existingCiphers[key]; exists { + logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) + continue + } + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + } + entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + ciphers.PushBack(&entry) + existingCiphers[key] = true + } + return &connHandler{ + ciphers: ciphers, + tcpTimeout: tcpTimeout, + natTimeout: natTimeout, + m: m, + replayCache: replayCache, + }, nil +} + +func (h *connHandler) NumCiphers() int { + return h.ciphers.Len() +} + +func (h *connHandler) AddCipher(entry *service.CipherEntry) { + h.ciphers.PushBack(entry) +} + +func (h *connHandler) Handle(ctx context.Context, conn any) { + switch c := conn.(type) { + case transport.StreamConn: + authFunc := service.NewShadowsocksStreamAuthenticator(h.ciphers, h.replayCache, h.m) + // TODO: Register initial data metrics at zero. + tcpHandler := service.NewTCPHandler(c.LocalAddr().String(), authFunc, h.m, h.tcpTimeout) + tcpHandler.Handle(ctx, c) + case net.PacketConn: + packetHandler := service.NewPacketHandler(h.natTimeout, h.ciphers, h.m) + packetHandler.Handle(ctx, c) + default: + logger.Errorf("unknown connection type: %v", c) + } +} + type SSServer struct { lnManager ListenerManager + lnSet ListenerSet natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache - services []*Service } func (s *SSServer) loadConfig(filename string) error { @@ -82,77 +151,80 @@ func (s *SSServer) loadConfig(filename string) error { // We hot swap the services by having them both live at the same time. This // means we create services for the new config first, and then take down the // services from the old config. - newServices := make([]*Service, 0) + oldListenerSet := s.lnSet + s.lnSet = s.lnManager.NewListenerSet() + var totalCipherCount int - legacyPortService := make(map[int]*Service) // Values are *List of *CipherEntry. + portHandlers := make(map[int]Handler) for _, legacyKeyServiceConfig := range config.Keys { - legacyService, ok := legacyPortService[legacyKeyServiceConfig.Port] + handler, ok := portHandlers[legacyKeyServiceConfig.Port] if !ok { - legacyService = &Service{ - lnManager: s.lnManager, + handler = &connHandler{ + ciphers: service.NewCipherList(), tcpTimeout: tcpReadTimeout, natTimeout: s.natTimeout, m: s.m, replayCache: &s.replayCache, - ciphers: list.New(), } - for _, network := range []string{"tcp", "udp"} { - addr := net.JoinHostPort("::", strconv.Itoa(legacyKeyServiceConfig.Port)) - if err := legacyService.AddListener(network, addr); err != nil { - return err - } - } - newServices = append(newServices, legacyService) - legacyPortService[legacyKeyServiceConfig.Port] = legacyService + portHandlers[legacyKeyServiceConfig.Port] = handler } cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyServiceConfig.Cipher, legacyKeyServiceConfig.Secret) if err != nil { return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyServiceConfig.ID, err) } entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) - legacyService.AddCipher(&entry) + handler.AddCipher(&entry) + } + for portNum, handler := range portHandlers { + totalCipherCount += handler.NumCiphers() + for _, network := range []string{"tcp", "udp"} { + addr := net.JoinHostPort("::", strconv.Itoa(portNum)) + listener, err := s.lnSet.Listen(context.TODO(), network, addr, net.ListenConfig{KeepAlive: 0}) + if err != nil { + return fmt.Errorf("%s service failed to start listening on address %s: %w", network, addr, err) + } + listener.SetHandler(handler) + } } for _, serviceConfig := range config.Services { - service, err := NewService(serviceConfig, s.lnManager, tcpReadTimeout, s.natTimeout, s.m, &s.replayCache) + handler, err := NewHandler(serviceConfig, tcpReadTimeout, s.natTimeout, s.m, &s.replayCache) if err != nil { - return fmt.Errorf("Failed to create new service: %v", err) + return fmt.Errorf("failed to create service handler: %w", err) + } + totalCipherCount += handler.NumCiphers() + for _, listenerConfig := range serviceConfig.Listeners { + network := string(listenerConfig.Type) + listener, err := s.lnSet.Listen(context.TODO(), network, listenerConfig.Address, net.ListenConfig{KeepAlive: 0}) + if err != nil { + return fmt.Errorf("%s service failed to start listening on address %s: %w", network, listenerConfig.Address, err) + } + listener.SetHandler(handler) } - newServices = append(newServices, service) } - logger.Infof("Loaded %d new services", len(newServices)) + logger.Infof("Loaded %d access keys over %d listeners", totalCipherCount, s.lnSet.Len()) + s.m.SetNumAccessKeys(totalCipherCount, s.lnSet.Len()) // Take down the old services now that the new ones are created and serving. - if err := s.Stop(); err != nil { - logger.Errorf("Failed to stop old services: %w", err) + if oldListenerSet != nil { + if err := oldListenerSet.Close(); err != nil { + logger.Errorf("Failed to stop old listeners: %w", err) + } + logger.Infof("Stopped %d old listeners", s.lnSet.Len()) } - s.services = newServices - var ( - listenerCount int - cipherCount int - ) - for _, service := range s.services { - listenerCount += service.NumListeners() - cipherCount += service.NumCiphers() - } - logger.Infof("%d services active: %d access keys over %d listeners", len(s.services), cipherCount, listenerCount) - s.m.SetNumAccessKeys(cipherCount, listenerCount) return nil } // Stop serving on all existing services. func (s *SSServer) Stop() error { - if len(s.services) == 0 { + if s.lnSet == nil { return nil } - for _, service := range s.services { - if err := service.Stop(); err != nil { - return err - } + if err := s.lnSet.Close(); err != nil { + logger.Errorf("Failed to stop all listeners: %w", err) } - logger.Infof("Stopped %d old services", len(s.services)) - s.services = nil + logger.Infof("Stopped %d listeners", s.lnSet.Len()) return nil } diff --git a/cmd/outline-ss-server/service.go b/cmd/outline-ss-server/service.go deleted file mode 100644 index d935b0de..00000000 --- a/cmd/outline-ss-server/service.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2024 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "container/list" - "context" - "fmt" - "net" - "time" - - "github.com/Jigsaw-Code/outline-sdk/transport" - "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" - "github.com/Jigsaw-Code/outline-ss-server/service" -) - -// The implementations of listeners for different network types are not -// interchangeable. The type of listener depends on the network type. -// TODO(sbruens): Create a custom `Listener` type so we can share serving logic, -// dispatching to the handlers based on connection type instead of on the -// listener type. -type Listener = any - -type Service struct { - lnManager ListenerManager - tcpTimeout time.Duration - natTimeout time.Duration - m *outlineMetrics - replayCache *service.ReplayCache - listeners []Listener - ciphers *list.List // Values are *List of *service.CipherEntry. -} - -func (s *Service) Serve(lnKey string, listener Listener, cipherList service.CipherList) error { - switch ln := listener.(type) { - case net.Listener: - authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, s.replayCache, s.m) - // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(lnKey, authFunc, s.m, s.tcpTimeout) - accept := func() (transport.StreamConn, error) { - c, err := ln.Accept() - if err == nil { - return c.(transport.StreamConn), err - } - return nil, err - } - go service.StreamServe(accept, tcpHandler.Handle) - case net.PacketConn: - packetHandler := service.NewPacketHandler(s.natTimeout, cipherList, s.m) - go packetHandler.Handle(ln) - default: - return fmt.Errorf("unknown listener type: %v", ln) - } - return nil -} - -func (s *Service) Stop() error { - for _, listener := range s.listeners { - switch ln := listener.(type) { - case net.Listener: - if err := ln.Close(); err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) - } - case net.PacketConn: - if err := ln.Close(); err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) - } - default: - return fmt.Errorf("unknown listener type: %v", ln) - } - } - return nil -} - -// AddListener adds a new listener to the service. -func (s *Service) AddListener(network string, addr string) error { - // Create new listeners based on the configured network addresses. - cipherList := service.NewCipherList() - cipherList.Update(s.ciphers) - - listener, err := s.lnManager.Listen(context.TODO(), network, addr, net.ListenConfig{KeepAlive: 0}) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks %s service failed to start on address %s: %w", network, addr, err) - } - s.listeners = append(s.listeners, listener) - logger.Infof("Shadowsocks %s service listening on %s", network, addr) - lnKey := network + "/" + addr - if err = s.Serve(lnKey, listener, cipherList); err != nil { - return fmt.Errorf("failed to serve on %s listener on address %s: %w", network, addr, err) - } - return nil -} - -func (s *Service) NumListeners() int { - return len(s.listeners) -} - -func (s *Service) AddCipher(entry *service.CipherEntry) { - s.ciphers.PushBack(entry) -} - -func (s *Service) NumCiphers() int { - return s.ciphers.Len() -} - -// NewService creates a new Service based on a config -func NewService(config ServiceConfig, lnManager ListenerManager, tcpTimeout time.Duration, natTimeout time.Duration, m *outlineMetrics, replayCache *service.ReplayCache) (*Service, error) { - s := Service{ - lnManager: lnManager, - tcpTimeout: tcpTimeout, - natTimeout: natTimeout, - m: m, - replayCache: replayCache, - ciphers: list.New(), - } - - type cipherKey struct { - cipher string - secret string - } - existingCiphers := make(map[cipherKey]bool) - for _, keyConfig := range config.Keys { - key := cipherKey{keyConfig.Cipher, keyConfig.Secret} - if _, exists := existingCiphers[key]; exists { - logger.Debugf("encryption key already exists for ID=`%v`. Skipping.", keyConfig.ID) - continue - } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) - if err != nil { - return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) - } - entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - s.AddCipher(&entry) - existingCiphers[key] = true - } - - for _, listener := range config.Listeners { - network := string(listener.Type) - if err := s.AddListener(network, listener.Address); err != nil { - return nil, err - } - } - - return &s, nil -} diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index f98319f4..2bfb0dfa 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -293,7 +293,7 @@ func TestUDPEcho(t *testing.T) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(proxyConn) + proxy.Handle(context.Background(), proxyConn) done <- struct{}{} }() @@ -525,7 +525,7 @@ func BenchmarkUDPEcho(b *testing.B) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(server) + proxy.Handle(context.Background(), server) done <- struct{}{} }() @@ -569,7 +569,7 @@ func BenchmarkUDPManyKeys(b *testing.B) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(proxyConn) + proxy.Handle(context.Background(), proxyConn) done <- struct{}{} }() diff --git a/service/cipher_list.go b/service/cipher_list.go index 171b4236..d84ab1ac 100644 --- a/service/cipher_list.go +++ b/service/cipher_list.go @@ -55,6 +55,7 @@ func MakeCipherEntry(id string, cryptoKey *shadowsocks.EncryptionKey, secret str // CipherList is a thread-safe collection of CipherEntry elements that allows for // snapshotting and moving to front. type CipherList interface { + Len() int // Returns a snapshot of the cipher list optimized for this client IP SnapshotForClientIP(clientIP netip.Addr) []*list.Element MarkUsedByClientIP(e *list.Element, clientIP netip.Addr) @@ -77,6 +78,10 @@ func NewCipherList() CipherList { return &cipherList{list: list.New()} } +func (cl *cipherList) Len() int { + return cl.list.Len() +} + func matchesIP(e *list.Element, clientIP netip.Addr) bool { c := e.Value.(*CipherEntry) return clientIP != netip.Addr{} && clientIP == c.lastClientIP diff --git a/service/udp.go b/service/udp.go index 4830e302..859c6c44 100644 --- a/service/udp.go +++ b/service/udp.go @@ -15,6 +15,7 @@ package service import ( + "context" "errors" "fmt" "net" @@ -101,7 +102,7 @@ type PacketHandler interface { // SetTargetIPValidator sets the function to be used to validate the target IP addresses. SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) // Handle returns after clientConn closes and all the sub goroutines return. - Handle(clientConn net.PacketConn) + Handle(ctx context.Context, clientConn net.PacketConn) } func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) { @@ -110,7 +111,7 @@ func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPVali // Listen on addr for encrypted packets and basically do UDP NAT. // We take the ciphers as a pointer because it gets replaced on config updates. -func (h *packetHandler) Handle(clientConn net.PacketConn) { +func (h *packetHandler) Handle(ctx context.Context, clientConn net.PacketConn) { var running sync.WaitGroup nm := newNATmap(h.natTimeout, h.m, &running) diff --git a/service/udp_test.go b/service/udp_test.go index f94238c5..90d880b5 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -16,6 +16,7 @@ package service import ( "bytes" + "context" "errors" "net" "net/netip" @@ -132,7 +133,7 @@ func sendToDiscard(payloads [][]byte, validator onet.TargetIPValidator) *natTest handler.SetTargetIPValidator(validator) done := make(chan struct{}) go func() { - handler.Handle(clientConn) + handler.Handle(context.Background(), clientConn) done <- struct{}{} }() @@ -488,7 +489,7 @@ func TestUDPEarlyClose(t *testing.T) { } require.Nil(t, clientConn.Close()) // This should return quickly without timing out. - s.Handle(clientConn) + s.Handle(context.Background(), clientConn) } // Makes sure the UDP listener returns [io.ErrClosed] on reads and writes after Close(). From cf9b7d2cd389c16c002f51331d076640228f59d4 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 11 Jul 2024 16:46:38 -0400 Subject: [PATCH 054/182] Refactor how we create listeners. We introduce shared listeners that allow us to keep an old config running while we set up a new config. This is done by keeping track of the usage of the listeners and only closing them when the last user is done with the shared listener. --- cmd/outline-ss-server/listeners.go | 300 ++++++++++++++++++ cmd/outline-ss-server/main.go | 167 +++++----- cmd/outline-ss-server/metrics.go | 14 +- go.mod | 4 +- internal/integration_test/integration_test.go | 6 +- service/cipher_list.go | 11 + service/cipher_list_testing.go | 7 +- service/tcp.go | 2 +- service/udp.go | 5 +- service/udp_test.go | 5 +- 10 files changed, 418 insertions(+), 103 deletions(-) create mode 100644 cmd/outline-ss-server/listeners.go diff --git a/cmd/outline-ss-server/listeners.go b/cmd/outline-ss-server/listeners.go new file mode 100644 index 00000000..f45339b1 --- /dev/null +++ b/cmd/outline-ss-server/listeners.go @@ -0,0 +1,300 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net" + "sync" + "sync/atomic" + + "github.com/Jigsaw-Code/outline-sdk/transport" + "github.com/Jigsaw-Code/outline-ss-server/service" +) + +type acceptResponse struct { + conn net.Conn + err error +} + +type SharedListener interface { + SetHandler(handler Handler) +} + +type sharedListener struct { + listener net.Listener + manager ListenerManager + key string + closed atomic.Int32 + usage *atomic.Int32 + acceptCh chan acceptResponse + closeCh chan struct{} +} + +func (sl *sharedListener) SetHandler(handler Handler) { + accept := func() (transport.StreamConn, error) { + c, err := sl.Accept() + if err == nil { + return c.(transport.StreamConn), err + } + return nil, err + } + handle := func(ctx context.Context, conn transport.StreamConn) { + handler.Handle(ctx, conn) + } + go service.StreamServe(accept, handle) +} + +// Accept accepts connections until Close() is called. +func (sl *sharedListener) Accept() (net.Conn, error) { + if sl.closed.Load() == 1 { + return nil, net.ErrClosed + } + select { + case acceptResponse := <-sl.acceptCh: + if acceptResponse.err != nil { + return nil, acceptResponse.err + } + return acceptResponse.conn, nil + case <-sl.closeCh: + return nil, net.ErrClosed + } +} + +// Close stops accepting new connections without closing the underlying socket. +// Only when the last user closes it, we actually close it. +func (sl *sharedListener) Close() error { + if sl.closed.CompareAndSwap(0, 1) { + close(sl.closeCh) + + // See if we need to actually close the underlying listener. + if sl.usage.Add(-1) == 0 { + sl.manager.Delete(sl.key) + err := sl.listener.Close() + if err != nil { + return err + } + } + } + + return nil +} + +func (sl *sharedListener) Addr() net.Addr { + return sl.listener.Addr() +} + +type sharedPacketConn struct { + net.PacketConn + manager ListenerManager + key string + closed atomic.Int32 + usage *atomic.Int32 +} + +func (spc *sharedPacketConn) Close() error { + if spc.closed.CompareAndSwap(0, 1) { + // See if we need to actually close the underlying listener. + if spc.usage.Add(-1) == 0 { + spc.manager.Delete(spc.key) + err := spc.PacketConn.Close() + if err != nil { + return err + } + } + } + + return nil +} + +func (spc *sharedPacketConn) SetHandler(handler Handler) { + go handler.Handle(context.TODO(), spc.PacketConn) +} + +type globalListener struct { + ln net.Listener + pc net.PacketConn + usage atomic.Int32 + acceptCh chan acceptResponse +} + +type ListenerSet interface { + Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) + Close() error + Len() int +} + +type listenerSet struct { + manager ListenerManager + listeners map[string]*SharedListener +} + +func (ls *listenerSet) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) { + lnKey := listenerKey(network, addr) + if _, exists := ls.listeners[lnKey]; exists { + return nil, fmt.Errorf("listener %s already exists", lnKey) + } + ln, err := ls.manager.Listen(ctx, network, addr, config) + if err != nil { + return nil, err + } + ls.listeners[lnKey] = &ln + return ln, nil +} + +func (ls *listenerSet) Close() error { + for _, listener := range ls.listeners { + switch ln := (*listener).(type) { + case net.Listener: + if err := ln.Close(); err != nil { + return fmt.Errorf("%s listener on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) + } + case net.PacketConn: + if err := ln.Close(); err != nil { + return fmt.Errorf("%s listener on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) + } + default: + return fmt.Errorf("unknown listener type: %v", ln) + } + } + return nil +} + +func (ls *listenerSet) Len() int { + return len(ls.listeners) +} + +// ListenerManager holds and manages the state of shared listeners. +type ListenerManager interface { + NewListenerSet() ListenerSet + Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) + Delete(key string) +} + +type listenerManager struct { + listeners map[string]*globalListener + listenersMu sync.Mutex +} + +func NewListenerManager() ListenerManager { + return &listenerManager{ + listeners: make(map[string]*globalListener), + } +} + +func (m *listenerManager) NewListenerSet() ListenerSet { + return &listenerSet{ + manager: m, + listeners: make(map[string]*SharedListener), + } +} + +// Listen creates a new listener for a given network and address. +// +// Listeners can overlap one another, because during config changes the new +// config is started before the old config is destroyed. This is done by using +// reusable listener wrappers, which do not actually close the underlying socket +// until all uses of the shared listener have been closed. +func (m *listenerManager) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) { + lnKey := listenerKey(network, addr) + + switch network { + + case "tcp": + m.listenersMu.Lock() + defer m.listenersMu.Unlock() + + if lnGlobal, ok := m.listeners[lnKey]; ok { + lnGlobal.usage.Add(1) + return &sharedListener{ + listener: lnGlobal.ln, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, + acceptCh: lnGlobal.acceptCh, + closeCh: make(chan struct{}), + }, nil + } + + ln, err := config.Listen(ctx, network, addr) + if err != nil { + return nil, err + } + + lnGlobal := &globalListener{ln: ln, acceptCh: make(chan acceptResponse)} + go func() { + for { + conn, err := lnGlobal.ln.Accept() + lnGlobal.acceptCh <- acceptResponse{conn, err} + } + }() + lnGlobal.usage.Store(1) + m.listeners[lnKey] = lnGlobal + + return &sharedListener{ + listener: ln, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, + acceptCh: lnGlobal.acceptCh, + closeCh: make(chan struct{}), + }, nil + + case "udp": + m.listenersMu.Lock() + defer m.listenersMu.Unlock() + + if lnGlobal, ok := m.listeners[lnKey]; ok { + lnGlobal.usage.Add(1) + return &sharedPacketConn{ + PacketConn: lnGlobal.pc, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, + }, nil + } + + pc, err := config.ListenPacket(ctx, network, addr) + if err != nil { + return nil, err + } + + lnGlobal := &globalListener{pc: pc} + lnGlobal.usage.Store(1) + m.listeners[lnKey] = lnGlobal + + return &sharedPacketConn{ + PacketConn: pc, + manager: m, + key: lnKey, + usage: &lnGlobal.usage, + }, nil + + default: + return nil, fmt.Errorf("unsupported network: %s", network) + + } +} + +func (m *listenerManager) Delete(key string) { + m.listenersMu.Lock() + delete(m.listeners, key) + m.listenersMu.Unlock() +} + +func listenerKey(network string, addr string) string { + return network + "/" + addr +} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 87860df7..be94cbdf 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -15,17 +15,19 @@ package main import ( - "container/list" + "context" "flag" "fmt" "net" "net/http" "os" "os/signal" + "strconv" "strings" "syscall" "time" + "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" @@ -58,62 +60,49 @@ func init() { logger = logging.MustGetLogger("") } -type ssPort struct { - tcpListener *net.TCPListener - packetConn net.PacketConn - cipherList service.CipherList +type Handler interface { + NumCiphers() int + AddCipher(entry *service.CipherEntry) + Handle(ctx context.Context, conn any) } -type SSServer struct { +type connHandler struct { + tcpTimeout time.Duration natTimeout time.Duration + replayCache *service.ReplayCache m *outlineMetrics - replayCache service.ReplayCache - ports map[int]*ssPort + ciphers service.CipherList } -func (s *SSServer) startPort(portNum int) error { - listener, err := net.ListenTCP("tcp", &net.TCPAddr{Port: portNum}) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks TCP service failed to start on port %v: %w", portNum, err) - } - logger.Infof("Shadowsocks TCP service listening on %v", listener.Addr().String()) - packetConn, err := net.ListenUDP("udp", &net.UDPAddr{Port: portNum}) - if err != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks UDP service failed to start on port %v: %w", portNum, err) - } - logger.Infof("Shadowsocks UDP service listening on %v", packetConn.LocalAddr().String()) - port := &ssPort{tcpListener: listener, packetConn: packetConn, cipherList: service.NewCipherList()} - authFunc := service.NewShadowsocksStreamAuthenticator(port.cipherList, &s.replayCache, s.m) - // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(listener.Addr().String(), authFunc, s.m, tcpReadTimeout) - packetHandler := service.NewPacketHandler(s.natTimeout, port.cipherList, s.m) - s.ports[portNum] = port - go service.StreamServe(service.WrapStreamListener(listener.AcceptTCP), tcpHandler.Handle) - go packetHandler.Handle(port.packetConn) - return nil +func (h *connHandler) NumCiphers() int { + return h.ciphers.Len() } -func (s *SSServer) removePort(portNum int) error { - port, ok := s.ports[portNum] - if !ok { - return fmt.Errorf("port %v doesn't exist", portNum) - } - tcpErr := port.tcpListener.Close() - udpErr := port.packetConn.Close() - delete(s.ports, portNum) - if tcpErr != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks TCP service on port %v failed to stop: %w", portNum, tcpErr) - } - logger.Infof("Shadowsocks TCP service on port %v stopped", portNum) - if udpErr != nil { - //lint:ignore ST1005 Shadowsocks is capitalized. - return fmt.Errorf("Shadowsocks UDP service on port %v failed to stop: %w", portNum, udpErr) +func (h *connHandler) AddCipher(entry *service.CipherEntry) { + h.ciphers.PushBack(entry) +} + +func (h *connHandler) Handle(ctx context.Context, conn any) { + switch c := conn.(type) { + case transport.StreamConn: + authFunc := service.NewShadowsocksStreamAuthenticator(h.ciphers, h.replayCache, h.m) + // TODO: Register initial data metrics at zero. + tcpHandler := service.NewTCPHandler(c.LocalAddr().String(), authFunc, h.m, h.tcpTimeout) + tcpHandler.Handle(ctx, c) + case net.PacketConn: + packetHandler := service.NewPacketHandler(h.natTimeout, h.ciphers, h.m) + packetHandler.Handle(ctx, c) + default: + logger.Errorf("unknown connection type: %v", c) } - logger.Infof("Shadowsocks UDP service on port %v stopped", portNum) - return nil +} + +type SSServer struct { + lnManager ListenerManager + lnSet ListenerSet + natTimeout time.Duration + m *outlineMetrics + replayCache service.ReplayCache } func (s *SSServer) loadConfig(filename string) error { @@ -122,65 +111,81 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("failed to load config (%v): %w", filename, err) } - portChanges := make(map[int]int) - portCiphers := make(map[int]*list.List) // Values are *List of *CipherEntry. - for _, keyConfig := range config.Keys { - portChanges[keyConfig.Port] = 1 - cipherList, ok := portCiphers[keyConfig.Port] + // We hot swap the services by having them both live at the same time. This + // means we create services for the new config first, and then take down the + // services from the old config. + oldListenerSet := s.lnSet + s.lnSet = s.lnManager.NewListenerSet() + var totalCipherCount int + + portHandlers := make(map[int]Handler) + for _, legacyKeyServiceConfig := range config.Keys { + handler, ok := portHandlers[legacyKeyServiceConfig.Port] if !ok { - cipherList = list.New() - portCiphers[keyConfig.Port] = cipherList + handler = &connHandler{ + ciphers: service.NewCipherList(), + tcpTimeout: tcpReadTimeout, + natTimeout: s.natTimeout, + m: s.m, + replayCache: &s.replayCache, + } + portHandlers[legacyKeyServiceConfig.Port] = handler } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyServiceConfig.Cipher, legacyKeyServiceConfig.Secret) if err != nil { - return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyServiceConfig.ID, err) } - entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - cipherList.PushBack(&entry) - } - for port := range s.ports { - portChanges[port] = portChanges[port] - 1 + entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) + handler.AddCipher(&entry) } - for portNum, count := range portChanges { - if count == -1 { - if err := s.removePort(portNum); err != nil { - return fmt.Errorf("failed to remove port %v: %w", portNum, err) - } - } else if count == +1 { - if err := s.startPort(portNum); err != nil { - return err + for portNum, handler := range portHandlers { + totalCipherCount += handler.NumCiphers() + for _, network := range []string{"tcp", "udp"} { + addr := net.JoinHostPort("::", strconv.Itoa(portNum)) + listener, err := s.lnSet.Listen(context.TODO(), network, addr, net.ListenConfig{KeepAlive: 0}) + if err != nil { + return fmt.Errorf("%s service failed to start listening on address %s: %w", network, addr, err) } + listener.SetHandler(handler) } } - for portNum, cipherList := range portCiphers { - s.ports[portNum].cipherList.Update(cipherList) + logger.Infof("Loaded %d access keys over %d listeners", totalCipherCount, s.lnSet.Len()) + s.m.SetNumAccessKeys(totalCipherCount, s.lnSet.Len()) + + // Take down the old services now that the new ones are created and serving. + if oldListenerSet != nil { + if err := oldListenerSet.Close(); err != nil { + logger.Errorf("Failed to stop old listeners: %w", err) + } + logger.Infof("Stopped %d old listeners", s.lnSet.Len()) } - logger.Infof("Loaded %v access keys over %v ports", len(config.Keys), len(s.ports)) - s.m.SetNumAccessKeys(len(config.Keys), len(portCiphers)) + return nil } -// Stop serving on all ports. +// Stop serving on all existing services. func (s *SSServer) Stop() error { - for portNum := range s.ports { - if err := s.removePort(portNum); err != nil { - return err - } + if s.lnSet == nil { + return nil + } + if err := s.lnSet.Close(); err != nil { + logger.Errorf("Failed to stop all listeners: %w", err) } + logger.Infof("Stopped %d listeners", s.lnSet.Len()) return nil } // RunSSServer starts a shadowsocks server running, and returns the server or an error. func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ + lnManager: NewListenerManager(), natTimeout: natTimeout, m: sm, replayCache: service.NewReplayCache(replayHistory), - ports: make(map[int]*ssPort), } err := server.loadConfig(filename) if err != nil { - return nil, fmt.Errorf("failed configure server: %w", err) + return nil, fmt.Errorf("failed to configure server: %w", err) } sigHup := make(chan os.Signal, 1) signal.Notify(sigHup, syscall.SIGHUP) diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index e95ceeb3..600cea16 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -38,7 +38,7 @@ type outlineMetrics struct { buildInfo *prometheus.GaugeVec accessKeys prometheus.Gauge - ports prometheus.Gauge + listeners prometheus.Gauge dataBytes *prometheus.CounterVec dataBytesPerLocation *prometheus.CounterVec timeToCipherMs *prometheus.HistogramVec @@ -183,10 +183,10 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus Name: "keys", Help: "Count of access keys", }), - ports: prometheus.NewGauge(prometheus.GaugeOpts{ + listeners: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, - Name: "ports", - Help: "Count of open Shadowsocks ports", + Name: "listeners", + Help: "Count of open Shadowsocks listeners", }), tcpProbes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, @@ -265,7 +265,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus m.tunnelTimeCollector = newTunnelTimeCollector(ip2info, registerer) // TODO: Is it possible to pass where to register the collectors? - registerer.MustRegister(m.buildInfo, m.accessKeys, m.ports, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, + registerer.MustRegister(m.buildInfo, m.accessKeys, m.listeners, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, m.dataBytes, m.dataBytesPerLocation, m.timeToCipherMs, m.udpPacketsFromClientPerLocation, m.udpAddedNatEntries, m.udpRemovedNatEntries, m.tunnelTimeCollector) return m @@ -275,9 +275,9 @@ func (m *outlineMetrics) SetBuildInfo(version string) { m.buildInfo.WithLabelValues(version).Set(1) } -func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) { +func (m *outlineMetrics) SetNumAccessKeys(numKeys int, listeners int) { m.accessKeys.Set(float64(numKeys)) - m.ports.Set(float64(ports)) + m.listeners.Set(float64(listeners)) } func (m *outlineMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { diff --git a/go.mod b/go.mod index 04a9ddab..5c1419d2 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/stretchr/testify v1.8.4 golang.org/x/crypto v0.17.0 golang.org/x/term v0.16.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -263,7 +263,7 @@ require ( gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect gopkg.in/src-d/go-git.v4 v4.13.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.90.0 // indirect mvdan.cc/sh/v3 v3.7.0 // indirect sigs.k8s.io/kind v0.17.0 // indirect diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index f98319f4..2bfb0dfa 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -293,7 +293,7 @@ func TestUDPEcho(t *testing.T) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(proxyConn) + proxy.Handle(context.Background(), proxyConn) done <- struct{}{} }() @@ -525,7 +525,7 @@ func BenchmarkUDPEcho(b *testing.B) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(server) + proxy.Handle(context.Background(), server) done <- struct{}{} }() @@ -569,7 +569,7 @@ func BenchmarkUDPManyKeys(b *testing.B) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(proxyConn) + proxy.Handle(context.Background(), proxyConn) done <- struct{}{} }() diff --git a/service/cipher_list.go b/service/cipher_list.go index 3b6f1957..d84ab1ac 100644 --- a/service/cipher_list.go +++ b/service/cipher_list.go @@ -55,6 +55,7 @@ func MakeCipherEntry(id string, cryptoKey *shadowsocks.EncryptionKey, secret str // CipherList is a thread-safe collection of CipherEntry elements that allows for // snapshotting and moving to front. type CipherList interface { + Len() int // Returns a snapshot of the cipher list optimized for this client IP SnapshotForClientIP(clientIP netip.Addr) []*list.Element MarkUsedByClientIP(e *list.Element, clientIP netip.Addr) @@ -62,6 +63,8 @@ type CipherList interface { // which is a List of *CipherEntry. Update takes ownership of `contents`, // which must not be read or written after this call. Update(contents *list.List) + // PushBack inserts a new cipher at the back of the list. + PushBack(entry *CipherEntry) *list.Element } type cipherList struct { @@ -75,6 +78,10 @@ func NewCipherList() CipherList { return &cipherList{list: list.New()} } +func (cl *cipherList) Len() int { + return cl.list.Len() +} + func matchesIP(e *list.Element, clientIP netip.Addr) bool { c := e.Value.(*CipherEntry) return clientIP != netip.Addr{} && clientIP == c.lastClientIP @@ -116,3 +123,7 @@ func (cl *cipherList) Update(src *list.List) { cl.list = src cl.mu.Unlock() } + +func (cl *cipherList) PushBack(entry *CipherEntry) *list.Element { + return cl.list.PushBack(entry) +} diff --git a/service/cipher_list_testing.go b/service/cipher_list_testing.go index a77427ed..d8532f79 100644 --- a/service/cipher_list_testing.go +++ b/service/cipher_list_testing.go @@ -15,7 +15,6 @@ package service import ( - "container/list" "fmt" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" @@ -24,7 +23,7 @@ import ( // MakeTestCiphers creates a CipherList containing one fresh AEAD cipher // for each secret in `secrets`. func MakeTestCiphers(secrets []string) (CipherList, error) { - l := list.New() + cipherList := NewCipherList() for i := 0; i < len(secrets); i++ { cipherID := fmt.Sprintf("id-%v", i) cipher, err := shadowsocks.NewEncryptionKey(shadowsocks.CHACHA20IETFPOLY1305, secrets[i]) @@ -32,10 +31,8 @@ func MakeTestCiphers(secrets []string) (CipherList, error) { return nil, fmt.Errorf("failed to create cipher %v: %w", i, err) } entry := MakeCipherEntry(cipherID, cipher, secrets[i]) - l.PushBack(&entry) + cipherList.PushBack(&entry) } - cipherList := NewCipherList() - cipherList.Update(l) return cipherList, nil } diff --git a/service/tcp.go b/service/tcp.go index 2195480b..ced85a54 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -236,7 +236,7 @@ func StreamServe(accept StreamListener, handle StreamHandler) { if errors.Is(err, net.ErrClosed) { break } - logger.Warningf("AcceptTCP failed: %v. Continuing to listen.", err) + logger.Warningf("Accept failed: %v. Continuing to listen.", err) continue } diff --git a/service/udp.go b/service/udp.go index 4830e302..859c6c44 100644 --- a/service/udp.go +++ b/service/udp.go @@ -15,6 +15,7 @@ package service import ( + "context" "errors" "fmt" "net" @@ -101,7 +102,7 @@ type PacketHandler interface { // SetTargetIPValidator sets the function to be used to validate the target IP addresses. SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) // Handle returns after clientConn closes and all the sub goroutines return. - Handle(clientConn net.PacketConn) + Handle(ctx context.Context, clientConn net.PacketConn) } func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) { @@ -110,7 +111,7 @@ func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPVali // Listen on addr for encrypted packets and basically do UDP NAT. // We take the ciphers as a pointer because it gets replaced on config updates. -func (h *packetHandler) Handle(clientConn net.PacketConn) { +func (h *packetHandler) Handle(ctx context.Context, clientConn net.PacketConn) { var running sync.WaitGroup nm := newNATmap(h.natTimeout, h.m, &running) diff --git a/service/udp_test.go b/service/udp_test.go index f94238c5..90d880b5 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -16,6 +16,7 @@ package service import ( "bytes" + "context" "errors" "net" "net/netip" @@ -132,7 +133,7 @@ func sendToDiscard(payloads [][]byte, validator onet.TargetIPValidator) *natTest handler.SetTargetIPValidator(validator) done := make(chan struct{}) go func() { - handler.Handle(clientConn) + handler.Handle(context.Background(), clientConn) done <- struct{}{} }() @@ -488,7 +489,7 @@ func TestUDPEarlyClose(t *testing.T) { } require.Nil(t, clientConn.Close()) // This should return quickly without timing out. - s.Handle(clientConn) + s.Handle(context.Background(), clientConn) } // Makes sure the UDP listener returns [io.ErrClosed] on reads and writes after Close(). From ae7f41d8a0632c406f7902bd81b81639d588fc16 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 11 Jul 2024 16:50:34 -0400 Subject: [PATCH 055/182] Update comments. --- cmd/outline-ss-server/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index be94cbdf..65d8acf9 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -111,9 +111,9 @@ func (s *SSServer) loadConfig(filename string) error { return fmt.Errorf("failed to load config (%v): %w", filename, err) } - // We hot swap the services by having them both live at the same time. This - // means we create services for the new config first, and then take down the - // services from the old config. + // We hot swap the config by having the old and new listeners both live at + // the same time. This means we create listeners for the new config first, + // and then close the old ones after. oldListenerSet := s.lnSet s.lnSet = s.lnManager.NewListenerSet() var totalCipherCount int @@ -152,7 +152,7 @@ func (s *SSServer) loadConfig(filename string) error { logger.Infof("Loaded %d access keys over %d listeners", totalCipherCount, s.lnSet.Len()) s.m.SetNumAccessKeys(totalCipherCount, s.lnSet.Len()) - // Take down the old services now that the new ones are created and serving. + // Take down the old listeners now that the new ones are created and serving. if oldListenerSet != nil { if err := oldListenerSet.Close(); err != nil { logger.Errorf("Failed to stop old listeners: %w", err) @@ -163,7 +163,7 @@ func (s *SSServer) loadConfig(filename string) error { return nil } -// Stop serving on all existing services. +// Stop serving on all existing listeners. func (s *SSServer) Stop() error { if s.lnSet == nil { return nil From 120db8e43f086e5fb3a1de168002d92bd50651ba Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 11 Jul 2024 16:51:34 -0400 Subject: [PATCH 056/182] `go mod tidy`. --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5c1419d2..04a9ddab 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/stretchr/testify v1.8.4 golang.org/x/crypto v0.17.0 golang.org/x/term v0.16.0 - gopkg.in/yaml.v3 v3.0.1 + gopkg.in/yaml.v2 v2.4.0 ) require ( @@ -263,7 +263,7 @@ require ( gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect gopkg.in/src-d/go-git.v4 v4.13.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.90.0 // indirect mvdan.cc/sh/v3 v3.7.0 // indirect sigs.k8s.io/kind v0.17.0 // indirect From d705603e1a70a15fd3234fc496610071633b6f5a Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 16 Jul 2024 16:50:36 -0400 Subject: [PATCH 057/182] refactor: don't link the TCP handler to a specific listener --- cmd/outline-ss-server/main.go | 2 +- internal/integration_test/integration_test.go | 8 ++++---- service/tcp.go | 7 +++---- service/tcp_test.go | 16 ++++++++-------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 87860df7..e73506a8 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -87,7 +87,7 @@ func (s *SSServer) startPort(portNum int) error { port := &ssPort{tcpListener: listener, packetConn: packetConn, cipherList: service.NewCipherList()} authFunc := service.NewShadowsocksStreamAuthenticator(port.cipherList, &s.replayCache, s.m) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(listener.Addr().String(), authFunc, s.m, tcpReadTimeout) + tcpHandler := service.NewTCPHandler(authFunc, s.m, tcpReadTimeout) packetHandler := service.NewPacketHandler(s.natTimeout, port.cipherList, s.m) s.ports[portNum] = port go service.StreamServe(service.WrapStreamListener(listener.AcceptTCP), tcpHandler.Handle) diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index f98319f4..43109b7a 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -133,7 +133,7 @@ func TestTCPEcho(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -202,7 +202,7 @@ func TestRestrictedAddresses(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { service.StreamServe(service.WrapStreamListener(proxyListener.AcceptTCP), handler.Handle) @@ -384,7 +384,7 @@ func BenchmarkTCPThroughput(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -448,7 +448,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(proxyListener.Addr().String(), authFunc, testMetrics, testTimeout) + handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { diff --git a/service/tcp.go b/service/tcp.go index 2195480b..bd459e20 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -170,9 +170,8 @@ type tcpHandler struct { } // NewTCPService creates a TCPService -func NewTCPHandler(listenerId string, authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { +func NewTCPHandler(authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { return &tcpHandler{ - listenerId: listenerId, m: m, readTimeout: timeout, authenticate: authenticate, @@ -370,12 +369,12 @@ func (h *tcpHandler) handleConnection(ctx context.Context, outerConn transport.S // Keep the connection open until we hit the authentication deadline to protect against probing attacks // `proxyMetrics` is a pointer because its value is being mutated by `clientConn`. -func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, status string, proxyMetrics *metrics.ProxyMetrics) { +func (h *tcpHandler) absorbProbe(clientConn transport.StreamConn, status string, proxyMetrics *metrics.ProxyMetrics) { // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) logger.Debugf("Drain error: %v, drain result: %v", drainErr, drainResult) - h.m.AddTCPProbe(status, drainResult, h.listenerId, proxyMetrics.ClientProxy) + h.m.AddTCPProbe(status, drainResult, clientConn.LocalAddr().String(), proxyMetrics.ClientProxy) } func drainErrToString(drainErr error) string { diff --git a/service/tcp_test.go b/service/tcp_test.go index 5c3bc9df..fbe80f7c 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -281,7 +281,7 @@ func TestProbeRandom(t *testing.T) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) @@ -358,7 +358,7 @@ func TestProbeClientBytesBasicTruncated(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -393,7 +393,7 @@ func TestProbeClientBytesBasicModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -429,7 +429,7 @@ func TestProbeClientBytesCoalescedModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -472,7 +472,7 @@ func TestProbeServerBytesModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, 200*time.Millisecond) + handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) @@ -503,7 +503,7 @@ func TestReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -582,7 +582,7 @@ func TestReverseReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -653,7 +653,7 @@ func probeExpectTimeout(t *testing.T, payloadSize int) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(listener.Addr().String(), authFunc, testMetrics, testTimeout) + handler := NewTCPHandler(authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { From d2ef46efbea2fc50a89191d1ea064327cd1f8fb3 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 16 Jul 2024 18:12:26 -0400 Subject: [PATCH 058/182] Protect new cipher handling methods with mutex. --- service/cipher_list.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/cipher_list.go b/service/cipher_list.go index d84ab1ac..beda57bc 100644 --- a/service/cipher_list.go +++ b/service/cipher_list.go @@ -79,6 +79,8 @@ func NewCipherList() CipherList { } func (cl *cipherList) Len() int { + cl.mu.Lock() + defer cl.mu.Unlock() return cl.list.Len() } @@ -125,5 +127,7 @@ func (cl *cipherList) Update(src *list.List) { } func (cl *cipherList) PushBack(entry *CipherEntry) *list.Element { + cl.mu.Lock() + defer cl.mu.Unlock() return cl.list.PushBack(entry) } From ab07400909c5ceeacbdeb2515b2953f0d74fb3b5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 16 Jul 2024 18:04:53 -0400 Subject: [PATCH 059/182] Move `listeners.go` under `/service`. --- cmd/outline-ss-server/main.go | 14 ++++---------- {cmd/outline-ss-server => service}/listeners.go | 11 ++++++++--- 2 files changed, 12 insertions(+), 13 deletions(-) rename {cmd/outline-ss-server => service}/listeners.go (97%) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 849b72a0..6e0d0e90 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -60,12 +60,6 @@ func init() { logger = logging.MustGetLogger("") } -type Handler interface { - NumCiphers() int - AddCipher(entry *service.CipherEntry) - Handle(ctx context.Context, conn any) -} - type connHandler struct { tcpTimeout time.Duration natTimeout time.Duration @@ -98,8 +92,8 @@ func (h *connHandler) Handle(ctx context.Context, conn any) { } type SSServer struct { - lnManager ListenerManager - lnSet ListenerSet + lnManager service.ListenerManager + lnSet service.ListenerSet natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache @@ -118,7 +112,7 @@ func (s *SSServer) loadConfig(filename string) error { s.lnSet = s.lnManager.NewListenerSet() var totalCipherCount int - portHandlers := make(map[int]Handler) + portHandlers := make(map[int]service.Handler) for _, legacyKeyServiceConfig := range config.Keys { handler, ok := portHandlers[legacyKeyServiceConfig.Port] if !ok { @@ -178,7 +172,7 @@ func (s *SSServer) Stop() error { // RunSSServer starts a shadowsocks server running, and returns the server or an error. func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ - lnManager: NewListenerManager(), + lnManager: service.NewListenerManager(), natTimeout: natTimeout, m: sm, replayCache: service.NewReplayCache(replayHistory), diff --git a/cmd/outline-ss-server/listeners.go b/service/listeners.go similarity index 97% rename from cmd/outline-ss-server/listeners.go rename to service/listeners.go index f45339b1..14615d9f 100644 --- a/cmd/outline-ss-server/listeners.go +++ b/service/listeners.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package service import ( "context" @@ -22,9 +22,14 @@ import ( "sync/atomic" "github.com/Jigsaw-Code/outline-sdk/transport" - "github.com/Jigsaw-Code/outline-ss-server/service" ) +type Handler interface { + NumCiphers() int + AddCipher(entry *CipherEntry) + Handle(ctx context.Context, conn any) +} + type acceptResponse struct { conn net.Conn err error @@ -55,7 +60,7 @@ func (sl *sharedListener) SetHandler(handler Handler) { handle := func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn) } - go service.StreamServe(accept, handle) + go StreamServe(accept, handle) } // Accept accepts connections until Close() is called. From 71d7140258031d36b4d724779bb7e8f06ebf8938 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 16 Jul 2024 18:19:47 -0400 Subject: [PATCH 060/182] Use callback instead of passing in key and manager. --- service/listeners.go | 47 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 14615d9f..58169362 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -40,13 +40,12 @@ type SharedListener interface { } type sharedListener struct { - listener net.Listener - manager ListenerManager - key string - closed atomic.Int32 - usage *atomic.Int32 - acceptCh chan acceptResponse - closeCh chan struct{} + listener net.Listener + closed atomic.Int32 + usage *atomic.Int32 + acceptCh chan acceptResponse + closeCh chan struct{} + closeFunc func() } func (sl *sharedListener) SetHandler(handler Handler) { @@ -87,7 +86,7 @@ func (sl *sharedListener) Close() error { // See if we need to actually close the underlying listener. if sl.usage.Add(-1) == 0 { - sl.manager.Delete(sl.key) + sl.closeFunc() err := sl.listener.Close() if err != nil { return err @@ -104,17 +103,16 @@ func (sl *sharedListener) Addr() net.Addr { type sharedPacketConn struct { net.PacketConn - manager ListenerManager - key string - closed atomic.Int32 - usage *atomic.Int32 + closed atomic.Int32 + usage *atomic.Int32 + closeFunc func() } func (spc *sharedPacketConn) Close() error { if spc.closed.CompareAndSwap(0, 1) { // See if we need to actually close the underlying listener. if spc.usage.Add(-1) == 0 { - spc.manager.Delete(spc.key) + spc.closeFunc() err := spc.PacketConn.Close() if err != nil { return err @@ -186,7 +184,6 @@ func (ls *listenerSet) Len() int { type ListenerManager interface { NewListenerSet() ListenerSet Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) - Delete(key string) } type listenerManager struct { @@ -226,11 +223,12 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin lnGlobal.usage.Add(1) return &sharedListener{ listener: lnGlobal.ln, - manager: m, - key: lnKey, usage: &lnGlobal.usage, acceptCh: lnGlobal.acceptCh, closeCh: make(chan struct{}), + closeFunc: func() { + m.delete(lnKey) + }, }, nil } @@ -251,11 +249,12 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin return &sharedListener{ listener: ln, - manager: m, - key: lnKey, usage: &lnGlobal.usage, acceptCh: lnGlobal.acceptCh, closeCh: make(chan struct{}), + closeFunc: func() { + m.delete(lnKey) + }, }, nil case "udp": @@ -266,9 +265,10 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin lnGlobal.usage.Add(1) return &sharedPacketConn{ PacketConn: lnGlobal.pc, - manager: m, - key: lnKey, usage: &lnGlobal.usage, + closeFunc: func() { + m.delete(lnKey) + }, }, nil } @@ -283,9 +283,10 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin return &sharedPacketConn{ PacketConn: pc, - manager: m, - key: lnKey, usage: &lnGlobal.usage, + closeFunc: func() { + m.delete(lnKey) + }, }, nil default: @@ -294,7 +295,7 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin } } -func (m *listenerManager) Delete(key string) { +func (m *listenerManager) delete(key string) { m.listenersMu.Lock() delete(m.listeners, key) m.listenersMu.Unlock() From 9dfa4e269bd3700963347e32b99ef92048dafd47 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 16 Jul 2024 18:02:19 -0400 Subject: [PATCH 061/182] Move config start into a go routine for easier cleanup. --- cmd/outline-ss-server/main.go | 177 ++++++++--------- internal/integration_test/integration_test.go | 6 +- service/listeners.go | 184 ++++++++---------- service/tcp.go | 24 +-- service/udp.go | 5 +- service/udp_test.go | 5 +- 6 files changed, 193 insertions(+), 208 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 6e0d0e90..13400c11 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -15,7 +15,6 @@ package main import ( - "context" "flag" "fmt" "net" @@ -60,40 +59,9 @@ func init() { logger = logging.MustGetLogger("") } -type connHandler struct { - tcpTimeout time.Duration - natTimeout time.Duration - replayCache *service.ReplayCache - m *outlineMetrics - ciphers service.CipherList -} - -func (h *connHandler) NumCiphers() int { - return h.ciphers.Len() -} - -func (h *connHandler) AddCipher(entry *service.CipherEntry) { - h.ciphers.PushBack(entry) -} - -func (h *connHandler) Handle(ctx context.Context, conn any) { - switch c := conn.(type) { - case transport.StreamConn: - authFunc := service.NewShadowsocksStreamAuthenticator(h.ciphers, h.replayCache, h.m) - // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(authFunc, h.m, h.tcpTimeout) - tcpHandler.Handle(ctx, c) - case net.PacketConn: - packetHandler := service.NewPacketHandler(h.natTimeout, h.ciphers, h.m) - packetHandler.Handle(ctx, c) - default: - logger.Errorf("unknown connection type: %v", c) - } -} - type SSServer struct { + stopConfig func() lnManager service.ListenerManager - lnSet service.ListenerSet natTimeout time.Duration m *outlineMetrics replayCache service.ReplayCache @@ -104,74 +72,109 @@ func (s *SSServer) loadConfig(filename string) error { if err != nil { return fmt.Errorf("failed to load config (%v): %w", filename, err) } - // We hot swap the config by having the old and new listeners both live at // the same time. This means we create listeners for the new config first, // and then close the old ones after. - oldListenerSet := s.lnSet - s.lnSet = s.lnManager.NewListenerSet() - var totalCipherCount int - - portHandlers := make(map[int]service.Handler) - for _, legacyKeyServiceConfig := range config.Keys { - handler, ok := portHandlers[legacyKeyServiceConfig.Port] - if !ok { - handler = &connHandler{ - ciphers: service.NewCipherList(), - tcpTimeout: tcpReadTimeout, - natTimeout: s.natTimeout, - m: s.m, - replayCache: &s.replayCache, - } - portHandlers[legacyKeyServiceConfig.Port] = handler - } - cryptoKey, err := shadowsocks.NewEncryptionKey(legacyKeyServiceConfig.Cipher, legacyKeyServiceConfig.Secret) - if err != nil { - return fmt.Errorf("failed to create encyption key for key %v: %w", legacyKeyServiceConfig.ID, err) - } - entry := service.MakeCipherEntry(legacyKeyServiceConfig.ID, cryptoKey, legacyKeyServiceConfig.Secret) - handler.AddCipher(&entry) - } - for portNum, handler := range portHandlers { - totalCipherCount += handler.NumCiphers() - for _, network := range []string{"tcp", "udp"} { - addr := net.JoinHostPort("::", strconv.Itoa(portNum)) - listener, err := s.lnSet.Listen(context.TODO(), network, addr, net.ListenConfig{KeepAlive: 0}) - if err != nil { - return fmt.Errorf("%s service failed to start listening on address %s: %w", network, addr, err) - } - listener.SetHandler(handler) - } + stopConfig, err := s.runConfig(*config) + if err != nil { + return err } - logger.Infof("Loaded %d access keys over %d listeners", totalCipherCount, s.lnSet.Len()) - s.m.SetNumAccessKeys(totalCipherCount, s.lnSet.Len()) + s.stopConfig() + s.stopConfig = stopConfig + return nil +} - // Take down the old listeners now that the new ones are created and serving. - if oldListenerSet != nil { - if err := oldListenerSet.Close(); err != nil { - logger.Errorf("Failed to stop old listeners: %w", err) - } - logger.Infof("Stopped %d old listeners", s.lnSet.Len()) - } +func (s *SSServer) NewShadowsocksStreamHandler(ciphers service.CipherList) service.StreamHandler { + authFunc := service.NewShadowsocksStreamAuthenticator(ciphers, &s.replayCache, s.m) + // TODO: Register initial data metrics at zero. + return service.NewStreamHandler(authFunc, s.m, tcpReadTimeout) +} - return nil +func (s *SSServer) NewShadowsocksPacketHandler(ciphers service.CipherList) service.PacketHandler { + return service.NewPacketHandler(s.natTimeout, ciphers, s.m) } -// Stop serving on all existing listeners. -func (s *SSServer) Stop() error { - if s.lnSet == nil { - return nil - } - if err := s.lnSet.Close(); err != nil { - logger.Errorf("Failed to stop all listeners: %w", err) +func (s *SSServer) runConfig(config Config) (func(), error) { + startErrCh := make(chan error) + stopCh := make(chan struct{}) + + go func() { + startErrCh <- func() error { + lnSet := s.lnManager.NewListenerSet() + defer lnSet.Close() + + var totalCipherCount int + + portCiphers := make(map[int]service.CipherList) + for _, keyConfig := range config.Keys { + ciphers, ok := portCiphers[keyConfig.Port] + if !ok { + ciphers = service.NewCipherList() + portCiphers[keyConfig.Port] = ciphers + } + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + } + entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + ciphers.PushBack(&entry) + } + for portNum, ciphers := range portCiphers { + addr := net.JoinHostPort("::", strconv.Itoa(portNum)) + + sh := s.NewShadowsocksStreamHandler(ciphers) + ln, err := lnSet.Listen("tcp", addr) + if err != nil { + return err + } + logger.Infof("Shadowsocks TCP service listening on %v", ln.Addr().String()) + accept := func() (transport.StreamConn, error) { + c, err := ln.Accept() + if err == nil { + return c.(transport.StreamConn), err + } + return nil, err + } + go service.StreamServe(accept, sh.Handle) + + pc, err := lnSet.ListenPacket("udp", addr) + if err != nil { + return err + } + logger.Infof("Shadowsocks UDP service listening on %v", pc.LocalAddr().String()) + ph := s.NewShadowsocksPacketHandler(ciphers) + go ph.Handle(pc) + + totalCipherCount += ciphers.Len() + } + logger.Infof("Loaded %d access keys over %d listeners", totalCipherCount, lnSet.Len()) + s.m.SetNumAccessKeys(totalCipherCount, lnSet.Len()) + return nil + }() + + <-stopCh + }() + + err := <-startErrCh + if err != nil { + return nil, err } - logger.Infof("Stopped %d listeners", s.lnSet.Len()) - return nil + return func() { + logger.Infof("Stopping running config.") + stopCh <- struct{}{} + }, nil +} + +// Stop serving the current config. +func (s *SSServer) Stop() { + s.stopConfig() + logger.Info("Stopped all listeners for running config") } // RunSSServer starts a shadowsocks server running, and returns the server or an error. func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ + stopConfig: func() {}, lnManager: service.NewListenerManager(), natTimeout: natTimeout, m: sm, diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index db1b82d5..43109b7a 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -293,7 +293,7 @@ func TestUDPEcho(t *testing.T) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(context.Background(), proxyConn) + proxy.Handle(proxyConn) done <- struct{}{} }() @@ -525,7 +525,7 @@ func BenchmarkUDPEcho(b *testing.B) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(context.Background(), server) + proxy.Handle(server) done <- struct{}{} }() @@ -569,7 +569,7 @@ func BenchmarkUDPManyKeys(b *testing.B) { proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { - proxy.Handle(context.Background(), proxyConn) + proxy.Handle(proxyConn) done <- struct{}{} }() diff --git a/service/listeners.go b/service/listeners.go index 58169362..355d36b3 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -15,30 +15,21 @@ package service import ( - "context" "fmt" "net" "sync" "sync/atomic" - - "github.com/Jigsaw-Code/outline-sdk/transport" ) -type Handler interface { - NumCiphers() int - AddCipher(entry *CipherEntry) - Handle(ctx context.Context, conn any) -} +// The implementations of listeners for different network types are not +// interchangeable. The type of listener depends on the network type. +type Listener = any type acceptResponse struct { conn net.Conn err error } -type SharedListener interface { - SetHandler(handler Handler) -} - type sharedListener struct { listener net.Listener closed atomic.Int32 @@ -48,20 +39,6 @@ type sharedListener struct { closeFunc func() } -func (sl *sharedListener) SetHandler(handler Handler) { - accept := func() (transport.StreamConn, error) { - c, err := sl.Accept() - if err == nil { - return c.(transport.StreamConn), err - } - return nil, err - } - handle := func(ctx context.Context, conn transport.StreamConn) { - handler.Handle(ctx, conn) - } - go StreamServe(accept, handle) -} - // Accept accepts connections until Close() is called. func (sl *sharedListener) Accept() (net.Conn, error) { if sl.closed.Load() == 1 { @@ -123,10 +100,6 @@ func (spc *sharedPacketConn) Close() error { return nil } -func (spc *sharedPacketConn) SetHandler(handler Handler) { - go handler.Handle(context.TODO(), spc.PacketConn) -} - type globalListener struct { ln net.Listener pc net.PacketConn @@ -135,32 +108,46 @@ type globalListener struct { } type ListenerSet interface { - Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) + Listen(network string, addr string) (net.Listener, error) + ListenPacket(network string, addr string) (net.PacketConn, error) Close() error Len() int } type listenerSet struct { manager ListenerManager - listeners map[string]*SharedListener + listeners map[string]Listener } -func (ls *listenerSet) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) { +func (ls *listenerSet) Listen(network string, addr string) (net.Listener, error) { lnKey := listenerKey(network, addr) if _, exists := ls.listeners[lnKey]; exists { return nil, fmt.Errorf("listener %s already exists", lnKey) } - ln, err := ls.manager.Listen(ctx, network, addr, config) + ln, err := ls.manager.Listen(network, addr) if err != nil { return nil, err } - ls.listeners[lnKey] = &ln + ls.listeners[lnKey] = ln + return ln, nil +} + +func (ls *listenerSet) ListenPacket(network string, addr string) (net.PacketConn, error) { + lnKey := listenerKey(network, addr) + if _, exists := ls.listeners[lnKey]; exists { + return nil, fmt.Errorf("listener %s already exists", lnKey) + } + ln, err := ls.manager.ListenPacket(network, addr) + if err != nil { + return nil, err + } + ls.listeners[lnKey] = ln return ln, nil } func (ls *listenerSet) Close() error { for _, listener := range ls.listeners { - switch ln := (*listener).(type) { + switch ln := listener.(type) { case net.Listener: if err := ln.Close(); err != nil { return fmt.Errorf("%s listener on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) @@ -183,7 +170,8 @@ func (ls *listenerSet) Len() int { // ListenerManager holds and manages the state of shared listeners. type ListenerManager interface { NewListenerSet() ListenerSet - Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) + Listen(network string, addr string) (net.Listener, error) + ListenPacket(network string, addr string) (net.PacketConn, error) } type listenerManager struct { @@ -200,55 +188,25 @@ func NewListenerManager() ListenerManager { func (m *listenerManager) NewListenerSet() ListenerSet { return &listenerSet{ manager: m, - listeners: make(map[string]*SharedListener), + listeners: make(map[string]Listener), } } -// Listen creates a new listener for a given network and address. +// ListenStream creates a new stream listener for a given network and address. // // Listeners can overlap one another, because during config changes the new // config is started before the old config is destroyed. This is done by using // reusable listener wrappers, which do not actually close the underlying socket // until all uses of the shared listener have been closed. -func (m *listenerManager) Listen(ctx context.Context, network string, addr string, config net.ListenConfig) (SharedListener, error) { - lnKey := listenerKey(network, addr) - - switch network { - - case "tcp": - m.listenersMu.Lock() - defer m.listenersMu.Unlock() - - if lnGlobal, ok := m.listeners[lnKey]; ok { - lnGlobal.usage.Add(1) - return &sharedListener{ - listener: lnGlobal.ln, - usage: &lnGlobal.usage, - acceptCh: lnGlobal.acceptCh, - closeCh: make(chan struct{}), - closeFunc: func() { - m.delete(lnKey) - }, - }, nil - } - - ln, err := config.Listen(ctx, network, addr) - if err != nil { - return nil, err - } - - lnGlobal := &globalListener{ln: ln, acceptCh: make(chan acceptResponse)} - go func() { - for { - conn, err := lnGlobal.ln.Accept() - lnGlobal.acceptCh <- acceptResponse{conn, err} - } - }() - lnGlobal.usage.Store(1) - m.listeners[lnKey] = lnGlobal +func (m *listenerManager) Listen(network string, addr string) (net.Listener, error) { + m.listenersMu.Lock() + defer m.listenersMu.Unlock() + lnKey := listenerKey(network, addr) + if lnGlobal, ok := m.listeners[lnKey]; ok { + lnGlobal.usage.Add(1) return &sharedListener{ - listener: ln, + listener: lnGlobal.ln, usage: &lnGlobal.usage, acceptCh: lnGlobal.acceptCh, closeCh: make(chan struct{}), @@ -256,43 +214,69 @@ func (m *listenerManager) Listen(ctx context.Context, network string, addr strin m.delete(lnKey) }, }, nil + } - case "udp": - m.listenersMu.Lock() - defer m.listenersMu.Unlock() - - if lnGlobal, ok := m.listeners[lnKey]; ok { - lnGlobal.usage.Add(1) - return &sharedPacketConn{ - PacketConn: lnGlobal.pc, - usage: &lnGlobal.usage, - closeFunc: func() { - m.delete(lnKey) - }, - }, nil - } + ln, err := net.Listen(network, addr) + if err != nil { + return nil, err + } - pc, err := config.ListenPacket(ctx, network, addr) - if err != nil { - return nil, err + lnGlobal := &globalListener{ln: ln, acceptCh: make(chan acceptResponse)} + go func() { + for { + conn, err := lnGlobal.ln.Accept() + lnGlobal.acceptCh <- acceptResponse{conn, err} } + }() + lnGlobal.usage.Store(1) + m.listeners[lnKey] = lnGlobal + + return &sharedListener{ + listener: ln, + usage: &lnGlobal.usage, + acceptCh: lnGlobal.acceptCh, + closeCh: make(chan struct{}), + closeFunc: func() { + m.delete(lnKey) + }, + }, nil +} - lnGlobal := &globalListener{pc: pc} - lnGlobal.usage.Store(1) - m.listeners[lnKey] = lnGlobal +// ListenPacket creates a new packet listener for a given network and address. +// +// See notes on [ListenStream]. +func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketConn, error) { + m.listenersMu.Lock() + defer m.listenersMu.Unlock() + lnKey := listenerKey(network, addr) + if lnGlobal, ok := m.listeners[lnKey]; ok { + lnGlobal.usage.Add(1) return &sharedPacketConn{ - PacketConn: pc, + PacketConn: lnGlobal.pc, usage: &lnGlobal.usage, closeFunc: func() { m.delete(lnKey) }, }, nil + } - default: - return nil, fmt.Errorf("unsupported network: %s", network) - + pc, err := net.ListenPacket(network, addr) + if err != nil { + return nil, err } + + lnGlobal := &globalListener{pc: pc} + lnGlobal.usage.Store(1) + m.listeners[lnKey] = lnGlobal + + return &sharedPacketConn{ + PacketConn: pc, + usage: &lnGlobal.usage, + closeFunc: func() { + m.delete(lnKey) + }, + }, nil } func (m *listenerManager) delete(key string) { diff --git a/service/tcp.go b/service/tcp.go index 10c2a4f2..d88f537a 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -161,7 +161,7 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa } } -type tcpHandler struct { +type streamHandler struct { listenerId string m TCPMetrics readTimeout time.Duration @@ -169,9 +169,9 @@ type tcpHandler struct { dialer transport.StreamDialer } -// NewTCPService creates a TCPService -func NewTCPHandler(authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { - return &tcpHandler{ +// NewStreamHandler creates a StreamHandler +func NewStreamHandler(authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) StreamHandler { + return &streamHandler{ m: m, readTimeout: timeout, authenticate: authenticate, @@ -188,14 +188,14 @@ func makeValidatingTCPStreamDialer(targetIPValidator onet.TargetIPValidator) tra }}} } -// TCPService is a Shadowsocks TCP service that can be started and stopped. -type TCPHandler interface { +// StreamHandler is a handler that handles stream connections. +type StreamHandler interface { Handle(ctx context.Context, conn transport.StreamConn) // SetTargetDialer sets the [transport.StreamDialer] to be used to connect to target addresses. SetTargetDialer(dialer transport.StreamDialer) } -func (s *tcpHandler) SetTargetDialer(dialer transport.StreamDialer) { +func (s *streamHandler) SetTargetDialer(dialer transport.StreamDialer) { s.dialer = dialer } @@ -219,12 +219,12 @@ func WrapStreamListener[T transport.StreamConn](f func() (T, error)) StreamListe } } -type StreamHandler func(ctx context.Context, conn transport.StreamConn) +type StreamHandleFunc func(ctx context.Context, conn transport.StreamConn) // StreamServe repeatedly calls `accept` to obtain connections and `handle` to handle them until // accept() returns [ErrClosed]. When that happens, all connection handlers will be notified // via their [context.Context]. StreamServe will return after all pending handlers return. -func StreamServe(accept StreamListener, handle StreamHandler) { +func StreamServe(accept StreamListener, handle StreamHandleFunc) { var running sync.WaitGroup defer running.Wait() ctx, contextCancel := context.WithCancel(context.Background()) @@ -253,7 +253,7 @@ func StreamServe(accept StreamListener, handle StreamHandler) { } } -func (h *tcpHandler) Handle(ctx context.Context, clientConn transport.StreamConn) { +func (h *streamHandler) Handle(ctx context.Context, clientConn transport.StreamConn) { clientInfo, err := ipinfo.GetIPInfoFromAddr(h.m, clientConn.RemoteAddr()) if err != nil { logger.Warningf("Failed client info lookup: %v", err) @@ -327,7 +327,7 @@ func proxyConnection(ctx context.Context, dialer transport.StreamDialer, tgtAddr return nil } -func (h *tcpHandler) handleConnection(ctx context.Context, outerConn transport.StreamConn, proxyMetrics *metrics.ProxyMetrics) (string, *onet.ConnectionError) { +func (h *streamHandler) handleConnection(ctx context.Context, outerConn transport.StreamConn, proxyMetrics *metrics.ProxyMetrics) (string, *onet.ConnectionError) { // Set a deadline to receive the address to the target. readDeadline := time.Now().Add(h.readTimeout) if deadline, ok := ctx.Deadline(); ok { @@ -369,7 +369,7 @@ func (h *tcpHandler) handleConnection(ctx context.Context, outerConn transport.S // Keep the connection open until we hit the authentication deadline to protect against probing attacks // `proxyMetrics` is a pointer because its value is being mutated by `clientConn`. -func (h *tcpHandler) absorbProbe(clientConn transport.StreamConn, status string, proxyMetrics *metrics.ProxyMetrics) { +func (h *streamHandler) absorbProbe(clientConn transport.StreamConn, status string, proxyMetrics *metrics.ProxyMetrics) { // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) diff --git a/service/udp.go b/service/udp.go index 859c6c44..4830e302 100644 --- a/service/udp.go +++ b/service/udp.go @@ -15,7 +15,6 @@ package service import ( - "context" "errors" "fmt" "net" @@ -102,7 +101,7 @@ type PacketHandler interface { // SetTargetIPValidator sets the function to be used to validate the target IP addresses. SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) // Handle returns after clientConn closes and all the sub goroutines return. - Handle(ctx context.Context, clientConn net.PacketConn) + Handle(clientConn net.PacketConn) } func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) { @@ -111,7 +110,7 @@ func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPVali // Listen on addr for encrypted packets and basically do UDP NAT. // We take the ciphers as a pointer because it gets replaced on config updates. -func (h *packetHandler) Handle(ctx context.Context, clientConn net.PacketConn) { +func (h *packetHandler) Handle(clientConn net.PacketConn) { var running sync.WaitGroup nm := newNATmap(h.natTimeout, h.m, &running) diff --git a/service/udp_test.go b/service/udp_test.go index 90d880b5..f94238c5 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -16,7 +16,6 @@ package service import ( "bytes" - "context" "errors" "net" "net/netip" @@ -133,7 +132,7 @@ func sendToDiscard(payloads [][]byte, validator onet.TargetIPValidator) *natTest handler.SetTargetIPValidator(validator) done := make(chan struct{}) go func() { - handler.Handle(context.Background(), clientConn) + handler.Handle(clientConn) done <- struct{}{} }() @@ -489,7 +488,7 @@ func TestUDPEarlyClose(t *testing.T) { } require.Nil(t, clientConn.Close()) // This should return quickly without timing out. - s.Handle(context.Background(), clientConn) + s.Handle(clientConn) } // Makes sure the UDP listener returns [io.ErrClosed] on reads and writes after Close(). From 0a63f5c01fc2e366b7d89197313faaccf8e22ab0 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 19 Jul 2024 13:06:26 -0400 Subject: [PATCH 062/182] Make a `StreamListener` type. --- cmd/outline-ss-server/main.go | 14 +--- internal/integration_test/integration_test.go | 14 ++-- service/listeners.go | 66 ++++++++++++++----- service/tcp.go | 12 ++-- service/tcp_test.go | 36 +++++----- 5 files changed, 84 insertions(+), 58 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 13400c11..6180106e 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -26,7 +26,6 @@ import ( "syscall" "time" - "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" @@ -123,21 +122,14 @@ func (s *SSServer) runConfig(config Config) (func(), error) { addr := net.JoinHostPort("::", strconv.Itoa(portNum)) sh := s.NewShadowsocksStreamHandler(ciphers) - ln, err := lnSet.Listen("tcp", addr) + ln, err := lnSet.ListenStream(addr) if err != nil { return err } logger.Infof("Shadowsocks TCP service listening on %v", ln.Addr().String()) - accept := func() (transport.StreamConn, error) { - c, err := ln.Accept() - if err == nil { - return c.(transport.StreamConn), err - } - return nil, err - } - go service.StreamServe(accept, sh.Handle) + go service.StreamServe(ln.AcceptStream, sh.Handle) - pc, err := lnSet.ListenPacket("udp", addr) + pc, err := lnSet.ListenPacket(addr) if err != nil { return err } diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index 43109b7a..f72d835b 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -133,7 +133,7 @@ func TestTCPEcho(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) + handler := service.NewStreamHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -202,10 +202,10 @@ func TestRestrictedAddresses(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) + handler := service.NewStreamHandler(authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { - service.StreamServe(service.WrapStreamListener(proxyListener.AcceptTCP), handler.Handle) + service.StreamServe(service.WrapStreamAcceptFunc(proxyListener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -384,11 +384,11 @@ func BenchmarkTCPThroughput(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) + handler := service.NewStreamHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { - service.StreamServe(service.WrapStreamListener(proxyListener.AcceptTCP), handler.Handle) + service.StreamServe(service.WrapStreamAcceptFunc(proxyListener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -448,11 +448,11 @@ func BenchmarkTCPMultiplexing(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) + handler := service.NewStreamHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { - service.StreamServe(service.WrapStreamListener(proxyListener.AcceptTCP), handler.Handle) + service.StreamServe(service.WrapStreamAcceptFunc(proxyListener.AcceptTCP), handler.Handle) done <- struct{}{} }() diff --git a/service/listeners.go b/service/listeners.go index 355d36b3..f36ecfd9 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -16,22 +16,37 @@ package service import ( "fmt" + "io" "net" "sync" "sync/atomic" + + "github.com/Jigsaw-Code/outline-sdk/transport" ) // The implementations of listeners for different network types are not // interchangeable. The type of listener depends on the network type. -type Listener = any +type Listener = io.Closer + +type StreamListener interface { + // Accept waits for and returns the next connection to the listener. + AcceptStream() (transport.StreamConn, error) + + // Close closes the listener. + // Any blocked Accept operations will be unblocked and return errors. + Close() error + + // Addr returns the listener's network address. + Addr() net.Addr +} type acceptResponse struct { - conn net.Conn + conn transport.StreamConn err error } type sharedListener struct { - listener net.Listener + listener net.TCPListener closed atomic.Int32 usage *atomic.Int32 acceptCh chan acceptResponse @@ -40,7 +55,7 @@ type sharedListener struct { } // Accept accepts connections until Close() is called. -func (sl *sharedListener) Accept() (net.Conn, error) { +func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { if sl.closed.Load() == 1 { return nil, net.ErrClosed } @@ -101,16 +116,24 @@ func (spc *sharedPacketConn) Close() error { } type globalListener struct { - ln net.Listener + ln net.TCPListener pc net.PacketConn usage atomic.Int32 acceptCh chan acceptResponse } +// ListenerSet represents a set of listeners listening on unique addresses. Trying +// to listen on the same address twice will result in an error. The set can be +// closed as a unit, which is useful if you want to bring down a group of +// listeners, such as when reloading a new config. type ListenerSet interface { - Listen(network string, addr string) (net.Listener, error) - ListenPacket(network string, addr string) (net.PacketConn, error) + // ListenStream announces on a given TCP network address. + ListenStream(addr string) (StreamListener, error) + // ListenStream announces on a given UDP network address. + ListenPacket(addr string) (net.PacketConn, error) + // Close closes all the listeners in the set. Close() error + // Len returns the number of listeners in the set. Len() int } @@ -119,12 +142,14 @@ type listenerSet struct { listeners map[string]Listener } -func (ls *listenerSet) Listen(network string, addr string) (net.Listener, error) { +// ListenStream announces on a given TCP network address. +func (ls *listenerSet) ListenStream(addr string) (StreamListener, error) { + network := "tcp" lnKey := listenerKey(network, addr) if _, exists := ls.listeners[lnKey]; exists { return nil, fmt.Errorf("listener %s already exists", lnKey) } - ln, err := ls.manager.Listen(network, addr) + ln, err := ls.manager.ListenStream(network, addr) if err != nil { return nil, err } @@ -132,7 +157,9 @@ func (ls *listenerSet) Listen(network string, addr string) (net.Listener, error) return ln, nil } -func (ls *listenerSet) ListenPacket(network string, addr string) (net.PacketConn, error) { +// ListenPacket announces on a given UDP network address. +func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { + network := "udp" lnKey := listenerKey(network, addr) if _, exists := ls.listeners[lnKey]; exists { return nil, fmt.Errorf("listener %s already exists", lnKey) @@ -145,6 +172,7 @@ func (ls *listenerSet) ListenPacket(network string, addr string) (net.PacketConn return ln, nil } +// Close closes all the listeners in the set. func (ls *listenerSet) Close() error { for _, listener := range ls.listeners { switch ln := listener.(type) { @@ -163,6 +191,7 @@ func (ls *listenerSet) Close() error { return nil } +// Len returns the number of listeners in the set. func (ls *listenerSet) Len() int { return len(ls.listeners) } @@ -170,7 +199,7 @@ func (ls *listenerSet) Len() int { // ListenerManager holds and manages the state of shared listeners. type ListenerManager interface { NewListenerSet() ListenerSet - Listen(network string, addr string) (net.Listener, error) + ListenStream(network string, addr string) (StreamListener, error) ListenPacket(network string, addr string) (net.PacketConn, error) } @@ -179,6 +208,7 @@ type listenerManager struct { listenersMu sync.Mutex } +// NewListenerManager creates a new [ListenerManger]. func NewListenerManager() ListenerManager { return &listenerManager{ listeners: make(map[string]*globalListener), @@ -198,7 +228,7 @@ func (m *listenerManager) NewListenerSet() ListenerSet { // config is started before the old config is destroyed. This is done by using // reusable listener wrappers, which do not actually close the underlying socket // until all uses of the shared listener have been closed. -func (m *listenerManager) Listen(network string, addr string) (net.Listener, error) { +func (m *listenerManager) ListenStream(network string, addr string) (StreamListener, error) { m.listenersMu.Lock() defer m.listenersMu.Unlock() @@ -216,15 +246,19 @@ func (m *listenerManager) Listen(network string, addr string) (net.Listener, err }, nil } - ln, err := net.Listen(network, addr) + tcpAddr, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + return nil, err + } + ln, err := net.ListenTCP(network, tcpAddr) if err != nil { return nil, err } - lnGlobal := &globalListener{ln: ln, acceptCh: make(chan acceptResponse)} + lnGlobal := &globalListener{ln: *ln, acceptCh: make(chan acceptResponse)} go func() { for { - conn, err := lnGlobal.ln.Accept() + conn, err := lnGlobal.ln.AcceptTCP() lnGlobal.acceptCh <- acceptResponse{conn, err} } }() @@ -232,7 +266,7 @@ func (m *listenerManager) Listen(network string, addr string) (net.Listener, err m.listeners[lnKey] = lnGlobal return &sharedListener{ - listener: ln, + listener: lnGlobal.ln, usage: &lnGlobal.usage, acceptCh: lnGlobal.acceptCh, closeCh: make(chan struct{}), diff --git a/service/tcp.go b/service/tcp.go index d88f537a..8138a24c 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -211,9 +211,9 @@ func ensureConnectionError(err error, fallbackStatus string, fallbackMsg string) } } -type StreamListener func() (transport.StreamConn, error) +type StreamAcceptFunc func() (transport.StreamConn, error) -func WrapStreamListener[T transport.StreamConn](f func() (T, error)) StreamListener { +func WrapStreamAcceptFunc[T transport.StreamConn](f func() (T, error)) StreamAcceptFunc { return func() (transport.StreamConn, error) { return f() } @@ -224,7 +224,7 @@ type StreamHandleFunc func(ctx context.Context, conn transport.StreamConn) // StreamServe repeatedly calls `accept` to obtain connections and `handle` to handle them until // accept() returns [ErrClosed]. When that happens, all connection handlers will be notified // via their [context.Context]. StreamServe will return after all pending handlers return. -func StreamServe(accept StreamListener, handle StreamHandleFunc) { +func StreamServe(accept StreamAcceptFunc, handle StreamHandleFunc) { var running sync.WaitGroup defer running.Wait() ctx, contextCancel := context.WithCancel(context.Background()) @@ -341,7 +341,7 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor id, innerConn, authErr := h.authenticate(outerConn) if authErr != nil { // Drain to protect against probing attacks. - h.absorbProbe(outerConn, authErr.Status, proxyMetrics) + h.absorbProbe(outerConn, outerConn.LocalAddr().String(), authErr.Status, proxyMetrics) return id, authErr } h.m.AddAuthenticatedTCPConnection(outerConn.RemoteAddr(), id) @@ -369,12 +369,12 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor // Keep the connection open until we hit the authentication deadline to protect against probing attacks // `proxyMetrics` is a pointer because its value is being mutated by `clientConn`. -func (h *streamHandler) absorbProbe(clientConn transport.StreamConn, status string, proxyMetrics *metrics.ProxyMetrics) { +func (h *streamHandler) absorbProbe(clientConn io.ReadCloser, addr, status string, proxyMetrics *metrics.ProxyMetrics) { // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) logger.Debugf("Drain error: %v, drain result: %v", drainErr, drainResult) - h.m.AddTCPProbe(status, drainResult, clientConn.LocalAddr().String(), proxyMetrics.ClientProxy) + h.m.AddTCPProbe(status, drainResult, addr, proxyMetrics.ClientProxy) } func drainErrToString(drainErr error) string { diff --git a/service/tcp_test.go b/service/tcp_test.go index fbe80f7c..428f70e0 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -281,10 +281,10 @@ func TestProbeRandom(t *testing.T) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -358,11 +358,11 @@ func TestProbeClientBytesBasicTruncated(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -393,11 +393,11 @@ func TestProbeClientBytesBasicModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -429,11 +429,11 @@ func TestProbeClientBytesCoalescedModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -472,10 +472,10 @@ func TestProbeServerBytesModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -503,7 +503,7 @@ func TestReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, testTimeout) + handler := NewStreamHandler(authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -528,7 +528,7 @@ func TestReplayDefense(t *testing.T) { done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -582,7 +582,7 @@ func TestReverseReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, testTimeout) + handler := NewStreamHandler(authFunc, testMetrics, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -598,7 +598,7 @@ func TestReverseReplayDefense(t *testing.T) { done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -653,11 +653,11 @@ func probeExpectTimeout(t *testing.T, payloadSize int) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) - handler := NewTCPHandler(authFunc, testMetrics, testTimeout) + handler := NewStreamHandler(authFunc, testMetrics, testTimeout) done := make(chan struct{}) go func() { - StreamServe(WrapStreamListener(listener.AcceptTCP), handler.Handle) + StreamServe(WrapStreamAcceptFunc(listener.AcceptTCP), handler.Handle) done <- struct{}{} }() @@ -717,14 +717,14 @@ func TestStreamServeEarlyClose(t *testing.T) { err = tcpListener.Close() require.NoError(t, err) // This should return quickly, without timing out or calling the handler. - StreamServe(WrapStreamListener(tcpListener.AcceptTCP), nil) + StreamServe(WrapStreamAcceptFunc(tcpListener.AcceptTCP), nil) } // Makes sure the TCP listener returns [io.ErrClosed] on Close(). func TestClosedTCPListenerError(t *testing.T) { tcpListener, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0}) require.NoError(t, err) - accept := WrapStreamListener(tcpListener.AcceptTCP) + accept := WrapStreamAcceptFunc(tcpListener.AcceptTCP) err = tcpListener.Close() require.NoError(t, err) _, err = accept() From f018d175a73f54d4d56cad98f43d51529be30d0a Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 19 Jul 2024 13:35:20 -0400 Subject: [PATCH 063/182] Rename `closeFunc` to `onCloseFunc`. --- service/listeners.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index f36ecfd9..3a3d3a83 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -46,12 +46,12 @@ type acceptResponse struct { } type sharedListener struct { - listener net.TCPListener - closed atomic.Int32 - usage *atomic.Int32 - acceptCh chan acceptResponse - closeCh chan struct{} - closeFunc func() + listener net.TCPListener + closed atomic.Int32 + usage *atomic.Int32 + acceptCh chan acceptResponse + closeCh chan struct{} + onCloseFunc func() } // Accept accepts connections until Close() is called. @@ -78,7 +78,7 @@ func (sl *sharedListener) Close() error { // See if we need to actually close the underlying listener. if sl.usage.Add(-1) == 0 { - sl.closeFunc() + sl.onCloseFunc() err := sl.listener.Close() if err != nil { return err @@ -95,16 +95,16 @@ func (sl *sharedListener) Addr() net.Addr { type sharedPacketConn struct { net.PacketConn - closed atomic.Int32 - usage *atomic.Int32 - closeFunc func() + closed atomic.Int32 + usage *atomic.Int32 + onCloseFunc func() } func (spc *sharedPacketConn) Close() error { if spc.closed.CompareAndSwap(0, 1) { // See if we need to actually close the underlying listener. if spc.usage.Add(-1) == 0 { - spc.closeFunc() + spc.onCloseFunc() err := spc.PacketConn.Close() if err != nil { return err @@ -240,7 +240,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe usage: &lnGlobal.usage, acceptCh: lnGlobal.acceptCh, closeCh: make(chan struct{}), - closeFunc: func() { + onCloseFunc: func() { m.delete(lnKey) }, }, nil @@ -270,7 +270,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe usage: &lnGlobal.usage, acceptCh: lnGlobal.acceptCh, closeCh: make(chan struct{}), - closeFunc: func() { + onCloseFunc: func() { m.delete(lnKey) }, }, nil @@ -289,7 +289,7 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return &sharedPacketConn{ PacketConn: lnGlobal.pc, usage: &lnGlobal.usage, - closeFunc: func() { + onCloseFunc: func() { m.delete(lnKey) }, }, nil @@ -307,7 +307,7 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return &sharedPacketConn{ PacketConn: pc, usage: &lnGlobal.usage, - closeFunc: func() { + onCloseFunc: func() { m.delete(lnKey) }, }, nil From 4295c45f3133d7715bdfb5efba92de2852ba1be8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 19 Jul 2024 13:40:09 -0400 Subject: [PATCH 064/182] Rename `globalListener`. --- service/listeners.go | 48 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 3a3d3a83..ce635217 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -115,7 +115,7 @@ func (spc *sharedPacketConn) Close() error { return nil } -type globalListener struct { +type concreteListener struct { ln net.TCPListener pc net.PacketConn usage atomic.Int32 @@ -204,14 +204,14 @@ type ListenerManager interface { } type listenerManager struct { - listeners map[string]*globalListener + listeners map[string]*concreteListener listenersMu sync.Mutex } // NewListenerManager creates a new [ListenerManger]. func NewListenerManager() ListenerManager { return &listenerManager{ - listeners: make(map[string]*globalListener), + listeners: make(map[string]*concreteListener), } } @@ -233,12 +233,12 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe defer m.listenersMu.Unlock() lnKey := listenerKey(network, addr) - if lnGlobal, ok := m.listeners[lnKey]; ok { - lnGlobal.usage.Add(1) + if lnConcrete, ok := m.listeners[lnKey]; ok { + lnConcrete.usage.Add(1) return &sharedListener{ - listener: lnGlobal.ln, - usage: &lnGlobal.usage, - acceptCh: lnGlobal.acceptCh, + listener: lnConcrete.ln, + usage: &lnConcrete.usage, + acceptCh: lnConcrete.acceptCh, closeCh: make(chan struct{}), onCloseFunc: func() { m.delete(lnKey) @@ -255,20 +255,20 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe return nil, err } - lnGlobal := &globalListener{ln: *ln, acceptCh: make(chan acceptResponse)} + lnConcrete := &concreteListener{ln: *ln, acceptCh: make(chan acceptResponse)} go func() { for { - conn, err := lnGlobal.ln.AcceptTCP() - lnGlobal.acceptCh <- acceptResponse{conn, err} + conn, err := lnConcrete.ln.AcceptTCP() + lnConcrete.acceptCh <- acceptResponse{conn, err} } }() - lnGlobal.usage.Store(1) - m.listeners[lnKey] = lnGlobal + lnConcrete.usage.Store(1) + m.listeners[lnKey] = lnConcrete return &sharedListener{ - listener: lnGlobal.ln, - usage: &lnGlobal.usage, - acceptCh: lnGlobal.acceptCh, + listener: lnConcrete.ln, + usage: &lnConcrete.usage, + acceptCh: lnConcrete.acceptCh, closeCh: make(chan struct{}), onCloseFunc: func() { m.delete(lnKey) @@ -284,11 +284,11 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC defer m.listenersMu.Unlock() lnKey := listenerKey(network, addr) - if lnGlobal, ok := m.listeners[lnKey]; ok { - lnGlobal.usage.Add(1) + if lnConcrete, ok := m.listeners[lnKey]; ok { + lnConcrete.usage.Add(1) return &sharedPacketConn{ - PacketConn: lnGlobal.pc, - usage: &lnGlobal.usage, + PacketConn: lnConcrete.pc, + usage: &lnConcrete.usage, onCloseFunc: func() { m.delete(lnKey) }, @@ -300,13 +300,13 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return nil, err } - lnGlobal := &globalListener{pc: pc} - lnGlobal.usage.Store(1) - m.listeners[lnKey] = lnGlobal + lnConcrete := &concreteListener{pc: pc} + lnConcrete.usage.Store(1) + m.listeners[lnKey] = lnConcrete return &sharedPacketConn{ PacketConn: pc, - usage: &lnGlobal.usage, + usage: &lnConcrete.usage, onCloseFunc: func() { m.delete(lnKey) }, From e6963f62e2b2e3fbe1f2ca7b3f2b8500917e262c Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 19 Jul 2024 14:45:09 -0400 Subject: [PATCH 065/182] Don't track usage in the shared listeners. --- service/listeners.go | 98 +++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index ce635217..b7a56aca 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -15,6 +15,7 @@ package service import ( + "errors" "fmt" "io" "net" @@ -46,19 +47,16 @@ type acceptResponse struct { } type sharedListener struct { - listener net.TCPListener - closed atomic.Int32 - usage *atomic.Int32 + listener net.TCPListener + once sync.Once + acceptCh chan acceptResponse closeCh chan struct{} - onCloseFunc func() + onCloseFunc func() error } // Accept accepts connections until Close() is called. func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { - if sl.closed.Load() == 1 { - return nil, net.ErrClosed - } select { case acceptResponse := <-sl.acceptCh: if acceptResponse.err != nil { @@ -73,20 +71,13 @@ func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { // Close stops accepting new connections without closing the underlying socket. // Only when the last user closes it, we actually close it. func (sl *sharedListener) Close() error { - if sl.closed.CompareAndSwap(0, 1) { - close(sl.closeCh) - - // See if we need to actually close the underlying listener. - if sl.usage.Add(-1) == 0 { - sl.onCloseFunc() - err := sl.listener.Close() - if err != nil { - return err - } - } - } + var err error + sl.once.Do(func() { - return nil + close(sl.closeCh) + err = sl.onCloseFunc() + }) + return err } func (sl *sharedListener) Addr() net.Addr { @@ -95,33 +86,43 @@ func (sl *sharedListener) Addr() net.Addr { type sharedPacketConn struct { net.PacketConn - closed atomic.Int32 - usage *atomic.Int32 - onCloseFunc func() + once sync.Once + onCloseFunc func() error } func (spc *sharedPacketConn) Close() error { - if spc.closed.CompareAndSwap(0, 1) { - // See if we need to actually close the underlying listener. - if spc.usage.Add(-1) == 0 { - spc.onCloseFunc() - err := spc.PacketConn.Close() - if err != nil { - return err - } - } - } - - return nil + var err error + spc.once.Do(func() { + err = spc.onCloseFunc() + }) + return err } type concreteListener struct { - ln net.TCPListener + ln *net.TCPListener pc net.PacketConn usage atomic.Int32 acceptCh chan acceptResponse } +func (cl *concreteListener) Close() error { + if cl.usage.Add(-1) == 0 { + if cl.ln != nil { + err := cl.ln.Close() + if err != nil { + return err + } + } + if cl.pc != nil { + err := cl.pc.Close() + if err != nil { + return err + } + } + } + return nil +} + // ListenerSet represents a set of listeners listening on unique addresses. Trying // to listen on the same address twice will result in an error. The set can be // closed as a unit, which is useful if you want to bring down a group of @@ -236,12 +237,12 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe if lnConcrete, ok := m.listeners[lnKey]; ok { lnConcrete.usage.Add(1) return &sharedListener{ - listener: lnConcrete.ln, - usage: &lnConcrete.usage, + listener: *lnConcrete.ln, acceptCh: lnConcrete.acceptCh, closeCh: make(chan struct{}), - onCloseFunc: func() { + onCloseFunc: func() error { m.delete(lnKey) + return lnConcrete.Close() }, }, nil } @@ -255,10 +256,13 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe return nil, err } - lnConcrete := &concreteListener{ln: *ln, acceptCh: make(chan acceptResponse)} + lnConcrete := &concreteListener{ln: ln, acceptCh: make(chan acceptResponse)} go func() { for { conn, err := lnConcrete.ln.AcceptTCP() + if errors.Is(err, net.ErrClosed) { + return + } lnConcrete.acceptCh <- acceptResponse{conn, err} } }() @@ -266,12 +270,12 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe m.listeners[lnKey] = lnConcrete return &sharedListener{ - listener: lnConcrete.ln, - usage: &lnConcrete.usage, + listener: *lnConcrete.ln, acceptCh: lnConcrete.acceptCh, closeCh: make(chan struct{}), - onCloseFunc: func() { + onCloseFunc: func() error { m.delete(lnKey) + return lnConcrete.Close() }, }, nil } @@ -288,9 +292,9 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC lnConcrete.usage.Add(1) return &sharedPacketConn{ PacketConn: lnConcrete.pc, - usage: &lnConcrete.usage, - onCloseFunc: func() { + onCloseFunc: func() error { m.delete(lnKey) + return lnConcrete.Close() }, }, nil } @@ -306,9 +310,9 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return &sharedPacketConn{ PacketConn: pc, - usage: &lnConcrete.usage, - onCloseFunc: func() { + onCloseFunc: func() error { m.delete(lnKey) + return lnConcrete.Close() }, }, nil } From 7113f02144e696c8d0581d56f3305a4be8951fe8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 19 Jul 2024 14:52:02 -0400 Subject: [PATCH 066/182] Add `getAddr()` to avoid some duplicate code. --- service/listeners.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index b7a56aca..f109cd6b 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -73,7 +73,6 @@ func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { func (sl *sharedListener) Close() error { var err error sl.once.Do(func() { - close(sl.closeCh) err = sl.onCloseFunc() }) @@ -176,17 +175,12 @@ func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { // Close closes all the listeners in the set. func (ls *listenerSet) Close() error { for _, listener := range ls.listeners { - switch ln := listener.(type) { - case net.Listener: - if err := ln.Close(); err != nil { - return fmt.Errorf("%s listener on address %s failed to stop: %w", ln.Addr().Network(), ln.Addr().String(), err) - } - case net.PacketConn: - if err := ln.Close(); err != nil { - return fmt.Errorf("%s listener on address %s failed to stop: %w", ln.LocalAddr().Network(), ln.LocalAddr().String(), err) - } - default: - return fmt.Errorf("unknown listener type: %v", ln) + addr, err := getAddr(listener) + if err != nil { + return err + } + if err := listener.Close(); err != nil { + return fmt.Errorf("%s listener on address %s failed to stop: %w", addr.Network(), addr.String(), err) } } return nil @@ -326,3 +320,14 @@ func (m *listenerManager) delete(key string) { func listenerKey(network string, addr string) string { return network + "/" + addr } + +func getAddr(listener Listener) (net.Addr, error) { + switch ln := listener.(type) { + case net.Listener: + return ln.Addr(), nil + case net.PacketConn: + return ln.LocalAddr(), nil + default: + return nil, fmt.Errorf("unknown listener type: %v", ln) + } +} From e4d679f05a4befb1f3e1bcde622fb56f13fcd5e7 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 11:54:42 -0400 Subject: [PATCH 067/182] Move listener set creation out of the inner function. --- cmd/outline-ss-server/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 6180106e..b97f14a5 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -98,10 +98,10 @@ func (s *SSServer) runConfig(config Config) (func(), error) { stopCh := make(chan struct{}) go func() { - startErrCh <- func() error { - lnSet := s.lnManager.NewListenerSet() - defer lnSet.Close() + lnSet := s.lnManager.NewListenerSet() + defer lnSet.Close() + startErrCh <- func() error { var totalCipherCount int portCiphers := make(map[int]service.CipherList) From be5f9b0ab0f4ded9d472bf8f30451bf18eb7632c Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 12:34:00 -0400 Subject: [PATCH 068/182] Remove `PushBack()` from `CipherList`. --- cmd/outline-ss-server/main.go | 24 ++++++++++++------------ service/cipher_list.go | 15 --------------- service/cipher_list_testing.go | 7 +++++-- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index b97f14a5..5040f25c 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -15,6 +15,7 @@ package main import ( + "container/list" "flag" "fmt" "net" @@ -102,25 +103,26 @@ func (s *SSServer) runConfig(config Config) (func(), error) { defer lnSet.Close() startErrCh <- func() error { - var totalCipherCount int - - portCiphers := make(map[int]service.CipherList) + portCiphers := make(map[int]*list.List) // Values are *List of *CipherEntry. for _, keyConfig := range config.Keys { - ciphers, ok := portCiphers[keyConfig.Port] + cipherList, ok := portCiphers[keyConfig.Port] if !ok { - ciphers = service.NewCipherList() - portCiphers[keyConfig.Port] = ciphers + cipherList = list.New() + portCiphers[keyConfig.Port] = cipherList } cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) if err != nil { return fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) } entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - ciphers.PushBack(&entry) + cipherList.PushBack(&entry) } - for portNum, ciphers := range portCiphers { + for portNum, cipherList := range portCiphers { addr := net.JoinHostPort("::", strconv.Itoa(portNum)) + ciphers := service.NewCipherList() + ciphers.Update(cipherList) + sh := s.NewShadowsocksStreamHandler(ciphers) ln, err := lnSet.ListenStream(addr) if err != nil { @@ -136,11 +138,9 @@ func (s *SSServer) runConfig(config Config) (func(), error) { logger.Infof("Shadowsocks UDP service listening on %v", pc.LocalAddr().String()) ph := s.NewShadowsocksPacketHandler(ciphers) go ph.Handle(pc) - - totalCipherCount += ciphers.Len() } - logger.Infof("Loaded %d access keys over %d listeners", totalCipherCount, lnSet.Len()) - s.m.SetNumAccessKeys(totalCipherCount, lnSet.Len()) + logger.Infof("Loaded %d access keys over %d listeners", len(config.Keys), lnSet.Len()) + s.m.SetNumAccessKeys(len(config.Keys), lnSet.Len()) return nil }() diff --git a/service/cipher_list.go b/service/cipher_list.go index beda57bc..3b6f1957 100644 --- a/service/cipher_list.go +++ b/service/cipher_list.go @@ -55,7 +55,6 @@ func MakeCipherEntry(id string, cryptoKey *shadowsocks.EncryptionKey, secret str // CipherList is a thread-safe collection of CipherEntry elements that allows for // snapshotting and moving to front. type CipherList interface { - Len() int // Returns a snapshot of the cipher list optimized for this client IP SnapshotForClientIP(clientIP netip.Addr) []*list.Element MarkUsedByClientIP(e *list.Element, clientIP netip.Addr) @@ -63,8 +62,6 @@ type CipherList interface { // which is a List of *CipherEntry. Update takes ownership of `contents`, // which must not be read or written after this call. Update(contents *list.List) - // PushBack inserts a new cipher at the back of the list. - PushBack(entry *CipherEntry) *list.Element } type cipherList struct { @@ -78,12 +75,6 @@ func NewCipherList() CipherList { return &cipherList{list: list.New()} } -func (cl *cipherList) Len() int { - cl.mu.Lock() - defer cl.mu.Unlock() - return cl.list.Len() -} - func matchesIP(e *list.Element, clientIP netip.Addr) bool { c := e.Value.(*CipherEntry) return clientIP != netip.Addr{} && clientIP == c.lastClientIP @@ -125,9 +116,3 @@ func (cl *cipherList) Update(src *list.List) { cl.list = src cl.mu.Unlock() } - -func (cl *cipherList) PushBack(entry *CipherEntry) *list.Element { - cl.mu.Lock() - defer cl.mu.Unlock() - return cl.list.PushBack(entry) -} diff --git a/service/cipher_list_testing.go b/service/cipher_list_testing.go index d8532f79..a77427ed 100644 --- a/service/cipher_list_testing.go +++ b/service/cipher_list_testing.go @@ -15,6 +15,7 @@ package service import ( + "container/list" "fmt" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" @@ -23,7 +24,7 @@ import ( // MakeTestCiphers creates a CipherList containing one fresh AEAD cipher // for each secret in `secrets`. func MakeTestCiphers(secrets []string) (CipherList, error) { - cipherList := NewCipherList() + l := list.New() for i := 0; i < len(secrets); i++ { cipherID := fmt.Sprintf("id-%v", i) cipher, err := shadowsocks.NewEncryptionKey(shadowsocks.CHACHA20IETFPOLY1305, secrets[i]) @@ -31,8 +32,10 @@ func MakeTestCiphers(secrets []string) (CipherList, error) { return nil, fmt.Errorf("failed to create cipher %v: %w", i, err) } entry := MakeCipherEntry(cipherID, cipher, secrets[i]) - cipherList.PushBack(&entry) + l.PushBack(&entry) } + cipherList := NewCipherList() + cipherList.Update(l) return cipherList, nil } From 343e4120fafb7b33fe843a98aeb0d165a3e5ef16 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 12:47:31 -0400 Subject: [PATCH 069/182] Move listener set to `main.go`. --- cmd/outline-ss-server/main.go | 63 ++++++++++++++++++++++++- service/listeners.go | 89 ----------------------------------- 2 files changed, 62 insertions(+), 90 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 5040f25c..90ff14fa 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -24,6 +24,7 @@ import ( "os/signal" "strconv" "strings" + "sync" "syscall" "time" @@ -94,12 +95,72 @@ func (s *SSServer) NewShadowsocksPacketHandler(ciphers service.CipherList) servi return service.NewPacketHandler(s.natTimeout, ciphers, s.m) } +type listenerSet struct { + manager service.ListenerManager + listeners map[string]service.Listener + listenersMu sync.Mutex +} + +// ListenStream announces on a given TCP network address. Trying to listen on +// the same address twice will result in an error. +func (ls *listenerSet) ListenStream(addr string) (service.StreamListener, error) { + ls.listenersMu.Lock() + defer ls.listenersMu.Unlock() + + lnKey := "tcp/" + addr + if _, exists := ls.listeners[lnKey]; exists { + return nil, fmt.Errorf("listener %s already exists", lnKey) + } + ln, err := ls.manager.ListenStream("tcp", addr) + if err != nil { + return nil, err + } + ls.listeners[lnKey] = ln + return ln, nil +} + +// ListenPacket announces on a given UDP network address. Trying to listen on +// the same address twice will result in an error. +func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { + ls.listenersMu.Lock() + defer ls.listenersMu.Unlock() + + lnKey := "udp/" + addr + if _, exists := ls.listeners[lnKey]; exists { + return nil, fmt.Errorf("listener %s already exists", lnKey) + } + ln, err := ls.manager.ListenPacket("udp", addr) + if err != nil { + return nil, err + } + ls.listeners[lnKey] = ln + return ln, nil +} + +// Close closes all the listeners in the set. +func (ls *listenerSet) Close() error { + for addr, listener := range ls.listeners { + if err := listener.Close(); err != nil { + return fmt.Errorf("listener on address %s failed to stop: %w", addr, err) + } + } + return nil +} + +// Len returns the number of listeners in the set. +func (ls *listenerSet) Len() int { + return len(ls.listeners) +} + func (s *SSServer) runConfig(config Config) (func(), error) { startErrCh := make(chan error) stopCh := make(chan struct{}) go func() { - lnSet := s.lnManager.NewListenerSet() + lnSet := &listenerSet{ + manager: s.lnManager, + listeners: make(map[string]service.Listener), + } defer lnSet.Close() startErrCh <- func() error { diff --git a/service/listeners.go b/service/listeners.go index f109cd6b..62f55a0f 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -16,7 +16,6 @@ package service import ( "errors" - "fmt" "io" "net" "sync" @@ -122,78 +121,8 @@ func (cl *concreteListener) Close() error { return nil } -// ListenerSet represents a set of listeners listening on unique addresses. Trying -// to listen on the same address twice will result in an error. The set can be -// closed as a unit, which is useful if you want to bring down a group of -// listeners, such as when reloading a new config. -type ListenerSet interface { - // ListenStream announces on a given TCP network address. - ListenStream(addr string) (StreamListener, error) - // ListenStream announces on a given UDP network address. - ListenPacket(addr string) (net.PacketConn, error) - // Close closes all the listeners in the set. - Close() error - // Len returns the number of listeners in the set. - Len() int -} - -type listenerSet struct { - manager ListenerManager - listeners map[string]Listener -} - -// ListenStream announces on a given TCP network address. -func (ls *listenerSet) ListenStream(addr string) (StreamListener, error) { - network := "tcp" - lnKey := listenerKey(network, addr) - if _, exists := ls.listeners[lnKey]; exists { - return nil, fmt.Errorf("listener %s already exists", lnKey) - } - ln, err := ls.manager.ListenStream(network, addr) - if err != nil { - return nil, err - } - ls.listeners[lnKey] = ln - return ln, nil -} - -// ListenPacket announces on a given UDP network address. -func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { - network := "udp" - lnKey := listenerKey(network, addr) - if _, exists := ls.listeners[lnKey]; exists { - return nil, fmt.Errorf("listener %s already exists", lnKey) - } - ln, err := ls.manager.ListenPacket(network, addr) - if err != nil { - return nil, err - } - ls.listeners[lnKey] = ln - return ln, nil -} - -// Close closes all the listeners in the set. -func (ls *listenerSet) Close() error { - for _, listener := range ls.listeners { - addr, err := getAddr(listener) - if err != nil { - return err - } - if err := listener.Close(); err != nil { - return fmt.Errorf("%s listener on address %s failed to stop: %w", addr.Network(), addr.String(), err) - } - } - return nil -} - -// Len returns the number of listeners in the set. -func (ls *listenerSet) Len() int { - return len(ls.listeners) -} - // ListenerManager holds and manages the state of shared listeners. type ListenerManager interface { - NewListenerSet() ListenerSet ListenStream(network string, addr string) (StreamListener, error) ListenPacket(network string, addr string) (net.PacketConn, error) } @@ -210,13 +139,6 @@ func NewListenerManager() ListenerManager { } } -func (m *listenerManager) NewListenerSet() ListenerSet { - return &listenerSet{ - manager: m, - listeners: make(map[string]Listener), - } -} - // ListenStream creates a new stream listener for a given network and address. // // Listeners can overlap one another, because during config changes the new @@ -320,14 +242,3 @@ func (m *listenerManager) delete(key string) { func listenerKey(network string, addr string) string { return network + "/" + addr } - -func getAddr(listener Listener) (net.Addr, error) { - switch ln := listener.(type) { - case net.Listener: - return ln.Addr(), nil - case net.PacketConn: - return ln.LocalAddr(), nil - default: - return nil, fmt.Errorf("unknown listener type: %v", ln) - } -} From 7f86ff17ae012c08f2cf96af77a2e1c7326661c1 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 13:15:14 -0400 Subject: [PATCH 070/182] Close the accept channel with an atomic value. --- service/listeners.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 62f55a0f..78523ac7 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -49,7 +49,7 @@ type sharedListener struct { listener net.TCPListener once sync.Once - acceptCh chan acceptResponse + acceptCh *atomic.Value // closed by first Close() call closeCh chan struct{} onCloseFunc func() error } @@ -57,7 +57,7 @@ type sharedListener struct { // Accept accepts connections until Close() is called. func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { select { - case acceptResponse := <-sl.acceptCh: + case acceptResponse := <-sl.acceptCh.Load().(chan acceptResponse): if acceptResponse.err != nil { return nil, acceptResponse.err } @@ -70,6 +70,7 @@ func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { // Close stops accepting new connections without closing the underlying socket. // Only when the last user closes it, we actually close it. func (sl *sharedListener) Close() error { + sl.acceptCh = nil var err error sl.once.Do(func() { close(sl.closeCh) @@ -152,15 +153,17 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe lnKey := listenerKey(network, addr) if lnConcrete, ok := m.listeners[lnKey]; ok { lnConcrete.usage.Add(1) - return &sharedListener{ + sl := &sharedListener{ listener: *lnConcrete.ln, - acceptCh: lnConcrete.acceptCh, closeCh: make(chan struct{}), onCloseFunc: func() error { m.delete(lnKey) return lnConcrete.Close() }, - }, nil + } + sl.acceptCh = &atomic.Value{} + sl.acceptCh.Store(lnConcrete.acceptCh) + return sl, nil } tcpAddr, err := net.ResolveTCPAddr("tcp", addr) @@ -185,15 +188,17 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe lnConcrete.usage.Store(1) m.listeners[lnKey] = lnConcrete - return &sharedListener{ + sl := &sharedListener{ listener: *lnConcrete.ln, - acceptCh: lnConcrete.acceptCh, closeCh: make(chan struct{}), onCloseFunc: func() error { m.delete(lnKey) return lnConcrete.Close() }, - }, nil + } + sl.acceptCh = &atomic.Value{} + sl.acceptCh.Store(lnConcrete.acceptCh) + return sl, nil } // ListenPacket creates a new packet listener for a given network and address. From e80b2c51d6f14d4643a8d7458f432b0f5903c10e Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 13:21:48 -0400 Subject: [PATCH 071/182] Update comment. --- cmd/outline-ss-server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 90ff14fa..7c27fcdc 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -218,7 +218,7 @@ func (s *SSServer) runConfig(config Config) (func(), error) { }, nil } -// Stop serving the current config. +// Stop stops serving the current config. func (s *SSServer) Stop() { s.stopConfig() logger.Info("Stopped all listeners for running config") From b1428edca64e02e6e40232fa21041aa7fb59cc1b Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 13:27:38 -0400 Subject: [PATCH 072/182] Address review comments. --- cmd/outline-ss-server/main.go | 2 +- cmd/outline-ss-server/metrics.go | 14 +++++++------- service/listeners.go | 19 ++++--------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 7c27fcdc..2a1c32ab 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -161,7 +161,7 @@ func (s *SSServer) runConfig(config Config) (func(), error) { manager: s.lnManager, listeners: make(map[string]service.Listener), } - defer lnSet.Close() + defer lnSet.Close() // This closes all the listeners in the set. startErrCh <- func() error { portCiphers := make(map[int]*list.List) // Values are *List of *CipherEntry. diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 600cea16..e95ceeb3 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -38,7 +38,7 @@ type outlineMetrics struct { buildInfo *prometheus.GaugeVec accessKeys prometheus.Gauge - listeners prometheus.Gauge + ports prometheus.Gauge dataBytes *prometheus.CounterVec dataBytesPerLocation *prometheus.CounterVec timeToCipherMs *prometheus.HistogramVec @@ -183,10 +183,10 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus Name: "keys", Help: "Count of access keys", }), - listeners: prometheus.NewGauge(prometheus.GaugeOpts{ + ports: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, - Name: "listeners", - Help: "Count of open Shadowsocks listeners", + Name: "ports", + Help: "Count of open Shadowsocks ports", }), tcpProbes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, @@ -265,7 +265,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus m.tunnelTimeCollector = newTunnelTimeCollector(ip2info, registerer) // TODO: Is it possible to pass where to register the collectors? - registerer.MustRegister(m.buildInfo, m.accessKeys, m.listeners, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, + registerer.MustRegister(m.buildInfo, m.accessKeys, m.ports, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, m.dataBytes, m.dataBytesPerLocation, m.timeToCipherMs, m.udpPacketsFromClientPerLocation, m.udpAddedNatEntries, m.udpRemovedNatEntries, m.tunnelTimeCollector) return m @@ -275,9 +275,9 @@ func (m *outlineMetrics) SetBuildInfo(version string) { m.buildInfo.WithLabelValues(version).Set(1) } -func (m *outlineMetrics) SetNumAccessKeys(numKeys int, listeners int) { +func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) { m.accessKeys.Set(float64(numKeys)) - m.listeners.Set(float64(listeners)) + m.ports.Set(float64(ports)) } func (m *outlineMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { diff --git a/service/listeners.go b/service/listeners.go index 78523ac7..15073e02 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -46,9 +46,7 @@ type acceptResponse struct { } type sharedListener struct { - listener net.TCPListener - once sync.Once - + listener net.TCPListener acceptCh *atomic.Value // closed by first Close() call closeCh chan struct{} onCloseFunc func() error @@ -71,12 +69,8 @@ func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { // Only when the last user closes it, we actually close it. func (sl *sharedListener) Close() error { sl.acceptCh = nil - var err error - sl.once.Do(func() { - close(sl.closeCh) - err = sl.onCloseFunc() - }) - return err + close(sl.closeCh) + return sl.onCloseFunc() } func (sl *sharedListener) Addr() net.Addr { @@ -85,16 +79,11 @@ func (sl *sharedListener) Addr() net.Addr { type sharedPacketConn struct { net.PacketConn - once sync.Once onCloseFunc func() error } func (spc *sharedPacketConn) Close() error { - var err error - spc.once.Do(func() { - err = spc.onCloseFunc() - }) - return err + return spc.onCloseFunc() } type concreteListener struct { From 1c16de86b18c3092e1ae9110203fd9a1b7ada4ef Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 13:31:53 -0400 Subject: [PATCH 073/182] Close before deleting key. --- service/listeners.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 15073e02..96e41085 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -146,8 +146,11 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe listener: *lnConcrete.ln, closeCh: make(chan struct{}), onCloseFunc: func() error { + if err := lnConcrete.Close(); err != nil { + return err + } m.delete(lnKey) - return lnConcrete.Close() + return nil }, } sl.acceptCh = &atomic.Value{} @@ -181,8 +184,11 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe listener: *lnConcrete.ln, closeCh: make(chan struct{}), onCloseFunc: func() error { + if err := lnConcrete.Close(); err != nil { + return err + } m.delete(lnKey) - return lnConcrete.Close() + return nil }, } sl.acceptCh = &atomic.Value{} @@ -203,8 +209,11 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return &sharedPacketConn{ PacketConn: lnConcrete.pc, onCloseFunc: func() error { + if err := lnConcrete.Close(); err != nil { + return err + } m.delete(lnKey) - return lnConcrete.Close() + return nil }, }, nil } @@ -221,8 +230,11 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return &sharedPacketConn{ PacketConn: pc, onCloseFunc: func() error { + if err := lnConcrete.Close(); err != nil { + return err + } m.delete(lnKey) - return lnConcrete.Close() + return nil }, }, nil } From ebc7053c6ca1fb72403e4940ab441c203063bf19 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 15:40:28 -0400 Subject: [PATCH 074/182] `server.Stop()` does not return a value --- cmd/outline-ss-server/server_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/outline-ss-server/server_test.go b/cmd/outline-ss-server/server_test.go index 0b7777b2..2ba0772e 100644 --- a/cmd/outline-ss-server/server_test.go +++ b/cmd/outline-ss-server/server_test.go @@ -27,7 +27,5 @@ func TestRunSSServer(t *testing.T) { if err != nil { t.Fatalf("RunSSServer() error = %v", err) } - if err := server.Stop(); err != nil { - t.Errorf("Error while stopping server: %v", err) - } + server.Stop() } From 67fc7fbf6ea596ef7b4cef7ced9c0667736ee31e Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 15:47:07 -0400 Subject: [PATCH 075/182] Add a comment for `StreamListener`. --- service/listeners.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/listeners.go b/service/listeners.go index 96e41085..2eda5481 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -28,6 +28,8 @@ import ( // interchangeable. The type of listener depends on the network type. type Listener = io.Closer +// StreamListener is a network listener for stream-oriented protocols that +// accepts [transport.StreamConn] connections. type StreamListener interface { // Accept waits for and returns the next connection to the listener. AcceptStream() (transport.StreamConn, error) From 7a15e7df8ebcfe5ed156dbe0839685b7ec1a9c38 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 16:11:04 -0400 Subject: [PATCH 076/182] Do not delete the listener from the manager until the last user has closed it. --- service/listeners.go | 49 +++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 2eda5481..ff93e25d 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -89,10 +89,11 @@ func (spc *sharedPacketConn) Close() error { } type concreteListener struct { - ln *net.TCPListener - pc net.PacketConn - usage atomic.Int32 - acceptCh chan acceptResponse + ln *net.TCPListener + pc net.PacketConn + usage atomic.Int32 + acceptCh chan acceptResponse + onCloseFunc func() // Called when the listener's last user closes it. } func (cl *concreteListener) Close() error { @@ -109,6 +110,7 @@ func (cl *concreteListener) Close() error { return err } } + cl.onCloseFunc() } return nil } @@ -148,11 +150,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe listener: *lnConcrete.ln, closeCh: make(chan struct{}), onCloseFunc: func() error { - if err := lnConcrete.Close(); err != nil { - return err - } - m.delete(lnKey) - return nil + return lnConcrete.Close() }, } sl.acceptCh = &atomic.Value{} @@ -169,7 +167,13 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe return nil, err } - lnConcrete := &concreteListener{ln: ln, acceptCh: make(chan acceptResponse)} + lnConcrete := &concreteListener{ + ln: ln, + acceptCh: make(chan acceptResponse), + onCloseFunc: func() { + m.delete(lnKey) + }, + } go func() { for { conn, err := lnConcrete.ln.AcceptTCP() @@ -186,11 +190,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe listener: *lnConcrete.ln, closeCh: make(chan struct{}), onCloseFunc: func() error { - if err := lnConcrete.Close(); err != nil { - return err - } - m.delete(lnKey) - return nil + return lnConcrete.Close() }, } sl.acceptCh = &atomic.Value{} @@ -211,11 +211,7 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return &sharedPacketConn{ PacketConn: lnConcrete.pc, onCloseFunc: func() error { - if err := lnConcrete.Close(); err != nil { - return err - } - m.delete(lnKey) - return nil + return lnConcrete.Close() }, }, nil } @@ -225,18 +221,19 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return nil, err } - lnConcrete := &concreteListener{pc: pc} + lnConcrete := &concreteListener{ + pc: pc, + onCloseFunc: func() { + m.delete(lnKey) + }, + } lnConcrete.usage.Store(1) m.listeners[lnKey] = lnConcrete return &sharedPacketConn{ PacketConn: pc, onCloseFunc: func() error { - if err := lnConcrete.Close(); err != nil { - return err - } - m.delete(lnKey) - return nil + return lnConcrete.Close() }, }, nil } From 499829e1462a03eb82281f1efb32970345fe1d31 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 16:28:03 -0400 Subject: [PATCH 077/182] Consolidate usage counting inside a `listenAddress` type. --- service/listeners.go | 111 +++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 63 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index ff93e25d..41aec8b0 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -88,7 +88,7 @@ func (spc *sharedPacketConn) Close() error { return spc.onCloseFunc() } -type concreteListener struct { +type listenAddr struct { ln *net.TCPListener pc net.PacketConn usage atomic.Int32 @@ -96,23 +96,42 @@ type concreteListener struct { onCloseFunc func() // Called when the listener's last user closes it. } -func (cl *concreteListener) Close() error { - if cl.usage.Add(-1) == 0 { - if cl.ln != nil { - err := cl.ln.Close() - if err != nil { - return err +func (cl *listenAddr) NewStreamListener() StreamListener { + cl.usage.Add(1) + sl := &sharedListener{ + listener: *cl.ln, + closeCh: make(chan struct{}), + onCloseFunc: func() error { + if cl.usage.Add(-1) == 0 { + err := cl.ln.Close() + if err != nil { + return err + } + cl.onCloseFunc() } - } - if cl.pc != nil { - err := cl.pc.Close() - if err != nil { - return err + return nil + }, + } + sl.acceptCh = &atomic.Value{} + sl.acceptCh.Store(cl.acceptCh) + return sl +} + +func (cl *listenAddr) NewPacketListener() net.PacketConn { + cl.usage.Add(1) + return &sharedPacketConn{ + PacketConn: cl.pc, + onCloseFunc: func() error { + if cl.usage.Add(-1) == 0 { + err := cl.pc.Close() + if err != nil { + return err + } + cl.onCloseFunc() } - } - cl.onCloseFunc() + return nil + }, } - return nil } // ListenerManager holds and manages the state of shared listeners. @@ -122,14 +141,14 @@ type ListenerManager interface { } type listenerManager struct { - listeners map[string]*concreteListener + listeners map[string]*listenAddr listenersMu sync.Mutex } // NewListenerManager creates a new [ListenerManger]. func NewListenerManager() ListenerManager { return &listenerManager{ - listeners: make(map[string]*concreteListener), + listeners: make(map[string]*listenAddr), } } @@ -144,18 +163,8 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe defer m.listenersMu.Unlock() lnKey := listenerKey(network, addr) - if lnConcrete, ok := m.listeners[lnKey]; ok { - lnConcrete.usage.Add(1) - sl := &sharedListener{ - listener: *lnConcrete.ln, - closeCh: make(chan struct{}), - onCloseFunc: func() error { - return lnConcrete.Close() - }, - } - sl.acceptCh = &atomic.Value{} - sl.acceptCh.Store(lnConcrete.acceptCh) - return sl, nil + if listenAddress, ok := m.listeners[lnKey]; ok { + return listenAddress.NewStreamListener(), nil } tcpAddr, err := net.ResolveTCPAddr("tcp", addr) @@ -167,7 +176,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe return nil, err } - lnConcrete := &concreteListener{ + listenAddress := &listenAddr{ ln: ln, acceptCh: make(chan acceptResponse), onCloseFunc: func() { @@ -176,26 +185,15 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe } go func() { for { - conn, err := lnConcrete.ln.AcceptTCP() + conn, err := listenAddress.ln.AcceptTCP() if errors.Is(err, net.ErrClosed) { return } - lnConcrete.acceptCh <- acceptResponse{conn, err} + listenAddress.acceptCh <- acceptResponse{conn, err} } }() - lnConcrete.usage.Store(1) - m.listeners[lnKey] = lnConcrete - - sl := &sharedListener{ - listener: *lnConcrete.ln, - closeCh: make(chan struct{}), - onCloseFunc: func() error { - return lnConcrete.Close() - }, - } - sl.acceptCh = &atomic.Value{} - sl.acceptCh.Store(lnConcrete.acceptCh) - return sl, nil + m.listeners[lnKey] = listenAddress + return listenAddress.NewStreamListener(), nil } // ListenPacket creates a new packet listener for a given network and address. @@ -206,14 +204,8 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC defer m.listenersMu.Unlock() lnKey := listenerKey(network, addr) - if lnConcrete, ok := m.listeners[lnKey]; ok { - lnConcrete.usage.Add(1) - return &sharedPacketConn{ - PacketConn: lnConcrete.pc, - onCloseFunc: func() error { - return lnConcrete.Close() - }, - }, nil + if listenAddress, ok := m.listeners[lnKey]; ok { + return listenAddress.NewPacketListener(), nil } pc, err := net.ListenPacket(network, addr) @@ -221,21 +213,14 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return nil, err } - lnConcrete := &concreteListener{ + listenAddress := &listenAddr{ pc: pc, onCloseFunc: func() { m.delete(lnKey) }, } - lnConcrete.usage.Store(1) - m.listeners[lnKey] = lnConcrete - - return &sharedPacketConn{ - PacketConn: pc, - onCloseFunc: func() error { - return lnConcrete.Close() - }, - }, nil + m.listeners[lnKey] = listenAddress + return listenAddress.NewPacketListener(), nil } func (m *listenerManager) delete(key string) { From f165dbd73dfb57f111b34ba79a75cddfee54ce5f Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 17:00:07 -0400 Subject: [PATCH 078/182] Remove `atomic.Value`. --- service/listeners.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 41aec8b0..6ee307f8 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -49,7 +49,7 @@ type acceptResponse struct { type sharedListener struct { listener net.TCPListener - acceptCh *atomic.Value // closed by first Close() call + acceptCh chan acceptResponse closeCh chan struct{} onCloseFunc func() error } @@ -57,7 +57,7 @@ type sharedListener struct { // Accept accepts connections until Close() is called. func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { select { - case acceptResponse := <-sl.acceptCh.Load().(chan acceptResponse): + case acceptResponse := <-sl.acceptCh: if acceptResponse.err != nil { return nil, acceptResponse.err } @@ -70,7 +70,6 @@ func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { // Close stops accepting new connections without closing the underlying socket. // Only when the last user closes it, we actually close it. func (sl *sharedListener) Close() error { - sl.acceptCh = nil close(sl.closeCh) return sl.onCloseFunc() } @@ -100,6 +99,7 @@ func (cl *listenAddr) NewStreamListener() StreamListener { cl.usage.Add(1) sl := &sharedListener{ listener: *cl.ln, + acceptCh: cl.acceptCh, closeCh: make(chan struct{}), onCloseFunc: func() error { if cl.usage.Add(-1) == 0 { @@ -112,8 +112,6 @@ func (cl *listenAddr) NewStreamListener() StreamListener { return nil }, } - sl.acceptCh = &atomic.Value{} - sl.acceptCh.Store(cl.acceptCh) return sl } From 2a2420a162dfc813e30b8a61c879a791b468f2e0 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 22 Jul 2024 17:18:37 -0400 Subject: [PATCH 079/182] Add some missing comments. --- service/listeners.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/service/listeners.go b/service/listeners.go index 6ee307f8..b9743ca3 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -74,6 +74,7 @@ func (sl *sharedListener) Close() error { return sl.onCloseFunc() } +// Addr returns the listener's network address. func (sl *sharedListener) Addr() net.Addr { return sl.listener.Addr() } @@ -95,6 +96,7 @@ type listenAddr struct { onCloseFunc func() // Called when the listener's last user closes it. } +// NewStreamListener creates a new [StreamListener]. func (cl *listenAddr) NewStreamListener() StreamListener { cl.usage.Add(1) sl := &sharedListener{ @@ -115,6 +117,7 @@ func (cl *listenAddr) NewStreamListener() StreamListener { return sl } +// NewStreamListener creates a new [net.PacketConn]. func (cl *listenAddr) NewPacketListener() net.PacketConn { cl.usage.Add(1) return &sharedPacketConn{ From cccba1a8312c0125785e516aa396349e48f7f3cb Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 25 Jul 2024 16:25:42 -0400 Subject: [PATCH 080/182] address review comments --- cmd/outline-ss-server/main.go | 5 ++++- service/listeners.go | 17 ++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 2a1c32ab..5dc28daf 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -137,13 +137,16 @@ func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { return ln, nil } -// Close closes all the listeners in the set. +// Close closes all the listeners in the set, after which the set can't be used again. func (ls *listenerSet) Close() error { for addr, listener := range ls.listeners { if err := listener.Close(); err != nil { return fmt.Errorf("listener on address %s failed to stop: %w", addr, err) } } + ls.listenersMu.Lock() + defer ls.listenersMu.Unlock() + ls.listeners = nil return nil } diff --git a/service/listeners.go b/service/listeners.go index b9743ca3..8c848e7d 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -35,7 +35,10 @@ type StreamListener interface { AcceptStream() (transport.StreamConn, error) // Close closes the listener. - // Any blocked Accept operations will be unblocked and return errors. + // Any blocked Accept operations will be unblocked and return errors. This + // stops the current listener from accepting new connections without closing + // the underlying socket. Only when the last user of the underlying socket + // closes it, do we actually close it. Close() error // Addr returns the listener's network address. @@ -57,18 +60,17 @@ type sharedListener struct { // Accept accepts connections until Close() is called. func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { select { - case acceptResponse := <-sl.acceptCh: - if acceptResponse.err != nil { - return nil, acceptResponse.err + case acceptResponse, ok := <-sl.acceptCh: + if !ok { + return nil, net.ErrClosed } - return acceptResponse.conn, nil + return acceptResponse.conn, acceptResponse.err case <-sl.closeCh: return nil, net.ErrClosed } } -// Close stops accepting new connections without closing the underlying socket. -// Only when the last user closes it, we actually close it. +// Close implements [StreamListener.Close]. func (sl *sharedListener) Close() error { close(sl.closeCh) return sl.onCloseFunc() @@ -188,6 +190,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe for { conn, err := listenAddress.ln.AcceptTCP() if errors.Is(err, net.ErrClosed) { + close(listenAddress.acceptCh) return } listenAddress.acceptCh <- acceptResponse{conn, err} From da4ccaadeef19de59a744df320b5aab0af9d2634 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 25 Jul 2024 16:38:11 -0400 Subject: [PATCH 081/182] Add type guard for `sharedListener`. --- service/listeners.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/listeners.go b/service/listeners.go index 8c848e7d..12253961 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -57,6 +57,8 @@ type sharedListener struct { onCloseFunc func() error } +var _ StreamListener = (*sharedListener)(nil) + // Accept accepts connections until Close() is called. func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { select { From d47f6120f060184eae1b70b62ae1e732a012c02c Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 25 Jul 2024 16:40:58 -0400 Subject: [PATCH 082/182] Stop the existing config in a goroutine. --- cmd/outline-ss-server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 5dc28daf..8b2c3049 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -80,7 +80,7 @@ func (s *SSServer) loadConfig(filename string) error { if err != nil { return err } - s.stopConfig() + go s.stopConfig() s.stopConfig = stopConfig return nil } From a928e2c9b9948a56c03a78248309f1de2a7f2be9 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 25 Jul 2024 16:44:36 -0400 Subject: [PATCH 083/182] Add a TODO to wait for all handlers to be stopped. --- cmd/outline-ss-server/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 8b2c3049..b2b28296 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -217,6 +217,8 @@ func (s *SSServer) runConfig(config Config) (func(), error) { } return func() { logger.Infof("Stopping running config.") + // TODO(sbruens): Actually wait for all handlers to be stopped, e.g. by + // using a https://pkg.go.dev/sync#WaitGroup. stopCh <- struct{}{} }, nil } From 98cc3a02181bb209869fcfe66eb8a04531c5b51e Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 25 Jul 2024 16:48:51 -0400 Subject: [PATCH 084/182] Run `stopConfig` in a goroutine in `Stop()` as well. --- cmd/outline-ss-server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index b2b28296..78bf2ef0 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -225,7 +225,7 @@ func (s *SSServer) runConfig(config Config) (func(), error) { // Stop stops serving the current config. func (s *SSServer) Stop() { - s.stopConfig() + go s.stopConfig() logger.Info("Stopped all listeners for running config") } From 48d0931e36e310be13b3160097b788fecbcc274b Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 25 Jul 2024 16:55:07 -0400 Subject: [PATCH 085/182] Create a `TCPListener` that implements a `StreamListener`. --- service/listeners.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 12253961..79e24158 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -45,13 +45,31 @@ type StreamListener interface { Addr() net.Addr } +type TCPListener struct { + ln *net.TCPListener +} + +var _ StreamListener = (*TCPListener)(nil) + +func (t *TCPListener) AcceptStream() (transport.StreamConn, error) { + return t.ln.AcceptTCP() +} + +func (t *TCPListener) Close() error { + return t.ln.Close() +} + +func (t *TCPListener) Addr() net.Addr { + return t.ln.Addr() +} + type acceptResponse struct { conn transport.StreamConn err error } type sharedListener struct { - listener net.TCPListener + listener StreamListener acceptCh chan acceptResponse closeCh chan struct{} onCloseFunc func() error @@ -93,7 +111,7 @@ func (spc *sharedPacketConn) Close() error { } type listenAddr struct { - ln *net.TCPListener + ln StreamListener pc net.PacketConn usage atomic.Int32 acceptCh chan acceptResponse @@ -104,7 +122,7 @@ type listenAddr struct { func (cl *listenAddr) NewStreamListener() StreamListener { cl.usage.Add(1) sl := &sharedListener{ - listener: *cl.ln, + listener: cl.ln, acceptCh: cl.acceptCh, closeCh: make(chan struct{}), onCloseFunc: func() error { @@ -181,8 +199,9 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe return nil, err } + streamLn := &TCPListener{ln} listenAddress := &listenAddr{ - ln: ln, + ln: streamLn, acceptCh: make(chan acceptResponse), onCloseFunc: func() { m.delete(lnKey) @@ -190,7 +209,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe } go func() { for { - conn, err := listenAddress.ln.AcceptTCP() + conn, err := streamLn.AcceptStream() if errors.Is(err, net.ErrClosed) { close(listenAddress.acceptCh) return From 2dec847096b1fcfbd7f0b12e3a6539a1fa8d2e17 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 25 Jul 2024 17:04:14 -0400 Subject: [PATCH 086/182] Track close functions instead of the entire listener, which is not needed. --- cmd/outline-ss-server/main.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 78bf2ef0..ec6b66b0 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -96,9 +96,9 @@ func (s *SSServer) NewShadowsocksPacketHandler(ciphers service.CipherList) servi } type listenerSet struct { - manager service.ListenerManager - listeners map[string]service.Listener - listenersMu sync.Mutex + manager service.ListenerManager + listenerCloseFuncs map[string]func() error + listenersMu sync.Mutex } // ListenStream announces on a given TCP network address. Trying to listen on @@ -108,14 +108,14 @@ func (ls *listenerSet) ListenStream(addr string) (service.StreamListener, error) defer ls.listenersMu.Unlock() lnKey := "tcp/" + addr - if _, exists := ls.listeners[lnKey]; exists { + if _, exists := ls.listenerCloseFuncs[lnKey]; exists { return nil, fmt.Errorf("listener %s already exists", lnKey) } ln, err := ls.manager.ListenStream("tcp", addr) if err != nil { return nil, err } - ls.listeners[lnKey] = ln + ls.listenerCloseFuncs[lnKey] = ln.Close return ln, nil } @@ -126,33 +126,33 @@ func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { defer ls.listenersMu.Unlock() lnKey := "udp/" + addr - if _, exists := ls.listeners[lnKey]; exists { + if _, exists := ls.listenerCloseFuncs[lnKey]; exists { return nil, fmt.Errorf("listener %s already exists", lnKey) } ln, err := ls.manager.ListenPacket("udp", addr) if err != nil { return nil, err } - ls.listeners[lnKey] = ln + ls.listenerCloseFuncs[lnKey] = ln.Close return ln, nil } // Close closes all the listeners in the set, after which the set can't be used again. func (ls *listenerSet) Close() error { - for addr, listener := range ls.listeners { - if err := listener.Close(); err != nil { + for addr, listenerCloseFunc := range ls.listenerCloseFuncs { + if err := listenerCloseFunc(); err != nil { return fmt.Errorf("listener on address %s failed to stop: %w", addr, err) } } ls.listenersMu.Lock() defer ls.listenersMu.Unlock() - ls.listeners = nil + ls.listenerCloseFuncs = nil return nil } // Len returns the number of listeners in the set. func (ls *listenerSet) Len() int { - return len(ls.listeners) + return len(ls.listenerCloseFuncs) } func (s *SSServer) runConfig(config Config) (func(), error) { @@ -161,8 +161,8 @@ func (s *SSServer) runConfig(config Config) (func(), error) { go func() { lnSet := &listenerSet{ - manager: s.lnManager, - listeners: make(map[string]service.Listener), + manager: s.lnManager, + listenerCloseFuncs: make(map[string]func() error), } defer lnSet.Close() // This closes all the listeners in the set. From ab22e47a3b68702fc9a2bd3fc6fac5f787332b8b Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 30 Jul 2024 17:22:29 -0400 Subject: [PATCH 087/182] Delegate usage tracking to a reference counter. --- service/listeners.go | 139 ++++++++++++++++++++++++-------------- service/listeners_test.go | 46 +++++++++++++ 2 files changed, 134 insertions(+), 51 deletions(-) create mode 100644 service/listeners_test.go diff --git a/service/listeners.go b/service/listeners.go index 79e24158..6fcc3b6d 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -113,50 +113,45 @@ func (spc *sharedPacketConn) Close() error { type listenAddr struct { ln StreamListener pc net.PacketConn - usage atomic.Int32 acceptCh chan acceptResponse - onCloseFunc func() // Called when the listener's last user closes it. + onCloseFunc func() error } // NewStreamListener creates a new [StreamListener]. func (cl *listenAddr) NewStreamListener() StreamListener { - cl.usage.Add(1) sl := &sharedListener{ - listener: cl.ln, - acceptCh: cl.acceptCh, - closeCh: make(chan struct{}), - onCloseFunc: func() error { - if cl.usage.Add(-1) == 0 { - err := cl.ln.Close() - if err != nil { - return err - } - cl.onCloseFunc() - } - return nil - }, + listener: cl.ln, + acceptCh: cl.acceptCh, + closeCh: make(chan struct{}), + onCloseFunc: cl.Close, } return sl } -// NewStreamListener creates a new [net.PacketConn]. +// NewPacketListener creates a new [net.PacketConn]. func (cl *listenAddr) NewPacketListener() net.PacketConn { - cl.usage.Add(1) return &sharedPacketConn{ - PacketConn: cl.pc, - onCloseFunc: func() error { - if cl.usage.Add(-1) == 0 { - err := cl.pc.Close() - if err != nil { - return err - } - cl.onCloseFunc() - } - return nil - }, + PacketConn: cl.pc, + onCloseFunc: cl.Close, } } +func (cl *listenAddr) Close() error { + if cl.ln != nil { + err := cl.ln.Close() + if err != nil { + return err + } + } + if cl.pc != nil { + err := cl.pc.Close() + if err != nil { + return err + } + } + return cl.onCloseFunc() +} + // ListenerManager holds and manages the state of shared listeners. type ListenerManager interface { ListenStream(network string, addr string) (StreamListener, error) @@ -164,14 +159,14 @@ type ListenerManager interface { } type listenerManager struct { - listeners map[string]*listenAddr + listeners map[string]RefCount[*listenAddr] listenersMu sync.Mutex } // NewListenerManager creates a new [ListenerManger]. func NewListenerManager() ListenerManager { return &listenerManager{ - listeners: make(map[string]*listenAddr), + listeners: make(map[string]RefCount[*listenAddr]), } } @@ -185,9 +180,10 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe m.listenersMu.Lock() defer m.listenersMu.Unlock() - lnKey := listenerKey(network, addr) - if listenAddress, ok := m.listeners[lnKey]; ok { - return listenAddress.NewStreamListener(), nil + lnKey := network + "/" + addr + if lnRefCount, ok := m.listeners[lnKey]; ok { + lnAddr := lnRefCount.Acquire().Get() + return lnAddr.NewStreamListener(), nil } tcpAddr, err := net.ResolveTCPAddr("tcp", addr) @@ -200,25 +196,27 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe } streamLn := &TCPListener{ln} - listenAddress := &listenAddr{ + lnRefCount := NewRefCount(&listenAddr{ ln: streamLn, acceptCh: make(chan acceptResponse), - onCloseFunc: func() { + onCloseFunc: func() error { m.delete(lnKey) + return nil }, - } + }) + lnAddr := lnRefCount.Get() go func() { for { conn, err := streamLn.AcceptStream() if errors.Is(err, net.ErrClosed) { - close(listenAddress.acceptCh) + close(lnAddr.acceptCh) return } - listenAddress.acceptCh <- acceptResponse{conn, err} + lnAddr.acceptCh <- acceptResponse{conn, err} } }() - m.listeners[lnKey] = listenAddress - return listenAddress.NewStreamListener(), nil + m.listeners[lnKey] = lnRefCount + return lnAddr.NewStreamListener(), nil } // ListenPacket creates a new packet listener for a given network and address. @@ -228,9 +226,10 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC m.listenersMu.Lock() defer m.listenersMu.Unlock() - lnKey := listenerKey(network, addr) - if listenAddress, ok := m.listeners[lnKey]; ok { - return listenAddress.NewPacketListener(), nil + lnKey := network + "/" + addr + if lnRefCount, ok := m.listeners[lnKey]; ok { + lnAddr := lnRefCount.Acquire().Get() + return lnAddr.NewPacketListener(), nil } pc, err := net.ListenPacket(network, addr) @@ -238,14 +237,15 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return nil, err } - listenAddress := &listenAddr{ + lnRefCount := NewRefCount(&listenAddr{ pc: pc, - onCloseFunc: func() { + onCloseFunc: func() error { m.delete(lnKey) + return nil }, - } - m.listeners[lnKey] = listenAddress - return listenAddress.NewPacketListener(), nil + }) + m.listeners[lnKey] = lnRefCount + return lnRefCount.Get().NewPacketListener(), nil } func (m *listenerManager) delete(key string) { @@ -254,6 +254,43 @@ func (m *listenerManager) delete(key string) { m.listenersMu.Unlock() } -func listenerKey(network string, addr string) string { - return network + "/" + addr +type RefCount[T io.Closer] interface { + io.Closer + + Acquire() RefCount[T] + Get() T +} + +func NewRefCount[T io.Closer](value T) RefCount[T] { + res := &refCount[T]{ + count: &atomic.Int32{}, + value: value, + } + res.count.Store(1) + return res +} + +type refCount[T io.Closer] struct { + count *atomic.Int32 + value T +} + +func (r refCount[T]) Close() error { + if count := r.count.Add(-1); count == 0 { + return r.value.Close() + } + + return nil +} + +func (r refCount[T]) Acquire() RefCount[T] { + r.count.Add(1) + return &refCount[T]{ + count: r.count, + value: r.value, + } +} + +func (r refCount[T]) Get() T { + return r.value } diff --git a/service/listeners_test.go b/service/listeners_test.go new file mode 100644 index 00000000..9005eb34 --- /dev/null +++ b/service/listeners_test.go @@ -0,0 +1,46 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type testRefCount struct { + onCloseFunc func() +} + +func (t *testRefCount) Close() error { + t.onCloseFunc() + return nil +} + +func TestRefCount(t *testing.T) { + var done bool + rc := NewRefCount[*testRefCount](&testRefCount{ + onCloseFunc: func() { + done = true + }, + }) + rc.Acquire() + + require.NoError(t, rc.Close()) + require.False(t, done) + + require.NoError(t, rc.Close()) + require.True(t, done) +} From 3c2a3efc5323888a752beff35cca903ff7a5e78d Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 12:13:17 -0400 Subject: [PATCH 088/182] Remove the `Get()` method from `refCount`. --- service/listeners.go | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 6fcc3b6d..7752fe83 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -182,7 +182,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe lnKey := network + "/" + addr if lnRefCount, ok := m.listeners[lnKey]; ok { - lnAddr := lnRefCount.Acquire().Get() + lnAddr := lnRefCount.Acquire() return lnAddr.NewStreamListener(), nil } @@ -196,15 +196,14 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe } streamLn := &TCPListener{ln} - lnRefCount := NewRefCount(&listenAddr{ + lnAddr := &listenAddr{ ln: streamLn, acceptCh: make(chan acceptResponse), onCloseFunc: func() error { m.delete(lnKey) return nil }, - }) - lnAddr := lnRefCount.Get() + } go func() { for { conn, err := streamLn.AcceptStream() @@ -215,7 +214,7 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe lnAddr.acceptCh <- acceptResponse{conn, err} } }() - m.listeners[lnKey] = lnRefCount + m.listeners[lnKey] = NewRefCount(lnAddr) return lnAddr.NewStreamListener(), nil } @@ -228,7 +227,7 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC lnKey := network + "/" + addr if lnRefCount, ok := m.listeners[lnKey]; ok { - lnAddr := lnRefCount.Acquire().Get() + lnAddr := lnRefCount.Acquire() return lnAddr.NewPacketListener(), nil } @@ -237,15 +236,15 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC return nil, err } - lnRefCount := NewRefCount(&listenAddr{ + lnAddr := &listenAddr{ pc: pc, onCloseFunc: func() error { m.delete(lnKey) return nil }, - }) - m.listeners[lnKey] = lnRefCount - return lnRefCount.Get().NewPacketListener(), nil + } + m.listeners[lnKey] = NewRefCount(lnAddr) + return lnAddr.NewPacketListener(), nil } func (m *listenerManager) delete(key string) { @@ -254,11 +253,18 @@ func (m *listenerManager) delete(key string) { m.listenersMu.Unlock() } +// RefCount is an atomic reference counter that can be used to track a shared +// [io.Closer] resource. type RefCount[T io.Closer] interface { io.Closer - Acquire() RefCount[T] - Get() T + // Acquire increases the ref count and returns the wrapped object. + Acquire() T +} + +type refCount[T io.Closer] struct { + count *atomic.Int32 + value T } func NewRefCount[T io.Closer](value T) RefCount[T] { @@ -270,11 +276,6 @@ func NewRefCount[T io.Closer](value T) RefCount[T] { return res } -type refCount[T io.Closer] struct { - count *atomic.Int32 - value T -} - func (r refCount[T]) Close() error { if count := r.count.Add(-1); count == 0 { return r.value.Close() @@ -283,14 +284,7 @@ func (r refCount[T]) Close() error { return nil } -func (r refCount[T]) Acquire() RefCount[T] { +func (r refCount[T]) Acquire() T { r.count.Add(1) - return &refCount[T]{ - count: r.count, - value: r.value, - } -} - -func (r refCount[T]) Get() T { return r.value } From 5e282f1debda6c247da82b09e52788dd0d42d3c8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 12:18:18 -0400 Subject: [PATCH 089/182] Return immediately. --- service/listeners.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 7752fe83..80873696 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -119,13 +119,12 @@ type listenAddr struct { // NewStreamListener creates a new [StreamListener]. func (cl *listenAddr) NewStreamListener() StreamListener { - sl := &sharedListener{ + return &sharedListener{ listener: cl.ln, acceptCh: cl.acceptCh, closeCh: make(chan struct{}), onCloseFunc: cl.Close, } - return sl } // NewPacketListener creates a new [net.PacketConn]. From 547e9e67484cad2ffc76257766eab1c29db0e804 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 13:26:53 -0400 Subject: [PATCH 090/182] Rename `shared` to `virtual` as they are not actually shared. --- service/listeners.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 80873696..680c006b 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -68,17 +68,17 @@ type acceptResponse struct { err error } -type sharedListener struct { +type virtualStreamListener struct { listener StreamListener acceptCh chan acceptResponse closeCh chan struct{} onCloseFunc func() error } -var _ StreamListener = (*sharedListener)(nil) +var _ StreamListener = (*virtualStreamListener)(nil) // Accept accepts connections until Close() is called. -func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { +func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { select { case acceptResponse, ok := <-sl.acceptCh: if !ok { @@ -91,22 +91,22 @@ func (sl *sharedListener) AcceptStream() (transport.StreamConn, error) { } // Close implements [StreamListener.Close]. -func (sl *sharedListener) Close() error { +func (sl *virtualStreamListener) Close() error { close(sl.closeCh) return sl.onCloseFunc() } // Addr returns the listener's network address. -func (sl *sharedListener) Addr() net.Addr { +func (sl *virtualStreamListener) Addr() net.Addr { return sl.listener.Addr() } -type sharedPacketConn struct { +type virtualPacketConn struct { net.PacketConn onCloseFunc func() error } -func (spc *sharedPacketConn) Close() error { +func (spc *virtualPacketConn) Close() error { return spc.onCloseFunc() } @@ -119,7 +119,7 @@ type listenAddr struct { // NewStreamListener creates a new [StreamListener]. func (cl *listenAddr) NewStreamListener() StreamListener { - return &sharedListener{ + return &virtualStreamListener{ listener: cl.ln, acceptCh: cl.acceptCh, closeCh: make(chan struct{}), @@ -129,7 +129,7 @@ func (cl *listenAddr) NewStreamListener() StreamListener { // NewPacketListener creates a new [net.PacketConn]. func (cl *listenAddr) NewPacketListener() net.PacketConn { - return &sharedPacketConn{ + return &virtualPacketConn{ PacketConn: cl.pc, onCloseFunc: cl.Close, } From c6774c8692197ab2b27ec4ddcf9a397a13d60e34 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 13:33:30 -0400 Subject: [PATCH 091/182] Simplify `listenAddr`. --- service/listeners.go | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 680c006b..9647285c 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -111,42 +111,38 @@ func (spc *virtualPacketConn) Close() error { } type listenAddr struct { - ln StreamListener - pc net.PacketConn + ln Listener acceptCh chan acceptResponse onCloseFunc func() error } // NewStreamListener creates a new [StreamListener]. func (cl *listenAddr) NewStreamListener() StreamListener { - return &virtualStreamListener{ - listener: cl.ln, - acceptCh: cl.acceptCh, - closeCh: make(chan struct{}), - onCloseFunc: cl.Close, + if ln, ok := cl.ln.(StreamListener); ok { + return &virtualStreamListener{ + listener: ln, + acceptCh: cl.acceptCh, + closeCh: make(chan struct{}), + onCloseFunc: cl.Close, + } } + return nil } // NewPacketListener creates a new [net.PacketConn]. func (cl *listenAddr) NewPacketListener() net.PacketConn { - return &virtualPacketConn{ - PacketConn: cl.pc, - onCloseFunc: cl.Close, + if ln, ok := cl.ln.(net.PacketConn); ok { + return &virtualPacketConn{ + PacketConn: ln, + onCloseFunc: cl.Close, + } } + return nil } func (cl *listenAddr) Close() error { - if cl.ln != nil { - err := cl.ln.Close() - if err != nil { - return err - } - } - if cl.pc != nil { - err := cl.pc.Close() - if err != nil { - return err - } + if err := cl.ln.Close(); err != nil { + return err } return cl.onCloseFunc() } @@ -236,7 +232,7 @@ func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketC } lnAddr := &listenAddr{ - pc: pc, + ln: pc, onCloseFunc: func() error { m.delete(lnKey) return nil From df2f9d0c3ecc15c07d6413460029ac9ee0af8023 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 14:24:56 -0400 Subject: [PATCH 092/182] Fix use of the ref count. --- service/listeners.go | 174 +++++++++++++++++++++++++------------------ 1 file changed, 103 insertions(+), 71 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 9647285c..e5d48e45 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -16,6 +16,7 @@ package service import ( "errors" + "fmt" "io" "net" "sync" @@ -68,16 +69,17 @@ type acceptResponse struct { err error } +type OnCloseFunc func() error + type virtualStreamListener struct { listener StreamListener acceptCh chan acceptResponse closeCh chan struct{} - onCloseFunc func() error + onCloseFunc OnCloseFunc } var _ StreamListener = (*virtualStreamListener)(nil) -// Accept accepts connections until Close() is called. func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { select { case acceptResponse, ok := <-sl.acceptCh: @@ -90,20 +92,18 @@ func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { } } -// Close implements [StreamListener.Close]. func (sl *virtualStreamListener) Close() error { close(sl.closeCh) return sl.onCloseFunc() } -// Addr returns the listener's network address. func (sl *virtualStreamListener) Addr() net.Addr { return sl.listener.Addr() } type virtualPacketConn struct { net.PacketConn - onCloseFunc func() error + onCloseFunc OnCloseFunc } func (spc *virtualPacketConn) Close() error { @@ -113,38 +113,50 @@ func (spc *virtualPacketConn) Close() error { type listenAddr struct { ln Listener acceptCh chan acceptResponse - onCloseFunc func() error + onCloseFunc OnCloseFunc +} + +type canCreateStreamListener interface { + NewStreamListener(onCloseFunc OnCloseFunc) StreamListener } +var _ canCreateStreamListener = (*listenAddr)(nil) + // NewStreamListener creates a new [StreamListener]. -func (cl *listenAddr) NewStreamListener() StreamListener { - if ln, ok := cl.ln.(StreamListener); ok { +func (la *listenAddr) NewStreamListener(onCloseFunc OnCloseFunc) StreamListener { + if ln, ok := la.ln.(StreamListener); ok { return &virtualStreamListener{ listener: ln, - acceptCh: cl.acceptCh, + acceptCh: la.acceptCh, closeCh: make(chan struct{}), - onCloseFunc: cl.Close, + onCloseFunc: onCloseFunc, } } return nil } +type canCreatePacketListener interface { + NewPacketListener(onCloseFunc OnCloseFunc) net.PacketConn +} + +var _ canCreatePacketListener = (*listenAddr)(nil) + // NewPacketListener creates a new [net.PacketConn]. -func (cl *listenAddr) NewPacketListener() net.PacketConn { +func (cl *listenAddr) NewPacketListener(onCloseFunc OnCloseFunc) net.PacketConn { if ln, ok := cl.ln.(net.PacketConn); ok { return &virtualPacketConn{ PacketConn: ln, - onCloseFunc: cl.Close, + onCloseFunc: onCloseFunc, } } return nil } -func (cl *listenAddr) Close() error { - if err := cl.ln.Close(); err != nil { +func (la *listenAddr) Close() error { + if err := la.ln.Close(); err != nil { return err } - return cl.onCloseFunc() + return la.onCloseFunc() } // ListenerManager holds and manages the state of shared listeners. @@ -154,92 +166,106 @@ type ListenerManager interface { } type listenerManager struct { - listeners map[string]RefCount[*listenAddr] + listeners map[string]RefCount[Listener] listenersMu sync.Mutex } // NewListenerManager creates a new [ListenerManger]. func NewListenerManager() ListenerManager { return &listenerManager{ - listeners: make(map[string]RefCount[*listenAddr]), + listeners: make(map[string]RefCount[Listener]), } } -// ListenStream creates a new stream listener for a given network and address. -// -// Listeners can overlap one another, because during config changes the new -// config is started before the old config is destroyed. This is done by using -// reusable listener wrappers, which do not actually close the underlying socket -// until all uses of the shared listener have been closed. -func (m *listenerManager) ListenStream(network string, addr string) (StreamListener, error) { +func (m *listenerManager) getOrCreate(key string, createFunc func() (Listener, error)) (RefCount[Listener], error) { m.listenersMu.Lock() defer m.listenersMu.Unlock() - lnKey := network + "/" + addr - if lnRefCount, ok := m.listeners[lnKey]; ok { - lnAddr := lnRefCount.Acquire() - return lnAddr.NewStreamListener(), nil + if lnRefCount, exists := m.listeners[key]; exists { + return lnRefCount.Acquire(), nil } - tcpAddr, err := net.ResolveTCPAddr("tcp", addr) + ln, err := createFunc() if err != nil { return nil, err } - ln, err := net.ListenTCP(network, tcpAddr) + lnRefCount := NewRefCount(ln) + m.listeners[key] = lnRefCount + return lnRefCount, nil +} + +// ListenStream creates a new stream listener for a given network and address. +// +// Listeners can overlap one another, because during config changes the new +// config is started before the old config is destroyed. This is done by using +// reusable listener wrappers, which do not actually close the underlying socket +// until all uses of the shared listener have been closed. +func (m *listenerManager) ListenStream(network string, addr string) (StreamListener, error) { + lnKey := network + "/" + addr + lnRefCount, err := m.getOrCreate(lnKey, func() (Listener, error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + return nil, err + } + ln, err := net.ListenTCP(network, tcpAddr) + if err != nil { + return nil, err + } + streamLn := &TCPListener{ln} + lnAddr := &listenAddr{ + ln: streamLn, + acceptCh: make(chan acceptResponse), + onCloseFunc: func() error { + m.delete(lnKey) + return nil + }, + } + go func() { + for { + conn, err := streamLn.AcceptStream() + if errors.Is(err, net.ErrClosed) { + close(lnAddr.acceptCh) + return + } + lnAddr.acceptCh <- acceptResponse{conn, err} + } + }() + return lnAddr, nil + }) if err != nil { return nil, err } - - streamLn := &TCPListener{ln} - lnAddr := &listenAddr{ - ln: streamLn, - acceptCh: make(chan acceptResponse), - onCloseFunc: func() error { - m.delete(lnKey) - return nil - }, + if lnAddr, ok := lnRefCount.Get().(canCreateStreamListener); ok { + return lnAddr.NewStreamListener(lnRefCount.Close), nil } - go func() { - for { - conn, err := streamLn.AcceptStream() - if errors.Is(err, net.ErrClosed) { - close(lnAddr.acceptCh) - return - } - lnAddr.acceptCh <- acceptResponse{conn, err} - } - }() - m.listeners[lnKey] = NewRefCount(lnAddr) - return lnAddr.NewStreamListener(), nil + return nil, fmt.Errorf("unable to create stream listener for %s", lnKey) } // ListenPacket creates a new packet listener for a given network and address. // // See notes on [ListenStream]. func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketConn, error) { - m.listenersMu.Lock() - defer m.listenersMu.Unlock() - lnKey := network + "/" + addr - if lnRefCount, ok := m.listeners[lnKey]; ok { - lnAddr := lnRefCount.Acquire() - return lnAddr.NewPacketListener(), nil - } - - pc, err := net.ListenPacket(network, addr) + lnRefCount, err := m.getOrCreate(lnKey, func() (Listener, error) { + pc, err := net.ListenPacket(network, addr) + if err != nil { + return nil, err + } + return &listenAddr{ + ln: pc, + onCloseFunc: func() error { + m.delete(lnKey) + return nil + }, + }, nil + }) if err != nil { return nil, err } - - lnAddr := &listenAddr{ - ln: pc, - onCloseFunc: func() error { - m.delete(lnKey) - return nil - }, + if lnAddr, ok := lnRefCount.Get().(canCreatePacketListener); ok { + return lnAddr.NewPacketListener(lnRefCount.Close), nil } - m.listeners[lnKey] = NewRefCount(lnAddr) - return lnAddr.NewPacketListener(), nil + return nil, fmt.Errorf("unable to create packet listener for %s", lnKey) } func (m *listenerManager) delete(key string) { @@ -254,7 +280,9 @@ type RefCount[T io.Closer] interface { io.Closer // Acquire increases the ref count and returns the wrapped object. - Acquire() T + Acquire() RefCount[T] + + Get() T } type refCount[T io.Closer] struct { @@ -279,7 +307,11 @@ func (r refCount[T]) Close() error { return nil } -func (r refCount[T]) Acquire() T { +func (r refCount[T]) Acquire() RefCount[T] { r.count.Add(1) + return r +} + +func (r refCount[T]) Get() T { return r.value } From c678372d7a9d0feeb12b5d92dd29d6caea91dfd5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 16:05:00 -0400 Subject: [PATCH 093/182] Add simple test case for early closing of stream listener. --- service/listeners.go | 19 ++++++++++--------- service/listeners_test.go | 12 ++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index e5d48e45..44fa7ec5 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -161,7 +161,17 @@ func (la *listenAddr) Close() error { // ListenerManager holds and manages the state of shared listeners. type ListenerManager interface { + // ListenStream creates a new stream listener for a given network and address. + // + // Listeners can overlap one another, because during config changes the new + // config is started before the old config is destroyed. This is done by using + // reusable listener wrappers, which do not actually close the underlying socket + // until all uses of the shared listener have been closed. ListenStream(network string, addr string) (StreamListener, error) + + // ListenPacket creates a new packet listener for a given network and address. + // + // See notes on [ListenStream]. ListenPacket(network string, addr string) (net.PacketConn, error) } @@ -194,12 +204,6 @@ func (m *listenerManager) getOrCreate(key string, createFunc func() (Listener, e return lnRefCount, nil } -// ListenStream creates a new stream listener for a given network and address. -// -// Listeners can overlap one another, because during config changes the new -// config is started before the old config is destroyed. This is done by using -// reusable listener wrappers, which do not actually close the underlying socket -// until all uses of the shared listener have been closed. func (m *listenerManager) ListenStream(network string, addr string) (StreamListener, error) { lnKey := network + "/" + addr lnRefCount, err := m.getOrCreate(lnKey, func() (Listener, error) { @@ -241,9 +245,6 @@ func (m *listenerManager) ListenStream(network string, addr string) (StreamListe return nil, fmt.Errorf("unable to create stream listener for %s", lnKey) } -// ListenPacket creates a new packet listener for a given network and address. -// -// See notes on [ListenStream]. func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketConn, error) { lnKey := network + "/" + addr lnRefCount, err := m.getOrCreate(lnKey, func() (Listener, error) { diff --git a/service/listeners_test.go b/service/listeners_test.go index 9005eb34..3e2c3e64 100644 --- a/service/listeners_test.go +++ b/service/listeners_test.go @@ -15,11 +15,23 @@ package service import ( + "net" "testing" "github.com/stretchr/testify/require" ) +func TestListenerManagerStreamListenerEarlyClose(t *testing.T) { + m := NewListenerManager() + ln, err := m.ListenStream("tcp", "127.0.0.1:0") + require.NoError(t, err) + + ln.Close() + _, err = ln.AcceptStream() + + require.ErrorIs(t, err, net.ErrClosed) +} + type testRefCount struct { onCloseFunc func() } From e41ababc48eaa64c2b838a4087ca8019743541a8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 16:28:32 -0400 Subject: [PATCH 094/182] Add tests for creating stream listeners. --- service/listeners_test.go | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/service/listeners_test.go b/service/listeners_test.go index 3e2c3e64..384126ad 100644 --- a/service/listeners_test.go +++ b/service/listeners_test.go @@ -15,6 +15,7 @@ package service import ( + "fmt" "net" "testing" @@ -32,6 +33,60 @@ func TestListenerManagerStreamListenerEarlyClose(t *testing.T) { require.ErrorIs(t, err, net.ErrClosed) } +func writeTestPayload(ln StreamListener) error { + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + return fmt.Errorf("Failed to dial %v: %v", ln.Addr().String(), err) + } + if _, err = conn.Write(makeTestPayload(50)); err != nil { + return fmt.Errorf("Failed to write to connection: %v", err) + } + conn.Close() + return nil +} + +func TestListenerManagerStreamListenerNotClosedIfStillInUse(t *testing.T) { + m := NewListenerManager() + ln, err := m.ListenStream("tcp", "127.0.0.1:0") + require.NoError(t, err) + ln2, err := m.ListenStream("tcp", "127.0.0.1:0") + require.NoError(t, err) + + // Close only the first listener. + ln.Close() + done := make(chan struct{}) + go func() { + ln2.AcceptStream() + done <- struct{}{} + }() + + err = writeTestPayload(ln2) + require.NoError(t, err) + + <-done +} + +func TestListenerManagerStreamListenerCreatesListenerOnDemand(t *testing.T) { + m := NewListenerManager() + // Create a listener and immediately close it. + ln, err := m.ListenStream("tcp", "127.0.0.1:0") + require.NoError(t, err) + ln.Close() + // Now create another listener on the same address. + ln2, err := m.ListenStream("tcp", "127.0.0.1:0") + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + ln2.AcceptStream() + done <- struct{}{} + }() + err = writeTestPayload(ln2) + require.NoError(t, err) + + <-done +} + type testRefCount struct { onCloseFunc func() } From f9432d23a51c29d7b3e7021fa7c0ac49675fb96f Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 31 Jul 2024 17:35:42 -0400 Subject: [PATCH 095/182] Create handlers on demand. --- cmd/outline-ss-server/main.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index b706ccb9..4a38ad40 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -256,16 +256,10 @@ func (s *SSServer) runConfig(config Config) (func(), error) { } for _, serviceConfig := range config.Services { - // TODO: Create the handlers on demand. - sh, err := s.NewShadowsocksStreamHandlerFromConfig(serviceConfig) - if err != nil { - return err - } - ph, err := s.NewShadowsocksPacketHandlerFromConfig(serviceConfig) - if err != nil { - return err - } - + var ( + sh service.StreamHandler + ph service.PacketHandler + ) for _, lnConfig := range serviceConfig.Listeners { switch lnConfig.Type { case listenerTypeTCP: @@ -274,6 +268,12 @@ func (s *SSServer) runConfig(config Config) (func(), error) { return err } logger.Infof("TCP service listening on %s", ln.Addr().String()) + if sh == nil { + sh, err = s.NewShadowsocksStreamHandlerFromConfig(serviceConfig) + if err != nil { + return err + } + } go service.StreamServe(ln.AcceptStream, sh.Handle) case listenerTypeUDP: pc, err := lnSet.ListenPacket(lnConfig.Address) @@ -281,6 +281,12 @@ func (s *SSServer) runConfig(config Config) (func(), error) { return err } logger.Infof("UDP service listening on %v", pc.LocalAddr().String()) + if ph == nil { + ph, err = s.NewShadowsocksPacketHandlerFromConfig(serviceConfig) + if err != nil { + return err + } + } go ph.Handle(pc) } } From 6b11f4ffc4bd2c9d25500a8512c35da742211715 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 2 Aug 2024 11:51:35 -0400 Subject: [PATCH 096/182] Refactor create methods. --- service/listeners.go | 117 ++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 51 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 44fa7ec5..8f0882ba 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -187,86 +187,97 @@ func NewListenerManager() ListenerManager { } } -func (m *listenerManager) getOrCreate(key string, createFunc func() (Listener, error)) (RefCount[Listener], error) { +func (m *listenerManager) newStreamListener(network string, addr string) (Listener, error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + return nil, err + } + ln, err := net.ListenTCP(network, tcpAddr) + if err != nil { + return nil, err + } + streamLn := &TCPListener{ln} + lnAddr := &listenAddr{ + ln: streamLn, + acceptCh: make(chan acceptResponse), + onCloseFunc: func() error { + m.delete(listenerKey(network, addr)) + return nil + }, + } + go func() { + for { + conn, err := streamLn.AcceptStream() + if errors.Is(err, net.ErrClosed) { + close(lnAddr.acceptCh) + return + } + lnAddr.acceptCh <- acceptResponse{conn, err} + } + }() + return lnAddr, nil +} + +func (m *listenerManager) newPacketListener(network string, addr string) (Listener, error) { + pc, err := net.ListenPacket(network, addr) + if err != nil { + return nil, err + } + return &listenAddr{ + ln: pc, + onCloseFunc: func() error { + m.delete(listenerKey(network, addr)) + return nil + }, + }, nil +} + +func (m *listenerManager) getListener(network string, addr string) (RefCount[Listener], error) { m.listenersMu.Lock() defer m.listenersMu.Unlock() - if lnRefCount, exists := m.listeners[key]; exists { + lnKey := listenerKey(network, addr) + if lnRefCount, exists := m.listeners[lnKey]; exists { return lnRefCount.Acquire(), nil } - ln, err := createFunc() + var ( + ln Listener + err error + ) + if network == "tcp" { + ln, err = m.newStreamListener(network, addr) + } else { + ln, err = m.newPacketListener(network, addr) + } if err != nil { return nil, err } lnRefCount := NewRefCount(ln) - m.listeners[key] = lnRefCount + m.listeners[lnKey] = lnRefCount return lnRefCount, nil } func (m *listenerManager) ListenStream(network string, addr string) (StreamListener, error) { - lnKey := network + "/" + addr - lnRefCount, err := m.getOrCreate(lnKey, func() (Listener, error) { - tcpAddr, err := net.ResolveTCPAddr("tcp", addr) - if err != nil { - return nil, err - } - ln, err := net.ListenTCP(network, tcpAddr) - if err != nil { - return nil, err - } - streamLn := &TCPListener{ln} - lnAddr := &listenAddr{ - ln: streamLn, - acceptCh: make(chan acceptResponse), - onCloseFunc: func() error { - m.delete(lnKey) - return nil - }, - } - go func() { - for { - conn, err := streamLn.AcceptStream() - if errors.Is(err, net.ErrClosed) { - close(lnAddr.acceptCh) - return - } - lnAddr.acceptCh <- acceptResponse{conn, err} - } - }() - return lnAddr, nil - }) + lnRefCount, err := m.getListener(network, addr) if err != nil { return nil, err } if lnAddr, ok := lnRefCount.Get().(canCreateStreamListener); ok { return lnAddr.NewStreamListener(lnRefCount.Close), nil } - return nil, fmt.Errorf("unable to create stream listener for %s", lnKey) + return nil, fmt.Errorf("unable to create stream listener for %s/%s", network, addr) } func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketConn, error) { - lnKey := network + "/" + addr - lnRefCount, err := m.getOrCreate(lnKey, func() (Listener, error) { - pc, err := net.ListenPacket(network, addr) - if err != nil { - return nil, err - } - return &listenAddr{ - ln: pc, - onCloseFunc: func() error { - m.delete(lnKey) - return nil - }, - }, nil - }) + lnRefCount, err := m.getListener(network, addr) if err != nil { return nil, err } if lnAddr, ok := lnRefCount.Get().(canCreatePacketListener); ok { return lnAddr.NewPacketListener(lnRefCount.Close), nil } - return nil, fmt.Errorf("unable to create packet listener for %s", lnKey) + return nil, fmt.Errorf("unable to create packet listener for %s/%s", network, addr) } func (m *listenerManager) delete(key string) { @@ -275,6 +286,10 @@ func (m *listenerManager) delete(key string) { m.listenersMu.Unlock() } +func listenerKey(network string, addr string) string { + return network + "/" + addr +} + // RefCount is an atomic reference counter that can be used to track a shared // [io.Closer] resource. type RefCount[T io.Closer] interface { From fe8bbdda37bbb3927761690bf5ded40ed7b11ff7 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 5 Aug 2024 15:22:13 -0400 Subject: [PATCH 097/182] Address review comments. --- cmd/outline-ss-server/main.go | 20 +++++++-------- service/listeners.go | 47 +++++++++++++++++++---------------- service/listeners_test.go | 10 ++++---- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index ec6b66b0..f180044c 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -101,17 +101,17 @@ type listenerSet struct { listenersMu sync.Mutex } -// ListenStream announces on a given TCP network address. Trying to listen on -// the same address twice will result in an error. +// ListenStream announces on a given network address. Trying to listen for stream connections +// on the same address twice will result in an error. func (ls *listenerSet) ListenStream(addr string) (service.StreamListener, error) { ls.listenersMu.Lock() defer ls.listenersMu.Unlock() - lnKey := "tcp/" + addr + lnKey := "stream-" + addr if _, exists := ls.listenerCloseFuncs[lnKey]; exists { - return nil, fmt.Errorf("listener %s already exists", lnKey) + return nil, fmt.Errorf("stream listener for %s already exists", addr) } - ln, err := ls.manager.ListenStream("tcp", addr) + ln, err := ls.manager.ListenStream(addr) if err != nil { return nil, err } @@ -119,17 +119,17 @@ func (ls *listenerSet) ListenStream(addr string) (service.StreamListener, error) return ln, nil } -// ListenPacket announces on a given UDP network address. Trying to listen on -// the same address twice will result in an error. +// ListenPacket announces on a given network address. Trying to listen for packet connections +// on the same address twice will result in an error. func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { ls.listenersMu.Lock() defer ls.listenersMu.Unlock() - lnKey := "udp/" + addr + lnKey := "packet-" + addr if _, exists := ls.listenerCloseFuncs[lnKey]; exists { - return nil, fmt.Errorf("listener %s already exists", lnKey) + return nil, fmt.Errorf("packet listener for %s already exists", addr) } - ln, err := ls.manager.ListenPacket("udp", addr) + ln, err := ls.manager.ListenPacket(addr) if err != nil { return nil, err } diff --git a/service/listeners.go b/service/listeners.go index 8f0882ba..d8d58192 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -72,8 +72,8 @@ type acceptResponse struct { type OnCloseFunc func() error type virtualStreamListener struct { - listener StreamListener - acceptCh chan acceptResponse + addr net.Addr + acceptCh <-chan acceptResponse closeCh chan struct{} onCloseFunc OnCloseFunc } @@ -93,12 +93,13 @@ func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { } func (sl *virtualStreamListener) Close() error { + sl.acceptCh = nil close(sl.closeCh) return sl.onCloseFunc() } func (sl *virtualStreamListener) Addr() net.Addr { - return sl.listener.Addr() + return sl.addr } type virtualPacketConn struct { @@ -126,7 +127,7 @@ var _ canCreateStreamListener = (*listenAddr)(nil) func (la *listenAddr) NewStreamListener(onCloseFunc OnCloseFunc) StreamListener { if ln, ok := la.ln.(StreamListener); ok { return &virtualStreamListener{ - listener: ln, + addr: ln.Addr(), acceptCh: la.acceptCh, closeCh: make(chan struct{}), onCloseFunc: onCloseFunc, @@ -161,18 +162,18 @@ func (la *listenAddr) Close() error { // ListenerManager holds and manages the state of shared listeners. type ListenerManager interface { - // ListenStream creates a new stream listener for a given network and address. + // ListenStream creates a new stream listener for a given address. // // Listeners can overlap one another, because during config changes the new // config is started before the old config is destroyed. This is done by using // reusable listener wrappers, which do not actually close the underlying socket // until all uses of the shared listener have been closed. - ListenStream(network string, addr string) (StreamListener, error) + ListenStream(addr string) (StreamListener, error) - // ListenPacket creates a new packet listener for a given network and address. + // ListenPacket creates a new packet listener for a given address. // // See notes on [ListenStream]. - ListenPacket(network string, addr string) (net.PacketConn, error) + ListenPacket(addr string) (net.PacketConn, error) } type listenerManager struct { @@ -187,12 +188,12 @@ func NewListenerManager() ListenerManager { } } -func (m *listenerManager) newStreamListener(network string, addr string) (Listener, error) { +func (m *listenerManager) newStreamListener(addr string) (Listener, error) { tcpAddr, err := net.ResolveTCPAddr("tcp", addr) if err != nil { return nil, err } - ln, err := net.ListenTCP(network, tcpAddr) + ln, err := net.ListenTCP("tcp", tcpAddr) if err != nil { return nil, err } @@ -201,7 +202,7 @@ func (m *listenerManager) newStreamListener(network string, addr string) (Listen ln: streamLn, acceptCh: make(chan acceptResponse), onCloseFunc: func() error { - m.delete(listenerKey(network, addr)) + m.delete(listenerKey("tcp", addr)) return nil }, } @@ -218,15 +219,15 @@ func (m *listenerManager) newStreamListener(network string, addr string) (Listen return lnAddr, nil } -func (m *listenerManager) newPacketListener(network string, addr string) (Listener, error) { - pc, err := net.ListenPacket(network, addr) +func (m *listenerManager) newPacketListener(addr string) (Listener, error) { + pc, err := net.ListenPacket("udp", addr) if err != nil { return nil, err } return &listenAddr{ ln: pc, onCloseFunc: func() error { - m.delete(listenerKey(network, addr)) + m.delete(listenerKey("udp", addr)) return nil }, }, nil @@ -246,9 +247,11 @@ func (m *listenerManager) getListener(network string, addr string) (RefCount[Lis err error ) if network == "tcp" { - ln, err = m.newStreamListener(network, addr) + ln, err = m.newStreamListener(addr) + } else if network == "udp" { + ln, err = m.newPacketListener(addr) } else { - ln, err = m.newPacketListener(network, addr) + return nil, fmt.Errorf("unable to get listener for unsupported network %s", network) } if err != nil { return nil, err @@ -258,26 +261,26 @@ func (m *listenerManager) getListener(network string, addr string) (RefCount[Lis return lnRefCount, nil } -func (m *listenerManager) ListenStream(network string, addr string) (StreamListener, error) { - lnRefCount, err := m.getListener(network, addr) +func (m *listenerManager) ListenStream(addr string) (StreamListener, error) { + lnRefCount, err := m.getListener("tcp", addr) if err != nil { return nil, err } if lnAddr, ok := lnRefCount.Get().(canCreateStreamListener); ok { return lnAddr.NewStreamListener(lnRefCount.Close), nil } - return nil, fmt.Errorf("unable to create stream listener for %s/%s", network, addr) + return nil, fmt.Errorf("unable to create stream listener for %s", addr) } -func (m *listenerManager) ListenPacket(network string, addr string) (net.PacketConn, error) { - lnRefCount, err := m.getListener(network, addr) +func (m *listenerManager) ListenPacket(addr string) (net.PacketConn, error) { + lnRefCount, err := m.getListener("udp", addr) if err != nil { return nil, err } if lnAddr, ok := lnRefCount.Get().(canCreatePacketListener); ok { return lnAddr.NewPacketListener(lnRefCount.Close), nil } - return nil, fmt.Errorf("unable to create packet listener for %s/%s", network, addr) + return nil, fmt.Errorf("unable to create packet listener for %s", addr) } func (m *listenerManager) delete(key string) { diff --git a/service/listeners_test.go b/service/listeners_test.go index 384126ad..da5aaa1e 100644 --- a/service/listeners_test.go +++ b/service/listeners_test.go @@ -24,7 +24,7 @@ import ( func TestListenerManagerStreamListenerEarlyClose(t *testing.T) { m := NewListenerManager() - ln, err := m.ListenStream("tcp", "127.0.0.1:0") + ln, err := m.ListenStream("127.0.0.1:0") require.NoError(t, err) ln.Close() @@ -47,9 +47,9 @@ func writeTestPayload(ln StreamListener) error { func TestListenerManagerStreamListenerNotClosedIfStillInUse(t *testing.T) { m := NewListenerManager() - ln, err := m.ListenStream("tcp", "127.0.0.1:0") + ln, err := m.ListenStream("127.0.0.1:0") require.NoError(t, err) - ln2, err := m.ListenStream("tcp", "127.0.0.1:0") + ln2, err := m.ListenStream("127.0.0.1:0") require.NoError(t, err) // Close only the first listener. @@ -69,11 +69,11 @@ func TestListenerManagerStreamListenerNotClosedIfStillInUse(t *testing.T) { func TestListenerManagerStreamListenerCreatesListenerOnDemand(t *testing.T) { m := NewListenerManager() // Create a listener and immediately close it. - ln, err := m.ListenStream("tcp", "127.0.0.1:0") + ln, err := m.ListenStream("127.0.0.1:0") require.NoError(t, err) ln.Close() // Now create another listener on the same address. - ln2, err := m.ListenStream("tcp", "127.0.0.1:0") + ln2, err := m.ListenStream("127.0.0.1:0") require.NoError(t, err) done := make(chan struct{}) From 36a0a1d9f43c25e023b7c9b70fd54dcdae2dcd5e Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 5 Aug 2024 18:32:20 -0400 Subject: [PATCH 098/182] Use a mutex to ensure another user doesn't acquire a new closer while we're closing it. --- service/listeners.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index d8d58192..648f315e 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -305,24 +305,28 @@ type RefCount[T io.Closer] interface { } type refCount[T io.Closer] struct { + mu sync.Mutex count *atomic.Int32 value T } func NewRefCount[T io.Closer](value T) RefCount[T] { - res := &refCount[T]{ + r := &refCount[T]{ count: &atomic.Int32{}, value: value, } - res.count.Store(1) - return res + r.count.Store(1) + return r } func (r refCount[T]) Close() error { + // Lock to prevent someone from acquiring while we close the value. + r.mu.Lock() + defer r.mu.Unlock() + if count := r.count.Add(-1); count == 0 { return r.value.Close() } - return nil } From aeb2652fb3da4ea7339c88fcba7e32c5b91a5454 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 6 Aug 2024 10:13:00 -0400 Subject: [PATCH 099/182] Move mutex up. --- cmd/outline-ss-server/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index f180044c..55a11acc 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -139,13 +139,14 @@ func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { // Close closes all the listeners in the set, after which the set can't be used again. func (ls *listenerSet) Close() error { + ls.listenersMu.Lock() + defer ls.listenersMu.Unlock() + for addr, listenerCloseFunc := range ls.listenerCloseFuncs { if err := listenerCloseFunc(); err != nil { return fmt.Errorf("listener on address %s failed to stop: %w", addr, err) } } - ls.listenersMu.Lock() - defer ls.listenersMu.Unlock() ls.listenerCloseFuncs = nil return nil } From 8873b107083fedfe2b6627f326405c039177e4a3 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 6 Aug 2024 17:33:18 -0400 Subject: [PATCH 100/182] Manage the ref counting next to the listener creation. --- service/listeners.go | 310 +++++++++++++++++++------------------- service/listeners_test.go | 24 ++- 2 files changed, 170 insertions(+), 164 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 648f315e..91aa877f 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -95,7 +95,10 @@ func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { func (sl *virtualStreamListener) Close() error { sl.acceptCh = nil close(sl.closeCh) - return sl.onCloseFunc() + if sl.onCloseFunc != nil { + return sl.onCloseFunc() + } + return nil } func (sl *virtualStreamListener) Addr() net.Addr { @@ -108,189 +111,180 @@ type virtualPacketConn struct { } func (spc *virtualPacketConn) Close() error { - return spc.onCloseFunc() + if spc.onCloseFunc != nil { + return spc.onCloseFunc() + } + return nil } -type listenAddr struct { - ln Listener +// MultiListener manages shared listeners. +type MultiListener[T Listener] interface { + // Acquire creates a new listener from the shared listener. Listeners can overlap + // one another (e.g. during config changes the new config is started before the + // old config is destroyed), which is done by creating virtual listeners that wrap + // the shared listener. These virtual listeners do not actually close the + // underlying socket until all uses of the shared listener have been closed. + Acquire() (T, error) +} + +type multiStreamListener struct { + mu sync.Mutex + addr string + ln RefCount[StreamListener] acceptCh chan acceptResponse onCloseFunc OnCloseFunc } -type canCreateStreamListener interface { - NewStreamListener(onCloseFunc OnCloseFunc) StreamListener +// NewMultiStreamListener creates a new stream-based [MultiListener]. +func NewMultiStreamListener(addr string, onCloseFunc OnCloseFunc) MultiListener[StreamListener] { + return &multiStreamListener{ + addr: addr, + acceptCh: make(chan acceptResponse), + onCloseFunc: onCloseFunc, + } } -var _ canCreateStreamListener = (*listenAddr)(nil) +func (m *multiStreamListener) Acquire() (StreamListener, error) { + m.mu.Lock() + defer m.mu.Unlock() -// NewStreamListener creates a new [StreamListener]. -func (la *listenAddr) NewStreamListener(onCloseFunc OnCloseFunc) StreamListener { - if ln, ok := la.ln.(StreamListener); ok { - return &virtualStreamListener{ - addr: ln.Addr(), - acceptCh: la.acceptCh, - closeCh: make(chan struct{}), - onCloseFunc: onCloseFunc, + var sl StreamListener + if m.ln == nil { + tcpAddr, err := net.ResolveTCPAddr("tcp", m.addr) + if err != nil { + return nil, err + } + ln, err := net.ListenTCP("tcp", tcpAddr) + if err != nil { + return nil, err } + sl = &TCPListener{ln} + m.ln = NewRefCount(sl, m.onCloseFunc) + go func() { + for { + conn, err := sl.AcceptStream() + if errors.Is(err, net.ErrClosed) { + close(m.acceptCh) + return + } + m.acceptCh <- acceptResponse{conn, err} + } + }() } - return nil -} -type canCreatePacketListener interface { - NewPacketListener(onCloseFunc OnCloseFunc) net.PacketConn + sl = m.ln.Acquire() + return &virtualStreamListener{ + addr: sl.Addr(), + acceptCh: m.acceptCh, + closeCh: make(chan struct{}), + onCloseFunc: m.ln.Close, + }, nil } -var _ canCreatePacketListener = (*listenAddr)(nil) +type multiPacketListener struct { + mu sync.Mutex + addr string + pc RefCount[net.PacketConn] + onCloseFunc OnCloseFunc +} -// NewPacketListener creates a new [net.PacketConn]. -func (cl *listenAddr) NewPacketListener(onCloseFunc OnCloseFunc) net.PacketConn { - if ln, ok := cl.ln.(net.PacketConn); ok { - return &virtualPacketConn{ - PacketConn: ln, - onCloseFunc: onCloseFunc, - } +// NewMultiPacketListener creates a new packet-based [MultiListener]. +func NewMultiPacketListener(addr string, onCloseFunc OnCloseFunc) MultiListener[net.PacketConn] { + return &multiPacketListener{ + addr: addr, + onCloseFunc: onCloseFunc, } - return nil } -func (la *listenAddr) Close() error { - if err := la.ln.Close(); err != nil { - return err +func (m *multiPacketListener) Acquire() (net.PacketConn, error) { + m.mu.Lock() + defer m.mu.Unlock() + + var pc net.PacketConn + if m.pc == nil { + pc, err := net.ListenPacket("udp", m.addr) + if err != nil { + return nil, err + } + m.pc = NewRefCount(pc, m.onCloseFunc) } - return la.onCloseFunc() + pc = m.pc.Acquire() + return &virtualPacketConn{ + PacketConn: pc, + onCloseFunc: m.pc.Close, + }, nil } -// ListenerManager holds and manages the state of shared listeners. +// ListenerManager holds the state of shared listeners. type ListenerManager interface { // ListenStream creates a new stream listener for a given address. - // - // Listeners can overlap one another, because during config changes the new - // config is started before the old config is destroyed. This is done by using - // reusable listener wrappers, which do not actually close the underlying socket - // until all uses of the shared listener have been closed. ListenStream(addr string) (StreamListener, error) // ListenPacket creates a new packet listener for a given address. - // - // See notes on [ListenStream]. ListenPacket(addr string) (net.PacketConn, error) } type listenerManager struct { - listeners map[string]RefCount[Listener] - listenersMu sync.Mutex + streamListeners map[string]MultiListener[StreamListener] + packetListeners map[string]MultiListener[net.PacketConn] + mu sync.Mutex } // NewListenerManager creates a new [ListenerManger]. func NewListenerManager() ListenerManager { return &listenerManager{ - listeners: make(map[string]RefCount[Listener]), - } -} - -func (m *listenerManager) newStreamListener(addr string) (Listener, error) { - tcpAddr, err := net.ResolveTCPAddr("tcp", addr) - if err != nil { - return nil, err - } - ln, err := net.ListenTCP("tcp", tcpAddr) - if err != nil { - return nil, err - } - streamLn := &TCPListener{ln} - lnAddr := &listenAddr{ - ln: streamLn, - acceptCh: make(chan acceptResponse), - onCloseFunc: func() error { - m.delete(listenerKey("tcp", addr)) - return nil - }, + streamListeners: make(map[string]MultiListener[StreamListener]), + packetListeners: make(map[string]MultiListener[net.PacketConn]), } - go func() { - for { - conn, err := streamLn.AcceptStream() - if errors.Is(err, net.ErrClosed) { - close(lnAddr.acceptCh) - return - } - lnAddr.acceptCh <- acceptResponse{conn, err} - } - }() - return lnAddr, nil } -func (m *listenerManager) newPacketListener(addr string) (Listener, error) { - pc, err := net.ListenPacket("udp", addr) - if err != nil { - return nil, err - } - return &listenAddr{ - ln: pc, - onCloseFunc: func() error { - m.delete(listenerKey("udp", addr)) - return nil - }, - }, nil -} - -func (m *listenerManager) getListener(network string, addr string) (RefCount[Listener], error) { - m.listenersMu.Lock() - defer m.listenersMu.Unlock() - - lnKey := listenerKey(network, addr) - if lnRefCount, exists := m.listeners[lnKey]; exists { - return lnRefCount.Acquire(), nil - } - - var ( - ln Listener - err error - ) - if network == "tcp" { - ln, err = m.newStreamListener(addr) - } else if network == "udp" { - ln, err = m.newPacketListener(addr) - } else { - return nil, fmt.Errorf("unable to get listener for unsupported network %s", network) +func (m *listenerManager) ListenStream(addr string) (StreamListener, error) { + m.mu.Lock() + defer m.mu.Unlock() + + streamLn, exists := m.streamListeners[addr] + if !exists { + streamLn = NewMultiStreamListener( + addr, + func() error { + m.mu.Lock() + delete(m.streamListeners, addr) + m.mu.Unlock() + return nil + }, + ) + m.streamListeners[addr] = streamLn } + ln, err := streamLn.Acquire() if err != nil { - return nil, err + return nil, fmt.Errorf("unable to create stream listener for %s: %v", addr, err) } - lnRefCount := NewRefCount(ln) - m.listeners[lnKey] = lnRefCount - return lnRefCount, nil + return ln, nil } -func (m *listenerManager) ListenStream(addr string) (StreamListener, error) { - lnRefCount, err := m.getListener("tcp", addr) - if err != nil { - return nil, err - } - if lnAddr, ok := lnRefCount.Get().(canCreateStreamListener); ok { - return lnAddr.NewStreamListener(lnRefCount.Close), nil +func (m *listenerManager) ListenPacket(addr string) (net.PacketConn, error) { + m.mu.Lock() + defer m.mu.Unlock() + + packetLn, exists := m.packetListeners[addr] + if !exists { + packetLn = NewMultiPacketListener( + addr, + func() error { + m.mu.Lock() + delete(m.packetListeners, addr) + m.mu.Unlock() + return nil + }, + ) + m.packetListeners[addr] = packetLn } - return nil, fmt.Errorf("unable to create stream listener for %s", addr) -} -func (m *listenerManager) ListenPacket(addr string) (net.PacketConn, error) { - lnRefCount, err := m.getListener("udp", addr) + ln, err := packetLn.Acquire() if err != nil { - return nil, err - } - if lnAddr, ok := lnRefCount.Get().(canCreatePacketListener); ok { - return lnAddr.NewPacketListener(lnRefCount.Close), nil + return nil, fmt.Errorf("unable to create packet listener for %s: %v", addr, err) } - return nil, fmt.Errorf("unable to create packet listener for %s", addr) -} - -func (m *listenerManager) delete(key string) { - m.listenersMu.Lock() - delete(m.listeners, key) - m.listenersMu.Unlock() -} - -func listenerKey(network string, addr string) string { - return network + "/" + addr + return ln, nil } // RefCount is an atomic reference counter that can be used to track a shared @@ -299,42 +293,44 @@ type RefCount[T io.Closer] interface { io.Closer // Acquire increases the ref count and returns the wrapped object. - Acquire() RefCount[T] - - Get() T + Acquire() T } type refCount[T io.Closer] struct { - mu sync.Mutex - count *atomic.Int32 - value T + mu sync.Mutex + count *atomic.Int32 + value T + onCloseFunc OnCloseFunc } -func NewRefCount[T io.Closer](value T) RefCount[T] { +func NewRefCount[T io.Closer](value T, onCloseFunc OnCloseFunc) RefCount[T] { r := &refCount[T]{ - count: &atomic.Int32{}, - value: value, + count: &atomic.Int32{}, + value: value, + onCloseFunc: onCloseFunc, } - r.count.Store(1) return r } +func (r refCount[T]) Acquire() T { + r.count.Add(1) + return r.value +} + func (r refCount[T]) Close() error { // Lock to prevent someone from acquiring while we close the value. r.mu.Lock() defer r.mu.Unlock() if count := r.count.Add(-1); count == 0 { - return r.value.Close() + err := r.value.Close() + if err != nil { + return err + } + if r.onCloseFunc != nil { + return r.onCloseFunc() + } + return nil } return nil } - -func (r refCount[T]) Acquire() RefCount[T] { - r.count.Add(1) - return r -} - -func (r refCount[T]) Get() T { - return r.value -} diff --git a/service/listeners_test.go b/service/listeners_test.go index da5aaa1e..0a840cff 100644 --- a/service/listeners_test.go +++ b/service/listeners_test.go @@ -97,17 +97,27 @@ func (t *testRefCount) Close() error { } func TestRefCount(t *testing.T) { - var done bool - rc := NewRefCount[*testRefCount](&testRefCount{ - onCloseFunc: func() { - done = true + var objectCloseDone bool + var onCloseFuncDone bool + rc := NewRefCount[*testRefCount]( + &testRefCount{ + onCloseFunc: func() { + objectCloseDone = true + }, }, - }) + func() error { + onCloseFuncDone = true + return nil + }, + ) + rc.Acquire() rc.Acquire() require.NoError(t, rc.Close()) - require.False(t, done) + require.False(t, objectCloseDone) + require.False(t, onCloseFuncDone) require.NoError(t, rc.Close()) - require.True(t, done) + require.True(t, objectCloseDone) + require.True(t, onCloseFuncDone) } From 899d13d80af21ed132b670916e8ec3fb22e1f762 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 6 Aug 2024 17:57:12 -0400 Subject: [PATCH 101/182] Do the lazy initialization inside an anonymous function. --- service/listeners.go | 87 +++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 91aa877f..9814adca 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -139,45 +139,50 @@ type multiStreamListener struct { func NewMultiStreamListener(addr string, onCloseFunc OnCloseFunc) MultiListener[StreamListener] { return &multiStreamListener{ addr: addr, - acceptCh: make(chan acceptResponse), onCloseFunc: onCloseFunc, } } func (m *multiStreamListener) Acquire() (StreamListener, error) { - m.mu.Lock() - defer m.mu.Unlock() - - var sl StreamListener - if m.ln == nil { - tcpAddr, err := net.ResolveTCPAddr("tcp", m.addr) - if err != nil { - return nil, err - } - ln, err := net.ListenTCP("tcp", tcpAddr) - if err != nil { - return nil, err - } - sl = &TCPListener{ln} - m.ln = NewRefCount(sl, m.onCloseFunc) - go func() { - for { - conn, err := sl.AcceptStream() - if errors.Is(err, net.ErrClosed) { - close(m.acceptCh) - return - } - m.acceptCh <- acceptResponse{conn, err} + refCount, err := func() (RefCount[StreamListener], error) { + m.mu.Lock() + defer m.mu.Unlock() + + if m.ln == nil { + tcpAddr, err := net.ResolveTCPAddr("tcp", m.addr) + if err != nil { + return nil, err } - }() + ln, err := net.ListenTCP("tcp", tcpAddr) + if err != nil { + return nil, err + } + sl := &TCPListener{ln} + m.ln = NewRefCount[StreamListener](sl, m.onCloseFunc) + m.acceptCh = make(chan acceptResponse) + go func() { + for { + conn, err := sl.AcceptStream() + if errors.Is(err, net.ErrClosed) { + close(m.acceptCh) + return + } + m.acceptCh <- acceptResponse{conn, err} + } + }() + } + return m.ln, nil + }() + if err != nil { + return nil, err } - sl = m.ln.Acquire() + sl := refCount.Acquire() return &virtualStreamListener{ addr: sl.Addr(), acceptCh: m.acceptCh, closeCh: make(chan struct{}), - onCloseFunc: m.ln.Close, + onCloseFunc: refCount.Close, }, nil } @@ -197,21 +202,27 @@ func NewMultiPacketListener(addr string, onCloseFunc OnCloseFunc) MultiListener[ } func (m *multiPacketListener) Acquire() (net.PacketConn, error) { - m.mu.Lock() - defer m.mu.Unlock() - - var pc net.PacketConn - if m.pc == nil { - pc, err := net.ListenPacket("udp", m.addr) - if err != nil { - return nil, err + refCount, err := func() (RefCount[net.PacketConn], error) { + m.mu.Lock() + defer m.mu.Unlock() + + if m.pc == nil { + pc, err := net.ListenPacket("udp", m.addr) + if err != nil { + return nil, err + } + m.pc = NewRefCount(pc, m.onCloseFunc) } - m.pc = NewRefCount(pc, m.onCloseFunc) + return m.pc, nil + }() + if err != nil { + return nil, err } - pc = m.pc.Acquire() + + pc := refCount.Acquire() return &virtualPacketConn{ PacketConn: pc, - onCloseFunc: m.pc.Close, + onCloseFunc: refCount.Close, }, nil } From 80e5d491c6c5027b45e8b6e45e76f2428cea3ab0 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 7 Aug 2024 10:40:33 -0400 Subject: [PATCH 102/182] Fix concurrent access to `acceptCh` and `closeCh`. --- service/listeners.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/service/listeners.go b/service/listeners.go index 9814adca..1cb09d06 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -72,17 +72,23 @@ type acceptResponse struct { type OnCloseFunc func() error type virtualStreamListener struct { + mu sync.Mutex // Mutex to protect access to the channels addr net.Addr acceptCh <-chan acceptResponse closeCh chan struct{} + closed bool onCloseFunc OnCloseFunc } var _ StreamListener = (*virtualStreamListener)(nil) func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { + sl.mu.Lock() + acceptCh := sl.acceptCh + sl.mu.Unlock() + select { - case acceptResponse, ok := <-sl.acceptCh: + case acceptResponse, ok := <-acceptCh: if !ok { return nil, net.ErrClosed } @@ -93,8 +99,16 @@ func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { } func (sl *virtualStreamListener) Close() error { + sl.mu.Lock() + if sl.closed { + sl.mu.Unlock() + return nil + } + sl.closed = true sl.acceptCh = nil close(sl.closeCh) + sl.mu.Unlock() + if sl.onCloseFunc != nil { return sl.onCloseFunc() } From aa00f2efe1d19632e30c2cd27f7a018e92d5d4a4 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 7 Aug 2024 11:42:56 -0400 Subject: [PATCH 103/182] Use `/` in key instead of `-`. --- cmd/outline-ss-server/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 55a11acc..9fd570f7 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -107,7 +107,7 @@ func (ls *listenerSet) ListenStream(addr string) (service.StreamListener, error) ls.listenersMu.Lock() defer ls.listenersMu.Unlock() - lnKey := "stream-" + addr + lnKey := "stream/" + addr if _, exists := ls.listenerCloseFuncs[lnKey]; exists { return nil, fmt.Errorf("stream listener for %s already exists", addr) } @@ -125,7 +125,7 @@ func (ls *listenerSet) ListenPacket(addr string) (net.PacketConn, error) { ls.listenersMu.Lock() defer ls.listenersMu.Unlock() - lnKey := "packet-" + addr + lnKey := "packet/" + addr if _, exists := ls.listenerCloseFuncs[lnKey]; exists { return nil, fmt.Errorf("packet listener for %s already exists", addr) } From e658b90573a79bd26cf19f468da47f73ca694a07 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 7 Aug 2024 11:51:19 -0400 Subject: [PATCH 104/182] Return error from stopping listeners. --- cmd/outline-ss-server/main.go | 34 ++++++++++++++++++++-------- cmd/outline-ss-server/server_test.go | 4 +++- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 9fd570f7..aa8bb55e 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -61,7 +61,7 @@ func init() { } type SSServer struct { - stopConfig func() + stopConfig func() error lnManager service.ListenerManager natTimeout time.Duration m *outlineMetrics @@ -76,12 +76,14 @@ func (s *SSServer) loadConfig(filename string) error { // We hot swap the config by having the old and new listeners both live at // the same time. This means we create listeners for the new config first, // and then close the old ones after. - stopConfig, err := s.runConfig(*config) + sopConfig, err := s.runConfig(*config) if err != nil { return err } - go s.stopConfig() - s.stopConfig = stopConfig + if err := s.Stop(); err != nil { + return fmt.Errorf("unable to stop old config: %v", err) + } + s.stopConfig = sopConfig return nil } @@ -156,8 +158,9 @@ func (ls *listenerSet) Len() int { return len(ls.listenerCloseFuncs) } -func (s *SSServer) runConfig(config Config) (func(), error) { +func (s *SSServer) runConfig(config Config) (func() error, error) { startErrCh := make(chan error) + stopErrCh := make(chan error) stopCh := make(chan struct{}) go func() { @@ -165,7 +168,9 @@ func (s *SSServer) runConfig(config Config) (func(), error) { manager: s.lnManager, listenerCloseFuncs: make(map[string]func() error), } - defer lnSet.Close() // This closes all the listeners in the set. + defer func() { + stopErrCh <- lnSet.Close() + }() startErrCh <- func() error { portCiphers := make(map[int]*list.List) // Values are *List of *CipherEntry. @@ -216,24 +221,33 @@ func (s *SSServer) runConfig(config Config) (func(), error) { if err != nil { return nil, err } - return func() { + return func() error { logger.Infof("Stopping running config.") // TODO(sbruens): Actually wait for all handlers to be stopped, e.g. by // using a https://pkg.go.dev/sync#WaitGroup. stopCh <- struct{}{} + stopErr := <-stopErrCh + return stopErr }, nil } // Stop stops serving the current config. -func (s *SSServer) Stop() { - go s.stopConfig() +func (s *SSServer) Stop() error { + stopFunc := s.stopConfig + if stopFunc == nil { + return nil + } + if err := stopFunc(); err != nil { + logger.Errorf("Error stopping config: %v", err) + return err + } logger.Info("Stopped all listeners for running config") + return nil } // RunSSServer starts a shadowsocks server running, and returns the server or an error. func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ - stopConfig: func() {}, lnManager: service.NewListenerManager(), natTimeout: natTimeout, m: sm, diff --git a/cmd/outline-ss-server/server_test.go b/cmd/outline-ss-server/server_test.go index 2ba0772e..0b7777b2 100644 --- a/cmd/outline-ss-server/server_test.go +++ b/cmd/outline-ss-server/server_test.go @@ -27,5 +27,7 @@ func TestRunSSServer(t *testing.T) { if err != nil { t.Fatalf("RunSSServer() error = %v", err) } - server.Stop() + if err := server.Stop(); err != nil { + t.Errorf("Error while stopping server: %v", err) + } } From fede4d8d7764e452951dc7f59a0a7bb38b828b41 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 7 Aug 2024 13:18:06 -0400 Subject: [PATCH 105/182] Use channels to ensure `virtualPacketConn`s get closed. --- service/listeners.go | 63 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 1cb09d06..788d651f 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -64,13 +64,13 @@ func (t *TCPListener) Addr() net.Addr { return t.ln.Addr() } +type OnCloseFunc func() error + type acceptResponse struct { conn transport.StreamConn err error } -type OnCloseFunc func() error - type virtualStreamListener struct { mu sync.Mutex // Mutex to protect access to the channels addr net.Addr @@ -119,14 +119,52 @@ func (sl *virtualStreamListener) Addr() net.Addr { return sl.addr } +type packetResponse struct { + n int + addr net.Addr + err error + data []byte +} + type virtualPacketConn struct { net.PacketConn + mu sync.Mutex // Mutex to protect access to the channels + readCh <-chan packetResponse + closeCh chan struct{} + closed bool onCloseFunc OnCloseFunc } -func (spc *virtualPacketConn) Close() error { - if spc.onCloseFunc != nil { - return spc.onCloseFunc() +func (pc *virtualPacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) { + pc.mu.Lock() + readCh := pc.readCh + pc.mu.Unlock() + + select { + case packetResponse, ok := <-readCh: + if !ok { + return 0, nil, net.ErrClosed + } + copy(p, packetResponse.data) + return packetResponse.n, packetResponse.addr, packetResponse.err + case <-pc.closeCh: + return 0, nil, net.ErrClosed + } +} + +func (pc *virtualPacketConn) Close() error { + pc.mu.Lock() + if pc.closed { + pc.mu.Unlock() + return nil + } + pc.closed = true + pc.readCh = nil + close(pc.closeCh) + pc.mu.Unlock() + + if pc.onCloseFunc != nil { + return pc.onCloseFunc() } return nil } @@ -204,6 +242,7 @@ type multiPacketListener struct { mu sync.Mutex addr string pc RefCount[net.PacketConn] + readCh chan packetResponse onCloseFunc OnCloseFunc } @@ -226,6 +265,18 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { return nil, err } m.pc = NewRefCount(pc, m.onCloseFunc) + m.readCh = make(chan packetResponse) + go func() { + for { + buffer := make([]byte, serverUDPBufferSize) + n, addr, err := pc.ReadFrom(buffer) + if err != nil { + close(m.readCh) + return + } + m.readCh <- packetResponse{n: n, addr: addr, err: err, data: buffer[:n]} + } + }() } return m.pc, nil }() @@ -236,6 +287,8 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { pc := refCount.Acquire() return &virtualPacketConn{ PacketConn: pc, + readCh: m.readCh, + closeCh: make(chan struct{}), onCloseFunc: refCount.Close, }, nil } From 4730d741237e5f697975d5f0f1ae26c3cebc0fce Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 7 Aug 2024 16:21:10 -0400 Subject: [PATCH 106/182] Add more test cases for packet listeners. --- service/listeners_test.go | 61 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/service/listeners_test.go b/service/listeners_test.go index 0a840cff..d627ec1a 100644 --- a/service/listeners_test.go +++ b/service/listeners_test.go @@ -51,18 +51,17 @@ func TestListenerManagerStreamListenerNotClosedIfStillInUse(t *testing.T) { require.NoError(t, err) ln2, err := m.ListenStream("127.0.0.1:0") require.NoError(t, err) - // Close only the first listener. ln.Close() + done := make(chan struct{}) go func() { ln2.AcceptStream() done <- struct{}{} }() - err = writeTestPayload(ln2) - require.NoError(t, err) + require.NoError(t, err) <-done } @@ -82,8 +81,64 @@ func TestListenerManagerStreamListenerCreatesListenerOnDemand(t *testing.T) { done <- struct{}{} }() err = writeTestPayload(ln2) + + require.NoError(t, err) + <-done +} + +func TestListenerManagerPacketListenerEarlyClose(t *testing.T) { + m := NewListenerManager() + pc, err := m.ListenPacket("127.0.0.1:0") + require.NoError(t, err) + + pc.Close() + _, _, readErr := pc.ReadFrom(nil) + _, writeErr := pc.WriteTo(nil, &net.UDPAddr{}) + + require.ErrorIs(t, readErr, net.ErrClosed) + require.ErrorIs(t, writeErr, net.ErrClosed) +} + +func TestListenerManagerPacketListenerNotClosedIfStillInUse(t *testing.T) { + m := NewListenerManager() + pc, err := m.ListenPacket("127.0.0.1:0") + require.NoError(t, err) + pc2, err := m.ListenPacket("127.0.0.1:0") + require.NoError(t, err) + // Close only the first listener. + pc.Close() + + done := make(chan struct{}) + go func() { + _, _, readErr := pc2.ReadFrom(nil) + require.NoError(t, readErr) + done <- struct{}{} + }() + _, err = pc.WriteTo(nil, pc2.LocalAddr()) + + require.NoError(t, err) + <-done +} + +func TestListenerManagerPacketListenerCreatesListenerOnDemand(t *testing.T) { + m := NewListenerManager() + // Create a listener and immediately close it. + pc, err := m.ListenPacket("127.0.0.1:0") require.NoError(t, err) + pc.Close() + // Now create another listener on the same address. + pc2, err := m.ListenPacket("127.0.0.1:0") + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + _, _, readErr := pc2.ReadFrom(nil) + require.NoError(t, readErr) + done <- struct{}{} + }() + _, err = pc2.WriteTo(nil, pc2.LocalAddr()) + require.NoError(t, err) <-done } From 458cf4141a7f783e960bf36bc83d7d1cb59ecc8d Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 9 Aug 2024 11:54:33 -0400 Subject: [PATCH 107/182] Only log errors from stopping old configs. --- cmd/outline-ss-server/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index aa8bb55e..c7212d61 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -76,14 +76,14 @@ func (s *SSServer) loadConfig(filename string) error { // We hot swap the config by having the old and new listeners both live at // the same time. This means we create listeners for the new config first, // and then close the old ones after. - sopConfig, err := s.runConfig(*config) + stopConfig, err := s.runConfig(*config) if err != nil { return err } if err := s.Stop(); err != nil { - return fmt.Errorf("unable to stop old config: %v", err) + logger.Warningf("Failed to stop old config: %v", err) } - s.stopConfig = sopConfig + s.stopConfig = stopConfig return nil } From 81bf20e013d4caf49b0a7cafabcbe5fe80b9bd16 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 9 Aug 2024 11:55:09 -0400 Subject: [PATCH 108/182] Remove the `closed` field from the virtual listeners. --- service/listeners.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 788d651f..19e342bc 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -76,7 +76,6 @@ type virtualStreamListener struct { addr net.Addr acceptCh <-chan acceptResponse closeCh chan struct{} - closed bool onCloseFunc OnCloseFunc } @@ -100,11 +99,10 @@ func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { func (sl *virtualStreamListener) Close() error { sl.mu.Lock() - if sl.closed { + if sl.acceptCh == nil { sl.mu.Unlock() return nil } - sl.closed = true sl.acceptCh = nil close(sl.closeCh) sl.mu.Unlock() @@ -131,7 +129,6 @@ type virtualPacketConn struct { mu sync.Mutex // Mutex to protect access to the channels readCh <-chan packetResponse closeCh chan struct{} - closed bool onCloseFunc OnCloseFunc } @@ -154,11 +151,10 @@ func (pc *virtualPacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error func (pc *virtualPacketConn) Close() error { pc.mu.Lock() - if pc.closed { + if pc.readCh == nil { pc.mu.Unlock() return nil } - pc.closed = true pc.readCh = nil close(pc.closeCh) pc.mu.Unlock() From 53b1e962af829f0b055ee327b0a568891446d909 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 9 Aug 2024 12:55:01 -0400 Subject: [PATCH 109/182] Remove the `RefCount`. --- service/listeners.go | 196 +++++++++++++++----------------------- service/listeners_test.go | 35 ------- 2 files changed, 79 insertions(+), 152 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 19e342bc..8b22d8a7 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -20,7 +20,6 @@ import ( "io" "net" "sync" - "sync/atomic" "github.com/Jigsaw-Code/outline-sdk/transport" ) @@ -178,8 +177,9 @@ type MultiListener[T Listener] interface { type multiStreamListener struct { mu sync.Mutex addr string - ln RefCount[StreamListener] + ln StreamListener acceptCh chan acceptResponse + count uint32 onCloseFunc OnCloseFunc } @@ -192,53 +192,58 @@ func NewMultiStreamListener(addr string, onCloseFunc OnCloseFunc) MultiListener[ } func (m *multiStreamListener) Acquire() (StreamListener, error) { - refCount, err := func() (RefCount[StreamListener], error) { - m.mu.Lock() - defer m.mu.Unlock() - - if m.ln == nil { - tcpAddr, err := net.ResolveTCPAddr("tcp", m.addr) - if err != nil { - return nil, err - } - ln, err := net.ListenTCP("tcp", tcpAddr) - if err != nil { - return nil, err - } - sl := &TCPListener{ln} - m.ln = NewRefCount[StreamListener](sl, m.onCloseFunc) - m.acceptCh = make(chan acceptResponse) - go func() { - for { - conn, err := sl.AcceptStream() - if errors.Is(err, net.ErrClosed) { - close(m.acceptCh) - return - } - m.acceptCh <- acceptResponse{conn, err} - } - }() + m.mu.Lock() + defer m.mu.Unlock() + + if m.ln == nil { + tcpAddr, err := net.ResolveTCPAddr("tcp", m.addr) + if err != nil { + return nil, err } - return m.ln, nil - }() - if err != nil { - return nil, err + ln, err := net.ListenTCP("tcp", tcpAddr) + if err != nil { + return nil, err + } + m.ln = &TCPListener{ln} + m.acceptCh = make(chan acceptResponse) + go func() { + for { + conn, err := m.ln.AcceptStream() + if errors.Is(err, net.ErrClosed) { + close(m.acceptCh) + return + } + m.acceptCh <- acceptResponse{conn, err} + } + }() } - sl := refCount.Acquire() + m.count++ return &virtualStreamListener{ - addr: sl.Addr(), - acceptCh: m.acceptCh, - closeCh: make(chan struct{}), - onCloseFunc: refCount.Close, + addr: m.ln.Addr(), + acceptCh: m.acceptCh, + closeCh: make(chan struct{}), + onCloseFunc: func() error { + m.mu.Lock() + defer m.mu.Unlock() + m.count-- + if m.count == 0 { + m.ln.Close() + if m.onCloseFunc != nil { + return m.onCloseFunc() + } + } + return nil + }, }, nil } type multiPacketListener struct { mu sync.Mutex addr string - pc RefCount[net.PacketConn] + pc net.PacketConn readCh chan packetResponse + count uint32 onCloseFunc OnCloseFunc } @@ -251,41 +256,46 @@ func NewMultiPacketListener(addr string, onCloseFunc OnCloseFunc) MultiListener[ } func (m *multiPacketListener) Acquire() (net.PacketConn, error) { - refCount, err := func() (RefCount[net.PacketConn], error) { - m.mu.Lock() - defer m.mu.Unlock() - - if m.pc == nil { - pc, err := net.ListenPacket("udp", m.addr) - if err != nil { - return nil, err - } - m.pc = NewRefCount(pc, m.onCloseFunc) - m.readCh = make(chan packetResponse) - go func() { - for { - buffer := make([]byte, serverUDPBufferSize) - n, addr, err := pc.ReadFrom(buffer) - if err != nil { - close(m.readCh) - return - } - m.readCh <- packetResponse{n: n, addr: addr, err: err, data: buffer[:n]} - } - }() + m.mu.Lock() + defer m.mu.Unlock() + + if m.pc == nil { + pc, err := net.ListenPacket("udp", m.addr) + if err != nil { + return nil, err } - return m.pc, nil - }() - if err != nil { - return nil, err + m.pc = pc + m.readCh = make(chan packetResponse) + go func() { + for { + buffer := make([]byte, serverUDPBufferSize) + n, addr, err := pc.ReadFrom(buffer) + if err != nil { + close(m.readCh) + return + } + m.readCh <- packetResponse{n: n, addr: addr, err: err, data: buffer[:n]} + } + }() } - pc := refCount.Acquire() + m.count++ return &virtualPacketConn{ - PacketConn: pc, - readCh: m.readCh, - closeCh: make(chan struct{}), - onCloseFunc: refCount.Close, + PacketConn: m.pc, + readCh: m.readCh, + closeCh: make(chan struct{}), + onCloseFunc: func() error { + m.mu.Lock() + defer m.mu.Unlock() + m.count-- + if m.count == 0 { + m.pc.Close() + if m.onCloseFunc != nil { + return m.onCloseFunc() + } + } + return nil + }, }, nil } @@ -360,51 +370,3 @@ func (m *listenerManager) ListenPacket(addr string) (net.PacketConn, error) { } return ln, nil } - -// RefCount is an atomic reference counter that can be used to track a shared -// [io.Closer] resource. -type RefCount[T io.Closer] interface { - io.Closer - - // Acquire increases the ref count and returns the wrapped object. - Acquire() T -} - -type refCount[T io.Closer] struct { - mu sync.Mutex - count *atomic.Int32 - value T - onCloseFunc OnCloseFunc -} - -func NewRefCount[T io.Closer](value T, onCloseFunc OnCloseFunc) RefCount[T] { - r := &refCount[T]{ - count: &atomic.Int32{}, - value: value, - onCloseFunc: onCloseFunc, - } - return r -} - -func (r refCount[T]) Acquire() T { - r.count.Add(1) - return r.value -} - -func (r refCount[T]) Close() error { - // Lock to prevent someone from acquiring while we close the value. - r.mu.Lock() - defer r.mu.Unlock() - - if count := r.count.Add(-1); count == 0 { - err := r.value.Close() - if err != nil { - return err - } - if r.onCloseFunc != nil { - return r.onCloseFunc() - } - return nil - } - return nil -} diff --git a/service/listeners_test.go b/service/listeners_test.go index d627ec1a..af98ba94 100644 --- a/service/listeners_test.go +++ b/service/listeners_test.go @@ -141,38 +141,3 @@ func TestListenerManagerPacketListenerCreatesListenerOnDemand(t *testing.T) { require.NoError(t, err) <-done } - -type testRefCount struct { - onCloseFunc func() -} - -func (t *testRefCount) Close() error { - t.onCloseFunc() - return nil -} - -func TestRefCount(t *testing.T) { - var objectCloseDone bool - var onCloseFuncDone bool - rc := NewRefCount[*testRefCount]( - &testRefCount{ - onCloseFunc: func() { - objectCloseDone = true - }, - }, - func() error { - onCloseFuncDone = true - return nil - }, - ) - rc.Acquire() - rc.Acquire() - - require.NoError(t, rc.Close()) - require.False(t, objectCloseDone) - require.False(t, onCloseFuncDone) - - require.NoError(t, rc.Close()) - require.True(t, objectCloseDone) - require.True(t, onCloseFuncDone) -} From 8f9f1eaf66dc78178940913cde99b076730928ba Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 9 Aug 2024 15:40:02 -0400 Subject: [PATCH 110/182] Implement channel-based packet read for virtual connections. --- service/listeners.go | 73 ++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 8b22d8a7..f20d485a 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -15,6 +15,7 @@ package service import ( + "context" "errors" "fmt" "io" @@ -98,14 +99,13 @@ func (sl *virtualStreamListener) AcceptStream() (transport.StreamConn, error) { func (sl *virtualStreamListener) Close() error { sl.mu.Lock() + defer sl.mu.Unlock() + if sl.acceptCh == nil { - sl.mu.Unlock() return nil } sl.acceptCh = nil close(sl.closeCh) - sl.mu.Unlock() - if sl.onCloseFunc != nil { return sl.onCloseFunc() } @@ -116,47 +116,54 @@ func (sl *virtualStreamListener) Addr() net.Addr { return sl.addr } -type packetResponse struct { - n int - addr net.Addr - err error - data []byte +type readRequest struct { + buffer []byte + respCh chan struct { + n int + addr net.Addr + err error + } } type virtualPacketConn struct { net.PacketConn mu sync.Mutex // Mutex to protect access to the channels - readCh <-chan packetResponse + readCh chan readRequest closeCh chan struct{} onCloseFunc OnCloseFunc } func (pc *virtualPacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) { - pc.mu.Lock() - readCh := pc.readCh - pc.mu.Unlock() + respCh := make(chan struct { + n int + addr net.Addr + err error + }, 1) - select { - case packetResponse, ok := <-readCh: - if !ok { - return 0, nil, net.ErrClosed - } - copy(p, packetResponse.data) - return packetResponse.n, packetResponse.addr, packetResponse.err - case <-pc.closeCh: + pc.mu.Lock() + if pc.readCh == nil { + pc.mu.Unlock() return 0, nil, net.ErrClosed } + pc.readCh <- readRequest{ + buffer: p, + respCh: respCh, + } + pc.mu.Unlock() + + resp := <-respCh + return resp.n, resp.addr, resp.err } func (pc *virtualPacketConn) Close() error { pc.mu.Lock() + defer pc.mu.Unlock() + if pc.readCh == nil { - pc.mu.Unlock() return nil } pc.readCh = nil close(pc.closeCh) - pc.mu.Unlock() if pc.onCloseFunc != nil { return pc.onCloseFunc() @@ -242,7 +249,8 @@ type multiPacketListener struct { mu sync.Mutex addr string pc net.PacketConn - readCh chan packetResponse + readCh chan readRequest + cancel context.CancelFunc count uint32 onCloseFunc OnCloseFunc } @@ -265,16 +273,22 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { return nil, err } m.pc = pc - m.readCh = make(chan packetResponse) + m.readCh = make(chan readRequest) + ctx, cancel := context.WithCancel(context.Background()) + m.cancel = cancel go func() { for { - buffer := make([]byte, serverUDPBufferSize) - n, addr, err := pc.ReadFrom(buffer) - if err != nil { - close(m.readCh) + select { + case req := <-m.readCh: + n, addr, err := pc.ReadFrom(req.buffer) + req.respCh <- struct { + n int + addr net.Addr + err error + }{n, addr, err} + case <-ctx.Done(): return } - m.readCh <- packetResponse{n: n, addr: addr, err: err, data: buffer[:n]} } }() } @@ -289,6 +303,7 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { defer m.mu.Unlock() m.count-- if m.count == 0 { + m.cancel() m.pc.Close() if m.onCloseFunc != nil { return m.onCloseFunc() From 1ac265db0ee92a573c68050beeeef5cac550e348 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 9 Aug 2024 15:55:07 -0400 Subject: [PATCH 111/182] Use a done channel. --- service/listeners.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index f20d485a..3fcb9688 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -15,7 +15,6 @@ package service import ( - "context" "errors" "fmt" "io" @@ -129,7 +128,6 @@ type virtualPacketConn struct { net.PacketConn mu sync.Mutex // Mutex to protect access to the channels readCh chan readRequest - closeCh chan struct{} onCloseFunc OnCloseFunc } @@ -163,7 +161,6 @@ func (pc *virtualPacketConn) Close() error { return nil } pc.readCh = nil - close(pc.closeCh) if pc.onCloseFunc != nil { return pc.onCloseFunc() @@ -185,8 +182,8 @@ type multiStreamListener struct { mu sync.Mutex addr string ln StreamListener - acceptCh chan acceptResponse count uint32 + acceptCh chan acceptResponse onCloseFunc OnCloseFunc } @@ -249,9 +246,9 @@ type multiPacketListener struct { mu sync.Mutex addr string pc net.PacketConn - readCh chan readRequest - cancel context.CancelFunc count uint32 + readCh chan readRequest + doneCh chan struct{} onCloseFunc OnCloseFunc } @@ -274,8 +271,7 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { } m.pc = pc m.readCh = make(chan readRequest) - ctx, cancel := context.WithCancel(context.Background()) - m.cancel = cancel + m.doneCh = make(chan struct{}) go func() { for { select { @@ -286,7 +282,7 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { addr net.Addr err error }{n, addr, err} - case <-ctx.Done(): + case <-m.doneCh: return } } @@ -297,13 +293,12 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { return &virtualPacketConn{ PacketConn: m.pc, readCh: m.readCh, - closeCh: make(chan struct{}), onCloseFunc: func() error { m.mu.Lock() defer m.mu.Unlock() m.count-- if m.count == 0 { - m.cancel() + close(m.doneCh) m.pc.Close() if m.onCloseFunc != nil { return m.onCloseFunc() From 1538a9ae1d25e79bb8598a047ce8fc40d5ffdd72 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 14 Aug 2024 18:00:04 -0400 Subject: [PATCH 112/182] Set listeners and `onCloseFunc`'s to nil when closing. --- service/listeners.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index 3fcb9688..c184e9ea 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -233,8 +233,11 @@ func (m *multiStreamListener) Acquire() (StreamListener, error) { m.count-- if m.count == 0 { m.ln.Close() + m.ln = nil if m.onCloseFunc != nil { - return m.onCloseFunc() + onCloseFunc := m.onCloseFunc + m.onCloseFunc = nil + return onCloseFunc() } } return nil @@ -300,8 +303,11 @@ func (m *multiPacketListener) Acquire() (net.PacketConn, error) { if m.count == 0 { close(m.doneCh) m.pc.Close() + m.pc = nil if m.onCloseFunc != nil { - return m.onCloseFunc() + onCloseFunc := m.onCloseFunc + m.onCloseFunc = nil + return onCloseFunc() } } return nil From 4df0b9fbf877d32770a57eba6bec6e44e52ab8df Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 14 Aug 2024 18:01:16 -0400 Subject: [PATCH 113/182] Set `onCloseFunc`'s to nil when closing. --- service/listeners.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/service/listeners.go b/service/listeners.go index c184e9ea..41bbaae2 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -106,7 +106,9 @@ func (sl *virtualStreamListener) Close() error { sl.acceptCh = nil close(sl.closeCh) if sl.onCloseFunc != nil { - return sl.onCloseFunc() + onCloseFunc := sl.onCloseFunc + sl.onCloseFunc = nil + return onCloseFunc() } return nil } @@ -163,7 +165,9 @@ func (pc *virtualPacketConn) Close() error { pc.readCh = nil if pc.onCloseFunc != nil { - return pc.onCloseFunc() + onCloseFunc := pc.onCloseFunc + pc.onCloseFunc = nil + return onCloseFunc() } return nil } From 16feaf9a70b972ee46b79e1299ae7f9195ffc34c Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 14 Aug 2024 18:33:35 -0400 Subject: [PATCH 114/182] Fix race condition. --- service/listeners.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/service/listeners.go b/service/listeners.go index 41bbaae2..55d6c7ef 100644 --- a/service/listeners.go +++ b/service/listeners.go @@ -216,7 +216,14 @@ func (m *multiStreamListener) Acquire() (StreamListener, error) { m.acceptCh = make(chan acceptResponse) go func() { for { - conn, err := m.ln.AcceptStream() + m.mu.Lock() + ln := m.ln + m.mu.Unlock() + + if ln == nil { + return + } + conn, err := ln.AcceptStream() if errors.Is(err, net.ErrClosed) { close(m.acceptCh) return From 288b88b80067b1eaccf8cbd613d223f892936cec Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 14 Aug 2024 18:34:57 -0400 Subject: [PATCH 115/182] Add some benchmarks for listener manager. --- service/listeners_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/service/listeners_test.go b/service/listeners_test.go index af98ba94..32468261 100644 --- a/service/listeners_test.go +++ b/service/listeners_test.go @@ -141,3 +141,27 @@ func TestListenerManagerPacketListenerCreatesListenerOnDemand(t *testing.T) { require.NoError(t, err) <-done } + +func BenchmarkMultiStreamListener_Acquire(b *testing.B) { + lm := NewListenerManager() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := lm.ListenStream("localhost:0") + if err != nil { + b.Fatalf("Failed to acquire stream listener: %v", err) + } + } +} + +func BenchmarkMultiPacketListener_Acquire(b *testing.B) { + lm := NewListenerManager() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := lm.ListenPacket("localhost:0") + if err != nil { + b.Fatalf("Failed to acquire packet listener: %v", err) + } + } +} From de64b8a2b39735a71ef9cbababeca881347e70b3 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 15 Aug 2024 17:55:15 -0400 Subject: [PATCH 116/182] Add structure logging with `slog`. --- cmd/outline-ss-server/main.go | 51 +++++++++++++++----------------- cmd/outline-ss-server/metrics.go | 5 ++-- go.mod | 3 +- go.sum | 2 ++ service/tcp.go | 33 +++++++++++---------- service/udp.go | 46 ++++++++++++++-------------- 6 files changed, 71 insertions(+), 69 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index e73506a8..48cd0bdc 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -18,25 +18,26 @@ import ( "container/list" "flag" "fmt" + "log/slog" "net" "net/http" "os" "os/signal" - "strings" "syscall" "time" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" - "github.com/op/go-logging" + "github.com/lmittmann/tint" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/term" "gopkg.in/yaml.v2" ) -var logger *logging.Logger +var logLevel = new(slog.LevelVar) // Info by default +var logHandler slog.Handler // Set by goreleaser default ldflags. See https://goreleaser.com/customization/build/ var version = "dev" @@ -48,14 +49,10 @@ const tcpReadTimeout time.Duration = 59 * time.Second const defaultNatTimeout time.Duration = 5 * time.Minute func init() { - var prefix = "%{level:.1s}%{time:2006-01-02T15:04:05.000Z07:00} %{pid} %{shortfile}]" - if term.IsTerminal(int(os.Stderr.Fd())) { - // Add color only if the output is the terminal - prefix = strings.Join([]string{"%{color}", prefix, "%{color:reset}"}, "") - } - logging.SetFormatter(logging.MustStringFormatter(strings.Join([]string{prefix, " %{message}"}, ""))) - logging.SetBackend(logging.NewLogBackend(os.Stderr, "", 0)) - logger = logging.MustGetLogger("") + logHandler = tint.NewHandler( + os.Stderr, + &tint.Options{NoColor: !term.IsTerminal(int(os.Stderr.Fd())), Level: logLevel}, + ) } type ssPort struct { @@ -77,13 +74,13 @@ func (s *SSServer) startPort(portNum int) error { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks TCP service failed to start on port %v: %w", portNum, err) } - logger.Infof("Shadowsocks TCP service listening on %v", listener.Addr().String()) + slog.Info("Shadowsocks TCP service started.", "address", listener.Addr().String()) packetConn, err := net.ListenUDP("udp", &net.UDPAddr{Port: portNum}) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks UDP service failed to start on port %v: %w", portNum, err) } - logger.Infof("Shadowsocks UDP service listening on %v", packetConn.LocalAddr().String()) + slog.Info("Shadowsocks UDP service started.", "address", packetConn.LocalAddr().String()) port := &ssPort{tcpListener: listener, packetConn: packetConn, cipherList: service.NewCipherList()} authFunc := service.NewShadowsocksStreamAuthenticator(port.cipherList, &s.replayCache, s.m) // TODO: Register initial data metrics at zero. @@ -107,12 +104,12 @@ func (s *SSServer) removePort(portNum int) error { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks TCP service on port %v failed to stop: %w", portNum, tcpErr) } - logger.Infof("Shadowsocks TCP service on port %v stopped", portNum) + slog.Info("Shadowsocks TCP service stopped.", "port", portNum) if udpErr != nil { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks UDP service on port %v failed to stop: %w", portNum, udpErr) } - logger.Infof("Shadowsocks UDP service on port %v stopped", portNum) + slog.Info("Shadowsocks UDP service stopped.", "port", portNum) return nil } @@ -155,7 +152,7 @@ func (s *SSServer) loadConfig(filename string) error { for portNum, cipherList := range portCiphers { s.ports[portNum].cipherList.Update(cipherList) } - logger.Infof("Loaded %v access keys over %v ports", len(config.Keys), len(s.ports)) + slog.Info("Loaded config.", "access keys", len(config.Keys), "ports", len(s.ports)) s.m.SetNumAccessKeys(len(config.Keys), len(portCiphers)) return nil } @@ -186,9 +183,9 @@ func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, signal.Notify(sigHup, syscall.SIGHUP) go func() { for range sigHup { - logger.Infof("SIGHUP received. Loading config from %v", filename) + slog.Info("SIGHUP received. Loading config.", "config", filename) if err := server.loadConfig(filename); err != nil { - logger.Errorf("Failed to update server: %v. Server state may be invalid. Fix the error and try the update again", err) + slog.Error("Failed to update server. Server state may be invalid. Fix the error and try the update again", "err", err) } } }() @@ -218,6 +215,8 @@ func readConfig(filename string) (*Config, error) { } func main() { + slog.SetDefault(slog.New(logHandler)) + var flags struct { ConfigFile string MetricsAddr string @@ -240,9 +239,7 @@ func main() { flag.Parse() if flags.Verbose { - logging.SetLevel(logging.DEBUG, "") - } else { - logging.SetLevel(logging.INFO, "") + logLevel.Set(slog.LevelDebug) } if flags.Version { @@ -258,21 +255,21 @@ func main() { if flags.MetricsAddr != "" { http.Handle("/metrics", promhttp.Handler()) go func() { - logger.Fatalf("Failed to run metrics server: %v. Aborting.", http.ListenAndServe(flags.MetricsAddr, nil)) + slog.Error("Failed to run metrics server. Aborting.", "err", http.ListenAndServe(flags.MetricsAddr, nil)) }() - logger.Infof("Prometheus metrics available at http://%v/metrics", flags.MetricsAddr) + slog.Info(fmt.Sprintf("Prometheus metrics available at http://%v/metrics.", flags.MetricsAddr)) } var err error if flags.IPCountryDB != "" { - logger.Infof("Using IP-Country database at %v", flags.IPCountryDB) + slog.Info("Using IP-Country database.", "db", flags.IPCountryDB) } if flags.IPASNDB != "" { - logger.Infof("Using IP-ASN database at %v", flags.IPASNDB) + slog.Info("Using IP-ASN database.", "db", flags.IPASNDB) } ip2info, err := ipinfo.NewMMDBIPInfoMap(flags.IPCountryDB, flags.IPASNDB) if err != nil { - logger.Fatalf("Could create IP info map: %v. Aborting", err) + slog.Error("Failed to create IP info map. Aborting.", "err", err) } defer ip2info.Close() @@ -280,7 +277,7 @@ func main() { m.SetBuildInfo(version) _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, m, flags.replayHistory) if err != nil { - logger.Fatalf("Server failed to start: %v. Aborting", err) + slog.Error("Server failed to start. Aborting.", "err", err) } sigCh := make(chan os.Signal, 1) diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index e95ceeb3..dfdeb1f2 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -16,6 +16,7 @@ package main import ( "fmt" + "log/slog" "net" "net/netip" "sync" @@ -112,7 +113,7 @@ func (c *tunnelTimeCollector) Collect(ch chan<- prometheus.Metric) { // Calculates and reports the tunnel time for a given active client. func (c *tunnelTimeCollector) reportTunnelTime(ipKey IPKey, client *activeClient, tNow time.Time) { tunnelTime := tNow.Sub(client.startTime) - logger.Debugf("Reporting tunnel time for key `%v`, duration: %v", ipKey.accessKey, tunnelTime) + slog.Debug("Reporting tunnel time.", "key", ipKey.accessKey, "duration", tunnelTime) c.tunnelTimePerKey.WithLabelValues(ipKey.accessKey).Add(tunnelTime.Seconds()) c.tunnelTimePerLocation.WithLabelValues(client.info.CountryCode.String(), asnLabel(client.info.ASN)).Add(tunnelTime.Seconds()) // Reset the start time now that the tunnel time has been reported. @@ -138,7 +139,7 @@ func (c *tunnelTimeCollector) stopConnection(ipKey IPKey) { defer c.mu.Unlock() client, exists := c.activeClients[ipKey] if !exists { - logger.Warningf("Failed to find active client") + slog.Warn("Failed to find active client.") return } client.connCount-- diff --git a/go.mod b/go.mod index 33104c55..e876f4b2 100644 --- a/go.mod +++ b/go.mod @@ -174,6 +174,7 @@ require ( github.com/klauspost/pgzip v1.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf // indirect + github.com/lmittmann/tint v1.0.5 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -272,4 +273,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -go 1.20 +go 1.21 diff --git a/go.sum b/go.sum index d6ba546c..d179f629 100644 --- a/go.sum +++ b/go.sum @@ -1657,6 +1657,8 @@ github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linode/linodego v1.4.0/go.mod h1:PVsRxSlOiJyvG4/scTszpmZDTdgS+to3X6eS8pRrWI8= github.com/linode/linodego v1.12.0/go.mod h1:NJlzvlNtdMRRkXb0oN6UWzUkj6t+IBsyveHgZ5Ppjyk= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lmittmann/tint v1.0.5 h1:NQclAutOfYsqs2F1Lenue6OoWCajs5wJcP3DfWVpePw= +github.com/lmittmann/tint v1.0.5/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= diff --git a/service/tcp.go b/service/tcp.go index ab74ce6a..e686b9c5 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net" "net/netip" "sync" @@ -32,7 +33,6 @@ import ( "github.com/Jigsaw-Code/outline-ss-server/ipinfo" onet "github.com/Jigsaw-Code/outline-ss-server/net" "github.com/Jigsaw-Code/outline-ss-server/service/metrics" - logging "github.com/op/go-logging" "github.com/shadowsocks/go-shadowsocks2/socks" ) @@ -62,12 +62,13 @@ func remoteIP(conn net.Conn) netip.Addr { return netip.Addr{} } -// Wrapper for logger.Debugf during TCP access key searches. -func debugTCP(cipherID, template string, val interface{}) { +// Wrapper for slog.Debug during TCP access key searches. +func debugTCP(template string, cipherID string, args ...any) { // This is an optimization to reduce unnecessary allocations due to an interaction - // between Go's inlining/escape analysis and varargs functions like logger.Debugf. - if logger.IsEnabledFor(logging.DEBUG) { - logger.Debugf("TCP(%s): "+template, cipherID, val) + // between Go's inlining/escape analysis and varargs functions like slog.Debug. + if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + args = append(args, slog.String("ID", cipherID)) + slog.Debug(fmt.Sprintf("TCP: %s", template), args...) } } @@ -108,10 +109,10 @@ func findEntry(firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list. cryptoKey := entry.CryptoKey _, err := shadowsocks.Unpack(chunkLenBuf[:0], firstBytes[:cryptoKey.SaltSize()+2+cryptoKey.TagSize()], cryptoKey) if err != nil { - debugTCP(entry.ID, "Failed to decrypt length: %v", err) + debugTCP("Failed to decrypt length.", entry.ID, slog.Any("err", err)) continue } - debugTCP(entry.ID, "Found cipher at index %d", ci) + debugTCP("Found cipher.", entry.ID, slog.Int("index", ci)) return entry, elt } return nil, nil @@ -235,7 +236,7 @@ func StreamServe(accept StreamListener, handle StreamHandler) { if errors.Is(err, net.ErrClosed) { break } - logger.Warningf("AcceptTCP failed: %v. Continuing to listen.", err) + slog.Warn("AcceptTCP failed. Continuing to listen.", "err", err) continue } @@ -245,7 +246,7 @@ func StreamServe(accept StreamListener, handle StreamHandler) { defer clientConn.Close() defer func() { if r := recover(); r != nil { - logger.Warningf("Panic in TCP handler: %v. Continuing to listen.", r) + slog.Warn("Panic in TCP handler. Continuing to listen.", "err", r) } }() handle(ctx, clientConn) @@ -256,9 +257,9 @@ func StreamServe(accept StreamListener, handle StreamHandler) { func (h *tcpHandler) Handle(ctx context.Context, clientConn transport.StreamConn) { clientInfo, err := ipinfo.GetIPInfoFromAddr(h.m, clientConn.RemoteAddr()) if err != nil { - logger.Warningf("Failed client info lookup: %v", err) + slog.Warn("Failed client info lookup", "err", err) } - logger.Debugf("Got info \"%#v\" for IP %v", clientInfo, clientConn.RemoteAddr().String()) + slog.Debug("Got info for IP.", "info", clientInfo, "IP", clientConn.RemoteAddr().String()) h.m.AddOpenTCPConnection(clientInfo) var proxyMetrics metrics.ProxyMetrics measuredClientConn := metrics.MeasureConn(clientConn, &proxyMetrics.ProxyClient, &proxyMetrics.ClientProxy) @@ -270,11 +271,11 @@ func (h *tcpHandler) Handle(ctx context.Context, clientConn transport.StreamConn status := "OK" if connError != nil { status = connError.Status - logger.Debugf("TCP Error: %v: %v", connError.Message, connError.Cause) + slog.Debug("TCP: Error", "msg", connError.Message, "cause", connError.Cause) } h.m.AddClosedTCPConnection(clientInfo, clientConn.RemoteAddr(), id, status, proxyMetrics, connDuration) measuredClientConn.Close() // Closing after the metrics are added aids integration testing. - logger.Debugf("Done with status %v, duration %v", status, connDuration) + slog.Debug("TCP: Done.", "status", status, "duration", connDuration) } func getProxyRequest(clientConn transport.StreamConn) (string, error) { @@ -296,7 +297,7 @@ func proxyConnection(ctx context.Context, dialer transport.StreamDialer, tgtAddr return ensureConnectionError(dialErr, "ERR_CONNECT", "Failed to connect to target") } defer tgtConn.Close() - logger.Debugf("proxy %s <-> %s", clientConn.RemoteAddr().String(), tgtConn.RemoteAddr().String()) + slog.Debug("proxy %s <-> %s", clientConn.RemoteAddr().String(), tgtConn.RemoteAddr().String()) fromClientErrCh := make(chan error) go func() { @@ -373,7 +374,7 @@ func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, addr, status string, // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) - logger.Debugf("Drain error: %v, drain result: %v", drainErr, drainResult) + slog.Debug("Drain error: %v, drain result: %v", drainErr, drainResult) h.m.AddTCPProbe(status, drainResult, addr, proxyMetrics.ClientProxy) } diff --git a/service/udp.go b/service/udp.go index 4830e302..f1aa0b99 100644 --- a/service/udp.go +++ b/service/udp.go @@ -15,8 +15,10 @@ package service import ( + "context" "errors" "fmt" + "log/slog" "net" "net/netip" "runtime/debug" @@ -26,7 +28,6 @@ import ( "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" onet "github.com/Jigsaw-Code/outline-ss-server/net" - logging "github.com/op/go-logging" "github.com/shadowsocks/go-shadowsocks2/socks" ) @@ -47,19 +48,20 @@ type UDPMetrics interface { // Max UDP buffer size for the server code. const serverUDPBufferSize = 64 * 1024 -// Wrapper for logger.Debugf during UDP proxying. -func debugUDP(tag string, template string, val interface{}) { +// Wrapper for slog.Debug during UDP proxying. +func debugUDP(template string, cipherID string, args ...any) { // This is an optimization to reduce unnecessary allocations due to an interaction - // between Go's inlining/escape analysis and varargs functions like logger.Debugf. - if logger.IsEnabledFor(logging.DEBUG) { - logger.Debugf("UDP(%s): "+template, tag, val) + // between Go's inlining/escape analysis and varargs functions like slog.Debug. + if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + args = append(args, slog.String("ID", cipherID)) + slog.Debug(fmt.Sprintf("UDP: %s", template), args...) } } -func debugUDPAddr(addr net.Addr, template string, val interface{}) { - if logger.IsEnabledFor(logging.DEBUG) { - // Avoid calling addr.String() unless debugging is enabled. - debugUDP(addr.String(), template, val) +func debugUDPAddr(template string, addr net.Addr, args ...any) { + if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + args = append(args, slog.String("address", addr.String())) + slog.Debug(fmt.Sprintf("UDP: %s", template), args...) } } @@ -73,10 +75,10 @@ func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherLis id, cryptoKey := entry.Value.(*CipherEntry).ID, entry.Value.(*CipherEntry).CryptoKey buf, err := shadowsocks.Unpack(dst, src, cryptoKey) if err != nil { - debugUDP(id, "Failed to unpack: %v", err) + debugUDP("Failed to unpack.", id, slog.Any("err", err)) continue } - debugUDP(id, "Found cipher at index %d", ci) + debugUDP("Found cipher.", id, slog.Int("index", ci)) // Move the active cipher to the front, so that the search is quicker next time. cipherList.MarkUsedByClientIP(entry, clientIP) return buf, id, cryptoKey, nil @@ -131,7 +133,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { connError := func() (connError *onet.ConnectionError) { defer func() { if r := recover(); r != nil { - logger.Errorf("Panic in UDP loop: %v. Continuing to listen.", r) + slog.Error("Panic in UDP loop: %v. Continuing to listen.", r) debug.PrintStack() } }() @@ -140,10 +142,8 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { if err != nil { return onet.NewConnectionError("ERR_READ", "Failed to read from client", err) } - if logger.IsEnabledFor(logging.DEBUG) { - defer logger.Debugf("UDP(%v): done", clientAddr) - logger.Debugf("UDP(%v): Outbound packet has %d bytes", clientAddr, clientProxyBytes) - } + defer slog.Debug("UDP: Done.", "address", clientAddr) + slog.Debug("UDP: Outbound packet.", "address", clientAddr, "bytes", clientProxyBytes) cipherData := cipherBuf[:clientProxyBytes] var payload []byte @@ -153,9 +153,9 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { var locErr error clientInfo, locErr = ipinfo.GetIPInfoFromAddr(h.m, clientAddr) if locErr != nil { - logger.Warningf("Failed client info lookup: %v", locErr) + slog.Warn("Failed client info lookup.", "err", locErr) } - debugUDPAddr(clientAddr, "Got info \"%#v\"", clientInfo) + debugUDPAddr("Got info for IP.", clientAddr, slog.Any("info", clientInfo)) ip := clientAddr.(*net.UDPAddr).AddrPort().Addr() var textData []byte @@ -200,7 +200,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { } } - debugUDPAddr(clientAddr, "Proxy exit %v", targetConn.LocalAddr()) + debugUDPAddr("Proxy exit.", clientAddr, slog.Any("target", targetConn.LocalAddr())) proxyTargetBytes, err = targetConn.WriteTo(payload, tgtUDPAddr) // accept only UDPAddr despite the signature if err != nil { return onet.NewConnectionError("ERR_WRITE", "Failed to write to target", err) @@ -210,7 +210,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { status := "OK" if connError != nil { - logger.Debugf("UDP Error: %v: %v", connError.Message, connError.Cause) + slog.Debug("UDP: Error.", "msg", connError.Message, "cause", connError.Cause) status = connError.Status } h.m.AddUDPPacketFromClient(clientInfo, keyID, status, clientProxyBytes, proxyTargetBytes) @@ -424,7 +424,7 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco return onet.NewConnectionError("ERR_READ", "Failed to read from target", err) } - debugUDPAddr(clientAddr, "Got response from %v", raddr) + debugUDPAddr("Got response.", clientAddr, slog.Any("target", raddr)) srcAddr := socks.ParseAddr(raddr.String()) addrStart := bodyStart - len(srcAddr) // `plainTextBuf` concatenates the SOCKS address and body: @@ -453,7 +453,7 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco }() status := "OK" if connError != nil { - logger.Debugf("UDP Error: %v: %v", connError.Message, connError.Cause) + slog.Debug("UDP: Error.", "msg", connError.Message, "cause", connError.Cause) status = connError.Status } if expired { From 38602b5ea434a2f8159635bcd9d167cdbeafe13d Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 16 Aug 2024 14:05:09 -0400 Subject: [PATCH 117/182] Structure forgotten log. --- service/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/tcp.go b/service/tcp.go index e686b9c5..81191b0e 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -297,7 +297,7 @@ func proxyConnection(ctx context.Context, dialer transport.StreamDialer, tgtAddr return ensureConnectionError(dialErr, "ERR_CONNECT", "Failed to connect to target") } defer tgtConn.Close() - slog.Debug("proxy %s <-> %s", clientConn.RemoteAddr().String(), tgtConn.RemoteAddr().String()) + slog.Debug("Proxy connection.", "client", clientConn.RemoteAddr().String(), "target", tgtConn.RemoteAddr().String()) fromClientErrCh := make(chan error) go func() { From 56c7b11c23634fbc07f8184a541d7992133d4e7e Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 16 Aug 2024 15:26:10 -0400 Subject: [PATCH 118/182] Another forgotten log. --- service/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/tcp.go b/service/tcp.go index 81191b0e..c3656aa7 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -374,7 +374,7 @@ func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, addr, status string, // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) - slog.Debug("Drain error: %v, drain result: %v", drainErr, drainResult) + slog.Debug("Drain error.", "err", drainErr, "result", drainResult) h.m.AddTCPProbe(status, drainResult, addr, proxyMetrics.ClientProxy) } From 27e28c7c99c75200e7b0c709db2d261a34b15863 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 16 Aug 2024 14:26:03 -0400 Subject: [PATCH 119/182] Remove IPInfo logic from TCP and UDP handling into the metrics collector. --- cmd/outline-ss-server/main.go | 4 +- cmd/outline-ss-server/metrics.go | 69 ++++++++++++++----- cmd/outline-ss-server/metrics_test.go | 38 +++++----- internal/integration_test/integration_test.go | 66 ++++++++---------- service/tcp.go | 43 +++++------- service/tcp_test.go | 37 +++++----- service/udp.go | 66 +++++++----------- service/udp_test.go | 34 ++++----- 8 files changed, 171 insertions(+), 186 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 48cd0bdc..f45c2339 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -63,7 +63,7 @@ type ssPort struct { type SSServer struct { natTimeout time.Duration - m *outlineMetrics + m *outlineMetricsCollector replayCache service.ReplayCache ports map[int]*ssPort } @@ -168,7 +168,7 @@ func (s *SSServer) Stop() error { } // RunSSServer starts a shadowsocks server running, and returns the server or an error. -func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { +func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetricsCollector, replayHistory int) (*SSServer, error) { server := &SSServer{ natTimeout: natTimeout, m: sm, diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index dfdeb1f2..8ddd502e 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -15,6 +15,7 @@ package main import ( + "context" "fmt" "log/slog" "net" @@ -33,8 +34,11 @@ const namespace = "shadowsocks" // `now` is stubbable for testing. var now = time.Now -type outlineMetrics struct { - ipinfo.IPInfoMap +type outlineMetricsCollector struct { + ip2info ipinfo.IPInfoMap + mu sync.Mutex // Protects the clientInfo map. + ipInfos map[net.Addr]ipinfo.IPInfo + *tunnelTimeCollector buildInfo *prometheus.GaugeVec @@ -55,8 +59,8 @@ type outlineMetrics struct { udpRemovedNatEntries prometheus.Counter } -var _ service.TCPMetrics = (*outlineMetrics)(nil) -var _ service.UDPMetrics = (*outlineMetrics)(nil) +var _ service.TCPMetricsCollector = (*outlineMetricsCollector)(nil) +var _ service.UDPMetricsCollector = (*outlineMetricsCollector)(nil) // Converts a [net.Addr] to an [IPKey]. func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { @@ -94,6 +98,8 @@ type tunnelTimeCollector struct { tunnelTimePerLocation *prometheus.CounterVec } +var _ prometheus.Collector = (*tunnelTimeCollector)(nil) + func (c *tunnelTimeCollector) Describe(ch chan<- *prometheus.Desc) { c.tunnelTimePerKey.Describe(ch) c.tunnelTimePerLocation.Describe(ch) @@ -171,9 +177,11 @@ func newTunnelTimeCollector(ip2info ipinfo.IPInfoMap, registerer prometheus.Regi // `ip2info` to convert IP addresses to countries, and reports all // metrics to Prometheus via `registerer`. `ip2info` may be nil, but // `registerer` must not be. -func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus.Registerer) *outlineMetrics { - m := &outlineMetrics{ - IPInfoMap: ip2info, +func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus.Registerer) *outlineMetricsCollector { + m := &outlineMetricsCollector{ + ip2info: ip2info, + ipInfos: make(map[net.Addr]ipinfo.IPInfo), + buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Name: "build_info", @@ -272,20 +280,40 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus return m } -func (m *outlineMetrics) SetBuildInfo(version string) { +func (m *outlineMetricsCollector) getIPInfoFromAddr(addr net.Addr) ipinfo.IPInfo { + m.mu.Lock() + defer m.mu.Unlock() + + ipInfo, exists := m.ipInfos[addr] + if !exists { + ipInfo, err := ipinfo.GetIPInfoFromAddr(m.ip2info, addr) + if err != nil { + slog.Warn("Failed client info lookup.", "err", err) + return ipInfo + } + m.ipInfos[addr] = ipInfo + } + if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + slog.Debug("Got IP info for address.", "address", addr, "info", ipInfo) + } + return ipInfo +} + +func (m *outlineMetricsCollector) SetBuildInfo(version string) { m.buildInfo.WithLabelValues(version).Set(1) } -func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) { +func (m *outlineMetricsCollector) SetNumAccessKeys(numKeys int, ports int) { m.accessKeys.Set(float64(numKeys)) m.ports.Set(float64(ports)) } -func (m *outlineMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { +func (m *outlineMetricsCollector) AddOpenTCPConnection(clientAddr net.Addr) { + clientInfo := m.getIPInfoFromAddr(clientAddr) m.tcpOpenConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)).Inc() } -func (m *outlineMetrics) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { +func (m *outlineMetricsCollector) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { ipKey, err := toIPKey(clientAddr, accessKey) if err == nil { m.tunnelTimeCollector.startConnection(*ipKey) @@ -306,7 +334,8 @@ func asnLabel(asn int) string { return fmt.Sprint(asn) } -func (m *outlineMetrics) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, clientAddr net.Addr, accessKey, status string, data metrics.ProxyMetrics, duration time.Duration) { +func (m *outlineMetricsCollector) AddClosedTCPConnection(clientAddr net.Addr, accessKey, status string, data metrics.ProxyMetrics, duration time.Duration) { + clientInfo := m.getIPInfoFromAddr(clientAddr) m.tcpClosedConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status, accessKey).Inc() m.tcpConnectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000) addIfNonZero(data.ClientProxy, m.dataBytes, "c>p", "tcp", accessKey) @@ -324,7 +353,8 @@ func (m *outlineMetrics) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, client } } -func (m *outlineMetrics) AddUDPPacketFromClient(clientInfo ipinfo.IPInfo, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { +func (m *outlineMetricsCollector) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { + clientInfo := m.getIPInfoFromAddr(clientAddr) m.udpPacketsFromClientPerLocation.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status).Inc() addIfNonZero(int64(clientProxyBytes), m.dataBytes, "c>p", "udp", accessKey) addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) @@ -332,14 +362,15 @@ func (m *outlineMetrics) AddUDPPacketFromClient(clientInfo ipinfo.IPInfo, access addIfNonZero(int64(proxyTargetBytes), m.dataBytesPerLocation, "p>t", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) } -func (m *outlineMetrics) AddUDPPacketFromTarget(clientInfo ipinfo.IPInfo, accessKey, status string, targetProxyBytes, proxyClientBytes int) { +func (m *outlineMetricsCollector) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { + clientInfo := m.getIPInfoFromAddr(clientAddr) addIfNonZero(int64(targetProxyBytes), m.dataBytes, "p Date: Fri, 16 Aug 2024 17:24:51 -0400 Subject: [PATCH 120/182] Refactor metrics into separate collectors. --- cmd/outline-ss-server/main.go | 8 +- cmd/outline-ss-server/metrics.go | 348 ++++++++++++++++---------- cmd/outline-ss-server/metrics_test.go | 18 +- 3 files changed, 231 insertions(+), 143 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index f45c2339..ecf7f1ac 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -273,9 +273,11 @@ func main() { } defer ip2info.Close() - m := newPrometheusOutlineMetrics(ip2info, prometheus.DefaultRegisterer) - m.SetBuildInfo(version) - _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, m, flags.replayHistory) + metrics := newPrometheusOutlineMetrics(ip2info) + r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) + r.MustRegister(metrics) + metrics.SetBuildInfo(version) + _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, metrics, flags.replayHistory) if err != nil { slog.Error("Server failed to start. Aborting.", "err", err) } diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 8ddd502e..f4dd2003 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -29,38 +29,136 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const namespace = "shadowsocks" - // `now` is stubbable for testing. var now = time.Now -type outlineMetricsCollector struct { - ip2info ipinfo.IPInfoMap - mu sync.Mutex // Protects the clientInfo map. - ipInfos map[net.Addr]ipinfo.IPInfo +type tcpCollector struct { + probes *prometheus.HistogramVec + openConnections *prometheus.CounterVec + closedConnections *prometheus.CounterVec + connectionDurationMs *prometheus.HistogramVec +} - *tunnelTimeCollector +var _ prometheus.Collector = (*tcpCollector)(nil) - buildInfo *prometheus.GaugeVec - accessKeys prometheus.Gauge - ports prometheus.Gauge - dataBytes *prometheus.CounterVec - dataBytesPerLocation *prometheus.CounterVec - timeToCipherMs *prometheus.HistogramVec - // TODO: Add time to first byte. +func newTcpCollector() *tcpCollector { + namespace := "tcp" + return &tcpCollector{ + probes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "probes", + Buckets: []float64{0, 49, 50, 51, 73, 91}, + Help: "Histogram of number of bytes from client to proxy, for detecting possible probes", + }, []string{"port", "status", "error"}), + openConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "connections_opened", + Help: "Count of open TCP connections", + }, []string{"location", "asn"}), + closedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "connections_closed", + Help: "Count of closed TCP connections", + }, []string{"location", "asn", "status", "access_key"}), + connectionDurationMs: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "connection_duration_ms", + Help: "TCP connection duration distributions.", + Buckets: []float64{ + 100, + float64(time.Second.Milliseconds()), + float64(time.Minute.Milliseconds()), + float64(time.Hour.Milliseconds()), + float64(24 * time.Hour.Milliseconds()), // Day + float64(7 * 24 * time.Hour.Milliseconds()), // Week + }, + }, []string{"status"}), + } +} - tcpProbes *prometheus.HistogramVec - tcpOpenConnections *prometheus.CounterVec - tcpClosedConnections *prometheus.CounterVec - tcpConnectionDurationMs *prometheus.HistogramVec +func (c *tcpCollector) Describe(ch chan<- *prometheus.Desc) { + c.probes.Describe(ch) + c.openConnections.Describe(ch) + c.closedConnections.Describe(ch) + c.connectionDurationMs.Describe(ch) +} - udpPacketsFromClientPerLocation *prometheus.CounterVec - udpAddedNatEntries prometheus.Counter - udpRemovedNatEntries prometheus.Counter +func (c *tcpCollector) Collect(ch chan<- prometheus.Metric) { + c.probes.Collect(ch) + c.openConnections.Collect(ch) + c.closedConnections.Collect(ch) + c.connectionDurationMs.Collect(ch) } -var _ service.TCPMetricsCollector = (*outlineMetricsCollector)(nil) -var _ service.UDPMetricsCollector = (*outlineMetricsCollector)(nil) +func (c *tcpCollector) openConnection(clientInfo ipinfo.IPInfo) { + c.openConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)).Inc() +} + +func (c *tcpCollector) closeConnection(clientInfo ipinfo.IPInfo, status, accessKey string, duration time.Duration) { + c.closedConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status, accessKey).Inc() + c.connectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000) +} + +func (c *tcpCollector) addProbe(listenerId, status, drainResult string, clientProxyBytes int64) { + c.probes.WithLabelValues(listenerId, status, drainResult).Observe(float64(clientProxyBytes)) +} + +type udpCollector struct { + packetsFromClientPerLocation *prometheus.CounterVec + addedNatEntries prometheus.Counter + removedNatEntries prometheus.Counter +} + +var _ prometheus.Collector = (*udpCollector)(nil) + +func newUdpCollector() *udpCollector { + namespace := "udp" + return &udpCollector{ + packetsFromClientPerLocation: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "packets_from_client_per_location", + Help: "Packets received from the client, per location and status", + }, []string{"location", "asn", "status"}), + addedNatEntries: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "nat_entries_added", + Help: "Entries added to the UDP NAT table", + }), + removedNatEntries: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "nat_entries_removed", + Help: "Entries removed from the UDP NAT table", + }), + } +} + +func (c *udpCollector) Describe(ch chan<- *prometheus.Desc) { + c.packetsFromClientPerLocation.Describe(ch) + c.addedNatEntries.Describe(ch) + c.removedNatEntries.Describe(ch) +} + +func (c *udpCollector) Collect(ch chan<- prometheus.Metric) { + c.packetsFromClientPerLocation.Collect(ch) + c.addedNatEntries.Collect(ch) + c.removedNatEntries.Collect(ch) +} + +func (c *udpCollector) addPacketFromClient(clientInfo ipinfo.IPInfo, status string) { + c.packetsFromClientPerLocation.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status).Inc() +} + +func (c *udpCollector) addNatEntry() { + c.addedNatEntries.Inc() +} + +func (c *udpCollector) removeNatEntry() { + c.removedNatEntries.Inc() +} // Converts a [net.Addr] to an [IPKey]. func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { @@ -100,6 +198,25 @@ type tunnelTimeCollector struct { var _ prometheus.Collector = (*tunnelTimeCollector)(nil) +func newTunnelTimeCollector(ip2info ipinfo.IPInfoMap) *tunnelTimeCollector { + namespace := "tunnel_time" + return &tunnelTimeCollector{ + ip2info: ip2info, + activeClients: make(map[IPKey]*activeClient), + + tunnelTimePerKey: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "seconds", + Help: "Tunnel time, per access key.", + }, []string{"access_key"}), + tunnelTimePerLocation: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "seconds_per_location", + Help: "Tunnel time, per location.", + }, []string{"location", "asn"}), + } +} + func (c *tunnelTimeCollector) Describe(ch chan<- *prometheus.Desc) { c.tunnelTimePerKey.Describe(ch) c.tunnelTimePerLocation.Describe(ch) @@ -155,129 +272,99 @@ func (c *tunnelTimeCollector) stopConnection(ipKey IPKey) { } } -func newTunnelTimeCollector(ip2info ipinfo.IPInfoMap, registerer prometheus.Registerer) *tunnelTimeCollector { - return &tunnelTimeCollector{ - ip2info: ip2info, - activeClients: make(map[IPKey]*activeClient), +type outlineMetricsCollector struct { + ip2info ipinfo.IPInfoMap + mu sync.Mutex // Protects the ipInfos map. + ipInfos map[net.Addr]ipinfo.IPInfo - tunnelTimePerKey: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "tunnel_time_seconds", - Help: "Tunnel time, per access key.", - }, []string{"access_key"}), - tunnelTimePerLocation: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "tunnel_time_seconds_per_location", - Help: "Tunnel time, per location.", - }, []string{"location", "asn"}), - } + *tcpCollector + *udpCollector + *tunnelTimeCollector + + buildInfo *prometheus.GaugeVec + accessKeys prometheus.Gauge + ports prometheus.Gauge + dataBytes *prometheus.CounterVec + dataBytesPerLocation *prometheus.CounterVec + timeToCipherMs *prometheus.HistogramVec + // TODO: Add time to first byte. } -// newPrometheusOutlineMetrics constructs a metrics object that uses -// `ip2info` to convert IP addresses to countries, and reports all -// metrics to Prometheus via `registerer`. `ip2info` may be nil, but -// `registerer` must not be. -func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus.Registerer) *outlineMetricsCollector { - m := &outlineMetricsCollector{ +var _ prometheus.Collector = (*outlineMetricsCollector)(nil) +var _ service.TCPMetricsCollector = (*outlineMetricsCollector)(nil) +var _ service.UDPMetricsCollector = (*outlineMetricsCollector)(nil) + +// newPrometheusOutlineMetrics constructs a Prometheus metrics collector that uses +// `ip2info` to convert IP addresses to countries. `ip2info` may be nil. +func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) *outlineMetricsCollector { + tcpCollector := newTcpCollector() + udpCollector := newUdpCollector() + tunnelTimeCollector := newTunnelTimeCollector(ip2info) + + return &outlineMetricsCollector{ ip2info: ip2info, ipInfos: make(map[net.Addr]ipinfo.IPInfo), + tcpCollector: tcpCollector, + udpCollector: udpCollector, + tunnelTimeCollector: tunnelTimeCollector, + buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "build_info", - Help: "Information on the outline-ss-server build", + Name: "build_info", + Help: "Information on the outline-ss-server build", }, []string{"version"}), accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "keys", - Help: "Count of access keys", + Name: "keys", + Help: "Count of access keys", }), ports: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "ports", - Help: "Count of open Shadowsocks ports", + Name: "ports", + Help: "Count of open Shadowsocks ports", }), - tcpProbes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Name: "tcp_probes", - Buckets: []float64{0, 49, 50, 51, 73, 91}, - Help: "Histogram of number of bytes from client to proxy, for detecting possible probes", - }, []string{"port", "status", "error"}), - tcpOpenConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "tcp", - Name: "connections_opened", - Help: "Count of open TCP connections", - }, []string{"location", "asn"}), - tcpClosedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "tcp", - Name: "connections_closed", - Help: "Count of closed TCP connections", - }, []string{"location", "asn", "status", "access_key"}), - tcpConnectionDurationMs: prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: "tcp", - Name: "connection_duration_ms", - Help: "TCP connection duration distributions.", - Buckets: []float64{ - 100, - float64(time.Second.Milliseconds()), - float64(time.Minute.Milliseconds()), - float64(time.Hour.Milliseconds()), - float64(24 * time.Hour.Milliseconds()), // Day - float64(7 * 24 * time.Hour.Milliseconds()), // Week - }, - }, []string{"status"}), dataBytes: prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: namespace, - Name: "data_bytes", - Help: "Bytes transferred by the proxy, per access key", + Name: "data_bytes", + Help: "Bytes transferred by the proxy, per access key", }, []string{"dir", "proto", "access_key"}), dataBytesPerLocation: prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: namespace, - Name: "data_bytes_per_location", - Help: "Bytes transferred by the proxy, per location", + Name: "data_bytes_per_location", + Help: "Bytes transferred by the proxy, per location", }, []string{"dir", "proto", "location", "asn"}), timeToCipherMs: prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: namespace, - Name: "time_to_cipher_ms", - Help: "Time needed to find the cipher", - Buckets: []float64{0.1, 1, 10, 100, 1000}, + Name: "time_to_cipher_ms", + Help: "Time needed to find the cipher", + Buckets: []float64{0.1, 1, 10, 100, 1000}, }, []string{"proto", "found_key"}), - udpPacketsFromClientPerLocation: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "udp", - Name: "packets_from_client_per_location", - Help: "Packets received from the client, per location and status", - }, []string{"location", "asn", "status"}), - udpAddedNatEntries: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "udp", - Name: "nat_entries_added", - Help: "Entries added to the UDP NAT table", - }), - udpRemovedNatEntries: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "udp", - Name: "nat_entries_removed", - Help: "Entries removed from the UDP NAT table", - }), } - m.tunnelTimeCollector = newTunnelTimeCollector(ip2info, registerer) +} - // TODO: Is it possible to pass where to register the collectors? - registerer.MustRegister(m.buildInfo, m.accessKeys, m.ports, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, - m.dataBytes, m.dataBytesPerLocation, m.timeToCipherMs, m.udpPacketsFromClientPerLocation, m.udpAddedNatEntries, m.udpRemovedNatEntries, - m.tunnelTimeCollector) - return m +func (m *outlineMetricsCollector) collectors() []prometheus.Collector { + return []prometheus.Collector{ + m.tcpCollector, + m.udpCollector, + m.tunnelTimeCollector, + + m.buildInfo, + m.accessKeys, + m.ports, + m.dataBytes, + m.dataBytesPerLocation, + m.timeToCipherMs, + } +} + +func (m *outlineMetricsCollector) Describe(ch chan<- *prometheus.Desc) { + for _, collector := range m.collectors() { + collector.Describe(ch) + } +} + +func (m *outlineMetricsCollector) Collect(ch chan<- prometheus.Metric) { + for _, collector := range m.collectors() { + collector.Collect(ch) + } } func (m *outlineMetricsCollector) getIPInfoFromAddr(addr net.Addr) ipinfo.IPInfo { @@ -310,7 +397,7 @@ func (m *outlineMetricsCollector) SetNumAccessKeys(numKeys int, ports int) { func (m *outlineMetricsCollector) AddOpenTCPConnection(clientAddr net.Addr) { clientInfo := m.getIPInfoFromAddr(clientAddr) - m.tcpOpenConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)).Inc() + m.tcpCollector.openConnection(clientInfo) } func (m *outlineMetricsCollector) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { @@ -336,8 +423,7 @@ func asnLabel(asn int) string { func (m *outlineMetricsCollector) AddClosedTCPConnection(clientAddr net.Addr, accessKey, status string, data metrics.ProxyMetrics, duration time.Duration) { clientInfo := m.getIPInfoFromAddr(clientAddr) - m.tcpClosedConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status, accessKey).Inc() - m.tcpConnectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000) + m.tcpCollector.closeConnection(clientInfo, status, accessKey, duration) addIfNonZero(data.ClientProxy, m.dataBytes, "c>p", "tcp", accessKey) addIfNonZero(data.ClientProxy, m.dataBytesPerLocation, "c>p", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) addIfNonZero(data.ProxyTarget, m.dataBytes, "p>t", "tcp", accessKey) @@ -355,7 +441,7 @@ func (m *outlineMetricsCollector) AddClosedTCPConnection(clientAddr net.Addr, ac func (m *outlineMetricsCollector) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { clientInfo := m.getIPInfoFromAddr(clientAddr) - m.udpPacketsFromClientPerLocation.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status).Inc() + m.udpCollector.addPacketFromClient(clientInfo, status) addIfNonZero(int64(clientProxyBytes), m.dataBytes, "c>p", "udp", accessKey) addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) addIfNonZero(int64(proxyTargetBytes), m.dataBytes, "p>t", "udp", accessKey) @@ -371,7 +457,7 @@ func (m *outlineMetricsCollector) AddUDPPacketFromTarget(clientAddr net.Addr, ac } func (m *outlineMetricsCollector) AddUDPNatEntry(clientAddr net.Addr, accessKey string) { - m.udpAddedNatEntries.Inc() + m.udpCollector.addNatEntry() ipKey, err := toIPKey(clientAddr, accessKey) if err == nil { @@ -380,7 +466,7 @@ func (m *outlineMetricsCollector) AddUDPNatEntry(clientAddr net.Addr, accessKey } func (m *outlineMetricsCollector) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string) { - m.udpRemovedNatEntries.Inc() + m.udpCollector.removeNatEntry() ipKey, err := toIPKey(clientAddr, accessKey) if err == nil { @@ -388,8 +474,8 @@ func (m *outlineMetricsCollector) RemoveUDPNatEntry(clientAddr net.Addr, accessK } } -func (m *outlineMetricsCollector) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { - m.tcpProbes.WithLabelValues(listenerId, status, drainResult).Observe(float64(clientProxyBytes)) +func (m *outlineMetricsCollector) AddTCPProbe(status, drainResult, listenerId string, clientProxyBytes int64) { + m.tcpCollector.addProbe(listenerId, status, drainResult, clientProxyBytes) } func (m *outlineMetricsCollector) AddTCPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { diff --git a/cmd/outline-ss-server/metrics_test.go b/cmd/outline-ss-server/metrics_test.go index 67f98ba3..7ec58429 100644 --- a/cmd/outline-ss-server/metrics_test.go +++ b/cmd/outline-ss-server/metrics_test.go @@ -87,14 +87,14 @@ func TestTunnelTimePerKey(t *testing.T) { setNow(time.Date(2010, 1, 2, 3, 4, 20, .0, time.Local)) expected := strings.NewReader(` - # HELP shadowsocks_tunnel_time_seconds Tunnel time, per access key. - # TYPE shadowsocks_tunnel_time_seconds counter - shadowsocks_tunnel_time_seconds{access_key="key-1"} 15 + # HELP tunnel_time_seconds Tunnel time, per access key. + # TYPE tunnel_time_seconds counter + tunnel_time_seconds{access_key="key-1"} 15 `) err := promtest.GatherAndCompare( reg, expected, - "shadowsocks_tunnel_time_seconds", + "tunnel_time_seconds", ) require.NoError(t, err, "unexpected metric value found") } @@ -108,14 +108,14 @@ func TestTunnelTimePerLocation(t *testing.T) { setNow(time.Date(2010, 1, 2, 3, 4, 10, .0, time.Local)) expected := strings.NewReader(` - # HELP shadowsocks_tunnel_time_seconds_per_location Tunnel time, per location. - # TYPE shadowsocks_tunnel_time_seconds_per_location counter - shadowsocks_tunnel_time_seconds_per_location{asn="",location="XL"} 5 + # HELP tunnel_time_seconds_per_location Tunnel time, per location. + # TYPE tunnel_time_seconds_per_location counter + tunnel_time_seconds_per_location{asn="",location="XL"} 5 `) err := promtest.GatherAndCompare( reg, expected, - "shadowsocks_tunnel_time_seconds_per_location", + "tunnel_time_seconds_per_location", ) require.NoError(t, err, "unexpected metric value found") } @@ -129,7 +129,7 @@ func TestTunnelTimePerKeyDoesNotPanicOnUnknownClosedConnection(t *testing.T) { err := promtest.GatherAndCompare( reg, strings.NewReader(""), - "shadowsocks_tunnel_time_seconds", + "tunnel_time_seconds", ) require.NoError(t, err, "unexpectedly found metric value") } From e6334e169c730cd1ac728ea1ae864460ad9f2a4f Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 16 Aug 2024 18:16:55 -0400 Subject: [PATCH 121/182] Rename some types to remove `Collector` suffix. --- cmd/outline-ss-server/metrics.go | 4 ++-- service/tcp.go | 24 ++++++++++++------------ service/tcp_test.go | 32 ++++++++++++++++---------------- service/udp.go | 30 +++++++++++++++--------------- service/udp_test.go | 24 ++++++++++++------------ 5 files changed, 57 insertions(+), 57 deletions(-) diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index f4dd2003..28148f86 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -291,8 +291,8 @@ type outlineMetricsCollector struct { } var _ prometheus.Collector = (*outlineMetricsCollector)(nil) -var _ service.TCPMetricsCollector = (*outlineMetricsCollector)(nil) -var _ service.UDPMetricsCollector = (*outlineMetricsCollector)(nil) +var _ service.TCPMetrics = (*outlineMetricsCollector)(nil) +var _ service.UDPMetrics = (*outlineMetricsCollector)(nil) // newPrometheusOutlineMetrics constructs a Prometheus metrics collector that uses // `ip2info` to convert IP addresses to countries. `ip2info` may be nil. diff --git a/service/tcp.go b/service/tcp.go index 0766f780..df15397c 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -35,8 +35,8 @@ import ( "github.com/shadowsocks/go-shadowsocks2/socks" ) -// TCPMetricsCollector is used to report metrics on TCP connections. -type TCPMetricsCollector interface { +// TCPMetrics is used to report metrics on TCP connections. +type TCPMetrics interface { AddOpenTCPConnection(clientAddr net.Addr) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) AddClosedTCPConnection(clientAddr net.Addr, accessKey string, status string, data metrics.ProxyMetrics, duration time.Duration) @@ -160,14 +160,14 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa type tcpHandler struct { listenerId string - m TCPMetricsCollector + m TCPMetrics readTimeout time.Duration authenticate StreamAuthenticateFunc dialer transport.StreamDialer } // NewTCPService creates a TCPService -func NewTCPHandler(authenticate StreamAuthenticateFunc, m TCPMetricsCollector, timeout time.Duration) TCPHandler { +func NewTCPHandler(authenticate StreamAuthenticateFunc, m TCPMetrics, timeout time.Duration) TCPHandler { return &tcpHandler{ m: m, readTimeout: timeout, @@ -381,18 +381,18 @@ func drainErrToString(drainErr error) string { } } -// NoOpTCPMetricsCollector is a [TCPMetricsCollector] that doesn't do anything. Useful in tests +// NoOpTCPMetrics is a [TCPMetrics] that doesn't do anything. Useful in tests // or if you don't want to track metrics. -type NoOpTCPMetricsCollector struct{} +type NoOpTCPMetrics struct{} -var _ TCPMetricsCollector = (*NoOpTCPMetricsCollector)(nil) +var _ TCPMetrics = (*NoOpTCPMetrics)(nil) -func (m *NoOpTCPMetricsCollector) AddClosedTCPConnection(clientAddr net.Addr, accessKey string, status string, data metrics.ProxyMetrics, duration time.Duration) { +func (m *NoOpTCPMetrics) AddClosedTCPConnection(clientAddr net.Addr, accessKey string, status string, data metrics.ProxyMetrics, duration time.Duration) { } -func (m *NoOpTCPMetricsCollector) AddOpenTCPConnection(clientAddr net.Addr) {} -func (m *NoOpTCPMetricsCollector) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { +func (m *NoOpTCPMetrics) AddOpenTCPConnection(clientAddr net.Addr) {} +func (m *NoOpTCPMetrics) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { } -func (m *NoOpTCPMetricsCollector) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { +func (m *NoOpTCPMetrics) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { } -func (m *NoOpTCPMetricsCollector) AddTCPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { +func (m *NoOpTCPMetrics) AddTCPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { } diff --git a/service/tcp_test.go b/service/tcp_test.go index 0171e53b..e93149dd 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -214,38 +214,38 @@ func BenchmarkTCPFindCipherRepeat(b *testing.B) { } // Stub metrics implementation for testing replay defense. -type probeTestMetricsCollector struct { +type probeTestMetrics struct { mu sync.Mutex probeData []int64 probeStatus []string closeStatus []string } -var _ TCPMetricsCollector = (*probeTestMetricsCollector)(nil) +var _ TCPMetrics = (*probeTestMetrics)(nil) -func (m *probeTestMetricsCollector) AddClosedTCPConnection(clientAddr net.Addr, accessKey string, status string, data metrics.ProxyMetrics, duration time.Duration) { +func (m *probeTestMetrics) AddClosedTCPConnection(clientAddr net.Addr, accessKey string, status string, data metrics.ProxyMetrics, duration time.Duration) { m.mu.Lock() m.closeStatus = append(m.closeStatus, status) m.mu.Unlock() } -func (m *probeTestMetricsCollector) AddOpenTCPConnection(clientAddr net.Addr) { +func (m *probeTestMetrics) AddOpenTCPConnection(clientAddr net.Addr) { } -func (m *probeTestMetricsCollector) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { +func (m *probeTestMetrics) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { } -func (m *probeTestMetricsCollector) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { +func (m *probeTestMetrics) AddTCPProbe(status, drainResult string, listenerId string, clientProxyBytes int64) { m.mu.Lock() m.probeData = append(m.probeData, clientProxyBytes) m.probeStatus = append(m.probeStatus, status) m.mu.Unlock() } -func (m *probeTestMetricsCollector) AddTCPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { +func (m *probeTestMetrics) AddTCPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { } -func (m *probeTestMetricsCollector) countStatuses() map[string]int { +func (m *probeTestMetrics) countStatuses() map[string]int { counts := make(map[string]int) for _, status := range m.closeStatus { counts[status] = counts[status] + 1 @@ -276,7 +276,7 @@ func TestProbeRandom(t *testing.T) { listener := makeLocalhostListener(t) cipherList, err := MakeTestCiphers(makeTestSecrets(1)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) @@ -353,7 +353,7 @@ func TestProbeClientBytesBasicTruncated(t *testing.T) { cipherList, err := MakeTestCiphers(makeTestSecrets(1)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) cipher := firstCipher(cipherList) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) @@ -388,7 +388,7 @@ func TestProbeClientBytesBasicModified(t *testing.T) { cipherList, err := MakeTestCiphers(makeTestSecrets(1)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) cipher := firstCipher(cipherList) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) @@ -424,7 +424,7 @@ func TestProbeClientBytesCoalescedModified(t *testing.T) { cipherList, err := MakeTestCiphers(makeTestSecrets(1)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) cipher := firstCipher(cipherList) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) @@ -467,7 +467,7 @@ func TestProbeServerBytesModified(t *testing.T) { cipherList, err := MakeTestCiphers(makeTestSecrets(1)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) cipher := firstCipher(cipherList) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, 200*time.Millisecond) done := make(chan struct{}) @@ -497,7 +497,7 @@ func TestReplayDefense(t *testing.T) { cipherList, err := MakeTestCiphers(makeTestSecrets(1)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) replayCache := NewReplayCache(5) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, testTimeout) @@ -576,7 +576,7 @@ func TestReverseReplayDefense(t *testing.T) { cipherList, err := MakeTestCiphers(makeTestSecrets(1)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) replayCache := NewReplayCache(5) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, testTimeout) @@ -648,7 +648,7 @@ func probeExpectTimeout(t *testing.T, payloadSize int) { listener := makeLocalhostListener(t) cipherList, err := MakeTestCiphers(makeTestSecrets(5)) require.NoError(t, err, "MakeTestCiphers failed: %v", err) - testMetrics := &probeTestMetricsCollector{} + testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) handler := NewTCPHandler(authFunc, testMetrics, testTimeout) diff --git a/service/udp.go b/service/udp.go index 0c929a93..399525d5 100644 --- a/service/udp.go +++ b/service/udp.go @@ -30,8 +30,8 @@ import ( "github.com/shadowsocks/go-shadowsocks2/socks" ) -// UDPMetricsCollector is used to report metrics on UDP connections. -type UDPMetricsCollector interface { +// UDPMetrics is used to report metrics on UDP connections. +type UDPMetrics interface { AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) AddUDPNatEntry(clientAddr net.Addr, accessKey string) @@ -85,12 +85,12 @@ func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherLis type packetHandler struct { natTimeout time.Duration ciphers CipherList - m UDPMetricsCollector + m UDPMetrics targetIPValidator onet.TargetIPValidator } // NewPacketHandler creates a UDPService -func NewPacketHandler(natTimeout time.Duration, cipherList CipherList, m UDPMetricsCollector) PacketHandler { +func NewPacketHandler(natTimeout time.Duration, cipherList CipherList, m UDPMetrics) PacketHandler { return &packetHandler{natTimeout: natTimeout, ciphers: cipherList, m: m, targetIPValidator: onet.RequirePublicIP} } @@ -293,11 +293,11 @@ type natmap struct { sync.RWMutex keyConn map[string]*natconn timeout time.Duration - metrics UDPMetricsCollector + metrics UDPMetrics running *sync.WaitGroup } -func newNATmap(timeout time.Duration, sm UDPMetricsCollector, running *sync.WaitGroup) *natmap { +func newNATmap(timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { m := &natmap{metrics: sm, running: running} m.keyConn = make(map[string]*natconn) m.timeout = timeout @@ -373,7 +373,7 @@ var maxAddrLen int = len(socks.ParseAddr("[2001:db8::1]:12345")) // copy from target to client until read timeout func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, - keyID string, sm UDPMetricsCollector) { + keyID string, sm UDPMetrics) { // pkt is used for in-place encryption of downstream UDP packets, with the layout // [padding?][salt][address][body][tag][extra] // Padding is only used if the address is IPv4. @@ -445,19 +445,19 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco } } -// NoOpUDPMetricsCollector is a [UDPMetricsCollector] that doesn't do anything. Useful in tests +// NoOpUDPMetrics is a [UDPMetrics] that doesn't do anything. Useful in tests // or if you don't want to track metrics. -type NoOpUDPMetricsCollector struct{} +type NoOpUDPMetrics struct{} -var _ UDPMetricsCollector = (*NoOpUDPMetricsCollector)(nil) +var _ UDPMetrics = (*NoOpUDPMetrics)(nil) -func (m *NoOpUDPMetricsCollector) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { +func (m *NoOpUDPMetrics) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { } -func (m *NoOpUDPMetricsCollector) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { +func (m *NoOpUDPMetrics) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { } -func (m *NoOpUDPMetricsCollector) AddUDPNatEntry(clientAddr net.Addr, accessKey string) { +func (m *NoOpUDPMetrics) AddUDPNatEntry(clientAddr net.Addr, accessKey string) { } -func (m *NoOpUDPMetricsCollector) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string) { +func (m *NoOpUDPMetrics) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string) { } -func (m *NoOpUDPMetricsCollector) AddUDPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { +func (m *NoOpUDPMetrics) AddUDPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { } diff --git a/service/udp_test.go b/service/udp_test.go index 19f52025..ccaa4f98 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -97,33 +97,33 @@ type udpReport struct { } // Stub metrics implementation for testing NAT behaviors. -type natTestMetricsCollector struct { +type natTestMetrics struct { natEntriesAdded int upstreamPackets []udpReport } -var _ UDPMetricsCollector = (*natTestMetricsCollector)(nil) +var _ UDPMetrics = (*natTestMetrics)(nil) -func (m *natTestMetricsCollector) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { +func (m *natTestMetrics) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { m.upstreamPackets = append(m.upstreamPackets, udpReport{accessKey, status, clientProxyBytes, proxyTargetBytes}) } -func (m *natTestMetricsCollector) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { +func (m *natTestMetrics) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { } -func (m *natTestMetricsCollector) AddUDPNatEntry(clientAddr net.Addr, accessKey string) { +func (m *natTestMetrics) AddUDPNatEntry(clientAddr net.Addr, accessKey string) { m.natEntriesAdded++ } -func (m *natTestMetricsCollector) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string) { +func (m *natTestMetrics) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string) { } -func (m *natTestMetricsCollector) AddUDPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { +func (m *natTestMetrics) AddUDPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { } // Takes a validation policy, and returns the metrics it // generates when localhost access is attempted -func sendToDiscard(payloads [][]byte, validator onet.TargetIPValidator) *natTestMetricsCollector { +func sendToDiscard(payloads [][]byte, validator onet.TargetIPValidator) *natTestMetrics { ciphers, _ := MakeTestCiphers([]string{"asdf"}) cipher := ciphers.SnapshotForClientIP(netip.Addr{})[0].Value.(*CipherEntry).CryptoKey clientConn := makePacketConn() - metrics := &natTestMetricsCollector{} + metrics := &natTestMetrics{} handler := NewPacketHandler(timeout, ciphers, metrics) handler.SetTargetIPValidator(validator) done := make(chan struct{}) @@ -201,14 +201,14 @@ func assertAlmostEqual(t *testing.T, a, b time.Time) { } func TestNATEmpty(t *testing.T) { - nat := newNATmap(timeout, &natTestMetricsCollector{}, &sync.WaitGroup{}) + nat := newNATmap(timeout, &natTestMetrics{}, &sync.WaitGroup{}) if nat.Get("foo") != nil { t.Error("Expected nil value from empty NAT map") } } func setupNAT() (*fakePacketConn, *fakePacketConn, *natconn) { - nat := newNATmap(timeout, &natTestMetricsCollector{}, &sync.WaitGroup{}) + nat := newNATmap(timeout, &natTestMetrics{}, &sync.WaitGroup{}) clientConn := makePacketConn() targetConn := makePacketConn() nat.Add(&clientAddr, clientConn, natCryptoKey, targetConn, "key id") @@ -474,7 +474,7 @@ func TestUDPEarlyClose(t *testing.T) { if err != nil { t.Fatal(err) } - testMetrics := &natTestMetricsCollector{} + testMetrics := &natTestMetrics{} const testTimeout = 200 * time.Millisecond s := NewPacketHandler(testTimeout, cipherList, testMetrics) From 1565ab5c759aaf2d8669940bcd8124cc542a87df Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 15:05:50 -0400 Subject: [PATCH 122/182] Use an LRU cache to manage the ipInfos for Prometheus metrics. --- cmd/outline-ss-server/lru_cache.go | 126 ++++++++++++++++++++++++ cmd/outline-ss-server/lru_cache_test.go | 78 +++++++++++++++ cmd/outline-ss-server/metrics.go | 16 ++- cmd/outline-ss-server/metrics_test.go | 100 ++++++++++--------- cmd/outline-ss-server/server_test.go | 4 +- 5 files changed, 263 insertions(+), 61 deletions(-) create mode 100644 cmd/outline-ss-server/lru_cache.go create mode 100644 cmd/outline-ss-server/lru_cache_test.go diff --git a/cmd/outline-ss-server/lru_cache.go b/cmd/outline-ss-server/lru_cache.go new file mode 100644 index 00000000..4cc084db --- /dev/null +++ b/cmd/outline-ss-server/lru_cache.go @@ -0,0 +1,126 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "container/list" + "sync" + "time" +) + +type LRUCache[K comparable, V any] struct { + capacity int + duration time.Duration + items map[K]*list.Element + lru *list.List + mu sync.RWMutex + done chan struct{} +} + +type entry[K comparable, V any] struct { + key K + value V + lastAccess time.Time +} + +func NewLRUCache[K comparable, V any](capacity int, duration time.Duration, cleanupInterval time.Duration) *LRUCache[K, V] { + c := &LRUCache[K, V]{ + capacity: capacity, + duration: duration, + items: make(map[K]*list.Element), + lru: list.New(), + done: make(chan struct{}), + } + go c.cleanup(cleanupInterval) + return c +} + +func (c *LRUCache[K, V]) Get(key K) (V, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + if elem, ok := c.items[key]; ok { + c.lru.MoveToFront(elem) + ent := elem.Value.(*entry[K, V]) + if time.Since(ent.lastAccess) > c.duration { + c.lru.Remove(elem) + delete(c.items, key) + var zero V + return zero, false + } + ent.lastAccess = time.Now() + return ent.value, true + } + var zero V + return zero, false +} + +func (c *LRUCache[K, V]) Set(key K, value V) { + c.mu.Lock() + defer c.mu.Unlock() + + if elem, ok := c.items[key]; ok { + c.lru.MoveToFront(elem) + ent := elem.Value.(*entry[K, V]) + ent.value = value + ent.lastAccess = time.Now() + return + } + + ent := &entry[K, V]{key, value, time.Now()} + elem := c.lru.PushFront(ent) + c.items[key] = elem + + if c.lru.Len() > c.capacity { + c.removeOldest() + } +} + +func (c *LRUCache[K, V]) removeOldest() { + elem := c.lru.Back() + if elem != nil { + c.lru.Remove(elem) + ent := elem.Value.(*entry[K, V]) + delete(c.items, ent.key) + } +} + +func (c *LRUCache[K, V]) cleanup(interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + c.mu.Lock() + for elem := c.lru.Back(); elem != nil; elem = c.lru.Back() { + ent := elem.Value.(*entry[K, V]) + if time.Since(ent.lastAccess) > c.duration { + c.lru.Remove(elem) + delete(c.items, ent.key) + } else { + break + } + } + c.mu.Unlock() + case <-c.done: + return + } + } +} + +func (c *LRUCache[K, V]) StopCleanup() { + close(c.done) +} diff --git a/cmd/outline-ss-server/lru_cache_test.go b/cmd/outline-ss-server/lru_cache_test.go new file mode 100644 index 00000000..bec29acb --- /dev/null +++ b/cmd/outline-ss-server/lru_cache_test.go @@ -0,0 +1,78 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLRUCache(t *testing.T) { + t.Run("BasicSetGet", func(t *testing.T) { + cache := NewLRUCache[string, int](2, time.Minute, time.Minute) + + cache.Set("a", 1) + cache.Set("b", 2) + a, aOk := cache.Get("a") + b, bOk := cache.Get("b") + + require.True(t, aOk) + require.Equal(t, 1, a) + require.True(t, bOk) + require.Equal(t, 2, b) + cache.StopCleanup() + }) + + t.Run("LRUEviction", func(t *testing.T) { + cache := NewLRUCache[string, int](2, time.Minute, time.Minute) + + cache.Set("a", 1) + cache.Set("b", 2) + cache.Set("c", 3) + + _, ok := cache.Get("a") + require.False(t, ok, "Expected `a` to have been evicted") + v, ok := cache.Get("b") + require.True(t, ok, "Did not expect `b` to have been evicted") + require.Equal(t, 2, v) + v, ok = cache.Get("c") + require.True(t, ok, "Did not expect `c` to have been evicted") + require.Equal(t, 3, v) + cache.StopCleanup() + }) + + t.Run("Expiration", func(t *testing.T) { + cache := NewLRUCache[string, int](2, 500*time.Millisecond, time.Minute) + cache.Set("a", 1) + + time.Sleep(600 * time.Millisecond) + + _, ok := cache.Get("a") + require.False(t, ok, "Expected `a` to have been evicted") + cache.StopCleanup() + }) + + t.Run("Cleanup", func(t *testing.T) { + cache := NewLRUCache[string, int](2, 500*time.Millisecond, 200*time.Millisecond) + cache.Set("a", 1) + + time.Sleep(600 * time.Millisecond) + + require.Len(t, cache.items, 0, "Expected `a` to have been evicted in cleanup") + cache.StopCleanup() + }) +} diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 28148f86..090cf867 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -273,9 +273,8 @@ func (c *tunnelTimeCollector) stopConnection(ipKey IPKey) { } type outlineMetricsCollector struct { - ip2info ipinfo.IPInfoMap - mu sync.Mutex // Protects the ipInfos map. - ipInfos map[net.Addr]ipinfo.IPInfo + ip2info ipinfo.IPInfoMap + ipInfoCache *LRUCache[net.Addr, ipinfo.IPInfo] *tcpCollector *udpCollector @@ -302,8 +301,8 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) *outlineMetricsCollec tunnelTimeCollector := newTunnelTimeCollector(ip2info) return &outlineMetricsCollector{ - ip2info: ip2info, - ipInfos: make(map[net.Addr]ipinfo.IPInfo), + ip2info: ip2info, + ipInfoCache: NewLRUCache[net.Addr, ipinfo.IPInfo](10000, 60*time.Second, 30*time.Second), tcpCollector: tcpCollector, udpCollector: udpCollector, @@ -368,17 +367,14 @@ func (m *outlineMetricsCollector) Collect(ch chan<- prometheus.Metric) { } func (m *outlineMetricsCollector) getIPInfoFromAddr(addr net.Addr) ipinfo.IPInfo { - m.mu.Lock() - defer m.mu.Unlock() - - ipInfo, exists := m.ipInfos[addr] + ipInfo, exists := m.ipInfoCache.Get(addr) if !exists { ipInfo, err := ipinfo.GetIPInfoFromAddr(m.ip2info, addr) if err != nil { slog.Warn("Failed client info lookup.", "err", err) return ipInfo } - m.ipInfos[addr] = ipInfo + m.ipInfoCache.Set(addr, ipInfo) } if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { slog.Debug("Got IP info for address.", "address", addr, "info", ipInfo) diff --git a/cmd/outline-ss-server/metrics_test.go b/cmd/outline-ss-server/metrics_test.go index 7ec58429..37cd158f 100644 --- a/cmd/outline-ss-server/metrics_test.go +++ b/cmd/outline-ss-server/metrics_test.go @@ -51,7 +51,7 @@ func init() { } func TestMethodsDontPanic(t *testing.T) { - ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewPedanticRegistry()) + ssMetrics := newPrometheusOutlineMetrics(nil) proxyMetrics := metrics.ProxyMetrics{ ClientProxy: 1, ProxyTarget: 2, @@ -78,51 +78,55 @@ func TestASNLabel(t *testing.T) { require.Equal(t, "100", asnLabel(100)) } -func TestTunnelTimePerKey(t *testing.T) { - setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) - reg := prometheus.NewPedanticRegistry() - ssMetrics := newPrometheusOutlineMetrics(nil, reg) - - ssMetrics.AddAuthenticatedTCPConnection(fakeAddr("127.0.0.1:9"), "key-1") - setNow(time.Date(2010, 1, 2, 3, 4, 20, .0, time.Local)) - - expected := strings.NewReader(` - # HELP tunnel_time_seconds Tunnel time, per access key. - # TYPE tunnel_time_seconds counter - tunnel_time_seconds{access_key="key-1"} 15 -`) - err := promtest.GatherAndCompare( - reg, - expected, - "tunnel_time_seconds", - ) - require.NoError(t, err, "unexpected metric value found") -} - -func TestTunnelTimePerLocation(t *testing.T) { - setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) - reg := prometheus.NewPedanticRegistry() - ssMetrics := newPrometheusOutlineMetrics(&noopMap{}, reg) - - ssMetrics.AddAuthenticatedTCPConnection(fakeAddr("127.0.0.1:9"), "key-1") - setNow(time.Date(2010, 1, 2, 3, 4, 10, .0, time.Local)) - - expected := strings.NewReader(` - # HELP tunnel_time_seconds_per_location Tunnel time, per location. - # TYPE tunnel_time_seconds_per_location counter - tunnel_time_seconds_per_location{asn="",location="XL"} 5 -`) - err := promtest.GatherAndCompare( - reg, - expected, - "tunnel_time_seconds_per_location", - ) - require.NoError(t, err, "unexpected metric value found") +func TestTunnelTime(t *testing.T) { + t.Run("PerKey", func(t *testing.T) { + setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) + ssMetrics := newPrometheusOutlineMetrics(nil) + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(ssMetrics) + + ssMetrics.AddAuthenticatedTCPConnection(fakeAddr("127.0.0.1:9"), "key-1") + setNow(time.Date(2010, 1, 2, 3, 4, 20, .0, time.Local)) + + expected := strings.NewReader(` + # HELP tunnel_time_seconds Tunnel time, per access key. + # TYPE tunnel_time_seconds counter + tunnel_time_seconds{access_key="key-1"} 15 + `) + err := promtest.GatherAndCompare( + reg, + expected, + "tunnel_time_seconds", + ) + require.NoError(t, err, "unexpected metric value found") + }) + + t.Run("PerLocation", func(t *testing.T) { + setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) + ssMetrics := newPrometheusOutlineMetrics(&noopMap{}) + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(ssMetrics) + + ssMetrics.AddAuthenticatedTCPConnection(fakeAddr("127.0.0.1:9"), "key-1") + setNow(time.Date(2010, 1, 2, 3, 4, 10, .0, time.Local)) + + expected := strings.NewReader(` + # HELP tunnel_time_seconds_per_location Tunnel time, per location. + # TYPE tunnel_time_seconds_per_location counter + tunnel_time_seconds_per_location{asn="",location="XL"} 5 + `) + err := promtest.GatherAndCompare( + reg, + expected, + "tunnel_time_seconds_per_location", + ) + require.NoError(t, err, "unexpected metric value found") + }) } func TestTunnelTimePerKeyDoesNotPanicOnUnknownClosedConnection(t *testing.T) { reg := prometheus.NewPedanticRegistry() - ssMetrics := newPrometheusOutlineMetrics(nil, reg) + ssMetrics := newPrometheusOutlineMetrics(nil) ssMetrics.AddClosedTCPConnection(fakeAddr("127.0.0.1:9"), "key-1", "OK", metrics.ProxyMetrics{}, time.Minute) @@ -135,7 +139,7 @@ func TestTunnelTimePerKeyDoesNotPanicOnUnknownClosedConnection(t *testing.T) { } func BenchmarkOpenTCP(b *testing.B) { - ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry()) + ssMetrics := newPrometheusOutlineMetrics(nil) addr := fakeAddr("127.0.0.1:9") b.ResetTimer() for i := 0; i < b.N; i++ { @@ -144,7 +148,7 @@ func BenchmarkOpenTCP(b *testing.B) { } func BenchmarkCloseTCP(b *testing.B) { - ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry()) + ssMetrics := newPrometheusOutlineMetrics(nil) addr := fakeAddr("127.0.0.1:9") accessKey := "key 1" status := "OK" @@ -160,7 +164,7 @@ func BenchmarkCloseTCP(b *testing.B) { } func BenchmarkProbe(b *testing.B) { - ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry()) + ssMetrics := newPrometheusOutlineMetrics(nil) status := "ERR_REPLAY" drainResult := "other" data := metrics.ProxyMetrics{} @@ -171,7 +175,7 @@ func BenchmarkProbe(b *testing.B) { } func BenchmarkClientUDP(b *testing.B) { - ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry()) + ssMetrics := newPrometheusOutlineMetrics(nil) addr := fakeAddr("127.0.0.1:9") accessKey := "key 1" status := "OK" @@ -185,7 +189,7 @@ func BenchmarkClientUDP(b *testing.B) { } func BenchmarkTargetUDP(b *testing.B) { - ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry()) + ssMetrics := newPrometheusOutlineMetrics(nil) addr := fakeAddr("127.0.0.1:9") accessKey := "key 1" status := "OK" @@ -197,7 +201,7 @@ func BenchmarkTargetUDP(b *testing.B) { } func BenchmarkNAT(b *testing.B) { - ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry()) + ssMetrics := newPrometheusOutlineMetrics(nil) addr := fakeAddr("127.0.0.1:9") b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/cmd/outline-ss-server/server_test.go b/cmd/outline-ss-server/server_test.go index 0b7777b2..e8a47f0f 100644 --- a/cmd/outline-ss-server/server_test.go +++ b/cmd/outline-ss-server/server_test.go @@ -17,12 +17,10 @@ package main import ( "testing" "time" - - "github.com/prometheus/client_golang/prometheus" ) func TestRunSSServer(t *testing.T) { - m := newPrometheusOutlineMetrics(nil, prometheus.DefaultRegisterer) + m := newPrometheusOutlineMetrics(nil) server, err := RunSSServer("config_example.yml", 30*time.Second, m, 10000) if err != nil { t.Fatalf("RunSSServer() error = %v", err) From d80ba6a88fbd60c76a8aa6349975c15fef41af96 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 15:07:20 -0400 Subject: [PATCH 123/182] Use `nil` instead of `context.TODO()`. --- service/tcp.go | 2 +- service/udp.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/service/tcp.go b/service/tcp.go index c3656aa7..6ea96d1f 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -66,7 +66,7 @@ func remoteIP(conn net.Conn) netip.Addr { func debugTCP(template string, cipherID string, args ...any) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. - if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + if slog.Default().Enabled(nil, slog.LevelDebug) { args = append(args, slog.String("ID", cipherID)) slog.Debug(fmt.Sprintf("TCP: %s", template), args...) } diff --git a/service/udp.go b/service/udp.go index f1aa0b99..76ec989a 100644 --- a/service/udp.go +++ b/service/udp.go @@ -15,7 +15,6 @@ package service import ( - "context" "errors" "fmt" "log/slog" @@ -52,14 +51,14 @@ const serverUDPBufferSize = 64 * 1024 func debugUDP(template string, cipherID string, args ...any) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. - if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + if slog.Default().Enabled(nil, slog.LevelDebug) { args = append(args, slog.String("ID", cipherID)) slog.Debug(fmt.Sprintf("UDP: %s", template), args...) } } func debugUDPAddr(template string, addr net.Addr, args ...any) { - if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + if slog.Default().Enabled(nil, slog.LevelDebug) { args = append(args, slog.String("address", addr.String())) slog.Debug(fmt.Sprintf("UDP: %s", template), args...) } From 30a285e6c485d372d9ad8611ed9b25511221457f Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 15:18:51 -0400 Subject: [PATCH 124/182] Use `LogAttrs` for `debug...()` log functions. --- service/tcp.go | 15 +++++++-------- service/udp.go | 18 ++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/service/tcp.go b/service/tcp.go index 6ea96d1f..e9e45941 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -63,12 +63,11 @@ func remoteIP(conn net.Conn) netip.Addr { } // Wrapper for slog.Debug during TCP access key searches. -func debugTCP(template string, cipherID string, args ...any) { +func debugTCP(template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. if slog.Default().Enabled(nil, slog.LevelDebug) { - args = append(args, slog.String("ID", cipherID)) - slog.Debug(fmt.Sprintf("TCP: %s", template), args...) + slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("TCP: %s", template), slog.String("ID", cipherID), attr) } } @@ -259,7 +258,7 @@ func (h *tcpHandler) Handle(ctx context.Context, clientConn transport.StreamConn if err != nil { slog.Warn("Failed client info lookup", "err", err) } - slog.Debug("Got info for IP.", "info", clientInfo, "IP", clientConn.RemoteAddr().String()) + slog.LogAttrs(nil, slog.LevelDebug, "Got info for IP.", slog.Any("info", clientInfo), slog.String("IP", clientConn.RemoteAddr().String())) h.m.AddOpenTCPConnection(clientInfo) var proxyMetrics metrics.ProxyMetrics measuredClientConn := metrics.MeasureConn(clientConn, &proxyMetrics.ProxyClient, &proxyMetrics.ClientProxy) @@ -271,11 +270,11 @@ func (h *tcpHandler) Handle(ctx context.Context, clientConn transport.StreamConn status := "OK" if connError != nil { status = connError.Status - slog.Debug("TCP: Error", "msg", connError.Message, "cause", connError.Cause) + slog.LogAttrs(nil, slog.LevelDebug, "TCP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) } h.m.AddClosedTCPConnection(clientInfo, clientConn.RemoteAddr(), id, status, proxyMetrics, connDuration) measuredClientConn.Close() // Closing after the metrics are added aids integration testing. - slog.Debug("TCP: Done.", "status", status, "duration", connDuration) + slog.LogAttrs(nil, slog.LevelDebug, "TCP: Done.", slog.String("status", status), slog.Duration("duration", connDuration)) } func getProxyRequest(clientConn transport.StreamConn) (string, error) { @@ -297,7 +296,7 @@ func proxyConnection(ctx context.Context, dialer transport.StreamDialer, tgtAddr return ensureConnectionError(dialErr, "ERR_CONNECT", "Failed to connect to target") } defer tgtConn.Close() - slog.Debug("Proxy connection.", "client", clientConn.RemoteAddr().String(), "target", tgtConn.RemoteAddr().String()) + slog.LogAttrs(nil, slog.LevelDebug, "Proxy connection.", slog.String("client", clientConn.RemoteAddr().String()), slog.String("target", tgtConn.RemoteAddr().String())) fromClientErrCh := make(chan error) go func() { @@ -374,7 +373,7 @@ func (h *tcpHandler) absorbProbe(clientConn io.ReadCloser, addr, status string, // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) - slog.Debug("Drain error.", "err", drainErr, "result", drainResult) + slog.LogAttrs(nil, slog.LevelDebug, "Drain error.", slog.Any("err", drainErr), slog.String("result", drainResult)) h.m.AddTCPProbe(status, drainResult, addr, proxyMetrics.ClientProxy) } diff --git a/service/udp.go b/service/udp.go index 76ec989a..119bd66f 100644 --- a/service/udp.go +++ b/service/udp.go @@ -48,19 +48,17 @@ type UDPMetrics interface { const serverUDPBufferSize = 64 * 1024 // Wrapper for slog.Debug during UDP proxying. -func debugUDP(template string, cipherID string, args ...any) { +func debugUDP(template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. if slog.Default().Enabled(nil, slog.LevelDebug) { - args = append(args, slog.String("ID", cipherID)) - slog.Debug(fmt.Sprintf("UDP: %s", template), args...) + slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("ID", cipherID), attr) } } -func debugUDPAddr(template string, addr net.Addr, args ...any) { +func debugUDPAddr(template string, addr net.Addr, attr slog.Attr) { if slog.Default().Enabled(nil, slog.LevelDebug) { - args = append(args, slog.String("address", addr.String())) - slog.Debug(fmt.Sprintf("UDP: %s", template), args...) + slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("address", addr.String()), attr) } } @@ -141,8 +139,8 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { if err != nil { return onet.NewConnectionError("ERR_READ", "Failed to read from client", err) } - defer slog.Debug("UDP: Done.", "address", clientAddr) - slog.Debug("UDP: Outbound packet.", "address", clientAddr, "bytes", clientProxyBytes) + defer slog.LogAttrs(nil, slog.LevelDebug, "UDP: Done", slog.String("address", clientAddr.String())) + debugUDPAddr("Outbound packet.", clientAddr, slog.Int("bytes", clientProxyBytes)) cipherData := cipherBuf[:clientProxyBytes] var payload []byte @@ -209,7 +207,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { status := "OK" if connError != nil { - slog.Debug("UDP: Error.", "msg", connError.Message, "cause", connError.Cause) + slog.LogAttrs(nil, slog.LevelDebug, "UDP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) status = connError.Status } h.m.AddUDPPacketFromClient(clientInfo, keyID, status, clientProxyBytes, proxyTargetBytes) @@ -452,7 +450,7 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco }() status := "OK" if connError != nil { - slog.Debug("UDP: Error.", "msg", connError.Message, "cause", connError.Cause) + slog.LogAttrs(nil, slog.LevelDebug, "UDP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) status = connError.Status } if expired { From 52331674603bf2d0cad09b51eef8cef1922b1d62 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 16:35:21 -0400 Subject: [PATCH 125/182] Update logging in `metrics.go`. --- cmd/outline-ss-server/metrics.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 090cf867..599b7a5d 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -15,7 +15,6 @@ package main import ( - "context" "fmt" "log/slog" "net" @@ -236,7 +235,7 @@ func (c *tunnelTimeCollector) Collect(ch chan<- prometheus.Metric) { // Calculates and reports the tunnel time for a given active client. func (c *tunnelTimeCollector) reportTunnelTime(ipKey IPKey, client *activeClient, tNow time.Time) { tunnelTime := tNow.Sub(client.startTime) - slog.Debug("Reporting tunnel time.", "key", ipKey.accessKey, "duration", tunnelTime) + slog.LogAttrs(nil, slog.LevelDebug, "Reporting tunnel time.", slog.String("key", ipKey.accessKey), slog.Duration("duration", tunnelTime)) c.tunnelTimePerKey.WithLabelValues(ipKey.accessKey).Add(tunnelTime.Seconds()) c.tunnelTimePerLocation.WithLabelValues(client.info.CountryCode.String(), asnLabel(client.info.ASN)).Add(tunnelTime.Seconds()) // Reset the start time now that the tunnel time has been reported. @@ -371,13 +370,13 @@ func (m *outlineMetricsCollector) getIPInfoFromAddr(addr net.Addr) ipinfo.IPInfo if !exists { ipInfo, err := ipinfo.GetIPInfoFromAddr(m.ip2info, addr) if err != nil { - slog.Warn("Failed client info lookup.", "err", err) + slog.LogAttrs(nil, slog.LevelWarn, "Failed client info lookup.", slog.Any("err", err)) return ipInfo } m.ipInfoCache.Set(addr, ipInfo) } - if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { - slog.Debug("Got IP info for address.", "address", addr, "info", ipInfo) + if slog.Default().Enabled(nil, slog.LevelDebug) { + slog.LogAttrs(nil, slog.LevelDebug, "Got info for IP.", slog.String("IP", addr.String()), slog.Any("info", ipInfo)) } return ipInfo } From feb30b6a3e7d65b39f4915c54aae3fa1f10037e9 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 16:41:39 -0400 Subject: [PATCH 126/182] Fix another race condition. --- cmd/outline-ss-server/lru_cache.go | 34 +++++++++++++++---------- cmd/outline-ss-server/lru_cache_test.go | 28 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/cmd/outline-ss-server/lru_cache.go b/cmd/outline-ss-server/lru_cache.go index 4cc084db..fd5991c5 100644 --- a/cmd/outline-ss-server/lru_cache.go +++ b/cmd/outline-ss-server/lru_cache.go @@ -49,22 +49,28 @@ func NewLRUCache[K comparable, V any](capacity int, duration time.Duration, clea func (c *LRUCache[K, V]) Get(key K) (V, bool) { c.mu.RLock() - defer c.mu.RUnlock() + elem, ok := c.items[key] + if !ok { + c.mu.RUnlock() + var zero V + return zero, false + } + ent := elem.Value.(*entry[K, V]) + c.mu.RUnlock() - if elem, ok := c.items[key]; ok { - c.lru.MoveToFront(elem) - ent := elem.Value.(*entry[K, V]) - if time.Since(ent.lastAccess) > c.duration { - c.lru.Remove(elem) - delete(c.items, key) - var zero V - return zero, false - } - ent.lastAccess = time.Now() - return ent.value, true + c.mu.Lock() + defer c.mu.Unlock() + + if time.Since(ent.lastAccess) > c.duration { + c.lru.Remove(elem) + delete(c.items, key) + var zero V + return zero, false } - var zero V - return zero, false + + c.lru.MoveToFront(elem) + ent.lastAccess = time.Now() + return ent.value, true } func (c *LRUCache[K, V]) Set(key K, value V) { diff --git a/cmd/outline-ss-server/lru_cache_test.go b/cmd/outline-ss-server/lru_cache_test.go index bec29acb..c91abf8c 100644 --- a/cmd/outline-ss-server/lru_cache_test.go +++ b/cmd/outline-ss-server/lru_cache_test.go @@ -15,6 +15,8 @@ package main import ( + "math/rand" + "sync" "testing" "time" @@ -76,3 +78,29 @@ func TestLRUCache(t *testing.T) { cache.StopCleanup() }) } + +func BenchmarkLRUCache(b *testing.B) { + cache := NewLRUCache[int, int](1000, time.Minute, time.Minute) + + keys := make([]int, b.N) + for i := 0; i < b.N; i++ { + keys[i] = rand.Intn(2000) + } + + b.ResetTimer() + + var wg sync.WaitGroup + for i := 0; i < b.N; i++ { + wg.Add(1) + go func(key int) { + defer wg.Done() + cache.Get(key) + if rand.Intn(2) == 0 { + cache.Set(key, rand.Int()) + } + }(keys[i]) + } + wg.Wait() + + cache.StopCleanup() +} From b74f5a0802a07a6a316ac2508a788cfc01c5f2da Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 16:46:02 -0400 Subject: [PATCH 127/182] Revert renaming. --- internal/integration_test/integration_test.go | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index 347db0b9..4ac503bd 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -130,7 +130,7 @@ func TestTCPEcho(t *testing.T) { } replayCache := service.NewReplayCache(5) const testTimeout = 200 * time.Millisecond - testMetrics := &service.NoOpTCPMetricsCollector{} + testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -181,7 +181,7 @@ func TestTCPEcho(t *testing.T) { } type statusMetrics struct { - service.NoOpTCPMetricsCollector + service.NoOpTCPMetrics sync.Mutex statuses []string } @@ -251,26 +251,26 @@ type udpRecord struct { } // Fake metrics implementation for UDP -type fakeUDPMetricsCollector struct { +type fakeUDPMetrics struct { up, down []udpRecord natAdded int } -var _ service.UDPMetricsCollector = (*fakeUDPMetricsCollector)(nil) +var _ service.UDPMetrics = (*fakeUDPMetrics)(nil) -func (m *fakeUDPMetricsCollector) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { +func (m *fakeUDPMetrics) AddUDPPacketFromClient(clientAddr net.Addr, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { m.up = append(m.up, udpRecord{clientAddr, accessKey, status, clientProxyBytes, proxyTargetBytes}) } -func (m *fakeUDPMetricsCollector) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { +func (m *fakeUDPMetrics) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { m.down = append(m.down, udpRecord{clientAddr, accessKey, status, targetProxyBytes, proxyClientBytes}) } -func (m *fakeUDPMetricsCollector) AddUDPNatEntry(clientAddr net.Addr, accessKey string) { +func (m *fakeUDPMetrics) AddUDPNatEntry(clientAddr net.Addr, accessKey string) { m.natAdded++ } -func (m *fakeUDPMetricsCollector) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string) { +func (m *fakeUDPMetrics) RemoveUDPNatEntry(clientAddr net.Addr, accessKey string) { // Not tested because it requires waiting for a long timeout. } -func (m *fakeUDPMetricsCollector) AddUDPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { +func (m *fakeUDPMetrics) AddUDPCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { } func TestUDPEcho(t *testing.T) { @@ -285,7 +285,7 @@ func TestUDPEcho(t *testing.T) { if err != nil { t.Fatal(err) } - testMetrics := &fakeUDPMetricsCollector{} + testMetrics := &fakeUDPMetrics{} proxy := service.NewPacketHandler(time.Hour, cipherList, testMetrics) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) @@ -374,7 +374,7 @@ func BenchmarkTCPThroughput(b *testing.B) { b.Fatal(err) } const testTimeout = 200 * time.Millisecond - testMetrics := &service.NoOpTCPMetricsCollector{} + testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, testMetrics) handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -438,7 +438,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { } replayCache := service.NewReplayCache(service.MaxCapacity) const testTimeout = 200 * time.Millisecond - testMetrics := &service.NoOpTCPMetricsCollector{} + testMetrics := &service.NoOpTCPMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) handler := service.NewTCPHandler(authFunc, testMetrics, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -513,7 +513,7 @@ func BenchmarkUDPEcho(b *testing.B) { if err != nil { b.Fatal(err) } - proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetricsCollector{}) + proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { @@ -557,7 +557,7 @@ func BenchmarkUDPManyKeys(b *testing.B) { if err != nil { b.Fatal(err) } - proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetricsCollector{}) + proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { From dcc99f6118f47771610709f13230149c93ef9bbc Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 17:09:45 -0400 Subject: [PATCH 128/182] Replace LRU cache with a simpler map that expires unused items. --- cmd/outline-ss-server/expiring_map.go | 109 +++++++++++++++++ cmd/outline-ss-server/expiring_map_test.go | 89 ++++++++++++++ cmd/outline-ss-server/lru_cache.go | 132 --------------------- cmd/outline-ss-server/lru_cache_test.go | 106 ----------------- cmd/outline-ss-server/metrics.go | 4 +- 5 files changed, 200 insertions(+), 240 deletions(-) create mode 100644 cmd/outline-ss-server/expiring_map.go create mode 100644 cmd/outline-ss-server/expiring_map_test.go delete mode 100644 cmd/outline-ss-server/lru_cache.go delete mode 100644 cmd/outline-ss-server/lru_cache_test.go diff --git a/cmd/outline-ss-server/expiring_map.go b/cmd/outline-ss-server/expiring_map.go new file mode 100644 index 00000000..78e4ee76 --- /dev/null +++ b/cmd/outline-ss-server/expiring_map.go @@ -0,0 +1,109 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "sync" + "sync/atomic" + "time" +) + +// ExpiringMap is a thread-safe, generic map that automatically removes +// key-value pairs after a specified duration of inactivity. +// It employs reference counting to ensure safe concurrent access and prevent +// premature deletion during cleanup. +type ExpiringMap[K comparable, V any] struct { + data map[K]*item[V] + mu sync.RWMutex + expiryTime time.Duration + done chan struct{} +} + +type item[V any] struct { + value V + lastAccess time.Time + refCount int32 +} + +func NewExpiringMap[K comparable, V any](expiryTime time.Duration) *ExpiringMap[K, V] { + em := &ExpiringMap[K, V]{ + data: make(map[K]*item[V]), + expiryTime: expiryTime, + done: make(chan struct{}), + } + go em.cleanupLoop() + return em +} + +func (em *ExpiringMap[K, V]) Set(key K, value V) { + em.mu.Lock() + defer em.mu.Unlock() + + em.data[key] = &item[V]{ + value: value, + lastAccess: time.Now(), + refCount: 0, + } +} + +func (em *ExpiringMap[K, V]) Get(key K) (V, bool) { + em.mu.RLock() + item, ok := em.data[key] + if !ok { + em.mu.RUnlock() + var zeroValue V + return zeroValue, false + } + + atomic.AddInt32(&item.refCount, 1) + em.mu.RUnlock() + + em.mu.Lock() + defer em.mu.Unlock() + + atomic.AddInt32(&item.refCount, -1) + + item.lastAccess = time.Now() + return item.value, true +} + +func (em *ExpiringMap[K, V]) cleanup() { + em.mu.Lock() + defer em.mu.Unlock() + + for key, item := range em.data { + if time.Since(item.lastAccess) > em.expiryTime && atomic.LoadInt32(&item.refCount) == 0 { + delete(em.data, key) + } + } +} + +func (em *ExpiringMap[K, V]) cleanupLoop() { + ticker := time.NewTicker(em.expiryTime / 2) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + em.cleanup() + case <-em.done: + return + } + } +} + +func (em *ExpiringMap[K, V]) StopCleanup() { + close(em.done) +} diff --git a/cmd/outline-ss-server/expiring_map_test.go b/cmd/outline-ss-server/expiring_map_test.go new file mode 100644 index 00000000..7560ef45 --- /dev/null +++ b/cmd/outline-ss-server/expiring_map_test.go @@ -0,0 +1,89 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestExpiringMap(t *testing.T) { + t.Run("BasicSetGet", func(t *testing.T) { + em := NewExpiringMap[string, int](2 * time.Second) + em.Set("key1", 10) + val, ok := em.Get("key1") + require.True(t, ok) + require.Equal(t, 10, val) + + time.Sleep(3 * time.Second) + + _, ok = em.Get("key1") + require.False(t, ok) + }) + + t.Run("Expiration", func(t *testing.T) { + em := NewExpiringMap[string, int](2 * time.Second) + em.Set("a", 1) + + time.Sleep(3 * time.Second) + + _, ok := em.Get("a") + require.False(t, ok, "Expected `a` to have been evicted") + em.StopCleanup() + }) + + t.Run("Concurrency", func(t *testing.T) { + em := NewExpiringMap[int, string](2 * time.Second) + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + em.Set(i, fmt.Sprintf("value%d", i)) + + time.Sleep(time.Duration(i) * time.Millisecond) + + val, ok := em.Get(i) + require.True(t, ok) + require.Equal(t, fmt.Sprintf("value%d", i), val) + }(i) + } + wg.Wait() + }) +} + +func BenchmarkExpiringMap(b *testing.B) { + b.Run("Set", func(b *testing.B) { + em := NewExpiringMap[int64, int64](10 * time.Second) + for i := 0; i < b.N; i++ { + em.Set(int64(i), int64(i)) + } + }) + + b.Run("Get", func(b *testing.B) { + em := NewExpiringMap[int64, int64](10 * time.Second) + for i := 0; i < b.N; i++ { + em.Set(int64(i), int64(i)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + em.Get(int64(i)) + } + }) +} diff --git a/cmd/outline-ss-server/lru_cache.go b/cmd/outline-ss-server/lru_cache.go deleted file mode 100644 index fd5991c5..00000000 --- a/cmd/outline-ss-server/lru_cache.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2024 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "container/list" - "sync" - "time" -) - -type LRUCache[K comparable, V any] struct { - capacity int - duration time.Duration - items map[K]*list.Element - lru *list.List - mu sync.RWMutex - done chan struct{} -} - -type entry[K comparable, V any] struct { - key K - value V - lastAccess time.Time -} - -func NewLRUCache[K comparable, V any](capacity int, duration time.Duration, cleanupInterval time.Duration) *LRUCache[K, V] { - c := &LRUCache[K, V]{ - capacity: capacity, - duration: duration, - items: make(map[K]*list.Element), - lru: list.New(), - done: make(chan struct{}), - } - go c.cleanup(cleanupInterval) - return c -} - -func (c *LRUCache[K, V]) Get(key K) (V, bool) { - c.mu.RLock() - elem, ok := c.items[key] - if !ok { - c.mu.RUnlock() - var zero V - return zero, false - } - ent := elem.Value.(*entry[K, V]) - c.mu.RUnlock() - - c.mu.Lock() - defer c.mu.Unlock() - - if time.Since(ent.lastAccess) > c.duration { - c.lru.Remove(elem) - delete(c.items, key) - var zero V - return zero, false - } - - c.lru.MoveToFront(elem) - ent.lastAccess = time.Now() - return ent.value, true -} - -func (c *LRUCache[K, V]) Set(key K, value V) { - c.mu.Lock() - defer c.mu.Unlock() - - if elem, ok := c.items[key]; ok { - c.lru.MoveToFront(elem) - ent := elem.Value.(*entry[K, V]) - ent.value = value - ent.lastAccess = time.Now() - return - } - - ent := &entry[K, V]{key, value, time.Now()} - elem := c.lru.PushFront(ent) - c.items[key] = elem - - if c.lru.Len() > c.capacity { - c.removeOldest() - } -} - -func (c *LRUCache[K, V]) removeOldest() { - elem := c.lru.Back() - if elem != nil { - c.lru.Remove(elem) - ent := elem.Value.(*entry[K, V]) - delete(c.items, ent.key) - } -} - -func (c *LRUCache[K, V]) cleanup(interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - c.mu.Lock() - for elem := c.lru.Back(); elem != nil; elem = c.lru.Back() { - ent := elem.Value.(*entry[K, V]) - if time.Since(ent.lastAccess) > c.duration { - c.lru.Remove(elem) - delete(c.items, ent.key) - } else { - break - } - } - c.mu.Unlock() - case <-c.done: - return - } - } -} - -func (c *LRUCache[K, V]) StopCleanup() { - close(c.done) -} diff --git a/cmd/outline-ss-server/lru_cache_test.go b/cmd/outline-ss-server/lru_cache_test.go deleted file mode 100644 index c91abf8c..00000000 --- a/cmd/outline-ss-server/lru_cache_test.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2024 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "math/rand" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestLRUCache(t *testing.T) { - t.Run("BasicSetGet", func(t *testing.T) { - cache := NewLRUCache[string, int](2, time.Minute, time.Minute) - - cache.Set("a", 1) - cache.Set("b", 2) - a, aOk := cache.Get("a") - b, bOk := cache.Get("b") - - require.True(t, aOk) - require.Equal(t, 1, a) - require.True(t, bOk) - require.Equal(t, 2, b) - cache.StopCleanup() - }) - - t.Run("LRUEviction", func(t *testing.T) { - cache := NewLRUCache[string, int](2, time.Minute, time.Minute) - - cache.Set("a", 1) - cache.Set("b", 2) - cache.Set("c", 3) - - _, ok := cache.Get("a") - require.False(t, ok, "Expected `a` to have been evicted") - v, ok := cache.Get("b") - require.True(t, ok, "Did not expect `b` to have been evicted") - require.Equal(t, 2, v) - v, ok = cache.Get("c") - require.True(t, ok, "Did not expect `c` to have been evicted") - require.Equal(t, 3, v) - cache.StopCleanup() - }) - - t.Run("Expiration", func(t *testing.T) { - cache := NewLRUCache[string, int](2, 500*time.Millisecond, time.Minute) - cache.Set("a", 1) - - time.Sleep(600 * time.Millisecond) - - _, ok := cache.Get("a") - require.False(t, ok, "Expected `a` to have been evicted") - cache.StopCleanup() - }) - - t.Run("Cleanup", func(t *testing.T) { - cache := NewLRUCache[string, int](2, 500*time.Millisecond, 200*time.Millisecond) - cache.Set("a", 1) - - time.Sleep(600 * time.Millisecond) - - require.Len(t, cache.items, 0, "Expected `a` to have been evicted in cleanup") - cache.StopCleanup() - }) -} - -func BenchmarkLRUCache(b *testing.B) { - cache := NewLRUCache[int, int](1000, time.Minute, time.Minute) - - keys := make([]int, b.N) - for i := 0; i < b.N; i++ { - keys[i] = rand.Intn(2000) - } - - b.ResetTimer() - - var wg sync.WaitGroup - for i := 0; i < b.N; i++ { - wg.Add(1) - go func(key int) { - defer wg.Done() - cache.Get(key) - if rand.Intn(2) == 0 { - cache.Set(key, rand.Int()) - } - }(keys[i]) - } - wg.Wait() - - cache.StopCleanup() -} diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 599b7a5d..77a94585 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -273,7 +273,7 @@ func (c *tunnelTimeCollector) stopConnection(ipKey IPKey) { type outlineMetricsCollector struct { ip2info ipinfo.IPInfoMap - ipInfoCache *LRUCache[net.Addr, ipinfo.IPInfo] + ipInfoCache *ExpiringMap[net.Addr, ipinfo.IPInfo] *tcpCollector *udpCollector @@ -301,7 +301,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) *outlineMetricsCollec return &outlineMetricsCollector{ ip2info: ip2info, - ipInfoCache: NewLRUCache[net.Addr, ipinfo.IPInfo](10000, 60*time.Second, 30*time.Second), + ipInfoCache: NewExpiringMap[net.Addr, ipinfo.IPInfo](20 * time.Second), tcpCollector: tcpCollector, udpCollector: udpCollector, From 26c462474d6d4aee762d62624a0f5ba3acd0ee41 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 17:39:56 -0400 Subject: [PATCH 129/182] Move `SetBuildInfo()` call up. --- cmd/outline-ss-server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index ecf7f1ac..a00de5cd 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -274,9 +274,9 @@ func main() { defer ip2info.Close() metrics := newPrometheusOutlineMetrics(ip2info) + metrics.SetBuildInfo(version) r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) r.MustRegister(metrics) - metrics.SetBuildInfo(version) _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, metrics, flags.replayHistory) if err != nil { slog.Error("Server failed to start. Aborting.", "err", err) From 80fe8d3b22f90071f56f7f7f0c3f2066ac92a02c Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 19 Aug 2024 17:51:52 -0400 Subject: [PATCH 130/182] refactor: change `outlineMetrics` to implement the `prometheus.Collector` interface --- cmd/outline-ss-server/main.go | 12 +- cmd/outline-ss-server/metrics.go | 372 ++++++++++++++++---------- cmd/outline-ss-server/metrics_test.go | 107 ++++---- cmd/outline-ss-server/server_test.go | 4 +- 4 files changed, 295 insertions(+), 200 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 48cd0bdc..a00de5cd 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -63,7 +63,7 @@ type ssPort struct { type SSServer struct { natTimeout time.Duration - m *outlineMetrics + m *outlineMetricsCollector replayCache service.ReplayCache ports map[int]*ssPort } @@ -168,7 +168,7 @@ func (s *SSServer) Stop() error { } // RunSSServer starts a shadowsocks server running, and returns the server or an error. -func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { +func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetricsCollector, replayHistory int) (*SSServer, error) { server := &SSServer{ natTimeout: natTimeout, m: sm, @@ -273,9 +273,11 @@ func main() { } defer ip2info.Close() - m := newPrometheusOutlineMetrics(ip2info, prometheus.DefaultRegisterer) - m.SetBuildInfo(version) - _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, m, flags.replayHistory) + metrics := newPrometheusOutlineMetrics(ip2info) + metrics.SetBuildInfo(version) + r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) + r.MustRegister(metrics) + _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, metrics, flags.replayHistory) if err != nil { slog.Error("Server failed to start. Aborting.", "err", err) } diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index dfdeb1f2..c9bf344b 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -28,35 +28,136 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const namespace = "shadowsocks" - // `now` is stubbable for testing. var now = time.Now -type outlineMetrics struct { - ipinfo.IPInfoMap - *tunnelTimeCollector +type tcpCollector struct { + probes *prometheus.HistogramVec + openConnections *prometheus.CounterVec + closedConnections *prometheus.CounterVec + connectionDurationMs *prometheus.HistogramVec +} - buildInfo *prometheus.GaugeVec - accessKeys prometheus.Gauge - ports prometheus.Gauge - dataBytes *prometheus.CounterVec - dataBytesPerLocation *prometheus.CounterVec - timeToCipherMs *prometheus.HistogramVec - // TODO: Add time to first byte. +var _ prometheus.Collector = (*tcpCollector)(nil) + +func newTcpCollector() *tcpCollector { + namespace := "tcp" + return &tcpCollector{ + probes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Name: "probes", + Buckets: []float64{0, 49, 50, 51, 73, 91}, + Help: "Histogram of number of bytes from client to proxy, for detecting possible probes", + }, []string{"port", "status", "error"}), + openConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "connections_opened", + Help: "Count of open TCP connections", + }, []string{"location", "asn"}), + closedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "connections_closed", + Help: "Count of closed TCP connections", + }, []string{"location", "asn", "status", "access_key"}), + connectionDurationMs: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "connection_duration_ms", + Help: "TCP connection duration distributions.", + Buckets: []float64{ + 100, + float64(time.Second.Milliseconds()), + float64(time.Minute.Milliseconds()), + float64(time.Hour.Milliseconds()), + float64(24 * time.Hour.Milliseconds()), // Day + float64(7 * 24 * time.Hour.Milliseconds()), // Week + }, + }, []string{"status"}), + } +} + +func (c *tcpCollector) Describe(ch chan<- *prometheus.Desc) { + c.probes.Describe(ch) + c.openConnections.Describe(ch) + c.closedConnections.Describe(ch) + c.connectionDurationMs.Describe(ch) +} + +func (c *tcpCollector) Collect(ch chan<- prometheus.Metric) { + c.probes.Collect(ch) + c.openConnections.Collect(ch) + c.closedConnections.Collect(ch) + c.connectionDurationMs.Collect(ch) +} - tcpProbes *prometheus.HistogramVec - tcpOpenConnections *prometheus.CounterVec - tcpClosedConnections *prometheus.CounterVec - tcpConnectionDurationMs *prometheus.HistogramVec +func (c *tcpCollector) openConnection(clientInfo ipinfo.IPInfo) { + c.openConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)).Inc() +} + +func (c *tcpCollector) closeConnection(clientInfo ipinfo.IPInfo, status, accessKey string, duration time.Duration) { + c.closedConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status, accessKey).Inc() + c.connectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000) +} - udpPacketsFromClientPerLocation *prometheus.CounterVec - udpAddedNatEntries prometheus.Counter - udpRemovedNatEntries prometheus.Counter +func (c *tcpCollector) addProbe(listenerId, status, drainResult string, clientProxyBytes int64) { + c.probes.WithLabelValues(listenerId, status, drainResult).Observe(float64(clientProxyBytes)) } -var _ service.TCPMetrics = (*outlineMetrics)(nil) -var _ service.UDPMetrics = (*outlineMetrics)(nil) +type udpCollector struct { + packetsFromClientPerLocation *prometheus.CounterVec + addedNatEntries prometheus.Counter + removedNatEntries prometheus.Counter +} + +var _ prometheus.Collector = (*udpCollector)(nil) + +func newUdpCollector() *udpCollector { + namespace := "udp" + return &udpCollector{ + packetsFromClientPerLocation: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "packets_from_client_per_location", + Help: "Packets received from the client, per location and status", + }, []string{"location", "asn", "status"}), + addedNatEntries: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "nat_entries_added", + Help: "Entries added to the UDP NAT table", + }), + removedNatEntries: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "nat_entries_removed", + Help: "Entries removed from the UDP NAT table", + }), + } +} + +func (c *udpCollector) Describe(ch chan<- *prometheus.Desc) { + c.packetsFromClientPerLocation.Describe(ch) + c.addedNatEntries.Describe(ch) + c.removedNatEntries.Describe(ch) +} + +func (c *udpCollector) Collect(ch chan<- prometheus.Metric) { + c.packetsFromClientPerLocation.Collect(ch) + c.addedNatEntries.Collect(ch) + c.removedNatEntries.Collect(ch) +} + +func (c *udpCollector) addPacketFromClient(clientInfo ipinfo.IPInfo, status string) { + c.packetsFromClientPerLocation.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status).Inc() +} + +func (c *udpCollector) addNatEntry() { + c.addedNatEntries.Inc() +} + +func (c *udpCollector) removeNatEntry() { + c.removedNatEntries.Inc() +} // Converts a [net.Addr] to an [IPKey]. func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { @@ -94,6 +195,27 @@ type tunnelTimeCollector struct { tunnelTimePerLocation *prometheus.CounterVec } +var _ prometheus.Collector = (*tunnelTimeCollector)(nil) + +func newTunnelTimeCollector(ip2info ipinfo.IPInfoMap) *tunnelTimeCollector { + namespace := "tunnel_time" + return &tunnelTimeCollector{ + ip2info: ip2info, + activeClients: make(map[IPKey]*activeClient), + + tunnelTimePerKey: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "seconds", + Help: "Tunnel time, per access key.", + }, []string{"access_key"}), + tunnelTimePerLocation: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "seconds_per_location", + Help: "Tunnel time, per location.", + }, []string{"location", "asn"}), + } +} + func (c *tunnelTimeCollector) Describe(ch chan<- *prometheus.Desc) { c.tunnelTimePerKey.Describe(ch) c.tunnelTimePerLocation.Describe(ch) @@ -113,7 +235,7 @@ func (c *tunnelTimeCollector) Collect(ch chan<- prometheus.Metric) { // Calculates and reports the tunnel time for a given active client. func (c *tunnelTimeCollector) reportTunnelTime(ipKey IPKey, client *activeClient, tNow time.Time) { tunnelTime := tNow.Sub(client.startTime) - slog.Debug("Reporting tunnel time.", "key", ipKey.accessKey, "duration", tunnelTime) + slog.LogAttrs(nil, slog.LevelDebug, "Reporting tunnel time.", slog.String("key", ipKey.accessKey), slog.Duration("duration", tunnelTime)) c.tunnelTimePerKey.WithLabelValues(ipKey.accessKey).Add(tunnelTime.Seconds()) c.tunnelTimePerLocation.WithLabelValues(client.info.CountryCode.String(), asnLabel(client.info.ASN)).Add(tunnelTime.Seconds()) // Reset the start time now that the tunnel time has been reported. @@ -149,143 +271,112 @@ func (c *tunnelTimeCollector) stopConnection(ipKey IPKey) { } } -func newTunnelTimeCollector(ip2info ipinfo.IPInfoMap, registerer prometheus.Registerer) *tunnelTimeCollector { - return &tunnelTimeCollector{ - ip2info: ip2info, - activeClients: make(map[IPKey]*activeClient), +type outlineMetricsCollector struct { + ipinfo.IPInfoMap - tunnelTimePerKey: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "tunnel_time_seconds", - Help: "Tunnel time, per access key.", - }, []string{"access_key"}), - tunnelTimePerLocation: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "tunnel_time_seconds_per_location", - Help: "Tunnel time, per location.", - }, []string{"location", "asn"}), - } + *tcpCollector + *udpCollector + *tunnelTimeCollector + + buildInfo *prometheus.GaugeVec + accessKeys prometheus.Gauge + ports prometheus.Gauge + dataBytes *prometheus.CounterVec + dataBytesPerLocation *prometheus.CounterVec + timeToCipherMs *prometheus.HistogramVec + // TODO: Add time to first byte. } -// newPrometheusOutlineMetrics constructs a metrics object that uses -// `ip2info` to convert IP addresses to countries, and reports all -// metrics to Prometheus via `registerer`. `ip2info` may be nil, but -// `registerer` must not be. -func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus.Registerer) *outlineMetrics { - m := &outlineMetrics{ +var _ prometheus.Collector = (*outlineMetricsCollector)(nil) +var _ service.TCPMetrics = (*outlineMetricsCollector)(nil) +var _ service.UDPMetrics = (*outlineMetricsCollector)(nil) + +// newPrometheusOutlineMetrics constructs a Prometheus metrics collector that uses +// `ip2info` to convert IP addresses to countries. `ip2info` may be nil. +func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) *outlineMetricsCollector { + tcpCollector := newTcpCollector() + udpCollector := newUdpCollector() + tunnelTimeCollector := newTunnelTimeCollector(ip2info) + + return &outlineMetricsCollector{ IPInfoMap: ip2info, + + tcpCollector: tcpCollector, + udpCollector: udpCollector, + tunnelTimeCollector: tunnelTimeCollector, + buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "build_info", - Help: "Information on the outline-ss-server build", + Name: "build_info", + Help: "Information on the outline-ss-server build", }, []string{"version"}), accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "keys", - Help: "Count of access keys", + Name: "keys", + Help: "Count of access keys", }), ports: prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "ports", - Help: "Count of open Shadowsocks ports", + Name: "ports", + Help: "Count of open Shadowsocks ports", }), - tcpProbes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Name: "tcp_probes", - Buckets: []float64{0, 49, 50, 51, 73, 91}, - Help: "Histogram of number of bytes from client to proxy, for detecting possible probes", - }, []string{"port", "status", "error"}), - tcpOpenConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "tcp", - Name: "connections_opened", - Help: "Count of open TCP connections", - }, []string{"location", "asn"}), - tcpClosedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "tcp", - Name: "connections_closed", - Help: "Count of closed TCP connections", - }, []string{"location", "asn", "status", "access_key"}), - tcpConnectionDurationMs: prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: "tcp", - Name: "connection_duration_ms", - Help: "TCP connection duration distributions.", - Buckets: []float64{ - 100, - float64(time.Second.Milliseconds()), - float64(time.Minute.Milliseconds()), - float64(time.Hour.Milliseconds()), - float64(24 * time.Hour.Milliseconds()), // Day - float64(7 * 24 * time.Hour.Milliseconds()), // Week - }, - }, []string{"status"}), dataBytes: prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: namespace, - Name: "data_bytes", - Help: "Bytes transferred by the proxy, per access key", + Name: "data_bytes", + Help: "Bytes transferred by the proxy, per access key", }, []string{"dir", "proto", "access_key"}), dataBytesPerLocation: prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: namespace, - Name: "data_bytes_per_location", - Help: "Bytes transferred by the proxy, per location", + Name: "data_bytes_per_location", + Help: "Bytes transferred by the proxy, per location", }, []string{"dir", "proto", "location", "asn"}), timeToCipherMs: prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: namespace, - Name: "time_to_cipher_ms", - Help: "Time needed to find the cipher", - Buckets: []float64{0.1, 1, 10, 100, 1000}, + Name: "time_to_cipher_ms", + Help: "Time needed to find the cipher", + Buckets: []float64{0.1, 1, 10, 100, 1000}, }, []string{"proto", "found_key"}), - udpPacketsFromClientPerLocation: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "udp", - Name: "packets_from_client_per_location", - Help: "Packets received from the client, per location and status", - }, []string{"location", "asn", "status"}), - udpAddedNatEntries: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "udp", - Name: "nat_entries_added", - Help: "Entries added to the UDP NAT table", - }), - udpRemovedNatEntries: prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: "udp", - Name: "nat_entries_removed", - Help: "Entries removed from the UDP NAT table", - }), } - m.tunnelTimeCollector = newTunnelTimeCollector(ip2info, registerer) +} + +func (m *outlineMetricsCollector) collectors() []prometheus.Collector { + return []prometheus.Collector{ + m.tcpCollector, + m.udpCollector, + m.tunnelTimeCollector, + + m.buildInfo, + m.accessKeys, + m.ports, + m.dataBytes, + m.dataBytesPerLocation, + m.timeToCipherMs, + } +} - // TODO: Is it possible to pass where to register the collectors? - registerer.MustRegister(m.buildInfo, m.accessKeys, m.ports, m.tcpProbes, m.tcpOpenConnections, m.tcpClosedConnections, m.tcpConnectionDurationMs, - m.dataBytes, m.dataBytesPerLocation, m.timeToCipherMs, m.udpPacketsFromClientPerLocation, m.udpAddedNatEntries, m.udpRemovedNatEntries, - m.tunnelTimeCollector) - return m +func (m *outlineMetricsCollector) Describe(ch chan<- *prometheus.Desc) { + for _, collector := range m.collectors() { + collector.Describe(ch) + } +} + +func (m *outlineMetricsCollector) Collect(ch chan<- prometheus.Metric) { + for _, collector := range m.collectors() { + collector.Collect(ch) + } } -func (m *outlineMetrics) SetBuildInfo(version string) { +func (m *outlineMetricsCollector) SetBuildInfo(version string) { m.buildInfo.WithLabelValues(version).Set(1) } -func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) { +func (m *outlineMetricsCollector) SetNumAccessKeys(numKeys int, ports int) { m.accessKeys.Set(float64(numKeys)) m.ports.Set(float64(ports)) } -func (m *outlineMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { - m.tcpOpenConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)).Inc() +func (m *outlineMetricsCollector) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) { + m.tcpCollector.openConnection(clientInfo) } -func (m *outlineMetrics) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { +func (m *outlineMetricsCollector) AddAuthenticatedTCPConnection(clientAddr net.Addr, accessKey string) { ipKey, err := toIPKey(clientAddr, accessKey) if err == nil { m.tunnelTimeCollector.startConnection(*ipKey) @@ -306,9 +397,8 @@ func asnLabel(asn int) string { return fmt.Sprint(asn) } -func (m *outlineMetrics) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, clientAddr net.Addr, accessKey, status string, data metrics.ProxyMetrics, duration time.Duration) { - m.tcpClosedConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status, accessKey).Inc() - m.tcpConnectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000) +func (m *outlineMetricsCollector) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, clientAddr net.Addr, accessKey, status string, data metrics.ProxyMetrics, duration time.Duration) { + m.tcpCollector.closeConnection(clientInfo, status, accessKey, duration) addIfNonZero(data.ClientProxy, m.dataBytes, "c>p", "tcp", accessKey) addIfNonZero(data.ClientProxy, m.dataBytesPerLocation, "c>p", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) addIfNonZero(data.ProxyTarget, m.dataBytes, "p>t", "tcp", accessKey) @@ -324,23 +414,23 @@ func (m *outlineMetrics) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, client } } -func (m *outlineMetrics) AddUDPPacketFromClient(clientInfo ipinfo.IPInfo, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { - m.udpPacketsFromClientPerLocation.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status).Inc() +func (m *outlineMetricsCollector) AddUDPPacketFromClient(clientInfo ipinfo.IPInfo, accessKey, status string, clientProxyBytes, proxyTargetBytes int) { + m.udpCollector.addPacketFromClient(clientInfo, status) addIfNonZero(int64(clientProxyBytes), m.dataBytes, "c>p", "udp", accessKey) addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) addIfNonZero(int64(proxyTargetBytes), m.dataBytes, "p>t", "udp", accessKey) addIfNonZero(int64(proxyTargetBytes), m.dataBytesPerLocation, "p>t", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) } -func (m *outlineMetrics) AddUDPPacketFromTarget(clientInfo ipinfo.IPInfo, accessKey, status string, targetProxyBytes, proxyClientBytes int) { +func (m *outlineMetricsCollector) AddUDPPacketFromTarget(clientInfo ipinfo.IPInfo, accessKey, status string, targetProxyBytes, proxyClientBytes int) { addIfNonZero(int64(targetProxyBytes), m.dataBytes, "p Date: Wed, 21 Aug 2024 18:03:19 -0400 Subject: [PATCH 131/182] Address review comments. --- cmd/outline-ss-server/metrics.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index c9bf344b..740bd489 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -32,6 +32,8 @@ import ( var now = time.Now type tcpCollector struct { + // NOTE: New metrics need to be added to `newTCPCollector()`, `Describe()` and + // `Collect()`. probes *prometheus.HistogramVec openConnections *prometheus.CounterVec closedConnections *prometheus.CounterVec @@ -40,7 +42,7 @@ type tcpCollector struct { var _ prometheus.Collector = (*tcpCollector)(nil) -func newTcpCollector() *tcpCollector { +func newTCPCollector() *tcpCollector { namespace := "tcp" return &tcpCollector{ probes: prometheus.NewHistogramVec(prometheus.HistogramOpts{ @@ -104,6 +106,8 @@ func (c *tcpCollector) addProbe(listenerId, status, drainResult string, clientPr } type udpCollector struct { + // NOTE: New metrics need to be added to `newUDPCollector()`, `Describe()` + // and `Collect()`. packetsFromClientPerLocation *prometheus.CounterVec addedNatEntries prometheus.Counter removedNatEntries prometheus.Counter @@ -111,7 +115,7 @@ type udpCollector struct { var _ prometheus.Collector = (*udpCollector)(nil) -func newUdpCollector() *udpCollector { +func newUDPCollector() *udpCollector { namespace := "udp" return &udpCollector{ packetsFromClientPerLocation: prometheus.NewCounterVec( @@ -191,6 +195,8 @@ type tunnelTimeCollector struct { mu sync.Mutex // Protects the activeClients map. activeClients map[IPKey]*activeClient + // NOTE: New metrics need to be added to `newTunnelTimeCollector()`, + // `Describe()` and `Collect()`. tunnelTimePerKey *prometheus.CounterVec tunnelTimePerLocation *prometheus.CounterVec } @@ -274,10 +280,12 @@ func (c *tunnelTimeCollector) stopConnection(ipKey IPKey) { type outlineMetricsCollector struct { ipinfo.IPInfoMap - *tcpCollector - *udpCollector - *tunnelTimeCollector + tcpCollector *tcpCollector + udpCollector *udpCollector + tunnelTimeCollector *tunnelTimeCollector + // NOTE: New metrics need to be added to `newPrometheusOutlineMetrics()` and + // `collectors()`. buildInfo *prometheus.GaugeVec accessKeys prometheus.Gauge ports prometheus.Gauge @@ -294,8 +302,8 @@ var _ service.UDPMetrics = (*outlineMetricsCollector)(nil) // newPrometheusOutlineMetrics constructs a Prometheus metrics collector that uses // `ip2info` to convert IP addresses to countries. `ip2info` may be nil. func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) *outlineMetricsCollector { - tcpCollector := newTcpCollector() - udpCollector := newUdpCollector() + tcpCollector := newTCPCollector() + udpCollector := newUDPCollector() tunnelTimeCollector := newTunnelTimeCollector(ip2info) return &outlineMetricsCollector{ From 9399e95e5fdb03509b2e152743006118c6ac1f84 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 22 Aug 2024 18:15:47 -0400 Subject: [PATCH 132/182] Refactor collectors so the connections/associations keep track of the connection metrics. --- cmd/outline-ss-server/expiring_map.go | 109 ----- cmd/outline-ss-server/expiring_map_test.go | 89 ---- cmd/outline-ss-server/main.go | 20 +- cmd/outline-ss-server/metrics.go | 423 +++++++++++------- cmd/outline-ss-server/metrics_test.go | 91 ++-- cmd/outline-ss-server/server_test.go | 5 +- internal/integration_test/integration_test.go | 115 +++-- service/shadowsocks.go | 22 + service/tcp.go | 61 +-- service/tcp_test.go | 92 ++-- service/udp.go | 72 +-- service/udp_test.go | 50 ++- 12 files changed, 581 insertions(+), 568 deletions(-) delete mode 100644 cmd/outline-ss-server/expiring_map.go delete mode 100644 cmd/outline-ss-server/expiring_map_test.go create mode 100644 service/shadowsocks.go diff --git a/cmd/outline-ss-server/expiring_map.go b/cmd/outline-ss-server/expiring_map.go deleted file mode 100644 index 78e4ee76..00000000 --- a/cmd/outline-ss-server/expiring_map.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2024 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "sync" - "sync/atomic" - "time" -) - -// ExpiringMap is a thread-safe, generic map that automatically removes -// key-value pairs after a specified duration of inactivity. -// It employs reference counting to ensure safe concurrent access and prevent -// premature deletion during cleanup. -type ExpiringMap[K comparable, V any] struct { - data map[K]*item[V] - mu sync.RWMutex - expiryTime time.Duration - done chan struct{} -} - -type item[V any] struct { - value V - lastAccess time.Time - refCount int32 -} - -func NewExpiringMap[K comparable, V any](expiryTime time.Duration) *ExpiringMap[K, V] { - em := &ExpiringMap[K, V]{ - data: make(map[K]*item[V]), - expiryTime: expiryTime, - done: make(chan struct{}), - } - go em.cleanupLoop() - return em -} - -func (em *ExpiringMap[K, V]) Set(key K, value V) { - em.mu.Lock() - defer em.mu.Unlock() - - em.data[key] = &item[V]{ - value: value, - lastAccess: time.Now(), - refCount: 0, - } -} - -func (em *ExpiringMap[K, V]) Get(key K) (V, bool) { - em.mu.RLock() - item, ok := em.data[key] - if !ok { - em.mu.RUnlock() - var zeroValue V - return zeroValue, false - } - - atomic.AddInt32(&item.refCount, 1) - em.mu.RUnlock() - - em.mu.Lock() - defer em.mu.Unlock() - - atomic.AddInt32(&item.refCount, -1) - - item.lastAccess = time.Now() - return item.value, true -} - -func (em *ExpiringMap[K, V]) cleanup() { - em.mu.Lock() - defer em.mu.Unlock() - - for key, item := range em.data { - if time.Since(item.lastAccess) > em.expiryTime && atomic.LoadInt32(&item.refCount) == 0 { - delete(em.data, key) - } - } -} - -func (em *ExpiringMap[K, V]) cleanupLoop() { - ticker := time.NewTicker(em.expiryTime / 2) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - em.cleanup() - case <-em.done: - return - } - } -} - -func (em *ExpiringMap[K, V]) StopCleanup() { - close(em.done) -} diff --git a/cmd/outline-ss-server/expiring_map_test.go b/cmd/outline-ss-server/expiring_map_test.go deleted file mode 100644 index 7560ef45..00000000 --- a/cmd/outline-ss-server/expiring_map_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2024 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestExpiringMap(t *testing.T) { - t.Run("BasicSetGet", func(t *testing.T) { - em := NewExpiringMap[string, int](2 * time.Second) - em.Set("key1", 10) - val, ok := em.Get("key1") - require.True(t, ok) - require.Equal(t, 10, val) - - time.Sleep(3 * time.Second) - - _, ok = em.Get("key1") - require.False(t, ok) - }) - - t.Run("Expiration", func(t *testing.T) { - em := NewExpiringMap[string, int](2 * time.Second) - em.Set("a", 1) - - time.Sleep(3 * time.Second) - - _, ok := em.Get("a") - require.False(t, ok, "Expected `a` to have been evicted") - em.StopCleanup() - }) - - t.Run("Concurrency", func(t *testing.T) { - em := NewExpiringMap[int, string](2 * time.Second) - var wg sync.WaitGroup - for i := 0; i < 100; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - em.Set(i, fmt.Sprintf("value%d", i)) - - time.Sleep(time.Duration(i) * time.Millisecond) - - val, ok := em.Get(i) - require.True(t, ok) - require.Equal(t, fmt.Sprintf("value%d", i), val) - }(i) - } - wg.Wait() - }) -} - -func BenchmarkExpiringMap(b *testing.B) { - b.Run("Set", func(b *testing.B) { - em := NewExpiringMap[int64, int64](10 * time.Second) - for i := 0; i < b.N; i++ { - em.Set(int64(i), int64(i)) - } - }) - - b.Run("Get", func(b *testing.B) { - em := NewExpiringMap[int64, int64](10 * time.Second) - for i := 0; i < b.N; i++ { - em.Set(int64(i), int64(i)) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - em.Get(int64(i)) - } - }) -} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index a00de5cd..6204f7e1 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -16,6 +16,7 @@ package main import ( "container/list" + "context" "flag" "fmt" "log/slog" @@ -26,6 +27,7 @@ import ( "syscall" "time" + "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" @@ -74,7 +76,7 @@ func (s *SSServer) startPort(portNum int) error { //lint:ignore ST1005 Shadowsocks is capitalized. return fmt.Errorf("Shadowsocks TCP service failed to start on port %v: %w", portNum, err) } - slog.Info("Shadowsocks TCP service started.", "address", listener.Addr().String()) + slog.Info("Shadowsocks TCP service stamed.", "address", listener.Addr().String()) packetConn, err := net.ListenUDP("udp", &net.UDPAddr{Port: portNum}) if err != nil { //lint:ignore ST1005 Shadowsocks is capitalized. @@ -82,12 +84,15 @@ func (s *SSServer) startPort(portNum int) error { } slog.Info("Shadowsocks UDP service started.", "address", packetConn.LocalAddr().String()) port := &ssPort{tcpListener: listener, packetConn: packetConn, cipherList: service.NewCipherList()} - authFunc := service.NewShadowsocksStreamAuthenticator(port.cipherList, &s.replayCache, s.m) + authFunc := service.NewShadowsocksStreamAuthenticator(port.cipherList, &s.replayCache, s.m.tcpCollector) // TODO: Register initial data metrics at zero. - tcpHandler := service.NewTCPHandler(authFunc, s.m, tcpReadTimeout) - packetHandler := service.NewPacketHandler(s.natTimeout, port.cipherList, s.m) + tcpHandler := service.NewTCPHandler(authFunc, tcpReadTimeout) + packetHandler := service.NewPacketHandler(s.natTimeout, port.cipherList, s.m, s.m.udpCollector) s.ports[portNum] = port - go service.StreamServe(service.WrapStreamListener(listener.AcceptTCP), tcpHandler.Handle) + go service.StreamServe(service.WrapStreamListener(listener.AcceptTCP), func(ctx context.Context, conn transport.StreamConn) { + connMetrics := s.m.AddOpenTCPConnection(conn) + tcpHandler.Handle(ctx, conn, connMetrics) + }) go packetHandler.Handle(port.packetConn) return nil } @@ -273,7 +278,10 @@ func main() { } defer ip2info.Close() - metrics := newPrometheusOutlineMetrics(ip2info) + metrics, err := newPrometheusOutlineMetrics(ip2info) + if err != nil { + slog.Error("Failed to create Outline Prometheus metrics. Aborting.", "err", err) + } metrics.SetBuildInfo(version) r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) r.MustRegister(metrics) diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index f150a1fa..ae9808c1 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -31,20 +31,138 @@ import ( // `now` is stubbable for testing. var now = time.Now +func NewTimeToCipherVec(proto string) (prometheus.ObserverVec, error) { + vec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "time_to_cipher_ms", + Help: "Time needed to find the cipher", + Buckets: []float64{0.1, 1, 10, 100, 1000}, + }, []string{"proto", "found_key"}) + return vec.CurryWith(map[string]string{"proto": proto}) +} + +type proxyCollector struct { + // NOTE: New metrics need to be added to `newProxyCollector()`, `Describe()` and `Collect()`. + dataBytesPerKey *prometheus.CounterVec + dataBytesPerLocation *prometheus.CounterVec +} + +func newProxyCollector(proto string) (*proxyCollector, error) { + dataBytesPerKey, err := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "data_bytes", + Help: "Bytes transferred by the proxy, per access key", + }, []string{"proto", "dir", "access_key"}).CurryWith(map[string]string{"proto": proto}) + if err != nil { + return nil, err + } + dataBytesPerLocation, err := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "data_bytes_per_location", + Help: "Bytes transferred by the proxy, per location", + }, []string{"proto", "dir", "location", "asn"}).CurryWith(map[string]string{"proto": proto}) + if err != nil { + return nil, err + } + return &proxyCollector{ + dataBytesPerKey: dataBytesPerKey, + dataBytesPerLocation: dataBytesPerLocation, + }, nil +} + +func (c *proxyCollector) Describe(ch chan<- *prometheus.Desc) { + c.dataBytesPerKey.Describe(ch) + c.dataBytesPerLocation.Describe(ch) +} + +func (c *proxyCollector) Collect(ch chan<- prometheus.Metric) { + c.dataBytesPerKey.Collect(ch) + c.dataBytesPerLocation.Collect(ch) +} + +func (c *proxyCollector) addOutbound(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { + addIfNonZero(clientProxyBytes, c.dataBytesPerKey, "c>p", accessKey) + addIfNonZero(clientProxyBytes, c.dataBytesPerLocation, "c>p", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) + addIfNonZero(proxyTargetBytes, c.dataBytesPerKey, "p>t", accessKey) + addIfNonZero(proxyTargetBytes, c.dataBytesPerLocation, "p>t", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) +} + +func (c *proxyCollector) addInbound(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { + addIfNonZero(targetProxyBytes, c.dataBytesPerKey, "pp", "tcp", accessKey) - addIfNonZero(data.ClientProxy, m.dataBytesPerLocation, "c>p", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) - addIfNonZero(data.ProxyTarget, m.dataBytes, "p>t", "tcp", accessKey) - addIfNonZero(data.ProxyTarget, m.dataBytesPerLocation, "p>t", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) - addIfNonZero(data.TargetProxy, m.dataBytes, "pp", "udp", accessKey) - addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) - addIfNonZero(int64(proxyTargetBytes), m.dataBytes, "p>t", "udp", accessKey) - addIfNonZero(int64(proxyTargetBytes), m.dataBytesPerLocation, "p>t", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) -} - -func (m *outlineMetricsCollector) AddUDPPacketFromTarget(clientAddr net.Addr, accessKey, status string, targetProxyBytes, proxyClientBytes int) { - clientInfo := m.getIPInfoFromAddr(clientAddr) - addIfNonZero(int64(targetProxyBytes), m.dataBytes, "p %d)", report.clientProxyBytes, report.proxyTargetBytes) assert.Equal(t, "id-0", report.accessKey, "Unexpected access key name: %s", report.accessKey) @@ -476,7 +482,7 @@ func TestUDPEarlyClose(t *testing.T) { } testMetrics := &natTestMetrics{} const testTimeout = 200 * time.Millisecond - s := NewPacketHandler(testTimeout, cipherList, testMetrics) + s := NewPacketHandler(testTimeout, cipherList, testMetrics, &fakeShadowsocksMetrics{}) clientConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0}) if err != nil { From dcc47cfd54ef707aa79e269f2ddcc7dfed97dbeb Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 28 Aug 2024 11:25:28 -0400 Subject: [PATCH 133/182] Address review comments. --- cmd/outline-ss-server/main.go | 4 +- cmd/outline-ss-server/metrics.go | 170 +++++++++--------- cmd/outline-ss-server/metrics_test.go | 10 +- internal/integration_test/integration_test.go | 2 +- service/tcp.go | 16 +- service/tcp_test.go | 2 +- 6 files changed, 103 insertions(+), 101 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 9eabd38d..22d280d5 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -87,13 +87,13 @@ func (s *SSServer) loadConfig(filename string) error { } func (s *SSServer) NewShadowsocksStreamHandler(ciphers service.CipherList) service.StreamHandler { - authFunc := service.NewShadowsocksStreamAuthenticator(ciphers, &s.replayCache, s.m.tcpCollector) + authFunc := service.NewShadowsocksStreamAuthenticator(ciphers, &s.replayCache, s.m.tcpServiceMetrics) // TODO: Register initial data metrics at zero. return service.NewStreamHandler(authFunc, tcpReadTimeout) } func (s *SSServer) NewShadowsocksPacketHandler(ciphers service.CipherList) service.PacketHandler { - return service.NewPacketHandler(s.natTimeout, ciphers, s.m, s.m.udpCollector) + return service.NewPacketHandler(s.natTimeout, ciphers, s.m, s.m.udpServiceMetrics) } type listenerSet struct { diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index ae9808c1..000f1f18 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -80,14 +80,14 @@ func (c *proxyCollector) Collect(ch chan<- prometheus.Metric) { c.dataBytesPerLocation.Collect(ch) } -func (c *proxyCollector) addOutbound(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { +func (c *proxyCollector) addClientTarget(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { addIfNonZero(clientProxyBytes, c.dataBytesPerKey, "c>p", accessKey) addIfNonZero(clientProxyBytes, c.dataBytesPerLocation, "c>p", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) addIfNonZero(proxyTargetBytes, c.dataBytesPerKey, "p>t", accessKey) addIfNonZero(proxyTargetBytes, c.dataBytesPerLocation, "p>t", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)) } -func (c *proxyCollector) addInbound(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { +func (c *proxyCollector) addTargetClient(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { addIfNonZero(targetProxyBytes, c.dataBytesPerKey, "p Date: Wed, 28 Aug 2024 12:37:56 -0400 Subject: [PATCH 134/182] Make metrics interfaces for bytes consistently use `int64`. --- cmd/outline-ss-server/metrics.go | 8 ++++---- cmd/outline-ss-server/metrics_test.go | 4 ++-- internal/integration_test/integration_test.go | 10 +++++----- service/udp.go | 12 ++++++------ service/udp_test.go | 8 ++++---- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 000f1f18..fa35cf82 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -263,12 +263,12 @@ func newUDPConnMetrics(udpServiceMetrics *udpServiceMetrics, tunnelTimeMetrics * } } -func (cm *udpConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int) { - cm.udpServiceMetrics.addPacketFromClient(status, int64(clientProxyBytes), int64(proxyTargetBytes), cm.accessKey, cm.clientInfo) +func (cm *udpConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int64) { + cm.udpServiceMetrics.addPacketFromClient(status, clientProxyBytes, proxyTargetBytes, cm.accessKey, cm.clientInfo) } -func (cm *udpConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int) { - cm.udpServiceMetrics.addPacketFromTarget(status, int64(targetProxyBytes), int64(proxyClientBytes), cm.accessKey, cm.clientInfo) +func (cm *udpConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int64) { + cm.udpServiceMetrics.addPacketFromTarget(status, targetProxyBytes, proxyClientBytes, cm.accessKey, cm.clientInfo) } func (cm *udpConnMetrics) RemoveNatEntry() { diff --git a/cmd/outline-ss-server/metrics_test.go b/cmd/outline-ss-server/metrics_test.go index 10e64238..44e1887b 100644 --- a/cmd/outline-ss-server/metrics_test.go +++ b/cmd/outline-ss-server/metrics_test.go @@ -197,7 +197,7 @@ func BenchmarkClientUDP(b *testing.B) { accessKey := "key 1" udpMetrics := ssMetrics.AddUDPNatEntry(addr, accessKey) status := "OK" - size := 1000 + size := int64(1000) b.ResetTimer() for i := 0; i < b.N; i++ { udpMetrics.AddPacketFromClient(status, size, size) @@ -210,7 +210,7 @@ func BenchmarkTargetUDP(b *testing.B) { accessKey := "key 1" udpMetrics := ssMetrics.AddUDPNatEntry(addr, accessKey) status := "OK" - size := 1000 + size := int64(1000) b.ResetTimer() for i := 0; i < b.N; i++ { udpMetrics.AddPacketFromTarget(status, size, size) diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index e59e86b0..f847f761 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -261,7 +261,7 @@ func TestRestrictedAddresses(t *testing.T) { type udpRecord struct { clientAddr net.Addr accessKey, status string - in, out int + in, out int64 } type fakeUDPConnMetrics struct { @@ -272,10 +272,10 @@ type fakeUDPConnMetrics struct { var _ service.UDPConnMetrics = (*fakeUDPConnMetrics)(nil) -func (m *fakeUDPConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int) { +func (m *fakeUDPConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int64) { m.up = append(m.up, udpRecord{m.clientAddr, m.accessKey, status, clientProxyBytes, proxyTargetBytes}) } -func (m *fakeUDPConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int) { +func (m *fakeUDPConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int64) { m.down = append(m.down, udpRecord{m.clientAddr, m.accessKey, status, targetProxyBytes, proxyClientBytes}) } func (m *fakeUDPConnMetrics) RemoveNatEntry() { @@ -372,7 +372,7 @@ func TestUDPEcho(t *testing.T) { require.Equal(t, keyID, record.accessKey, "Bad upstream metrics") require.Equal(t, "OK", record.status, "Bad upstream metrics") require.Greater(t, record.in, record.out, "Bad upstream metrics") - require.Equal(t, N, record.out, "Bad upstream metrics") + require.Equal(t, int64(N), record.out, "Bad upstream metrics") } if len(testMetrics.connMetrics[0].down) != 1 { t.Errorf("Wrong number of packets received: %v", testMetrics.connMetrics[0].down) @@ -382,7 +382,7 @@ func TestUDPEcho(t *testing.T) { require.Equal(t, keyID, record.accessKey, "Bad downstream metrics") require.Equal(t, "OK", record.status, "Bad downstream metrics") require.Greater(t, record.out, record.in, "Bad downstream metrics") - require.Equal(t, N, record.in, "Bad downstream metrics") + require.Equal(t, int64(N), record.in, "Bad downstream metrics") } } diff --git a/service/udp.go b/service/udp.go index 19fc43b9..2d08a709 100644 --- a/service/udp.go +++ b/service/udp.go @@ -31,8 +31,8 @@ import ( // UDPConnMetrics is used to report metrics on UDP connections. type UDPConnMetrics interface { - AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int) - AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int) + AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int64) + AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int64) RemoveNatEntry() } @@ -199,7 +199,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { status = connError.Status } if targetConn != nil { - targetConn.metrics.AddPacketFromClient(status, clientProxyBytes, proxyTargetBytes) + targetConn.metrics.AddPacketFromClient(status, int64(clientProxyBytes), int64(proxyTargetBytes)) } } } @@ -443,7 +443,7 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco if expired { break } - targetConn.metrics.AddPacketFromTarget(status, bodyLen, proxyClientBytes) + targetConn.metrics.AddPacketFromTarget(status, int64(bodyLen), int64(proxyClientBytes)) } } @@ -453,9 +453,9 @@ type NoOpUDPConnMetrics struct{} var _ UDPConnMetrics = (*NoOpUDPConnMetrics)(nil) -func (m *NoOpUDPConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int) { +func (m *NoOpUDPConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int64) { } -func (m *NoOpUDPConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int) { +func (m *NoOpUDPConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int64) { } func (m *NoOpUDPConnMetrics) RemoveNatEntry() {} diff --git a/service/udp_test.go b/service/udp_test.go index 933d080c..8ba00af3 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -94,7 +94,7 @@ func (conn *fakePacketConn) Close() error { type udpReport struct { clientAddr net.Addr accessKey, status string - clientProxyBytes, proxyTargetBytes int + clientProxyBytes, proxyTargetBytes int64 } // Stub metrics implementation for testing NAT behaviors. @@ -106,10 +106,10 @@ type fakeUDPConnMetrics struct { var _ UDPConnMetrics = (*fakeUDPConnMetrics)(nil) -func (m *fakeUDPConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int) { +func (m *fakeUDPConnMetrics) AddPacketFromClient(status string, clientProxyBytes, proxyTargetBytes int64) { m.upstreamPackets = append(m.upstreamPackets, udpReport{m.clientAddr, m.accessKey, status, clientProxyBytes, proxyTargetBytes}) } -func (m *fakeUDPConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int) { +func (m *fakeUDPConnMetrics) AddPacketFromTarget(status string, targetProxyBytes, proxyClientBytes int64) { } func (m *fakeUDPConnMetrics) RemoveNatEntry() { } @@ -191,7 +191,7 @@ func TestUpstreamMetrics(t *testing.T) { assert.Equal(t, N, len(metrics.connMetrics[0].upstreamPackets), "Expected %d reports, not %v", N, metrics.connMetrics[0].upstreamPackets) for i, report := range metrics.connMetrics[0].upstreamPackets { - assert.Equal(t, i+1, report.proxyTargetBytes, "Expected %d payload bytes, not %d", i+1, report.proxyTargetBytes) + assert.Equal(t, int64(i+1), report.proxyTargetBytes, "Expected %d payload bytes, not %d", i+1, report.proxyTargetBytes) assert.Greater(t, report.clientProxyBytes, report.proxyTargetBytes, "Expected nonzero input overhead (%d > %d)", report.clientProxyBytes, report.proxyTargetBytes) assert.Equal(t, "id-0", report.accessKey, "Unexpected access key name: %s", report.accessKey) assert.Equal(t, "OK", report.status, "Wrong status: %s", report.status) From cf5a6760be1ccfdfe8ae2954adaeffaa1b01d032 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 28 Aug 2024 12:45:48 -0400 Subject: [PATCH 135/182] Add license header. --- .../config_example.deprecated.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/outline-ss-server/config_example.deprecated.yml b/cmd/outline-ss-server/config_example.deprecated.yml index 8895b86d..1186d555 100644 --- a/cmd/outline-ss-server/config_example.deprecated.yml +++ b/cmd/outline-ss-server/config_example.deprecated.yml @@ -1,3 +1,17 @@ +# Copyright 2024 The Outline Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + keys: - id: user-0 port: 9000 From 0e55a6f01614df11b54a1ef294eeb12d63f81ae8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 28 Aug 2024 21:44:32 -0400 Subject: [PATCH 136/182] Support multi-module workspaces so we can develop Caddy and ss-server at the same time. --- .github/workflows/go.yml | 2 +- .gitignore | 4 ++++ .goreleaser.yml | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 24822c5e..e05d7899 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -46,7 +46,7 @@ jobs: git submodule update --init - name: Build - run: go build -v ./... + run: GOWORK=off go build -v ./... - name: Test run: go test -race -benchmem -bench=. ./... -benchtime=100ms diff --git a/.gitignore b/.gitignore index 82e9d403..7fc155d1 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ # Go task /.task/ + +# Go workspace +go.work +go.work.sum diff --git a/.goreleaser.yml b/.goreleaser.yml index 29842233..b07a73b4 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -21,6 +21,7 @@ builds: main: ./cmd/outline-ss-server env: - CGO_ENABLED=0 + - GOWORK=off goos: - darwin - windows From 74d3c671e2514a03c160ed6adb9dea30d259d395 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 28 Aug 2024 22:56:19 -0400 Subject: [PATCH 137/182] Rename `Collector` to `Metrics`. --- cmd/outline-ss-server/main.go | 4 ++-- cmd/outline-ss-server/metrics.go | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 22d280d5..2bd9c8b7 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -63,7 +63,7 @@ type SSServer struct { stopConfig func() error lnManager service.ListenerManager natTimeout time.Duration - m *outlineMetricsCollector + m *outlineMetrics replayCache service.ReplayCache } @@ -248,7 +248,7 @@ func (s *SSServer) Stop() error { } // RunSSServer starts a shadowsocks server running, and returns the server or an error. -func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetricsCollector, replayHistory int) (*SSServer, error) { +func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ lnManager: service.NewListenerManager(), natTimeout: natTimeout, diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 85d99024..c5641411 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -467,7 +467,7 @@ func (c *tunnelTimeMetrics) stopConnection(ipKey IPKey) { } } -type outlineMetricsCollector struct { +type outlineMetrics struct { ip2info ipinfo.IPInfoMap tcpServiceMetrics *tcpServiceMetrics @@ -481,12 +481,12 @@ type outlineMetricsCollector struct { // TODO: Add time to first byte. } -var _ prometheus.Collector = (*outlineMetricsCollector)(nil) -var _ service.UDPMetrics = (*outlineMetricsCollector)(nil) +var _ prometheus.Collector = (*outlineMetrics)(nil) +var _ service.UDPMetrics = (*outlineMetrics)(nil) // newPrometheusOutlineMetrics constructs a Prometheus metrics collector that uses // `ip2info` to convert IP addresses to countries. `ip2info` may be nil. -func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) (*outlineMetricsCollector, error) { +func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) (*outlineMetrics, error) { tcpServiceMetrics, err := newTCPCollector() if err != nil { return nil, err @@ -497,7 +497,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) (*outlineMetricsColle } tunnelTimeMetrics := newTunnelTimeMetrics(ip2info) - return &outlineMetricsCollector{ + return &outlineMetrics{ ip2info: ip2info, tcpServiceMetrics: tcpServiceMetrics, @@ -519,7 +519,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap) (*outlineMetricsColle }, nil } -func (m *outlineMetricsCollector) Describe(ch chan<- *prometheus.Desc) { +func (m *outlineMetrics) Describe(ch chan<- *prometheus.Desc) { m.tcpServiceMetrics.Describe(ch) m.udpServiceMetrics.Describe(ch) m.tunnelTimeMetrics.Describe(ch) @@ -528,7 +528,7 @@ func (m *outlineMetricsCollector) Describe(ch chan<- *prometheus.Desc) { m.ports.Describe(ch) } -func (m *outlineMetricsCollector) Collect(ch chan<- prometheus.Metric) { +func (m *outlineMetrics) Collect(ch chan<- prometheus.Metric) { m.tcpServiceMetrics.Collect(ch) m.udpServiceMetrics.Collect(ch) m.tunnelTimeMetrics.Collect(ch) @@ -537,7 +537,7 @@ func (m *outlineMetricsCollector) Collect(ch chan<- prometheus.Metric) { m.ports.Collect(ch) } -func (m *outlineMetricsCollector) getIPInfoFromAddr(addr net.Addr) ipinfo.IPInfo { +func (m *outlineMetrics) getIPInfoFromAddr(addr net.Addr) ipinfo.IPInfo { ipInfo, err := ipinfo.GetIPInfoFromAddr(m.ip2info, addr) if err != nil { slog.LogAttrs(nil, slog.LevelWarn, "Failed client info lookup.", slog.Any("err", err)) @@ -549,22 +549,22 @@ func (m *outlineMetricsCollector) getIPInfoFromAddr(addr net.Addr) ipinfo.IPInfo return ipInfo } -func (m *outlineMetricsCollector) SetBuildInfo(version string) { +func (m *outlineMetrics) SetBuildInfo(version string) { m.buildInfo.WithLabelValues(version).Set(1) } -func (m *outlineMetricsCollector) SetNumAccessKeys(numKeys int, ports int) { +func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) { m.accessKeys.Set(float64(numKeys)) m.ports.Set(float64(ports)) } -func (m *outlineMetricsCollector) AddOpenTCPConnection(clientConn net.Conn) *tcpConnMetrics { +func (m *outlineMetrics) AddOpenTCPConnection(clientConn net.Conn) *tcpConnMetrics { clientAddr := clientConn.RemoteAddr() clientInfo := m.getIPInfoFromAddr(clientAddr) return newTCPConnMetrics(m.tcpServiceMetrics, m.tunnelTimeMetrics, clientConn, clientInfo) } -func (m *outlineMetricsCollector) AddUDPNatEntry(clientAddr net.Addr, accessKey string) service.UDPConnMetrics { +func (m *outlineMetrics) AddUDPNatEntry(clientAddr net.Addr, accessKey string) service.UDPConnMetrics { clientInfo := m.getIPInfoFromAddr(clientAddr) return newUDPConnMetrics(m.udpServiceMetrics, m.tunnelTimeMetrics, accessKey, clientAddr, clientInfo) } From 7c061e4f6a392f1277000399ea580eab3dca1e0e Mon Sep 17 00:00:00 2001 From: sbruens Date: Sat, 31 Aug 2024 00:57:06 -0400 Subject: [PATCH 138/182] Move service creation into the service package so it can be re-used by Caddy. --- cmd/outline-ss-server/config_test.go | 167 ----- cmd/outline-ss-server/main.go | 145 ++--- cmd/outline-ss-server/main_test.go | 113 ++++ cmd/outline-ss-server/metrics.go | 548 +---------------- {cmd/outline-ss-server => service}/config.go | 30 +- service/config_test.go | 89 +++ service/metrics.go | 573 ++++++++++++++++++ .../metrics_test.go | 2 +- service/service.go | 175 ++++++ 9 files changed, 1019 insertions(+), 823 deletions(-) delete mode 100644 cmd/outline-ss-server/config_test.go create mode 100644 cmd/outline-ss-server/main_test.go rename {cmd/outline-ss-server => service}/config.go (79%) create mode 100644 service/config_test.go create mode 100644 service/metrics.go rename {cmd/outline-ss-server => service}/metrics_test.go (99%) create mode 100644 service/service.go diff --git a/cmd/outline-ss-server/config_test.go b/cmd/outline-ss-server/config_test.go deleted file mode 100644 index f183ff5a..00000000 --- a/cmd/outline-ss-server/config_test.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2024 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestValidateConfigFails(t *testing.T) { - tests := []struct { - name string - cfg *Config - }{ - { - name: "WithUnknownListenerType", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: "foo", Address: "[::]:9000"}, - }, - }, - }, - }, - }, - { - name: "WithInvalidListenerAddress", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "tcp/[::]:9000"}, - }, - }, - }, - }, - }, - { - name: "WithHostnameAddress", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "example.com:9000"}, - }, - }, - }, - }, - }, - { - name: "WithDuplicateListeners", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, - }, - }, - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, - }, - }, - }, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.cfg.Validate() - require.Error(t, err) - }) - } -} - -func TestReadConfig(t *testing.T) { - config, err := readConfigFile("./config_example.yml") - - require.NoError(t, err) - expected := Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, - ListenerConfig{Type: listenerTypeUDP, Address: "[::]:9000"}, - }, - Keys: []KeyConfig{ - KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, - KeyConfig{"user-1", "chacha20-ietf-poly1305", "Secret1"}, - }, - }, - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9001"}, - ListenerConfig{Type: listenerTypeUDP, Address: "[::]:9001"}, - }, - Keys: []KeyConfig{ - KeyConfig{"user-2", "chacha20-ietf-poly1305", "Secret2"}, - }, - }, - }, - } - require.Equal(t, expected, *config) -} - -func TestReadConfigParsesDeprecatedFormat(t *testing.T) { - config, err := readConfigFile("./config_example.deprecated.yml") - - require.NoError(t, err) - expected := Config{ - Keys: []LegacyKeyServiceConfig{ - LegacyKeyServiceConfig{ - KeyConfig: KeyConfig{ID: "user-0", Cipher: "chacha20-ietf-poly1305", Secret: "Secret0"}, - Port: 9000, - }, - LegacyKeyServiceConfig{ - KeyConfig: KeyConfig{ID: "user-1", Cipher: "chacha20-ietf-poly1305", Secret: "Secret1"}, - Port: 9000, - }, - LegacyKeyServiceConfig{ - KeyConfig: KeyConfig{ID: "user-2", Cipher: "chacha20-ietf-poly1305", Secret: "Secret2"}, - Port: 9001, - }, - }, - } - require.Equal(t, expected, *config) -} - -func TestReadConfigFromEmptyFile(t *testing.T) { - file, _ := os.CreateTemp("", "empty.yaml") - - config, err := readConfigFile(file.Name()) - - require.NoError(t, err) - require.ElementsMatch(t, Config{}, config) -} - -func TestReadConfigFromIncorrectFormatFails(t *testing.T) { - file, _ := os.CreateTemp("", "empty.yaml") - file.WriteString("foo") - - config, err := readConfigFile(file.Name()) - - require.Error(t, err) - require.ElementsMatch(t, Config{}, config) -} - -func readConfigFile(filename string) (*Config, error) { - configData, _ := os.ReadFile(filename) - return readConfig(configData) -} diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index e61150c7..64d5145d 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -16,7 +16,6 @@ package main import ( "container/list" - "context" "flag" "fmt" "log/slog" @@ -29,7 +28,6 @@ import ( "syscall" "time" - "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" "github.com/Jigsaw-Code/outline-ss-server/service" @@ -59,11 +57,12 @@ func init() { } type SSServer struct { - stopConfig func() error - lnManager service.ListenerManager - natTimeout time.Duration - m *outlineMetrics - replayCache service.ReplayCache + stopConfig func() error + lnManager service.ListenerManager + natTimeout time.Duration + serverMetrics *serverMetrics + serviceMetrics service.ServiceMetrics + replayCache service.ReplayCache } func (s *SSServer) loadConfig(filename string) error { @@ -71,8 +70,8 @@ func (s *SSServer) loadConfig(filename string) error { if err != nil { return fmt.Errorf("failed to read config file %s: %w", filename, err) } - config, err := readConfig(configData) - if err != nil { + config := &service.Config{} + if err := config.LoadFrom(configData); err != nil { return fmt.Errorf("failed to load config (%v): %w", filename, err) } if err := config.Validate(); err != nil { @@ -93,59 +92,6 @@ func (s *SSServer) loadConfig(filename string) error { return nil } -func newCipherListFromConfig(config ServiceConfig) (service.CipherList, error) { - type cipherKey struct { - cipher string - secret string - } - cipherList := list.New() - existingCiphers := make(map[cipherKey]bool) - for _, keyConfig := range config.Keys { - key := cipherKey{keyConfig.Cipher, keyConfig.Secret} - if _, exists := existingCiphers[key]; exists { - slog.Debug("Encryption key already exists. Skipping.", "id", keyConfig.ID) - continue - } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) - if err != nil { - return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) - } - entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - cipherList.PushBack(&entry) - existingCiphers[key] = true - } - ciphers := service.NewCipherList() - ciphers.Update(cipherList) - - return ciphers, nil -} - -func (s *SSServer) NewShadowsocksStreamHandler(ciphers service.CipherList) service.StreamHandler { - authFunc := service.NewShadowsocksStreamAuthenticator(ciphers, &s.replayCache, s.m.tcpServiceMetrics) - // TODO: Register initial data metrics at zero. - return service.NewStreamHandler(authFunc, tcpReadTimeout) -} - -func (s *SSServer) NewShadowsocksPacketHandler(ciphers service.CipherList) service.PacketHandler { - return service.NewPacketHandler(s.natTimeout, ciphers, s.m, s.m.udpServiceMetrics) -} - -func (s *SSServer) NewShadowsocksStreamHandlerFromConfig(config ServiceConfig) (service.StreamHandler, error) { - ciphers, err := newCipherListFromConfig(config) - if err != nil { - return nil, err - } - return s.NewShadowsocksStreamHandler(ciphers), nil -} - -func (s *SSServer) NewShadowsocksPacketHandlerFromConfig(config ServiceConfig) (service.PacketHandler, error) { - ciphers, err := newCipherListFromConfig(config) - if err != nil { - return nil, err - } - return s.NewShadowsocksPacketHandler(ciphers), nil -} - type listenerSet struct { manager service.ListenerManager listenerCloseFuncs map[string]func() error @@ -207,7 +153,7 @@ func (ls *listenerSet) Len() int { return len(ls.listenerCloseFuncs) } -func (s *SSServer) runConfig(config Config) (func() error, error) { +func (s *SSServer) runConfig(config service.Config) (func() error, error) { startErrCh := make(chan error) stopErrCh := make(chan error) stopCh := make(chan struct{}) @@ -243,69 +189,59 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { ciphers := service.NewCipherList() ciphers.Update(cipherList) - sh := s.NewShadowsocksStreamHandler(ciphers) + ssService, err := service.NewService( + service.WithCiphers(ciphers), + service.WithNatTimeout(s.natTimeout), + service.WithMetrics(s.serviceMetrics), + service.WithReplayCache(&s.replayCache), + ) ln, err := lnSet.ListenStream(addr) if err != nil { return err } slog.Info("TCP service started.", "address", ln.Addr().String()) - go service.StreamServe(ln.AcceptStream, func(ctx context.Context, conn transport.StreamConn) { - connMetrics := s.m.AddOpenTCPConnection(conn) - sh.Handle(ctx, conn, connMetrics) - }) + go service.StreamServe(ln.AcceptStream, ssService.HandleStream) pc, err := lnSet.ListenPacket(addr) if err != nil { return err } slog.Info("UDP service started.", "address", pc.LocalAddr().String()) - ph := s.NewShadowsocksPacketHandler(ciphers) - go ph.Handle(pc) + go ssService.HandlePacket(pc) } for _, serviceConfig := range config.Services { - var ( - sh service.StreamHandler - ph service.PacketHandler + ssService, err := service.NewService( + service.WithConfig(serviceConfig), + service.WithMetrics(s.serviceMetrics), + service.WithReplayCache(&s.replayCache), ) + if err != nil { + return err + } for _, lnConfig := range serviceConfig.Listeners { switch lnConfig.Type { - case listenerTypeTCP: + case service.ListenerTypeTCP: ln, err := lnSet.ListenStream(lnConfig.Address) if err != nil { return err } slog.Info("TCP service started.", "address", ln.Addr().String()) - if sh == nil { - sh, err = s.NewShadowsocksStreamHandlerFromConfig(serviceConfig) - if err != nil { - return err - } - } - go service.StreamServe(ln.AcceptStream, func(ctx context.Context, conn transport.StreamConn) { - connMetrics := s.m.AddOpenTCPConnection(conn) - sh.Handle(ctx, conn, connMetrics) - }) - case listenerTypeUDP: + go service.StreamServe(ln.AcceptStream, ssService.HandleStream) + case service.ListenerTypeUDP: pc, err := lnSet.ListenPacket(lnConfig.Address) if err != nil { return err } slog.Info("UDP service started.", "address", pc.LocalAddr().String()) - if ph == nil { - ph, err = s.NewShadowsocksPacketHandlerFromConfig(serviceConfig) - if err != nil { - return err - } - } - go ph.Handle(pc) + go ssService.HandlePacket(pc) } } totalCipherCount += len(serviceConfig.Keys) } slog.Info("Loaded config.", "access_keys", totalCipherCount, "listeners", lnSet.Len()) - s.m.SetNumAccessKeys(totalCipherCount, lnSet.Len()) + s.serverMetrics.SetNumAccessKeys(totalCipherCount, lnSet.Len()) return nil }() @@ -341,12 +277,13 @@ func (s *SSServer) Stop() error { } // RunSSServer starts a shadowsocks server running, and returns the server or an error. -func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { +func RunSSServer(filename string, natTimeout time.Duration, serverMetrics *serverMetrics, serviceMetrics service.ServiceMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ - lnManager: service.NewListenerManager(), - natTimeout: natTimeout, - m: sm, - replayCache: service.NewReplayCache(replayHistory), + lnManager: service.NewListenerManager(), + natTimeout: natTimeout, + serverMetrics: serverMetrics, + serviceMetrics: serviceMetrics, + replayCache: service.NewReplayCache(replayHistory), } err := server.loadConfig(filename) if err != nil { @@ -424,14 +361,16 @@ func main() { } defer ip2info.Close() - metrics, err := newPrometheusOutlineMetrics(ip2info) + serverMetrics := newPrometheusServerMetrics() + serverMetrics.SetVersion(version) + serviceMetrics, err := service.NewPrometheusServiceMetrics(ip2info) if err != nil { - slog.Error("Failed to create Outline Prometheus metrics. Aborting.", "err", err) + slog.Error("Failed to create Outline Prometheus service metrics. Aborting.", "err", err) } - metrics.SetBuildInfo(version) r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) - r.MustRegister(metrics) - _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, metrics, flags.replayHistory) + r.MustRegister(serverMetrics, serviceMetrics) + + _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, serverMetrics, serviceMetrics, flags.replayHistory) if err != nil { slog.Error("Server failed to start. Aborting.", "err", err) } diff --git a/cmd/outline-ss-server/main_test.go b/cmd/outline-ss-server/main_test.go new file mode 100644 index 00000000..60587cbd --- /dev/null +++ b/cmd/outline-ss-server/main_test.go @@ -0,0 +1,113 @@ +// Copyright 2020 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "testing" + "time" + + "github.com/Jigsaw-Code/outline-ss-server/service" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestRunSSServer(t *testing.T) { + m := service.NewPrometheusOutlineMetrics(nil, prometheus.DefaultRegisterer) + server, err := RunSSServer("config_example.yml", m, 30*time.Second, 10000) + if err != nil { + t.Fatalf("RunSSServer() error = %v", err) + } + if err := server.Stop(); err != nil { + t.Errorf("Error while stopping server: %v", err) + } +} + +func TestReadConfig(t *testing.T) { + config, err := readConfigFile("./config_example.yml") + + require.NoError(t, err) + expected := service.Config{ + Services: []service.ServiceConfig{ + service.ServiceConfig{ + Listeners: []service.ListenerConfig{ + service.ListenerConfig{Type: "tcp", Address: "[::]:9000"}, + service.ListenerConfig{Type: "udp", Address: "[::]:9000"}, + }, + Keys: []service.KeyConfig{ + service.KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, + service.KeyConfig{"user-1", "chacha20-ietf-poly1305", "Secret1"}, + }, + }, + service.ServiceConfig{ + Listeners: []service.ListenerConfig{ + service.ListenerConfig{Type: "tcp", Address: "[::]:9001"}, + service.ListenerConfig{Type: "udp", Address: "[::]:9001"}, + }, + Keys: []service.KeyConfig{ + service.KeyConfig{"user-2", "chacha20-ietf-poly1305", "Secret2"}, + }, + }, + }, + } + require.Equal(t, expected, *config) +} + +func TestReadConfigParsesDeprecatedFormat(t *testing.T) { + config, err := readConfigFile("./config_example.deprecated.yml") + + require.NoError(t, err) + expected := service.Config{ + Keys: []service.LegacyKeyServiceConfig{ + service.LegacyKeyServiceConfig{ + KeyConfig: service.KeyConfig{ID: "user-0", Cipher: "chacha20-ietf-poly1305", Secret: "Secret0"}, + Port: 9000, + }, + service.LegacyKeyServiceConfig{ + KeyConfig: service.KeyConfig{ID: "user-1", Cipher: "chacha20-ietf-poly1305", Secret: "Secret1"}, + Port: 9000, + }, + service.LegacyKeyServiceConfig{ + KeyConfig: service.KeyConfig{ID: "user-2", Cipher: "chacha20-ietf-poly1305", Secret: "Secret2"}, + Port: 9001, + }, + }, + } + require.Equal(t, expected, *config) +} + +func TestReadConfigFromEmptyFile(t *testing.T) { + file, _ := os.CreateTemp("", "empty.yaml") + + config, err := readConfigFile(file.Name()) + + require.NoError(t, err) + require.ElementsMatch(t, service.Config{}, config) +} + +func TestReadConfigFromIncorrectFormatFails(t *testing.T) { + file, _ := os.CreateTemp("", "empty.yaml") + file.WriteString("foo") + + config, err := readConfigFile(file.Name()) + + require.Error(t, err) + require.ElementsMatch(t, service.Config{}, config) +} + +func readConfigFile(filename string) (*service.Config, error) { + configData, _ := os.ReadFile(filename) + return readConfig(configData) +} diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 5766163a..780274a4 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -15,495 +15,27 @@ package main import ( - "fmt" - "log/slog" - "net" - "net/netip" - "sync" "time" - "github.com/Jigsaw-Code/outline-ss-server/ipinfo" - "github.com/Jigsaw-Code/outline-ss-server/service" - "github.com/Jigsaw-Code/outline-ss-server/service/metrics" "github.com/prometheus/client_golang/prometheus" ) // `now` is stubbable for testing. var now = time.Now -func NewTimeToCipherVec(proto string) (prometheus.ObserverVec, error) { - vec := prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "time_to_cipher_ms", - Help: "Time needed to find the cipher", - Buckets: []float64{0.1, 1, 10, 100, 1000}, - }, []string{"proto", "found_key"}) - return vec.CurryWith(map[string]string{"proto": proto}) -} - -type proxyCollector struct { - // NOTE: New metrics need to be added to `newProxyCollector()`, `Describe()` and `Collect()`. - dataBytesPerKey *prometheus.CounterVec - dataBytesPerLocation *prometheus.CounterVec -} - -func newProxyCollector(proto string) (*proxyCollector, error) { - dataBytesPerKey, err := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "data_bytes", - Help: "Bytes transferred by the proxy, per access key", - }, []string{"proto", "dir", "access_key"}).CurryWith(map[string]string{"proto": proto}) - if err != nil { - return nil, err - } - dataBytesPerLocation, err := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "data_bytes_per_location", - Help: "Bytes transferred by the proxy, per location", - }, []string{"proto", "dir", "location", "asn", "asorg"}).CurryWith(map[string]string{"proto": proto}) - if err != nil { - return nil, err - } - return &proxyCollector{ - dataBytesPerKey: dataBytesPerKey, - dataBytesPerLocation: dataBytesPerLocation, - }, nil -} - -func (c *proxyCollector) Describe(ch chan<- *prometheus.Desc) { - c.dataBytesPerKey.Describe(ch) - c.dataBytesPerLocation.Describe(ch) -} - -func (c *proxyCollector) Collect(ch chan<- prometheus.Metric) { - c.dataBytesPerKey.Collect(ch) - c.dataBytesPerLocation.Collect(ch) -} - -func (c *proxyCollector) addClientTarget(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { - addIfNonZero(clientProxyBytes, c.dataBytesPerKey, "c>p", accessKey) - addIfNonZero(clientProxyBytes, c.dataBytesPerLocation, "c>p", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) - addIfNonZero(proxyTargetBytes, c.dataBytesPerKey, "p>t", accessKey) - addIfNonZero(proxyTargetBytes, c.dataBytesPerLocation, "p>t", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) -} - -func (c *proxyCollector) addTargetClient(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { - addIfNonZero(targetProxyBytes, c.dataBytesPerKey, "p 0 { - counterVec.WithLabelValues(lvs...).Add(float64(value)) - } -} - -func asnLabel(asn int) string { - if asn == 0 { - return "" - } - return fmt.Sprint(asn) -} - -// Converts a [net.Addr] to an [IPKey]. -func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { - hostname, _, err := net.SplitHostPort(addr.String()) - if err != nil { - return nil, fmt.Errorf("failed to create IPKey: %w", err) - } - ip, err := netip.ParseAddr(hostname) - if err != nil { - return nil, fmt.Errorf("failed to create IPKey: %w", err) - } - return &IPKey{ip, accessKey}, nil -} diff --git a/cmd/outline-ss-server/config.go b/service/config.go similarity index 79% rename from cmd/outline-ss-server/config.go rename to service/config.go index d85fd72a..4fa242e9 100644 --- a/cmd/outline-ss-server/config.go +++ b/service/config.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package service import ( "fmt" @@ -22,14 +22,15 @@ import ( ) type ServiceConfig struct { - Listeners []ListenerConfig - Keys []KeyConfig + Listeners []ListenerConfig + Keys []KeyConfig + NatTimeoutSec int } type ListenerType string -const listenerTypeTCP ListenerType = "tcp" -const listenerTypeUDP ListenerType = "udp" +const ListenerTypeTCP ListenerType = "tcp" +const ListenerTypeUDP ListenerType = "udp" type ListenerConfig struct { Type ListenerType @@ -55,13 +56,21 @@ type Config struct { Keys []LegacyKeyServiceConfig } +// LoadFrom attempts to load and parse config yaml bytes as a [Config]. +func (c *Config) LoadFrom(configData []byte) error { + if err := yaml.Unmarshal(configData, &c); err != nil { + return fmt.Errorf("failed to parse config: %w", err) + } + return nil +} + // Validate checks that the config is valid. func (c *Config) Validate() error { existingListeners := make(map[string]bool) for _, serviceConfig := range c.Services { for _, lnConfig := range serviceConfig.Listeners { // TODO: Support more listener types. - if lnConfig.Type != listenerTypeTCP && lnConfig.Type != listenerTypeUDP { + if lnConfig.Type != ListenerTypeTCP && lnConfig.Type != ListenerTypeUDP { return fmt.Errorf("unsupported listener type: %s", lnConfig.Type) } host, _, err := net.SplitHostPort(lnConfig.Address) @@ -80,12 +89,3 @@ func (c *Config) Validate() error { } return nil } - -// readConfig attempts to read a config from a filename and parses it as a [Config]. -func readConfig(configData []byte) (*Config, error) { - config := Config{} - if err := yaml.Unmarshal(configData, &config); err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - return &config, nil -} diff --git a/service/config_test.go b/service/config_test.go new file mode 100644 index 00000000..d31e8ff6 --- /dev/null +++ b/service/config_test.go @@ -0,0 +1,89 @@ +// Copyright 2024 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateConfigFails(t *testing.T) { + tests := []struct { + name string + cfg *Config + }{ + { + name: "WithUnknownListenerType", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: "foo", Address: "[::]:9000"}, + }, + }, + }, + }, + }, + { + name: "WithInvalidListenerAddress", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeTCP, Address: "tcp/[::]:9000"}, + }, + }, + }, + }, + }, + { + name: "WithHostnameAddress", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeTCP, Address: "example.com:9000"}, + }, + }, + }, + }, + }, + { + name: "WithDuplicateListeners", + cfg: &Config{ + Services: []ServiceConfig{ + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, + }, + }, + ServiceConfig{ + Listeners: []ListenerConfig{ + ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.Validate() + require.Error(t, err) + }) + } +} diff --git a/service/metrics.go b/service/metrics.go new file mode 100644 index 00000000..de118444 --- /dev/null +++ b/service/metrics.go @@ -0,0 +1,573 @@ +// Copyright 2023 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "fmt" + "log/slog" + "net" + "net/netip" + "sync" + "time" + + "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + "github.com/Jigsaw-Code/outline-ss-server/service/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +// `now` is stubbable for testing. +var now = time.Now + +func NewTimeToCipherVec(proto string) (prometheus.ObserverVec, error) { + vec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "time_to_cipher_ms", + Help: "Time needed to find the cipher", + Buckets: []float64{0.1, 1, 10, 100, 1000}, + }, []string{"proto", "found_key"}) + return vec.CurryWith(map[string]string{"proto": proto}) +} + +type proxyCollector struct { + // NOTE: New metrics need to be added to `newProxyCollector()`, `Describe()` and `Collect()`. + dataBytesPerKey *prometheus.CounterVec + dataBytesPerLocation *prometheus.CounterVec +} + +func newProxyCollector(proto string) (*proxyCollector, error) { + dataBytesPerKey, err := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "data_bytes", + Help: "Bytes transferred by the proxy, per access key", + }, []string{"proto", "dir", "access_key"}).CurryWith(map[string]string{"proto": proto}) + if err != nil { + return nil, err + } + dataBytesPerLocation, err := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "data_bytes_per_location", + Help: "Bytes transferred by the proxy, per location", + }, []string{"proto", "dir", "location", "asn", "asorg"}).CurryWith(map[string]string{"proto": proto}) + if err != nil { + return nil, err + } + return &proxyCollector{ + dataBytesPerKey: dataBytesPerKey, + dataBytesPerLocation: dataBytesPerLocation, + }, nil +} + +func (c *proxyCollector) Describe(ch chan<- *prometheus.Desc) { + c.dataBytesPerKey.Describe(ch) + c.dataBytesPerLocation.Describe(ch) +} + +func (c *proxyCollector) Collect(ch chan<- prometheus.Metric) { + c.dataBytesPerKey.Collect(ch) + c.dataBytesPerLocation.Collect(ch) +} + +func (c *proxyCollector) addClientTarget(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { + addIfNonZero(clientProxyBytes, c.dataBytesPerKey, "c>p", accessKey) + addIfNonZero(clientProxyBytes, c.dataBytesPerLocation, "c>p", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) + addIfNonZero(proxyTargetBytes, c.dataBytesPerKey, "p>t", accessKey) + addIfNonZero(proxyTargetBytes, c.dataBytesPerLocation, "p>t", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) +} + +func (c *proxyCollector) addTargetClient(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { + addIfNonZero(targetProxyBytes, c.dataBytesPerKey, "p 0 { + counterVec.WithLabelValues(lvs...).Add(float64(value)) + } +} + +func asnLabel(asn int) string { + if asn == 0 { + return "" + } + return fmt.Sprint(asn) +} + +// Converts a [net.Addr] to an [IPKey]. +func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { + hostname, _, err := net.SplitHostPort(addr.String()) + if err != nil { + return nil, fmt.Errorf("failed to create IPKey: %w", err) + } + ip, err := netip.ParseAddr(hostname) + if err != nil { + return nil, fmt.Errorf("failed to create IPKey: %w", err) + } + return &IPKey{ip, accessKey}, nil +} diff --git a/cmd/outline-ss-server/metrics_test.go b/service/metrics_test.go similarity index 99% rename from cmd/outline-ss-server/metrics_test.go rename to service/metrics_test.go index 15a112bf..1e2c8b95 100644 --- a/cmd/outline-ss-server/metrics_test.go +++ b/service/metrics_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package service import ( "net" diff --git a/service/service.go b/service/service.go new file mode 100644 index 00000000..d2d56916 --- /dev/null +++ b/service/service.go @@ -0,0 +1,175 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "container/list" + "context" + "fmt" + "log/slog" + "net" + "time" + + "github.com/Jigsaw-Code/outline-sdk/transport" + "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" +) + +const ( + // 59 seconds is most common timeout for servers that do not respond to invalid requests + tcpReadTimeout time.Duration = 59 * time.Second + + // A UDP NAT timeout of at least 5 minutes is recommended in RFC 4787 Section 4.3. + defaultNatTimeout time.Duration = 5 * time.Minute +) + +type ServiceMetrics interface { + UDPMetrics + AddOpenTCPConnection(conn net.Conn) TCPConnMetrics + AddCipherSearch(proto string, accessKeyFound bool, timeToCipher time.Duration) +} + +type Service interface { + HandleStream(ctx context.Context, conn transport.StreamConn) + HandlePacket(conn net.PacketConn) +} + +// Option user's option. +type Option func(s *ssService) error + +type ssService struct { + m ServiceMetrics + ciphers CipherList + natTimeout time.Duration + replayCache *ReplayCache + + sh StreamHandler + ph PacketHandler +} + +func NewService(opts ...Option) (Service, error) { + s := &ssService{} + + for _, opt := range opts { + if err := opt(s); err != nil { + return nil, fmt.Errorf("failed to create new service: %v", err) + } + } + + if s.natTimeout == 0 { + s.natTimeout = defaultNatTimeout + } + return s, nil +} + +// WithConfig option function. +func WithConfig(config ServiceConfig) Option { + return func(s *ssService) error { + ciphers, err := newCipherListFromConfig(config) + if err != nil { + return fmt.Errorf("failed to create cipher list from config: %v", err) + } + s.ciphers = ciphers + + s.natTimeout = time.Duration(config.NatTimeoutSec) * time.Second + + return nil + } +} + +// WithCiphers option function. +func WithCiphers(ciphers CipherList) Option { + return func(s *ssService) error { + s.ciphers = ciphers + return nil + } +} + +// WithMetrics option function. +func WithMetrics(metrics ServiceMetrics) Option { + return func(s *ssService) error { + s.m = metrics + return nil + } +} + +// WithReplayCache option function. +func WithReplayCache(replayCache *ReplayCache) Option { + return func(s *ssService) error { + s.replayCache = replayCache + return nil + } +} + +func WithNatTimeout(natTimeout time.Duration) Option { + return func(s *ssService) error { + s.natTimeout = natTimeout + return nil + } +} + +func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { + if s.sh == nil { + authFunc := NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}) + // TODO: Register initial data metrics at zero. + s.sh = NewStreamHandler(authFunc, tcpReadTimeout) + } + connMetrics := s.m.AddOpenTCPConnection(conn) + s.sh.Handle(ctx, conn, connMetrics) +} + +func (s *ssService) HandlePacket(conn net.PacketConn) { + if s.ph == nil { + s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) + } + s.ph.Handle(conn) +} + +func newCipherListFromConfig(config ServiceConfig) (CipherList, error) { + type cipherKey struct { + cipher string + secret string + } + cipherList := list.New() + existingCiphers := make(map[cipherKey]bool) + for _, keyConfig := range config.Keys { + key := cipherKey{keyConfig.Cipher, keyConfig.Secret} + if _, exists := existingCiphers[key]; exists { + slog.Debug("Encryption key already exists. Skipping.", "ID", keyConfig.ID) + continue + } + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + } + entry := MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + cipherList.PushBack(&entry) + existingCiphers[key] = true + } + ciphers := NewCipherList() + ciphers.Update(cipherList) + + return ciphers, nil +} + +type ssConnMetrics struct { + ServiceMetrics + proto string +} + +var _ ShadowsocksConnMetrics = (*ssConnMetrics)(nil) + +func (cm *ssConnMetrics) AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { + cm.ServiceMetrics.AddCipherSearch(cm.proto, accessKeyFound, timeToCipher) +} From 76e320e7b9e3893941f1c41ce7f0d47d0f96ee82 Mon Sep 17 00:00:00 2001 From: sbruens Date: Sat, 31 Aug 2024 01:00:32 -0400 Subject: [PATCH 139/182] Ignore custom Caddy binary. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7fc155d1..cfb902b3 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ # Go workspace go.work go.work.sum + +# Custom caddy binary +/caddy/caddy From e80ccc3f1cd8c73ade665aee5463933ecf6f55a0 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 6 Sep 2024 12:16:48 -0400 Subject: [PATCH 140/182] refactor: create re-usable service that can be re-used by Caddy --- cmd/outline-ss-server/main.go | 114 ++--- cmd/outline-ss-server/metrics.go | 550 +----------------------- cmd/outline-ss-server/metrics_test.go | 184 ++------- cmd/outline-ss-server/server_test.go | 9 +- prometheus/metrics.go | 574 ++++++++++++++++++++++++++ prometheus/metrics_test.go | 226 ++++++++++ service/service.go | 130 ++++++ 7 files changed, 1025 insertions(+), 762 deletions(-) create mode 100644 prometheus/metrics.go create mode 100644 prometheus/metrics_test.go create mode 100644 service/service.go diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index e61150c7..6fd84dfa 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -16,7 +16,6 @@ package main import ( "container/list" - "context" "flag" "fmt" "log/slog" @@ -29,9 +28,9 @@ import ( "syscall" "time" - "github.com/Jigsaw-Code/outline-sdk/transport" "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + outline_prometheus "github.com/Jigsaw-Code/outline-ss-server/prometheus" "github.com/Jigsaw-Code/outline-ss-server/service" "github.com/lmittmann/tint" "github.com/prometheus/client_golang/prometheus" @@ -59,11 +58,12 @@ func init() { } type SSServer struct { - stopConfig func() error - lnManager service.ListenerManager - natTimeout time.Duration - m *outlineMetrics - replayCache service.ReplayCache + stopConfig func() error + lnManager service.ListenerManager + natTimeout time.Duration + serverMetrics *serverMetrics + serviceMetrics service.ServiceMetrics + replayCache service.ReplayCache } func (s *SSServer) loadConfig(filename string) error { @@ -120,32 +120,6 @@ func newCipherListFromConfig(config ServiceConfig) (service.CipherList, error) { return ciphers, nil } -func (s *SSServer) NewShadowsocksStreamHandler(ciphers service.CipherList) service.StreamHandler { - authFunc := service.NewShadowsocksStreamAuthenticator(ciphers, &s.replayCache, s.m.tcpServiceMetrics) - // TODO: Register initial data metrics at zero. - return service.NewStreamHandler(authFunc, tcpReadTimeout) -} - -func (s *SSServer) NewShadowsocksPacketHandler(ciphers service.CipherList) service.PacketHandler { - return service.NewPacketHandler(s.natTimeout, ciphers, s.m, s.m.udpServiceMetrics) -} - -func (s *SSServer) NewShadowsocksStreamHandlerFromConfig(config ServiceConfig) (service.StreamHandler, error) { - ciphers, err := newCipherListFromConfig(config) - if err != nil { - return nil, err - } - return s.NewShadowsocksStreamHandler(ciphers), nil -} - -func (s *SSServer) NewShadowsocksPacketHandlerFromConfig(config ServiceConfig) (service.PacketHandler, error) { - ciphers, err := newCipherListFromConfig(config) - if err != nil { - return nil, err - } - return s.NewShadowsocksPacketHandler(ciphers), nil -} - type listenerSet struct { manager service.ListenerManager listenerCloseFuncs map[string]func() error @@ -243,31 +217,41 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { ciphers := service.NewCipherList() ciphers.Update(cipherList) - sh := s.NewShadowsocksStreamHandler(ciphers) + ssService, err := service.NewService( + service.WithCiphers(ciphers), + service.WithNatTimeout(s.natTimeout), + service.WithMetrics(s.serviceMetrics), + service.WithReplayCache(&s.replayCache), + ) ln, err := lnSet.ListenStream(addr) if err != nil { return err } slog.Info("TCP service started.", "address", ln.Addr().String()) - go service.StreamServe(ln.AcceptStream, func(ctx context.Context, conn transport.StreamConn) { - connMetrics := s.m.AddOpenTCPConnection(conn) - sh.Handle(ctx, conn, connMetrics) - }) + go service.StreamServe(ln.AcceptStream, ssService.HandleStream) pc, err := lnSet.ListenPacket(addr) if err != nil { return err } slog.Info("UDP service started.", "address", pc.LocalAddr().String()) - ph := s.NewShadowsocksPacketHandler(ciphers) - go ph.Handle(pc) + go ssService.HandlePacket(pc) } for _, serviceConfig := range config.Services { - var ( - sh service.StreamHandler - ph service.PacketHandler + ciphers, err := newCipherListFromConfig(serviceConfig) + if err != nil { + return fmt.Errorf("failed to create cipher list from config: %v", err) + } + ssService, err := service.NewService( + service.WithCiphers(ciphers), + service.WithNatTimeout(s.natTimeout), + service.WithMetrics(s.serviceMetrics), + service.WithReplayCache(&s.replayCache), ) + if err != nil { + return err + } for _, lnConfig := range serviceConfig.Listeners { switch lnConfig.Type { case listenerTypeTCP: @@ -276,36 +260,21 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { return err } slog.Info("TCP service started.", "address", ln.Addr().String()) - if sh == nil { - sh, err = s.NewShadowsocksStreamHandlerFromConfig(serviceConfig) - if err != nil { - return err - } - } - go service.StreamServe(ln.AcceptStream, func(ctx context.Context, conn transport.StreamConn) { - connMetrics := s.m.AddOpenTCPConnection(conn) - sh.Handle(ctx, conn, connMetrics) - }) + go service.StreamServe(ln.AcceptStream, ssService.HandleStream) case listenerTypeUDP: pc, err := lnSet.ListenPacket(lnConfig.Address) if err != nil { return err } slog.Info("UDP service started.", "address", pc.LocalAddr().String()) - if ph == nil { - ph, err = s.NewShadowsocksPacketHandlerFromConfig(serviceConfig) - if err != nil { - return err - } - } - go ph.Handle(pc) + go ssService.HandlePacket(pc) } } totalCipherCount += len(serviceConfig.Keys) } slog.Info("Loaded config.", "access_keys", totalCipherCount, "listeners", lnSet.Len()) - s.m.SetNumAccessKeys(totalCipherCount, lnSet.Len()) + s.serverMetrics.SetNumAccessKeys(totalCipherCount, lnSet.Len()) return nil }() @@ -341,12 +310,13 @@ func (s *SSServer) Stop() error { } // RunSSServer starts a shadowsocks server running, and returns the server or an error. -func RunSSServer(filename string, natTimeout time.Duration, sm *outlineMetrics, replayHistory int) (*SSServer, error) { +func RunSSServer(filename string, natTimeout time.Duration, serverMetrics *serverMetrics, serviceMetrics service.ServiceMetrics, replayHistory int) (*SSServer, error) { server := &SSServer{ - lnManager: service.NewListenerManager(), - natTimeout: natTimeout, - m: sm, - replayCache: service.NewReplayCache(replayHistory), + lnManager: service.NewListenerManager(), + natTimeout: natTimeout, + serverMetrics: serverMetrics, + serviceMetrics: serviceMetrics, + replayCache: service.NewReplayCache(replayHistory), } err := server.loadConfig(filename) if err != nil { @@ -424,14 +394,16 @@ func main() { } defer ip2info.Close() - metrics, err := newPrometheusOutlineMetrics(ip2info) + serverMetrics := newPrometheusServerMetrics() + serverMetrics.SetVersion(version) + serviceMetrics, err := outline_prometheus.NewServiceMetrics(ip2info) if err != nil { - slog.Error("Failed to create Outline Prometheus metrics. Aborting.", "err", err) + slog.Error("Failed to create Outline Prometheus service metrics. Aborting.", "err", err) } - metrics.SetBuildInfo(version) r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) - r.MustRegister(metrics) - _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, metrics, flags.replayHistory) + r.MustRegister(serverMetrics, serviceMetrics) + + _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, serverMetrics, serviceMetrics, flags.replayHistory) if err != nil { slog.Error("Server failed to start. Aborting.", "err", err) } diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go index 5766163a..32c9b0aa 100644 --- a/cmd/outline-ss-server/metrics.go +++ b/cmd/outline-ss-server/metrics.go @@ -15,495 +15,27 @@ package main import ( - "fmt" - "log/slog" - "net" - "net/netip" - "sync" "time" - "github.com/Jigsaw-Code/outline-ss-server/ipinfo" - "github.com/Jigsaw-Code/outline-ss-server/service" - "github.com/Jigsaw-Code/outline-ss-server/service/metrics" "github.com/prometheus/client_golang/prometheus" ) // `now` is stubbable for testing. var now = time.Now -func NewTimeToCipherVec(proto string) (prometheus.ObserverVec, error) { - vec := prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "time_to_cipher_ms", - Help: "Time needed to find the cipher", - Buckets: []float64{0.1, 1, 10, 100, 1000}, - }, []string{"proto", "found_key"}) - return vec.CurryWith(map[string]string{"proto": proto}) -} - -type proxyCollector struct { - // NOTE: New metrics need to be added to `newProxyCollector()`, `Describe()` and `Collect()`. - dataBytesPerKey *prometheus.CounterVec - dataBytesPerLocation *prometheus.CounterVec -} - -func newProxyCollector(proto string) (*proxyCollector, error) { - dataBytesPerKey, err := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "data_bytes", - Help: "Bytes transferred by the proxy, per access key", - }, []string{"proto", "dir", "access_key"}).CurryWith(map[string]string{"proto": proto}) - if err != nil { - return nil, err - } - dataBytesPerLocation, err := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "data_bytes_per_location", - Help: "Bytes transferred by the proxy, per location", - }, []string{"proto", "dir", "location", "asn", "asorg"}).CurryWith(map[string]string{"proto": proto}) - if err != nil { - return nil, err - } - return &proxyCollector{ - dataBytesPerKey: dataBytesPerKey, - dataBytesPerLocation: dataBytesPerLocation, - }, nil -} - -func (c *proxyCollector) Describe(ch chan<- *prometheus.Desc) { - c.dataBytesPerKey.Describe(ch) - c.dataBytesPerLocation.Describe(ch) -} - -func (c *proxyCollector) Collect(ch chan<- prometheus.Metric) { - c.dataBytesPerKey.Collect(ch) - c.dataBytesPerLocation.Collect(ch) -} - -func (c *proxyCollector) addClientTarget(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { - addIfNonZero(clientProxyBytes, c.dataBytesPerKey, "c>p", accessKey) - addIfNonZero(clientProxyBytes, c.dataBytesPerLocation, "c>p", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) - addIfNonZero(proxyTargetBytes, c.dataBytesPerKey, "p>t", accessKey) - addIfNonZero(proxyTargetBytes, c.dataBytesPerLocation, "p>t", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) -} - -func (c *proxyCollector) addTargetClient(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { - addIfNonZero(targetProxyBytes, c.dataBytesPerKey, "p 0 { - counterVec.WithLabelValues(lvs...).Add(float64(value)) - } -} - -func asnLabel(asn int) string { - if asn == 0 { - return "" - } - return fmt.Sprint(asn) -} - -// Converts a [net.Addr] to an [IPKey]. -func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { - hostname, _, err := net.SplitHostPort(addr.String()) - if err != nil { - return nil, fmt.Errorf("failed to create IPKey: %w", err) - } - ip, err := netip.ParseAddr(hostname) - if err != nil { - return nil, fmt.Errorf("failed to create IPKey: %w", err) - } - return &IPKey{ip, accessKey}, nil -} diff --git a/cmd/outline-ss-server/metrics_test.go b/cmd/outline-ss-server/metrics_test.go index 15a112bf..93cce446 100644 --- a/cmd/outline-ss-server/metrics_test.go +++ b/cmd/outline-ss-server/metrics_test.go @@ -21,7 +21,6 @@ import ( "time" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" - "github.com/Jigsaw-Code/outline-ss-server/service/metrics" "github.com/op/go-logging" "github.com/prometheus/client_golang/prometheus" promtest "github.com/prometheus/client_golang/prometheus/testutil" @@ -63,166 +62,49 @@ func (c *fakeConn) RemoteAddr() net.Addr { } func TestMethodsDontPanic(t *testing.T) { - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - proxyMetrics := metrics.ProxyMetrics{ - ClientProxy: 1, - ProxyTarget: 2, - TargetProxy: 3, - ProxyClient: 4, - } - addr := fakeAddr("127.0.0.1:9") - ssMetrics.SetBuildInfo("0.0.0-test") - ssMetrics.SetNumAccessKeys(20, 2) - - tcpMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) - tcpMetrics.AddAuthenticated("0") - tcpMetrics.AddClosed("OK", proxyMetrics, 10*time.Millisecond) - tcpMetrics.AddProbe("ERR_CIPHER", "eof", proxyMetrics.ClientProxy) - - udpMetrics := ssMetrics.AddUDPNatEntry(addr, "key-1") - udpMetrics.AddPacketFromClient("OK", 10, 20) - udpMetrics.AddPacketFromTarget("OK", 10, 20) - udpMetrics.RemoveNatEntry() - - ssMetrics.tcpServiceMetrics.AddCipherSearch(true, 10*time.Millisecond) - ssMetrics.udpServiceMetrics.AddCipherSearch(true, 10*time.Millisecond) -} - -func TestASNLabel(t *testing.T) { - require.Equal(t, "", asnLabel(0)) - require.Equal(t, "100", asnLabel(100)) + m := newPrometheusServerMetrics() + m.SetVersion("0.0.0-test") + m.SetNumAccessKeys(20, 2) } -func TestTunnelTime(t *testing.T) { - t.Run("PerKey", func(t *testing.T) { - setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - reg := prometheus.NewPedanticRegistry() - reg.MustRegister(ssMetrics) - - connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) - connMetrics.AddAuthenticated("key-1") - setNow(time.Date(2010, 1, 2, 3, 4, 20, .0, time.Local)) - - expected := strings.NewReader(` - # HELP tunnel_time_seconds Tunnel time, per access key. - # TYPE tunnel_time_seconds counter - tunnel_time_seconds{access_key="key-1"} 15 - `) - err := promtest.GatherAndCompare( - reg, - expected, - "tunnel_time_seconds", - ) - require.NoError(t, err, "unexpected metric value found") - }) - - t.Run("PerLocation", func(t *testing.T) { - setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) - ssMetrics, _ := newPrometheusOutlineMetrics(&noopMap{}) - reg := prometheus.NewPedanticRegistry() - reg.MustRegister(ssMetrics) - - connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) - connMetrics.AddAuthenticated("key-1") - setNow(time.Date(2010, 1, 2, 3, 4, 10, .0, time.Local)) - - expected := strings.NewReader(` - # HELP tunnel_time_seconds_per_location Tunnel time, per location. - # TYPE tunnel_time_seconds_per_location counter - tunnel_time_seconds_per_location{asn="",asorg="",location="XL"} 5 - `) - err := promtest.GatherAndCompare( - reg, - expected, - "tunnel_time_seconds_per_location", - ) - require.NoError(t, err, "unexpected metric value found") - }) -} - -func TestTunnelTimePerKeyDoesNotPanicOnUnknownClosedConnection(t *testing.T) { +func TestSetVersion(t *testing.T) { + m := newPrometheusServerMetrics() reg := prometheus.NewPedanticRegistry() - ssMetrics, _ := newPrometheusOutlineMetrics(nil) + reg.MustRegister(m) - connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) - connMetrics.AddClosed("OK", metrics.ProxyMetrics{}, time.Minute) + m.SetVersion("0.0.0-test") err := promtest.GatherAndCompare( reg, - strings.NewReader(""), - "tunnel_time_seconds", + strings.NewReader(` + # HELP build_info Information on the outline-ss-server build + # TYPE build_info gauge + build_info{version="0.0.0-test"} 1 + `), + "build_info", ) - require.NoError(t, err, "unexpectedly found metric value") -} - -func BenchmarkOpenTCP(b *testing.B) { - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - conn := &fakeConn{} - b.ResetTimer() - for i := 0; i < b.N; i++ { - ssMetrics.AddOpenTCPConnection(conn) - } -} - -func BenchmarkCloseTCP(b *testing.B) { - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) - accessKey := "key 1" - status := "OK" - data := metrics.ProxyMetrics{} - duration := time.Minute - b.ResetTimer() - for i := 0; i < b.N; i++ { - connMetrics.AddAuthenticated(accessKey) - connMetrics.AddClosed(status, data, duration) - } + require.NoError(t, err, "unexpected metric value found") } -func BenchmarkProbe(b *testing.B) { - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) - status := "ERR_REPLAY" - drainResult := "other" - data := metrics.ProxyMetrics{} - b.ResetTimer() - for i := 0; i < b.N; i++ { - connMetrics.AddProbe(status, drainResult, data.ClientProxy) - } -} +func TestSetNumAccessKeys(t *testing.T) { + m := newPrometheusServerMetrics() + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(m) -func BenchmarkClientUDP(b *testing.B) { - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - addr := fakeAddr("127.0.0.1:9") - accessKey := "key 1" - udpMetrics := ssMetrics.AddUDPNatEntry(addr, accessKey) - status := "OK" - size := int64(1000) - b.ResetTimer() - for i := 0; i < b.N; i++ { - udpMetrics.AddPacketFromClient(status, size, size) - } -} + m.SetNumAccessKeys(1, 2) -func BenchmarkTargetUDP(b *testing.B) { - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - addr := fakeAddr("127.0.0.1:9") - accessKey := "key 1" - udpMetrics := ssMetrics.AddUDPNatEntry(addr, accessKey) - status := "OK" - size := int64(1000) - b.ResetTimer() - for i := 0; i < b.N; i++ { - udpMetrics.AddPacketFromTarget(status, size, size) - } -} - -func BenchmarkNAT(b *testing.B) { - ssMetrics, _ := newPrometheusOutlineMetrics(nil) - addr := fakeAddr("127.0.0.1:9") - b.ResetTimer() - for i := 0; i < b.N; i++ { - udpMetrics := ssMetrics.AddUDPNatEntry(addr, "key-0") - udpMetrics.RemoveNatEntry() - } + err := promtest.GatherAndCompare( + reg, + strings.NewReader(` + # HELP keys Count of access keys + # TYPE keys gauge + keys 1 + # HELP ports Count of open ports + # TYPE ports gauge + ports 2 + `), + "keys", + "ports", + ) + require.NoError(t, err, "unexpected metric value found") } diff --git a/cmd/outline-ss-server/server_test.go b/cmd/outline-ss-server/server_test.go index 20729a06..05999486 100644 --- a/cmd/outline-ss-server/server_test.go +++ b/cmd/outline-ss-server/server_test.go @@ -17,14 +17,17 @@ package main import ( "testing" "time" + + "github.com/Jigsaw-Code/outline-ss-server/prometheus" ) func TestRunSSServer(t *testing.T) { - m, err := newPrometheusOutlineMetrics(nil) + serverMetrics := newPrometheusServerMetrics() + serviceMetrics, err := prometheus.NewServiceMetrics(nil) if err != nil { - t.Fatalf("Failed to create Prometheus metrics: %v", err) + t.Fatalf("Failed to create Prometheus service metrics: %v", err) } - server, err := RunSSServer("config_example.yml", 30*time.Second, m, 10000) + server, err := RunSSServer("config_example.yml", 30*time.Second, serverMetrics, serviceMetrics, 10000) if err != nil { t.Fatalf("RunSSServer() error = %v", err) } diff --git a/prometheus/metrics.go b/prometheus/metrics.go new file mode 100644 index 00000000..186cba7c --- /dev/null +++ b/prometheus/metrics.go @@ -0,0 +1,574 @@ +// Copyright 2023 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "fmt" + "log/slog" + "net" + "net/netip" + "sync" + "time" + + "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + "github.com/Jigsaw-Code/outline-ss-server/service" + "github.com/Jigsaw-Code/outline-ss-server/service/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +// `now` is stubbable for testing. +var now = time.Now + +func newTimeToCipherVec(proto string) (prometheus.ObserverVec, error) { + vec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "time_to_cipher_ms", + Help: "Time needed to find the cipher", + Buckets: []float64{0.1, 1, 10, 100, 1000}, + }, []string{"proto", "found_key"}) + return vec.CurryWith(map[string]string{"proto": proto}) +} + +type proxyCollector struct { + // NOTE: New metrics need to be added to `newProxyCollector()`, `Describe()` and `Collect()`. + dataBytesPerKey *prometheus.CounterVec + dataBytesPerLocation *prometheus.CounterVec +} + +func newProxyCollector(proto string) (*proxyCollector, error) { + dataBytesPerKey, err := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "data_bytes", + Help: "Bytes transferred by the proxy, per access key", + }, []string{"proto", "dir", "access_key"}).CurryWith(map[string]string{"proto": proto}) + if err != nil { + return nil, err + } + dataBytesPerLocation, err := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "data_bytes_per_location", + Help: "Bytes transferred by the proxy, per location", + }, []string{"proto", "dir", "location", "asn", "asorg"}).CurryWith(map[string]string{"proto": proto}) + if err != nil { + return nil, err + } + return &proxyCollector{ + dataBytesPerKey: dataBytesPerKey, + dataBytesPerLocation: dataBytesPerLocation, + }, nil +} + +func (c *proxyCollector) Describe(ch chan<- *prometheus.Desc) { + c.dataBytesPerKey.Describe(ch) + c.dataBytesPerLocation.Describe(ch) +} + +func (c *proxyCollector) Collect(ch chan<- prometheus.Metric) { + c.dataBytesPerKey.Collect(ch) + c.dataBytesPerLocation.Collect(ch) +} + +func (c *proxyCollector) addClientTarget(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { + addIfNonZero(clientProxyBytes, c.dataBytesPerKey, "c>p", accessKey) + addIfNonZero(clientProxyBytes, c.dataBytesPerLocation, "c>p", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) + addIfNonZero(proxyTargetBytes, c.dataBytesPerKey, "p>t", accessKey) + addIfNonZero(proxyTargetBytes, c.dataBytesPerLocation, "p>t", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) +} + +func (c *proxyCollector) addTargetClient(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { + addIfNonZero(targetProxyBytes, c.dataBytesPerKey, "p 0 { + counterVec.WithLabelValues(lvs...).Add(float64(value)) + } +} + +func asnLabel(asn int) string { + if asn == 0 { + return "" + } + return fmt.Sprint(asn) +} + +// Converts a [net.Addr] to an [IPKey]. +func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { + hostname, _, err := net.SplitHostPort(addr.String()) + if err != nil { + return nil, fmt.Errorf("failed to create IPKey: %w", err) + } + ip, err := netip.ParseAddr(hostname) + if err != nil { + return nil, fmt.Errorf("failed to create IPKey: %w", err) + } + return &IPKey{ip, accessKey}, nil +} diff --git a/prometheus/metrics_test.go b/prometheus/metrics_test.go new file mode 100644 index 00000000..5dfcf05a --- /dev/null +++ b/prometheus/metrics_test.go @@ -0,0 +1,226 @@ +// Copyright 2023 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "net" + "strings" + "testing" + "time" + + "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + "github.com/Jigsaw-Code/outline-ss-server/service/metrics" + "github.com/op/go-logging" + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +type noopMap struct{} + +func (*noopMap) GetIPInfo(ip net.IP) (ipinfo.IPInfo, error) { + return ipinfo.IPInfo{}, nil +} + +type fakeAddr string + +func (a fakeAddr) String() string { return string(a) } +func (a fakeAddr) Network() string { return "" } + +// Sets the processing clock to be t until changed. +func setNow(t time.Time) { + now = func() time.Time { + return t + } +} + +func init() { + logging.SetLevel(logging.INFO, "") +} + +type fakeConn struct { + net.Conn +} + +func (c *fakeConn) LocalAddr() net.Addr { + return fakeAddr("127.0.0.1:9") +} + +func (c *fakeConn) RemoteAddr() net.Addr { + return fakeAddr("127.0.0.1:10") +} + +func TestMethodsDontPanic(t *testing.T) { + ssMetrics, _ := NewServiceMetrics(nil) + proxyMetrics := metrics.ProxyMetrics{ + ClientProxy: 1, + ProxyTarget: 2, + TargetProxy: 3, + ProxyClient: 4, + } + addr := fakeAddr("127.0.0.1:9") + + tcpMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) + tcpMetrics.AddAuthenticated("0") + tcpMetrics.AddClosed("OK", proxyMetrics, 10*time.Millisecond) + tcpMetrics.AddProbe("ERR_CIPHER", "eof", proxyMetrics.ClientProxy) + + udpMetrics := ssMetrics.AddUDPNatEntry(addr, "key-1") + udpMetrics.AddPacketFromClient("OK", 10, 20) + udpMetrics.AddPacketFromTarget("OK", 10, 20) + udpMetrics.RemoveNatEntry() + + ssMetrics.tcpServiceMetrics.AddCipherSearch(true, 10*time.Millisecond) + ssMetrics.udpServiceMetrics.AddCipherSearch(true, 10*time.Millisecond) +} + +func TestASNLabel(t *testing.T) { + require.Equal(t, "", asnLabel(0)) + require.Equal(t, "100", asnLabel(100)) +} + +func TestTunnelTime(t *testing.T) { + t.Run("PerKey", func(t *testing.T) { + setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) + ssMetrics, _ := NewServiceMetrics(nil) + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(ssMetrics) + + connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) + connMetrics.AddAuthenticated("key-1") + setNow(time.Date(2010, 1, 2, 3, 4, 20, .0, time.Local)) + + expected := strings.NewReader(` + # HELP tunnel_time_seconds Tunnel time, per access key. + # TYPE tunnel_time_seconds counter + tunnel_time_seconds{access_key="key-1"} 15 + `) + err := promtest.GatherAndCompare( + reg, + expected, + "tunnel_time_seconds", + ) + require.NoError(t, err, "unexpected metric value found") + }) + + t.Run("PerLocation", func(t *testing.T) { + setNow(time.Date(2010, 1, 2, 3, 4, 5, .0, time.Local)) + ssMetrics, _ := NewServiceMetrics(&noopMap{}) + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(ssMetrics) + + connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) + connMetrics.AddAuthenticated("key-1") + setNow(time.Date(2010, 1, 2, 3, 4, 10, .0, time.Local)) + + expected := strings.NewReader(` + # HELP tunnel_time_seconds_per_location Tunnel time, per location. + # TYPE tunnel_time_seconds_per_location counter + tunnel_time_seconds_per_location{asn="",asorg="",location="XL"} 5 + `) + err := promtest.GatherAndCompare( + reg, + expected, + "tunnel_time_seconds_per_location", + ) + require.NoError(t, err, "unexpected metric value found") + }) +} + +func TestTunnelTimePerKeyDoesNotPanicOnUnknownClosedConnection(t *testing.T) { + reg := prometheus.NewPedanticRegistry() + ssMetrics, _ := NewServiceMetrics(nil) + + connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) + connMetrics.AddClosed("OK", metrics.ProxyMetrics{}, time.Minute) + + err := promtest.GatherAndCompare( + reg, + strings.NewReader(""), + "tunnel_time_seconds", + ) + require.NoError(t, err, "unexpectedly found metric value") +} + +func BenchmarkOpenTCP(b *testing.B) { + ssMetrics, _ := NewServiceMetrics(nil) + conn := &fakeConn{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + ssMetrics.AddOpenTCPConnection(conn) + } +} + +func BenchmarkCloseTCP(b *testing.B) { + ssMetrics, _ := NewServiceMetrics(nil) + connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) + accessKey := "key 1" + status := "OK" + data := metrics.ProxyMetrics{} + duration := time.Minute + b.ResetTimer() + for i := 0; i < b.N; i++ { + connMetrics.AddAuthenticated(accessKey) + connMetrics.AddClosed(status, data, duration) + } +} + +func BenchmarkProbe(b *testing.B) { + ssMetrics, _ := NewServiceMetrics(nil) + connMetrics := ssMetrics.AddOpenTCPConnection(&fakeConn{}) + status := "ERR_REPLAY" + drainResult := "other" + data := metrics.ProxyMetrics{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + connMetrics.AddProbe(status, drainResult, data.ClientProxy) + } +} + +func BenchmarkClientUDP(b *testing.B) { + ssMetrics, _ := NewServiceMetrics(nil) + addr := fakeAddr("127.0.0.1:9") + accessKey := "key 1" + udpMetrics := ssMetrics.AddUDPNatEntry(addr, accessKey) + status := "OK" + size := int64(1000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + udpMetrics.AddPacketFromClient(status, size, size) + } +} + +func BenchmarkTargetUDP(b *testing.B) { + ssMetrics, _ := NewServiceMetrics(nil) + addr := fakeAddr("127.0.0.1:9") + accessKey := "key 1" + udpMetrics := ssMetrics.AddUDPNatEntry(addr, accessKey) + status := "OK" + size := int64(1000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + udpMetrics.AddPacketFromTarget(status, size, size) + } +} + +func BenchmarkNAT(b *testing.B) { + ssMetrics, _ := NewServiceMetrics(nil) + addr := fakeAddr("127.0.0.1:9") + b.ResetTimer() + for i := 0; i < b.N; i++ { + udpMetrics := ssMetrics.AddUDPNatEntry(addr, "key-0") + udpMetrics.RemoveNatEntry() + } +} diff --git a/service/service.go b/service/service.go new file mode 100644 index 00000000..ae8f625d --- /dev/null +++ b/service/service.go @@ -0,0 +1,130 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/Jigsaw-Code/outline-sdk/transport" +) + +const ( + // 59 seconds is most common timeout for servers that do not respond to invalid requests + tcpReadTimeout time.Duration = 59 * time.Second + + // A UDP NAT timeout of at least 5 minutes is recommended in RFC 4787 Section 4.3. + defaultNatTimeout time.Duration = 5 * time.Minute +) + +type ServiceMetrics interface { + UDPMetrics + AddOpenTCPConnection(conn net.Conn) TCPConnMetrics + AddCipherSearch(proto string, accessKeyFound bool, timeToCipher time.Duration) +} + +type Service interface { + HandleStream(ctx context.Context, conn transport.StreamConn) + HandlePacket(conn net.PacketConn) +} + +// Option user's option. +type Option func(s *ssService) error + +type ssService struct { + m ServiceMetrics + ciphers CipherList + natTimeout time.Duration + replayCache *ReplayCache + + sh StreamHandler + ph PacketHandler +} + +func NewService(opts ...Option) (Service, error) { + s := &ssService{} + + for _, opt := range opts { + if err := opt(s); err != nil { + return nil, fmt.Errorf("failed to create new service: %v", err) + } + } + + if s.natTimeout == 0 { + s.natTimeout = defaultNatTimeout + } + return s, nil +} + +// WithCiphers option function. +func WithCiphers(ciphers CipherList) Option { + return func(s *ssService) error { + s.ciphers = ciphers + return nil + } +} + +// WithMetrics option function. +func WithMetrics(metrics ServiceMetrics) Option { + return func(s *ssService) error { + s.m = metrics + return nil + } +} + +// WithReplayCache option function. +func WithReplayCache(replayCache *ReplayCache) Option { + return func(s *ssService) error { + s.replayCache = replayCache + return nil + } +} + +func WithNatTimeout(natTimeout time.Duration) Option { + return func(s *ssService) error { + s.natTimeout = natTimeout + return nil + } +} + +func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { + if s.sh == nil { + authFunc := NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}) + // TODO: Register initial data metrics at zero. + s.sh = NewStreamHandler(authFunc, tcpReadTimeout) + } + connMetrics := s.m.AddOpenTCPConnection(conn) + s.sh.Handle(ctx, conn, connMetrics) +} + +func (s *ssService) HandlePacket(conn net.PacketConn) { + if s.ph == nil { + s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) + } + s.ph.Handle(conn) +} + +type ssConnMetrics struct { + ServiceMetrics + proto string +} + +var _ ShadowsocksConnMetrics = (*ssConnMetrics)(nil) + +func (cm *ssConnMetrics) AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { + cm.ServiceMetrics.AddCipherSearch(cm.proto, accessKeyFound, timeToCipher) +} From 6e330fac5763c9d7ef1e23e724a3eb027c2a9c77 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 6 Sep 2024 12:45:32 -0400 Subject: [PATCH 141/182] Remove need to return errors in opt functions. --- service/service.go | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/service/service.go b/service/service.go index ae8f625d..065fc22d 100644 --- a/service/service.go +++ b/service/service.go @@ -16,7 +16,6 @@ package service import ( "context" - "fmt" "net" "time" @@ -43,7 +42,7 @@ type Service interface { } // Option user's option. -type Option func(s *ssService) error +type Option func(s *ssService) type ssService struct { m ServiceMetrics @@ -59,9 +58,7 @@ func NewService(opts ...Option) (Service, error) { s := &ssService{} for _, opt := range opts { - if err := opt(s); err != nil { - return nil, fmt.Errorf("failed to create new service: %v", err) - } + opt(s) } if s.natTimeout == 0 { @@ -72,32 +69,28 @@ func NewService(opts ...Option) (Service, error) { // WithCiphers option function. func WithCiphers(ciphers CipherList) Option { - return func(s *ssService) error { + return func(s *ssService) { s.ciphers = ciphers - return nil } } // WithMetrics option function. func WithMetrics(metrics ServiceMetrics) Option { - return func(s *ssService) error { + return func(s *ssService) { s.m = metrics - return nil } } // WithReplayCache option function. func WithReplayCache(replayCache *ReplayCache) Option { - return func(s *ssService) error { + return func(s *ssService) { s.replayCache = replayCache - return nil } } func WithNatTimeout(natTimeout time.Duration) Option { - return func(s *ssService) error { + return func(s *ssService) { s.natTimeout = natTimeout - return nil } } From 2738b45f5749b1c1e05692a1b19743aeead79cc4 Mon Sep 17 00:00:00 2001 From: sbruens Date: Fri, 6 Sep 2024 12:52:38 -0400 Subject: [PATCH 142/182] Move the service into `shadowsocks.go`. --- cmd/outline-ss-server/main.go | 4 +- service/service.go | 123 ---------------------------------- service/shadowsocks.go | 112 ++++++++++++++++++++++++++++++- 3 files changed, 113 insertions(+), 126 deletions(-) delete mode 100644 service/service.go diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 6fd84dfa..543d036d 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -217,7 +217,7 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { ciphers := service.NewCipherList() ciphers.Update(cipherList) - ssService, err := service.NewService( + ssService, err := service.NewShadowsocksService( service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), @@ -243,7 +243,7 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { if err != nil { return fmt.Errorf("failed to create cipher list from config: %v", err) } - ssService, err := service.NewService( + ssService, err := service.NewShadowsocksService( service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), diff --git a/service/service.go b/service/service.go deleted file mode 100644 index 065fc22d..00000000 --- a/service/service.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2024 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "context" - "net" - "time" - - "github.com/Jigsaw-Code/outline-sdk/transport" -) - -const ( - // 59 seconds is most common timeout for servers that do not respond to invalid requests - tcpReadTimeout time.Duration = 59 * time.Second - - // A UDP NAT timeout of at least 5 minutes is recommended in RFC 4787 Section 4.3. - defaultNatTimeout time.Duration = 5 * time.Minute -) - -type ServiceMetrics interface { - UDPMetrics - AddOpenTCPConnection(conn net.Conn) TCPConnMetrics - AddCipherSearch(proto string, accessKeyFound bool, timeToCipher time.Duration) -} - -type Service interface { - HandleStream(ctx context.Context, conn transport.StreamConn) - HandlePacket(conn net.PacketConn) -} - -// Option user's option. -type Option func(s *ssService) - -type ssService struct { - m ServiceMetrics - ciphers CipherList - natTimeout time.Duration - replayCache *ReplayCache - - sh StreamHandler - ph PacketHandler -} - -func NewService(opts ...Option) (Service, error) { - s := &ssService{} - - for _, opt := range opts { - opt(s) - } - - if s.natTimeout == 0 { - s.natTimeout = defaultNatTimeout - } - return s, nil -} - -// WithCiphers option function. -func WithCiphers(ciphers CipherList) Option { - return func(s *ssService) { - s.ciphers = ciphers - } -} - -// WithMetrics option function. -func WithMetrics(metrics ServiceMetrics) Option { - return func(s *ssService) { - s.m = metrics - } -} - -// WithReplayCache option function. -func WithReplayCache(replayCache *ReplayCache) Option { - return func(s *ssService) { - s.replayCache = replayCache - } -} - -func WithNatTimeout(natTimeout time.Duration) Option { - return func(s *ssService) { - s.natTimeout = natTimeout - } -} - -func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { - if s.sh == nil { - authFunc := NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}) - // TODO: Register initial data metrics at zero. - s.sh = NewStreamHandler(authFunc, tcpReadTimeout) - } - connMetrics := s.m.AddOpenTCPConnection(conn) - s.sh.Handle(ctx, conn, connMetrics) -} - -func (s *ssService) HandlePacket(conn net.PacketConn) { - if s.ph == nil { - s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) - } - s.ph.Handle(conn) -} - -type ssConnMetrics struct { - ServiceMetrics - proto string -} - -var _ ShadowsocksConnMetrics = (*ssConnMetrics)(nil) - -func (cm *ssConnMetrics) AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { - cm.ServiceMetrics.AddCipherSearch(cm.proto, accessKeyFound, timeToCipher) -} diff --git a/service/shadowsocks.go b/service/shadowsocks.go index 97329c3a..87814df8 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -14,9 +14,119 @@ package service -import "time" +import ( + "context" + "net" + "time" + + "github.com/Jigsaw-Code/outline-sdk/transport" +) + +const ( + // 59 seconds is most common timeout for servers that do not respond to invalid requests + tcpReadTimeout time.Duration = 59 * time.Second + + // A UDP NAT timeout of at least 5 minutes is recommended in RFC 4787 Section 4.3. + defaultNatTimeout time.Duration = 5 * time.Minute +) // ShadowsocksConnMetrics is used to report Shadowsocks related metrics on connections. type ShadowsocksConnMetrics interface { AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) } + +type ServiceMetrics interface { + UDPMetrics + AddOpenTCPConnection(conn net.Conn) TCPConnMetrics + AddCipherSearch(proto string, accessKeyFound bool, timeToCipher time.Duration) +} + +type Service interface { + HandleStream(ctx context.Context, conn transport.StreamConn) + HandlePacket(conn net.PacketConn) +} + +// Option is a Shadowsocks service constructor option. +type Option func(s *ssService) + +type ssService struct { + m ServiceMetrics + ciphers CipherList + natTimeout time.Duration + replayCache *ReplayCache + + sh StreamHandler + ph PacketHandler +} + +// NewShadowsocksService creates a new service +func NewShadowsocksService(opts ...Option) (Service, error) { + s := &ssService{} + + for _, opt := range opts { + opt(s) + } + + if s.natTimeout == 0 { + s.natTimeout = defaultNatTimeout + } + return s, nil +} + +// WithCiphers option function. +func WithCiphers(ciphers CipherList) Option { + return func(s *ssService) { + s.ciphers = ciphers + } +} + +// WithMetrics option function. +func WithMetrics(metrics ServiceMetrics) Option { + return func(s *ssService) { + s.m = metrics + } +} + +// WithReplayCache option function. +func WithReplayCache(replayCache *ReplayCache) Option { + return func(s *ssService) { + s.replayCache = replayCache + } +} + +// WithNatTimeout option function. +func WithNatTimeout(natTimeout time.Duration) Option { + return func(s *ssService) { + s.natTimeout = natTimeout + } +} + +// HandleStream handles a Shadowsocks stream-based connection. +func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { + if s.sh == nil { + authFunc := NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}) + // TODO: Register initial data metrics at zero. + s.sh = NewStreamHandler(authFunc, tcpReadTimeout) + } + connMetrics := s.m.AddOpenTCPConnection(conn) + s.sh.Handle(ctx, conn, connMetrics) +} + +// HandlePacket handles a Shadowsocks packet connection. +func (s *ssService) HandlePacket(conn net.PacketConn) { + if s.ph == nil { + s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) + } + s.ph.Handle(conn) +} + +type ssConnMetrics struct { + ServiceMetrics + proto string +} + +var _ ShadowsocksConnMetrics = (*ssConnMetrics)(nil) + +func (cm *ssConnMetrics) AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { + cm.ServiceMetrics.AddCipherSearch(cm.proto, accessKeyFound, timeToCipher) +} From 89354651d6c4ed80445cbdbead4401d44692dac8 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 13:57:57 -0400 Subject: [PATCH 143/182] Add Caddy module with app and handler. --- caddy/README.md | 25 ++ caddy/app.go | 126 ++++++ caddy/config_example.json | 76 ++++ caddy/go.mod | 126 ++++++ caddy/go.sum | 619 +++++++++++++++++++++++++++ caddy/logger.go | 61 +++ caddy/shadowsocks_handler.go | 125 ++++++ cmd/outline-ss-server/main.go | 55 ++- cmd/outline-ss-server/main_test.go | 113 ----- cmd/outline-ss-server/metrics.go | 73 ---- cmd/outline-ss-server/server_test.go | 2 +- prometheus/metrics.go | 49 +++ service/config.go | 91 ---- service/config_test.go | 89 ---- service/logger.go | 22 +- service/metrics.go | 573 ------------------------- service/metrics_test.go | 110 ----- service/shadowsocks.go | 67 +-- service/tcp.go | 40 +- service/udp.go | 42 +- 20 files changed, 1339 insertions(+), 1145 deletions(-) create mode 100644 caddy/README.md create mode 100644 caddy/app.go create mode 100644 caddy/config_example.json create mode 100644 caddy/go.mod create mode 100644 caddy/go.sum create mode 100644 caddy/logger.go create mode 100644 caddy/shadowsocks_handler.go delete mode 100644 cmd/outline-ss-server/main_test.go delete mode 100644 cmd/outline-ss-server/metrics.go delete mode 100644 service/config.go delete mode 100644 service/config_test.go delete mode 100644 service/metrics.go delete mode 100644 service/metrics_test.go diff --git a/caddy/README.md b/caddy/README.md new file mode 100644 index 00000000..cc2ffb9f --- /dev/null +++ b/caddy/README.md @@ -0,0 +1,25 @@ +# Caddy Module + +The Caddy module provides an app and handler for Caddy Server +(https://caddyserver.com/) allowing it to turn any Caddy Server into an Outline +Shadowsocks backend. + +## Prerequisites + +- [xcaddy](https://github.com/caddyserver/xcaddy) + +## Usage + +From this directory, build and run a custom binary with `xcaddy`: + +```sh +xcaddy run --config config_example.json +``` + +In a separate window, confirm you can fetch a page using this server: + +```sh +go run github.com/Jigsaw-Code/outline-sdk/x/examples/fetch -transport "ss://chacha20-ietf-poly1305:Secret1@:9000" http://ipinfo.io +``` + +Prometheus metrics are exposed on http://localhost:2019/metrics. diff --git a/caddy/app.go b/caddy/app.go new file mode 100644 index 00000000..6ca00335 --- /dev/null +++ b/caddy/app.go @@ -0,0 +1,126 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package caddy provides an app and handler for Caddy Server (https://caddyserver.com/) +// allowing it to turn any handler into one supporting the Vulcain protocol. + +package caddy + +import ( + "errors" + + outline_prometheus "github.com/Jigsaw-Code/outline-ss-server/prometheus" + outline "github.com/Jigsaw-Code/outline-ss-server/service" + "github.com/caddyserver/caddy/v2" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +func init() { + caddy.RegisterModule(OutlineApp{}) +} + +const moduleName = "outline" + +type ShadowsocksConfig struct { + ReplayHistory int `json:"replay_history,omitempty"` +} + +type OutlineApp struct { + Version string `json:"version,omitempty"` + ShadowsocksConfig *ShadowsocksConfig `json:"shadowsocks,omitempty"` + + Metrics outline.ServiceMetrics + ReplayCache outline.ReplayCache + logger *zap.Logger +} + +func (OutlineApp) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: moduleName, + New: func() caddy.Module { return new(OutlineApp) }, + } +} + +// Provision sets up Outline. +func (app *OutlineApp) Provision(ctx caddy.Context) error { + app.logger = ctx.Logger(app) + defer app.logger.Sync() + + app.logger.Info("provisioning app instance") + + if app.Version == "" { + app.Version = "dev" + } + if app.ShadowsocksConfig != nil { + app.ReplayCache = outline.NewReplayCache(app.ShadowsocksConfig.ReplayHistory) + } + + if err := app.defineMetrics(); err != nil { + app.logger.Error("failed to create Prometheus metrics", zap.Error(err)) + } + + return nil +} + +func (app *OutlineApp) defineMetrics() error { + // TODO: Use `once.Do()` instead of catching already registered collectors? + // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? + r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) + + serverMetrics := outline_prometheus.NewServerMetrics() + registeredServerMetrics := registerCollector(r, serverMetrics) + registeredServerMetrics.SetVersion(app.Version) + // TODO: Call `registeredServerMetrics.SetNumAccessKeys()`. + + // TODO: Allow the configuration of ip2info. + serviceMetrics, err := outline_prometheus.NewServiceMetrics(nil) + if err != nil { + return err + } + registeredServiceMetrics := registerCollector(r, serviceMetrics) + app.Metrics = registeredServiceMetrics + + return nil +} + +func registerCollector[T prometheus.Collector](registerer prometheus.Registerer, coll T) T { + if err := registerer.Register(coll); err != nil { + are := &prometheus.AlreadyRegisteredError{} + if errors.As(err, are) { + // This collector has been registered before. This is expected during a config reload. + coll = are.ExistingCollector.(T) + } else { + panic(err) + } + } + return coll +} + +// Start starts the App. +func (app *OutlineApp) Start() error { + app.logger.Debug("started app instance") + return nil +} + +// Stop stops the App. +func (app *OutlineApp) Stop() error { + app.logger.Debug("stopped app instance") + return nil +} + +var ( + _ caddy.App = (*OutlineApp)(nil) + _ caddy.Provisioner = (*OutlineApp)(nil) +) diff --git a/caddy/config_example.json b/caddy/config_example.json new file mode 100644 index 00000000..bcbf5345 --- /dev/null +++ b/caddy/config_example.json @@ -0,0 +1,76 @@ +{ + "logging": { + "logs": { + "default": {"level":"DEBUG", "encoder": {"format":"console"}} + } + }, + "apps": { + "http": { + "servers": { + "": { + "metrics": {} + } + } + }, + "layer4": { + "servers": { + + "1": { + "listen": [ + "tcp/[::]:9000", + "udp/[::]:9000" + ], + "routes": [ + { + "handle": [ + { + "handler": "shadowsocks", + "keys": [ + { + "id": "user-0", + "cipher": "chacha20-ietf-poly1305", + "secret": "Secret0" + }, + { + "id": "user-1", + "cipher": "chacha20-ietf-poly1305", + "secret": "Secret1" + } + ] + } + ] + } + ] + }, + "2": { + "listen": [ + "tcp/[::]:9001", + "udp/[::]:9001" + ], + "routes": [ + { + "handle": [ + { + "handler": "shadowsocks", + "keys": [ + { + "id": "user-2", + "cipher": "chacha20-ietf-poly1305", + "secret": "Secret2" + } + ] + } + ] + } + ] + } + } + }, + "outline": { + "version": "0.0.0", + "shadowsocks": { + "replay_history": 10000 + } + } + } +} \ No newline at end of file diff --git a/caddy/go.mod b/caddy/go.mod new file mode 100644 index 00000000..2f2fa737 --- /dev/null +++ b/caddy/go.mod @@ -0,0 +1,126 @@ +module github.com/Jigsaw-Code/outline-ss-server/caddy + +go 1.23 + +require ( + github.com/Jigsaw-Code/outline-sdk v0.0.16 + github.com/Jigsaw-Code/outline-ss-server v1.5.0 + github.com/caddyserver/caddy/v2 v2.8.4 + github.com/mholt/caddy-l4 v0.0.0-20240812213304-afa78d72257b + github.com/prometheus/client_golang v1.20.0 + go.uber.org/zap v1.27.0 +) + +replace github.com/mholt/caddy-l4 v0.0.0-20240812213304-afa78d72257b => ../../caddy-l4 + +require ( + filippo.io/edwards25519 v1.1.0 // indirect + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/Masterminds/sprig/v3 v3.2.3 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/caddyserver/certmagic v0.21.3 // indirect + github.com/caddyserver/zerossl v0.1.3 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chzyer/readline v1.5.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/dgraph-io/badger v1.6.2 // indirect + github.com/dgraph-io/badger/v2 v2.2007.4 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-jose/go-jose/v3 v3.0.3 // indirect + github.com/go-kit/kit v0.13.0 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-sql-driver/mysql v1.7.1 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/golang/glog v1.2.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/cel-go v0.20.1 // indirect + github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/huandu/xstrings v1.4.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.14.3 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect + github.com/jackc/pgx/v4 v4.18.3 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/libdns/libdns v0.2.2 // indirect + github.com/manifoldco/promptui v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/mholt/acmez/v2 v2.0.1 // indirect + github.com/miekg/dns v1.1.59 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-ps v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.15.0 // indirect + github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect + github.com/oschwald/geoip2-golang v1.8.0 // indirect + github.com/oschwald/maxminddb-golang v1.10.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/quic-go v0.44.0 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shadowsocks/go-shadowsocks2 v0.1.5 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/slackhq/nebula v1.7.2 // indirect + github.com/smallstep/certificates v0.26.1 // indirect + github.com/smallstep/nosql v0.6.1 // indirect + github.com/smallstep/pkcs7 v0.0.0-20231024181729-3b98ecc1ca81 // indirect + github.com/smallstep/scep v0.0.0-20231024192529-aee96d7ad34d // indirect + github.com/smallstep/truststore v0.13.0 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/tailscale/tscert v0.0.0-20240517230440-bbccfbf48933 // indirect + github.com/urfave/cli v1.22.14 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect + go.etcd.io/bbolt v1.3.9 // indirect + go.step.sm/cli-utils v0.9.0 // indirect + go.step.sm/crypto v0.45.0 // indirect + go.step.sm/linkedca v0.20.1 // indirect + go.uber.org/automaxprocs v1.5.3 // indirect + go.uber.org/mock v0.4.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap/exp v0.2.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto/x509roots/fallback v0.0.0-20240507223354-67b13616a595 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + howett.net/plist v1.0.0 // indirect +) diff --git a/caddy/go.sum b/caddy/go.sum new file mode 100644 index 00000000..60fce3ee --- /dev/null +++ b/caddy/go.sum @@ -0,0 +1,619 @@ +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= +cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/kms v1.16.0 h1:1yZsRPhmargZOmY+fVAh8IKiR9HzCb0U1zsxb5g2nRY= +cloud.google.com/go/kms v1.16.0/go.mod h1:olQUXy2Xud+1GzYfiBO9N0RhjsJk5IJLU6n/ethLXVc= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/Jigsaw-Code/outline-sdk v0.0.16 h1:WbHmv80FKDIpzEmR3GehTbq5CibYTLvcxIIpMMILiEs= +github.com/Jigsaw-Code/outline-sdk v0.0.16/go.mod h1:e1oQZbSdLJBBuHgfeQsgEkvkuyIePPwstUeZRGq0KO8= +github.com/Jigsaw-Code/outline-ss-server v1.5.0 h1:Vz+iS0xR7i3PrLD82pzFFwZ9fsh6zrNawMeYERR8VTc= +github.com/Jigsaw-Code/outline-ss-server v1.5.0/go.mod h1:KaebwBiCWDSkgsJrJIbGH0szON8CZq4LgQaFV8v3RM4= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b h1:uUXgbcPDK3KpW29o4iy7GtuappbWT0l5NaMo9H9pJDw= +github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/config v1.27.13 h1:WbKW8hOzrWoOA/+35S5okqO/2Ap8hkkFUzoW8Hzq24A= +github.com/aws/aws-sdk-go-v2/config v1.27.13/go.mod h1:XLiyiTMnguytjRER7u5RIkhIqS8Nyz41SwAWb4xEjxs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.13 h1:XDCJDzk/u5cN7Aple7D/MiAhx1Rjo/0nueJ0La8mRuE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.13/go.mod h1:FMNcjQrmuBYvOTZDtOLCIu0esmxjF7RuA/89iSXWzQI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/kms v1.31.1 h1:5wtyAwuUiJiM3DHYeGZmP5iMonM7DFBWAEaaVPHYZA0= +github.com/aws/aws-sdk-go-v2/service/kms v1.31.1/go.mod h1:2snWQJQUKsbN66vAawJuOGX7dr37pfOq9hb0tZDGIqQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 h1:o5cTaeunSpfXiLTIBx5xo2enQmiChtu1IBbzXnfU9Hs= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.6/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0 h1:Qe0r0lVURDDeBQJ4yP+BOrJkvkiCo/3FH/t+wY11dmw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 h1:et3Ta53gotFR4ERLXXHIHl/Uuk1qYpP5uU7cvNql8ns= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.7/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/caddyserver/caddy/v2 v2.8.4 h1:q3pe0wpBj1OcHFZ3n/1nl4V4bxBrYoSoab7rL9BMYNk= +github.com/caddyserver/caddy/v2 v2.8.4/go.mod h1:vmDAHp3d05JIvuhc24LmnxVlsZmWnUwbP5WMjzcMPWw= +github.com/caddyserver/certmagic v0.21.3 h1:pqRRry3yuB4CWBVq9+cUqu+Y6E2z8TswbhNx1AZeYm0= +github.com/caddyserver/certmagic v0.21.3/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI= +github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= +github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= +github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-kit/kit v0.4.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= +github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= +github.com/google/certificate-transparency-go v1.1.8-0.20240110162603-74a5dd331745 h1:heyoXNxkRT155x4jTAiSv5BVSVkueifPUm+Q8LUXMRo= +github.com/google/certificate-transparency-go v1.1.8-0.20240110162603-74a5dd331745/go.mod h1:zN0wUQgV9LjwLZeFHnrAbQi8hzMVvEWePyk+MhPOk7k= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= +github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= +github.com/google/go-tpm-tools v0.4.4 h1:oiQfAIkc6xTy9Fl5NKTeTJkBTlXdHsxAofmQyxBKY98= +github.com/google/go-tpm-tools v0.4.4/go.mod h1:T8jXkp2s+eltnCDIsXR84/MTcVU9Ja7bh3Mit0pa4AY= +github.com/google/go-tspi v0.3.0 h1:ADtq8RKfP+jrTyIWIZDIYcKOMecRqNJFOew2IT0Inus= +github.com/google/go-tspi v0.3.0/go.mod h1:xfMGI3G0PhxCdNVcYr1C4C+EizojDg/TXuX5by8CiHI= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= +github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= +github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= +github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k= +github.com/mholt/acmez/v2 v2.0.1/go.mod h1:fX4c9r5jYwMyMsC+7tkYRxHibkOTgta5DIFGoe67e1U= +github.com/mholt/caddy-l4 v0.0.0-20240812213304-afa78d72257b h1:NPIvgpGOH1Pn0Pcz+UKNn8x/npjk8zttw1W67QpzWoc= +github.com/mholt/caddy-l4 v0.0.0-20240812213304-afa78d72257b/go.mod h1:xwlIq08dbs2FdmZZkAqrL5IGtLXP2OYUjmdIGxie/mI= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/oschwald/geoip2-golang v1.8.0 h1:KfjYB8ojCEn/QLqsDU0AzrJ3R5Qa9vFlx3z6SLNcKTs= +github.com/oschwald/geoip2-golang v1.8.0/go.mod h1:R7bRvYjOeaoenAp9sKRS8GX5bJWcZ0laWO5+DauEktw= +github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd9181uj2MQ5Vndg= +github.com/oschwald/maxminddb-golang v1.10.0/go.mod h1:Y2ELenReaLAZ0b400URyGwvYxHV1dLIxBuyOsyYjHK0= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU= +github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/quic-go v0.44.0 h1:So5wOr7jyO4vzL2sd8/pD9Kesciv91zSk8BoFngItQ0= +github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQlpnPcWmek= +github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 h1:f/FNXud6gA3MNr8meMVVGxhp+QBTqY91tM8HjEuMjGg= +github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3/go.mod h1:HgjTstvQsPGkxUsCd2KWxErBblirPizecHcpD3ffK+s= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/schollz/jsonstore v1.1.0 h1:WZBDjgezFS34CHI+myb4s8GGpir3UMpy7vWoCeO0n6E= +github.com/schollz/jsonstore v1.1.0/go.mod h1:15c6+9guw8vDRyozGjN3FoILt0wpruJk9Pi66vjaZfg= +github.com/shadowsocks/go-shadowsocks2 v0.1.5 h1:PDSQv9y2S85Fl7VBeOMF9StzeXZyK1HakRm86CUbr28= +github.com/shadowsocks/go-shadowsocks2 v0.1.5/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slackhq/nebula v1.7.2 h1:Rko1Mlksz/nC0c919xjGpB8uOSrTJ5e6KPgZx+lVfYw= +github.com/slackhq/nebula v1.7.2/go.mod h1:cnaoahkUipDs1vrNoIszyp0QPRIQN9Pm68ppQEW1Fhg= +github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY= +github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262/go.mod h1:MyOHs9Po2fbM1LHej6sBUT8ozbxmMOFG+E+rx/GSGuc= +github.com/smallstep/certificates v0.26.1 h1:FIUliEBcExSfJJDhRFA/s8aZgMIFuorexnRSKQd884o= +github.com/smallstep/certificates v0.26.1/go.mod h1:OQMrW39IrGKDViKSHrKcgSQArMZ8c7EcjhYKK7mYqis= +github.com/smallstep/go-attestation v0.4.4-0.20240109183208-413678f90935 h1:kjYvkvS/Wdy0PVRDUAA0gGJIVSEZYhiAJtfwYgOYoGA= +github.com/smallstep/go-attestation v0.4.4-0.20240109183208-413678f90935/go.mod h1:vNAduivU014fubg6ewygkAvQC0IQVXqdc8vaGl/0er4= +github.com/smallstep/nosql v0.6.1 h1:X8IBZFTRIp1gmuf23ne/jlD/BWKJtDQbtatxEn7Et1Y= +github.com/smallstep/nosql v0.6.1/go.mod h1:vrN+CftYYNnDM+DQqd863ATynvYFm/6FuY9D4TeAm2Y= +github.com/smallstep/pkcs7 v0.0.0-20231024181729-3b98ecc1ca81 h1:B6cED3iLJTgxpdh4tuqByDjRRKan2EvtnOfHr2zHJVg= +github.com/smallstep/pkcs7 v0.0.0-20231024181729-3b98ecc1ca81/go.mod h1:SoUAr/4M46rZ3WaLstHxGhLEgoYIDRqxQEXLOmOEB0Y= +github.com/smallstep/scep v0.0.0-20231024192529-aee96d7ad34d h1:06LUHn4Ia2X6syjIaCMNaXXDNdU+1N/oOHynJbWgpXw= +github.com/smallstep/scep v0.0.0-20231024192529-aee96d7ad34d/go.mod h1:4d0ub42ut1mMtvGyMensjuHYEUpRrASvkzLEJvoRQcU= +github.com/smallstep/truststore v0.13.0 h1:90if9htAOblavbMeWlqNLnO9bsjjgVv2hQeQJCi/py4= +github.com/smallstep/truststore v0.13.0/go.mod h1:3tmMp2aLKZ/OA/jnFUB0cYPcho402UG2knuJoPh4j7A= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tailscale/tscert v0.0.0-20240517230440-bbccfbf48933 h1:pV0H+XIvFoP7pl1MRtyPXh5hqoxB5I7snOtTHgrn6HU= +github.com/tailscale/tscert v0.0.0-20240517230440-bbccfbf48933/go.mod h1:kNGUQ3VESx3VZwRwA9MSCUegIl6+saPL8Noq82ozCaU= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= +go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.step.sm/cli-utils v0.9.0 h1:55jYcsQbnArNqepZyAwcato6Zy2MoZDRkWW+jF+aPfQ= +go.step.sm/cli-utils v0.9.0/go.mod h1:Y/CRoWl1FVR9j+7PnAewufAwKmBOTzR6l9+7EYGAnp8= +go.step.sm/crypto v0.45.0 h1:Z0WYAaaOYrJmKP9sJkPW+6wy3pgN3Ija8ek/D4serjc= +go.step.sm/crypto v0.45.0/go.mod h1:6IYlT0L2jfj81nVyCPpvA5cORy0EVHPhieSgQyuwHIY= +go.step.sm/linkedca v0.20.1 h1:bHDn1+UG1NgRrERkWbbCiAIvv4lD5NOFaswPDTyO5vU= +go.step.sm/linkedca v0.20.1/go.mod h1:Vaq4+Umtjh7DLFI1KuIxeo598vfBzgSYZUjgVJ7Syxw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.2.0 h1:FtGenNNeCATRB3CmB/yEUnjEFeJWpB/pMcy7e2bKPYs= +go.uber.org/zap/exp v0.2.0/go.mod h1:t0gqAIdh1MfKv9EwN/dLwfZnJxe9ITAZN78HEWPFWDQ= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto/x509roots/fallback v0.0.0-20240507223354-67b13616a595 h1:TgSqweA595vD0Zt86JzLv3Pb/syKg8gd5KMGGbJPYFw= +golang.org/x/crypto/x509roots/fallback v0.0.0-20240507223354-67b13616a595/go.mod h1:kNa9WdvYnzFwC79zRpLRMJbdEFlhyM5RPFBBZp/wWH8= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.180.0 h1:M2D87Yo0rGBPWpo1orwfCLehUUL6E7/TYe5gvMQWDh4= +google.golang.org/api v0.180.0/go.mod h1:51AiyoEg1MJPSZ9zvklA8VnRILPXxn1iVen9v25XHAE= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= +google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= +google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= +howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= diff --git a/caddy/logger.go b/caddy/logger.go new file mode 100644 index 00000000..95cc2c7e --- /dev/null +++ b/caddy/logger.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package caddy + +import ( + "context" + "log/slog" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type logger struct { + zap *zap.Logger +} + +func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { + return l.zap.Check(toZapLevel(level), "") != nil +} + +func (l *logger) LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { + fields := toZapFields(attrs) + l.zap.Log(toZapLevel(level), msg, fields...) + +} + +func toZapLevel(level slog.Level) zapcore.Level { + switch level { + case slog.LevelInfo: + return zapcore.InfoLevel + case slog.LevelWarn: + return zapcore.WarnLevel + case slog.LevelError: + return zapcore.ErrorLevel + default: + return zapcore.DebugLevel + } +} + +func toZapFields(attrs []slog.Attr) []zapcore.Field { + fields := make([]zapcore.Field, 0, len(attrs)) + var field zapcore.Field + for _, attr := range attrs { + field = zap.Any(attr.Key, attr.Value) + fields = append(fields, field) + } + + return fields +} diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go new file mode 100644 index 00000000..176f33ea --- /dev/null +++ b/caddy/shadowsocks_handler.go @@ -0,0 +1,125 @@ +// Copyright 2024 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package caddy + +import ( + "container/list" + "fmt" + "net" + + "github.com/Jigsaw-Code/outline-sdk/transport" + "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" + outline "github.com/Jigsaw-Code/outline-ss-server/service" + "github.com/caddyserver/caddy/v2" + "github.com/mholt/caddy-l4/layer4" + "go.uber.org/zap" +) + +func init() { + caddy.RegisterModule(&ShadowsocksHandler{}) +} + +type KeyConfig struct { + ID string + Cipher string + Secret string +} + +type ShadowsocksHandler struct { + Keys []KeyConfig `json:"keys,omitempty"` + NatTimeoutSec int `json:"nat_timeout_sec,omitempty"` + + service outline.Service + logger *zap.Logger +} + +func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "layer4.handlers.shadowsocks", + New: func() caddy.Module { return new(ShadowsocksHandler) }, + } +} + +// Provision implements caddy.Provisioner. +func (h *ShadowsocksHandler) Provision(ctx caddy.Context) error { + h.logger = ctx.Logger(h) + defer h.logger.Sync() + + ctx.App(moduleName) + if _, err := ctx.AppIfConfigured(moduleName); err != nil { + return fmt.Errorf("outline app configure error: %w", err) + } + mod, err := ctx.App(moduleName) + if err != nil { + return err + } + app := mod.(*OutlineApp) + + if len(h.Keys) == 0 { + h.logger.Warn("no keys configured") + } + type cipherKey struct { + cipher string + secret string + } + cipherList := list.New() + existingCiphers := make(map[cipherKey]bool) + for _, cfg := range h.Keys { + key := cipherKey{cfg.Cipher, cfg.Secret} + if _, exists := existingCiphers[key]; exists { + h.logger.Debug("Encryption key already exists. Skipping.", zap.String("id", cfg.ID)) + continue + } + cryptoKey, err := shadowsocks.NewEncryptionKey(cfg.Cipher, cfg.Secret) + if err != nil { + return fmt.Errorf("failed to create encyption key for key %v: %w", cfg.ID, err) + } + entry := outline.MakeCipherEntry(cfg.ID, cryptoKey, cfg.Secret) + cipherList.PushBack(&entry) + existingCiphers[key] = true + } + ciphers := outline.NewCipherList() + ciphers.Update(cipherList) + + service, err := outline.NewShadowsocksService( + outline.WithLogger(&logger{app.logger}), + outline.WithCiphers(ciphers), + outline.WithMetrics(app.Metrics), + outline.WithReplayCache(&app.ReplayCache), + ) + if err != nil { + return err + } + h.service = service + return nil +} + +// Handle implements layer4.NextHandler. +func (h *ShadowsocksHandler) Handle(cx *layer4.Connection, _ layer4.Handler) error { + switch conn := cx.Conn.(type) { + case transport.StreamConn: + h.service.HandleStream(cx.Context, conn) + case net.PacketConn: + h.service.HandlePacket(conn) + default: + return fmt.Errorf("failed to handle unknown connection type: %t", conn) + } + return nil +} + +var ( + _ caddy.Provisioner = (*ShadowsocksHandler)(nil) + _ layer4.NextHandler = (*ShadowsocksHandler)(nil) +) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 64d5145d..06241452 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -30,6 +30,7 @@ import ( "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + outline_prometheus "github.com/Jigsaw-Code/outline-ss-server/prometheus" "github.com/Jigsaw-Code/outline-ss-server/service" "github.com/lmittmann/tint" "github.com/prometheus/client_golang/prometheus" @@ -70,8 +71,8 @@ func (s *SSServer) loadConfig(filename string) error { if err != nil { return fmt.Errorf("failed to read config file %s: %w", filename, err) } - config := &service.Config{} - if err := config.LoadFrom(configData); err != nil { + config, err := readConfig(configData) + if err != nil { return fmt.Errorf("failed to load config (%v): %w", filename, err) } if err := config.Validate(); err != nil { @@ -92,6 +93,33 @@ func (s *SSServer) loadConfig(filename string) error { return nil } +func newCipherListFromConfig(config ServiceConfig) (service.CipherList, error) { + type cipherKey struct { + cipher string + secret string + } + cipherList := list.New() + existingCiphers := make(map[cipherKey]bool) + for _, keyConfig := range config.Keys { + key := cipherKey{keyConfig.Cipher, keyConfig.Secret} + if _, exists := existingCiphers[key]; exists { + slog.Debug("Encryption key already exists. Skipping.", "id", keyConfig.ID) + continue + } + cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) + if err != nil { + return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) + } + entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) + cipherList.PushBack(&entry) + existingCiphers[key] = true + } + ciphers := service.NewCipherList() + ciphers.Update(cipherList) + + return ciphers, nil +} + type listenerSet struct { manager service.ListenerManager listenerCloseFuncs map[string]func() error @@ -153,7 +181,7 @@ func (ls *listenerSet) Len() int { return len(ls.listenerCloseFuncs) } -func (s *SSServer) runConfig(config service.Config) (func() error, error) { +func (s *SSServer) runConfig(config Config) (func() error, error) { startErrCh := make(chan error) stopErrCh := make(chan error) stopCh := make(chan struct{}) @@ -189,7 +217,8 @@ func (s *SSServer) runConfig(config service.Config) (func() error, error) { ciphers := service.NewCipherList() ciphers.Update(cipherList) - ssService, err := service.NewService( + ssService, err := service.NewShadowsocksService( + service.WithLogger(slog.Default()), service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), @@ -211,8 +240,14 @@ func (s *SSServer) runConfig(config service.Config) (func() error, error) { } for _, serviceConfig := range config.Services { - ssService, err := service.NewService( - service.WithConfig(serviceConfig), + ciphers, err := newCipherListFromConfig(serviceConfig) + if err != nil { + return fmt.Errorf("failed to create cipher list from config: %v", err) + } + ssService, err := service.NewShadowsocksService( + service.WithLogger(slog.Default()), + service.WithCiphers(ciphers), + service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), service.WithReplayCache(&s.replayCache), ) @@ -221,14 +256,14 @@ func (s *SSServer) runConfig(config service.Config) (func() error, error) { } for _, lnConfig := range serviceConfig.Listeners { switch lnConfig.Type { - case service.ListenerTypeTCP: + case listenerTypeTCP: ln, err := lnSet.ListenStream(lnConfig.Address) if err != nil { return err } slog.Info("TCP service started.", "address", ln.Addr().String()) go service.StreamServe(ln.AcceptStream, ssService.HandleStream) - case service.ListenerTypeUDP: + case listenerTypeUDP: pc, err := lnSet.ListenPacket(lnConfig.Address) if err != nil { return err @@ -361,9 +396,9 @@ func main() { } defer ip2info.Close() - serverMetrics := newPrometheusServerMetrics() + serverMetrics := outline_prometheus.NewServerMetrics() serverMetrics.SetVersion(version) - serviceMetrics, err := service.NewPrometheusServiceMetrics(ip2info) + serviceMetrics, err := outline_prometheus.NewServiceMetrics(ip2info) if err != nil { slog.Error("Failed to create Outline Prometheus service metrics. Aborting.", "err", err) } diff --git a/cmd/outline-ss-server/main_test.go b/cmd/outline-ss-server/main_test.go deleted file mode 100644 index 60587cbd..00000000 --- a/cmd/outline-ss-server/main_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2020 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "testing" - "time" - - "github.com/Jigsaw-Code/outline-ss-server/service" - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" -) - -func TestRunSSServer(t *testing.T) { - m := service.NewPrometheusOutlineMetrics(nil, prometheus.DefaultRegisterer) - server, err := RunSSServer("config_example.yml", m, 30*time.Second, 10000) - if err != nil { - t.Fatalf("RunSSServer() error = %v", err) - } - if err := server.Stop(); err != nil { - t.Errorf("Error while stopping server: %v", err) - } -} - -func TestReadConfig(t *testing.T) { - config, err := readConfigFile("./config_example.yml") - - require.NoError(t, err) - expected := service.Config{ - Services: []service.ServiceConfig{ - service.ServiceConfig{ - Listeners: []service.ListenerConfig{ - service.ListenerConfig{Type: "tcp", Address: "[::]:9000"}, - service.ListenerConfig{Type: "udp", Address: "[::]:9000"}, - }, - Keys: []service.KeyConfig{ - service.KeyConfig{"user-0", "chacha20-ietf-poly1305", "Secret0"}, - service.KeyConfig{"user-1", "chacha20-ietf-poly1305", "Secret1"}, - }, - }, - service.ServiceConfig{ - Listeners: []service.ListenerConfig{ - service.ListenerConfig{Type: "tcp", Address: "[::]:9001"}, - service.ListenerConfig{Type: "udp", Address: "[::]:9001"}, - }, - Keys: []service.KeyConfig{ - service.KeyConfig{"user-2", "chacha20-ietf-poly1305", "Secret2"}, - }, - }, - }, - } - require.Equal(t, expected, *config) -} - -func TestReadConfigParsesDeprecatedFormat(t *testing.T) { - config, err := readConfigFile("./config_example.deprecated.yml") - - require.NoError(t, err) - expected := service.Config{ - Keys: []service.LegacyKeyServiceConfig{ - service.LegacyKeyServiceConfig{ - KeyConfig: service.KeyConfig{ID: "user-0", Cipher: "chacha20-ietf-poly1305", Secret: "Secret0"}, - Port: 9000, - }, - service.LegacyKeyServiceConfig{ - KeyConfig: service.KeyConfig{ID: "user-1", Cipher: "chacha20-ietf-poly1305", Secret: "Secret1"}, - Port: 9000, - }, - service.LegacyKeyServiceConfig{ - KeyConfig: service.KeyConfig{ID: "user-2", Cipher: "chacha20-ietf-poly1305", Secret: "Secret2"}, - Port: 9001, - }, - }, - } - require.Equal(t, expected, *config) -} - -func TestReadConfigFromEmptyFile(t *testing.T) { - file, _ := os.CreateTemp("", "empty.yaml") - - config, err := readConfigFile(file.Name()) - - require.NoError(t, err) - require.ElementsMatch(t, service.Config{}, config) -} - -func TestReadConfigFromIncorrectFormatFails(t *testing.T) { - file, _ := os.CreateTemp("", "empty.yaml") - file.WriteString("foo") - - config, err := readConfigFile(file.Name()) - - require.Error(t, err) - require.ElementsMatch(t, service.Config{}, config) -} - -func readConfigFile(filename string) (*service.Config, error) { - configData, _ := os.ReadFile(filename) - return readConfig(configData) -} diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go deleted file mode 100644 index 32c9b0aa..00000000 --- a/cmd/outline-ss-server/metrics.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2023 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" -) - -// `now` is stubbable for testing. -var now = time.Now - -type serverMetrics struct { - // NOTE: New metrics need to be added to `newPrometheusServerMetrics()`, `Describe()` and `Collect()`. - buildInfo *prometheus.GaugeVec - accessKeys prometheus.Gauge - ports prometheus.Gauge -} - -var _ prometheus.Collector = (*serverMetrics)(nil) - -// newPrometheusServerMetrics constructs a Prometheus metrics collector for server -// related metrics. -func newPrometheusServerMetrics() *serverMetrics { - return &serverMetrics{ - buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "build_info", - Help: "Information on the outline-ss-server build", - }, []string{"version"}), - accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "keys", - Help: "Count of access keys", - }), - ports: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "ports", - Help: "Count of open ports", - }), - } -} - -func (m *serverMetrics) Describe(ch chan<- *prometheus.Desc) { - m.buildInfo.Describe(ch) - m.accessKeys.Describe(ch) - m.ports.Describe(ch) -} - -func (m *serverMetrics) Collect(ch chan<- prometheus.Metric) { - m.buildInfo.Collect(ch) - m.accessKeys.Collect(ch) - m.ports.Collect(ch) -} - -func (m *serverMetrics) SetVersion(version string) { - m.buildInfo.WithLabelValues(version).Set(1) -} - -func (m *serverMetrics) SetNumAccessKeys(numKeys int, ports int) { - m.accessKeys.Set(float64(numKeys)) - m.ports.Set(float64(ports)) -} diff --git a/cmd/outline-ss-server/server_test.go b/cmd/outline-ss-server/server_test.go index 05999486..9b8400a0 100644 --- a/cmd/outline-ss-server/server_test.go +++ b/cmd/outline-ss-server/server_test.go @@ -22,7 +22,7 @@ import ( ) func TestRunSSServer(t *testing.T) { - serverMetrics := newPrometheusServerMetrics() + serverMetrics := prometheus.NewServerMetrics() serviceMetrics, err := prometheus.NewServiceMetrics(nil) if err != nil { t.Fatalf("Failed to create Prometheus service metrics: %v", err) diff --git a/prometheus/metrics.go b/prometheus/metrics.go index 186cba7c..8c05140e 100644 --- a/prometheus/metrics.go +++ b/prometheus/metrics.go @@ -572,3 +572,52 @@ func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { } return &IPKey{ip, accessKey}, nil } + +type serverMetrics struct { + // NOTE: New metrics need to be added to `NewServerMetrics()`, `Describe()` and `Collect()`. + buildInfo *prometheus.GaugeVec + accessKeys prometheus.Gauge + ports prometheus.Gauge +} + +var _ prometheus.Collector = (*serverMetrics)(nil) + +// NewServerMetrics constructs a Prometheus metrics collector for server +// related metrics. +func NewServerMetrics() *serverMetrics { + return &serverMetrics{ + buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "build_info", + Help: "Information on the outline-ss-server build", + }, []string{"version"}), + accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "keys", + Help: "Count of access keys", + }), + ports: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "ports", + Help: "Count of open ports", + }), + } +} + +func (m *serverMetrics) Describe(ch chan<- *prometheus.Desc) { + m.buildInfo.Describe(ch) + m.accessKeys.Describe(ch) + m.ports.Describe(ch) +} + +func (m *serverMetrics) Collect(ch chan<- prometheus.Metric) { + m.buildInfo.Collect(ch) + m.accessKeys.Collect(ch) + m.ports.Collect(ch) +} + +func (m *serverMetrics) SetVersion(version string) { + m.buildInfo.WithLabelValues(version).Set(1) +} + +func (m *serverMetrics) SetNumAccessKeys(numKeys int, ports int) { + m.accessKeys.Set(float64(numKeys)) + m.ports.Set(float64(ports)) +} diff --git a/service/config.go b/service/config.go deleted file mode 100644 index 4fa242e9..00000000 --- a/service/config.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2024 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "fmt" - "net" - - "gopkg.in/yaml.v3" -) - -type ServiceConfig struct { - Listeners []ListenerConfig - Keys []KeyConfig - NatTimeoutSec int -} - -type ListenerType string - -const ListenerTypeTCP ListenerType = "tcp" -const ListenerTypeUDP ListenerType = "udp" - -type ListenerConfig struct { - Type ListenerType - Address string -} - -type KeyConfig struct { - ID string - Cipher string - Secret string -} - -type LegacyKeyServiceConfig struct { - KeyConfig `yaml:",inline"` - Port int -} - -type Config struct { - Services []ServiceConfig - - // Deprecated: `keys` exists for backward compatibility. Prefer to configure - // using the newer `services` format. - Keys []LegacyKeyServiceConfig -} - -// LoadFrom attempts to load and parse config yaml bytes as a [Config]. -func (c *Config) LoadFrom(configData []byte) error { - if err := yaml.Unmarshal(configData, &c); err != nil { - return fmt.Errorf("failed to parse config: %w", err) - } - return nil -} - -// Validate checks that the config is valid. -func (c *Config) Validate() error { - existingListeners := make(map[string]bool) - for _, serviceConfig := range c.Services { - for _, lnConfig := range serviceConfig.Listeners { - // TODO: Support more listener types. - if lnConfig.Type != ListenerTypeTCP && lnConfig.Type != ListenerTypeUDP { - return fmt.Errorf("unsupported listener type: %s", lnConfig.Type) - } - host, _, err := net.SplitHostPort(lnConfig.Address) - if err != nil { - return fmt.Errorf("invalid listener address `%s`: %v", lnConfig.Address, err) - } - if ip := net.ParseIP(host); ip == nil { - return fmt.Errorf("address must be IP, found: %s", host) - } - key := string(lnConfig.Type) + "/" + lnConfig.Address - if _, exists := existingListeners[key]; exists { - return fmt.Errorf("listener of type %s with address %s already exists.", lnConfig.Type, lnConfig.Address) - } - existingListeners[key] = true - } - } - return nil -} diff --git a/service/config_test.go b/service/config_test.go deleted file mode 100644 index d31e8ff6..00000000 --- a/service/config_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2024 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestValidateConfigFails(t *testing.T) { - tests := []struct { - name string - cfg *Config - }{ - { - name: "WithUnknownListenerType", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: "foo", Address: "[::]:9000"}, - }, - }, - }, - }, - }, - { - name: "WithInvalidListenerAddress", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "tcp/[::]:9000"}, - }, - }, - }, - }, - }, - { - name: "WithHostnameAddress", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "example.com:9000"}, - }, - }, - }, - }, - }, - { - name: "WithDuplicateListeners", - cfg: &Config{ - Services: []ServiceConfig{ - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, - }, - }, - ServiceConfig{ - Listeners: []ListenerConfig{ - ListenerConfig{Type: listenerTypeTCP, Address: "[::]:9000"}, - }, - }, - }, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.cfg.Validate() - require.Error(t, err) - }) - } -} diff --git a/service/logger.go b/service/logger.go index 79b8ee3a..2e8cab9e 100644 --- a/service/logger.go +++ b/service/logger.go @@ -14,6 +14,24 @@ package service -import logging "github.com/op/go-logging" +import ( + "context" + "log/slog" +) -var logger = logging.MustGetLogger("shadowsocks") +type logger interface { + Enabled(ctx context.Context, level slog.Level) bool + LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) +} + +type noopLogger struct { +} + +var _ logger = (*noopLogger)(nil) + +func (l *noopLogger) Enabled(ctx context.Context, level slog.Level) bool { + return false +} + +func (l *noopLogger) LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { +} diff --git a/service/metrics.go b/service/metrics.go deleted file mode 100644 index de118444..00000000 --- a/service/metrics.go +++ /dev/null @@ -1,573 +0,0 @@ -// Copyright 2023 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "fmt" - "log/slog" - "net" - "net/netip" - "sync" - "time" - - "github.com/Jigsaw-Code/outline-ss-server/ipinfo" - "github.com/Jigsaw-Code/outline-ss-server/service/metrics" - "github.com/prometheus/client_golang/prometheus" -) - -// `now` is stubbable for testing. -var now = time.Now - -func NewTimeToCipherVec(proto string) (prometheus.ObserverVec, error) { - vec := prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "time_to_cipher_ms", - Help: "Time needed to find the cipher", - Buckets: []float64{0.1, 1, 10, 100, 1000}, - }, []string{"proto", "found_key"}) - return vec.CurryWith(map[string]string{"proto": proto}) -} - -type proxyCollector struct { - // NOTE: New metrics need to be added to `newProxyCollector()`, `Describe()` and `Collect()`. - dataBytesPerKey *prometheus.CounterVec - dataBytesPerLocation *prometheus.CounterVec -} - -func newProxyCollector(proto string) (*proxyCollector, error) { - dataBytesPerKey, err := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "data_bytes", - Help: "Bytes transferred by the proxy, per access key", - }, []string{"proto", "dir", "access_key"}).CurryWith(map[string]string{"proto": proto}) - if err != nil { - return nil, err - } - dataBytesPerLocation, err := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "data_bytes_per_location", - Help: "Bytes transferred by the proxy, per location", - }, []string{"proto", "dir", "location", "asn", "asorg"}).CurryWith(map[string]string{"proto": proto}) - if err != nil { - return nil, err - } - return &proxyCollector{ - dataBytesPerKey: dataBytesPerKey, - dataBytesPerLocation: dataBytesPerLocation, - }, nil -} - -func (c *proxyCollector) Describe(ch chan<- *prometheus.Desc) { - c.dataBytesPerKey.Describe(ch) - c.dataBytesPerLocation.Describe(ch) -} - -func (c *proxyCollector) Collect(ch chan<- prometheus.Metric) { - c.dataBytesPerKey.Collect(ch) - c.dataBytesPerLocation.Collect(ch) -} - -func (c *proxyCollector) addClientTarget(clientProxyBytes, proxyTargetBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { - addIfNonZero(clientProxyBytes, c.dataBytesPerKey, "c>p", accessKey) - addIfNonZero(clientProxyBytes, c.dataBytesPerLocation, "c>p", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) - addIfNonZero(proxyTargetBytes, c.dataBytesPerKey, "p>t", accessKey) - addIfNonZero(proxyTargetBytes, c.dataBytesPerLocation, "p>t", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN.Number), clientInfo.ASN.Organization) -} - -func (c *proxyCollector) addTargetClient(targetProxyBytes, proxyClientBytes int64, accessKey string, clientInfo ipinfo.IPInfo) { - addIfNonZero(targetProxyBytes, c.dataBytesPerKey, "p 0 { - counterVec.WithLabelValues(lvs...).Add(float64(value)) - } -} - -func asnLabel(asn int) string { - if asn == 0 { - return "" - } - return fmt.Sprint(asn) -} - -// Converts a [net.Addr] to an [IPKey]. -func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { - hostname, _, err := net.SplitHostPort(addr.String()) - if err != nil { - return nil, fmt.Errorf("failed to create IPKey: %w", err) - } - ip, err := netip.ParseAddr(hostname) - if err != nil { - return nil, fmt.Errorf("failed to create IPKey: %w", err) - } - return &IPKey{ip, accessKey}, nil -} diff --git a/service/metrics_test.go b/service/metrics_test.go deleted file mode 100644 index 288b695c..00000000 --- a/service/metrics_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2023 Jigsaw Operations LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "net" - "strings" - "testing" - "time" - - "github.com/Jigsaw-Code/outline-ss-server/ipinfo" - "github.com/op/go-logging" - "github.com/prometheus/client_golang/prometheus" - promtest "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/stretchr/testify/require" -) - -type noopMap struct{} - -func (*noopMap) GetIPInfo(ip net.IP) (ipinfo.IPInfo, error) { - return ipinfo.IPInfo{}, nil -} - -type fakeAddr string - -func (a fakeAddr) String() string { return string(a) } -func (a fakeAddr) Network() string { return "" } - -// Sets the processing clock to be t until changed. -func setNow(t time.Time) { - now = func() time.Time { - return t - } -} - -func init() { - logging.SetLevel(logging.INFO, "") -} - -type fakeConn struct { - net.Conn -} - -func (c *fakeConn) LocalAddr() net.Addr { - return fakeAddr("127.0.0.1:9") -} - -func (c *fakeConn) RemoteAddr() net.Addr { - return fakeAddr("127.0.0.1:10") -} - -func TestMethodsDontPanic(t *testing.T) { - m := newPrometheusServerMetrics() - m.SetVersion("0.0.0-test") - m.SetNumAccessKeys(20, 2) -} - -func TestSetVersion(t *testing.T) { - m := newPrometheusServerMetrics() - reg := prometheus.NewPedanticRegistry() - reg.MustRegister(m) - - m.SetVersion("0.0.0-test") - - err := promtest.GatherAndCompare( - reg, - strings.NewReader(` - # HELP build_info Information on the outline-ss-server build - # TYPE build_info gauge - build_info{version="0.0.0-test"} 1 - `), - "build_info", - ) - require.NoError(t, err, "unexpected metric value found") -} - -func TestSetNumAccessKeys(t *testing.T) { - m := newPrometheusServerMetrics() - reg := prometheus.NewPedanticRegistry() - reg.MustRegister(m) - - m.SetNumAccessKeys(1, 2) - - err := promtest.GatherAndCompare( - reg, - strings.NewReader(` - # HELP keys Count of access keys - # TYPE keys gauge - keys 1 - # HELP ports Count of open ports - # TYPE ports gauge - ports 2 - `), - "keys", - "ports", - ) - require.NoError(t, err, "unexpected metric value found") -} diff --git a/service/shadowsocks.go b/service/shadowsocks.go index 8c43c50a..fcca785a 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -15,15 +15,12 @@ package service import ( - "container/list" "context" "fmt" - "log/slog" "net" "time" "github.com/Jigsaw-Code/outline-sdk/transport" - "github.com/Jigsaw-Code/outline-sdk/transport/shadowsocks" ) const ( @@ -54,6 +51,7 @@ type Service interface { type Option func(s *ssService) error type ssService struct { + logger logger m ServiceMetrics ciphers CipherList natTimeout time.Duration @@ -63,7 +61,7 @@ type ssService struct { ph PacketHandler } -func NewService(opts ...Option) (Service, error) { +func NewShadowsocksService(opts ...Option) (Service, error) { s := &ssService{} for _, opt := range opts { @@ -72,23 +70,21 @@ func NewService(opts ...Option) (Service, error) { } } + if s.logger == nil { + s.logger = &noopLogger{} + } + if s.natTimeout == 0 { s.natTimeout = defaultNatTimeout } return s, nil } -// WithConfig option function. -func WithConfig(config ServiceConfig) Option { +// WithLogger can be used to provide a custom log target. +// Defaults to io.Discard. +func WithLogger(l logger) Option { return func(s *ssService) error { - ciphers, err := newCipherListFromConfig(config) - if err != nil { - return fmt.Errorf("failed to create cipher list from config: %v", err) - } - s.ciphers = ciphers - - s.natTimeout = time.Duration(config.NatTimeoutSec) * time.Second - + s.logger = l return nil } } @@ -128,9 +124,13 @@ func WithNatTimeout(natTimeout time.Duration) Option { // HandleStream handles a Shadowsocks stream-based connection. func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { if s.sh == nil { - authFunc := NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}) + authFunc := NewShadowsocksStreamAuthenticator( + s.ciphers, + s.replayCache, + &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}, + ) // TODO: Register initial data metrics at zero. - s.sh = NewStreamHandler(authFunc, tcpReadTimeout) + s.sh = NewStreamHandler(s.logger, authFunc, tcpReadTimeout) } connMetrics := s.m.AddOpenTCPConnection(conn) s.sh.Handle(ctx, conn, connMetrics) @@ -139,38 +139,17 @@ func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) // HandlePacket handles a Shadowsocks packet connection. func (s *ssService) HandlePacket(conn net.PacketConn) { if s.ph == nil { - s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) + s.ph = NewPacketHandler( + s.logger, + s.natTimeout, + s.ciphers, + s.m, + &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}, + ) } s.ph.Handle(conn) } -func newCipherListFromConfig(config ServiceConfig) (CipherList, error) { - type cipherKey struct { - cipher string - secret string - } - cipherList := list.New() - existingCiphers := make(map[cipherKey]bool) - for _, keyConfig := range config.Keys { - key := cipherKey{keyConfig.Cipher, keyConfig.Secret} - if _, exists := existingCiphers[key]; exists { - slog.Debug("Encryption key already exists. Skipping.", "ID", keyConfig.ID) - continue - } - cryptoKey, err := shadowsocks.NewEncryptionKey(keyConfig.Cipher, keyConfig.Secret) - if err != nil { - return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) - } - entry := MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - cipherList.PushBack(&entry) - existingCiphers[key] = true - } - ciphers := NewCipherList() - ciphers.Update(cipherList) - - return ciphers, nil -} - type ssConnMetrics struct { ServiceMetrics proto string diff --git a/service/tcp.go b/service/tcp.go index 8637663b..7a490bb5 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -58,11 +58,11 @@ func remoteIP(conn net.Conn) netip.Addr { } // Wrapper for slog.Debug during TCP access key searches. -func debugTCP(template string, cipherID string, attr slog.Attr) { +func debugTCP(l logger, template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. - if slog.Default().Enabled(nil, slog.LevelDebug) { - slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("TCP: %s", template), slog.String("ID", cipherID), attr) + if l.Enabled(nil, slog.LevelDebug) { + l.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("TCP: %s", template), slog.String("ID", cipherID), attr) } } @@ -72,7 +72,7 @@ func debugTCP(template string, cipherID string, attr slog.Attr) { // required = saltSize + 2 + cipher.TagSize, the number of bytes needed to authenticate the connection. const bytesForKeyFinding = 50 -func findAccessKey(clientReader io.Reader, clientIP netip.Addr, cipherList CipherList) (*CipherEntry, io.Reader, []byte, time.Duration, error) { +func findAccessKey(l logger, clientReader io.Reader, clientIP netip.Addr, cipherList CipherList) (*CipherEntry, io.Reader, []byte, time.Duration, error) { // We snapshot the list because it may be modified while we use it. ciphers := cipherList.SnapshotForClientIP(clientIP) firstBytes := make([]byte, bytesForKeyFinding) @@ -81,7 +81,7 @@ func findAccessKey(clientReader io.Reader, clientIP netip.Addr, cipherList Ciphe } findStartTime := time.Now() - entry, elt := findEntry(firstBytes, ciphers) + entry, elt := findEntry(l, firstBytes, ciphers) timeToCipher := time.Since(findStartTime) if entry == nil { // TODO: Ban and log client IPs with too many failures too quick to protect against DoS. @@ -95,7 +95,7 @@ func findAccessKey(clientReader io.Reader, clientIP netip.Addr, cipherList Ciphe } // Implements a trial decryption search. This assumes that all ciphers are AEAD. -func findEntry(firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list.Element) { +func findEntry(l logger, firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list.Element) { // To hold the decrypted chunk length. chunkLenBuf := [2]byte{} for ci, elt := range ciphers { @@ -103,23 +103,23 @@ func findEntry(firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list. cryptoKey := entry.CryptoKey _, err := shadowsocks.Unpack(chunkLenBuf[:0], firstBytes[:cryptoKey.SaltSize()+2+cryptoKey.TagSize()], cryptoKey) if err != nil { - debugTCP("Failed to decrypt length.", entry.ID, slog.Any("err", err)) + debugTCP(l, "Failed to decrypt length.", entry.ID, slog.Any("err", err)) continue } - debugTCP("Found cipher.", entry.ID, slog.Int("index", ci)) + debugTCP(l, "Found cipher.", entry.ID, slog.Int("index", ci)) return entry, elt } return nil, nil } -type StreamAuthenticateFunc func(clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) +type StreamAuthenticateFunc func(l logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) // NewShadowsocksStreamAuthenticator creates a stream authenticator that uses Shadowsocks. // TODO(fortuna): Offer alternative transports. func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCache, metrics ShadowsocksConnMetrics) StreamAuthenticateFunc { - return func(clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { + return func(l logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { // Find the cipher and acess key id. - cipherEntry, clientReader, clientSalt, timeToCipher, keyErr := findAccessKey(clientConn, remoteIP(clientConn), ciphers) + cipherEntry, clientReader, clientSalt, timeToCipher, keyErr := findAccessKey(l, clientConn, remoteIP(clientConn), ciphers) metrics.AddCipherSearch(keyErr == nil, timeToCipher) if keyErr != nil { const status = "ERR_CIPHER" @@ -151,6 +151,7 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa } type streamHandler struct { + l logger listenerId string readTimeout time.Duration authenticate StreamAuthenticateFunc @@ -158,8 +159,9 @@ type streamHandler struct { } // NewStreamHandler creates a StreamHandler -func NewStreamHandler(authenticate StreamAuthenticateFunc, timeout time.Duration) StreamHandler { +func NewStreamHandler(l logger, authenticate StreamAuthenticateFunc, timeout time.Duration) StreamHandler { return &streamHandler{ + l: l, readTimeout: timeout, authenticate: authenticate, dialer: defaultDialer, @@ -251,11 +253,11 @@ func (h *streamHandler) Handle(ctx context.Context, clientConn transport.StreamC status := "OK" if connError != nil { status = connError.Status - slog.LogAttrs(nil, slog.LevelDebug, "TCP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) + h.l.LogAttrs(nil, slog.LevelDebug, "TCP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) } connMetrics.AddClosed(status, proxyMetrics, connDuration) measuredClientConn.Close() // Closing after the metrics are added aids integration testing. - slog.LogAttrs(nil, slog.LevelDebug, "TCP: Done.", slog.String("status", status), slog.Duration("duration", connDuration)) + h.l.LogAttrs(nil, slog.LevelDebug, "TCP: Done.", slog.String("status", status), slog.Duration("duration", connDuration)) } func getProxyRequest(clientConn transport.StreamConn) (string, error) { @@ -270,14 +272,14 @@ func getProxyRequest(clientConn transport.StreamConn) (string, error) { return tgtAddr.String(), nil } -func proxyConnection(ctx context.Context, dialer transport.StreamDialer, tgtAddr string, clientConn transport.StreamConn) *onet.ConnectionError { +func proxyConnection(l logger, ctx context.Context, dialer transport.StreamDialer, tgtAddr string, clientConn transport.StreamConn) *onet.ConnectionError { tgtConn, dialErr := dialer.DialStream(ctx, tgtAddr) if dialErr != nil { // We don't drain so dial errors and invalid addresses are communicated quickly. return ensureConnectionError(dialErr, "ERR_CONNECT", "Failed to connect to target") } defer tgtConn.Close() - slog.LogAttrs(nil, slog.LevelDebug, "Proxy connection.", slog.String("client", clientConn.RemoteAddr().String()), slog.String("target", tgtConn.RemoteAddr().String())) + l.LogAttrs(nil, slog.LevelDebug, "Proxy connection.", slog.String("client", clientConn.RemoteAddr().String()), slog.String("target", tgtConn.RemoteAddr().String())) fromClientErrCh := make(chan error) go func() { @@ -319,7 +321,7 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor } outerConn.SetReadDeadline(readDeadline) - id, innerConn, authErr := h.authenticate(outerConn) + id, innerConn, authErr := h.authenticate(h.l, outerConn) if authErr != nil { // Drain to protect against probing attacks. h.absorbProbe(outerConn, connMetrics, authErr.Status, proxyMetrics) @@ -345,7 +347,7 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor tgtConn = metrics.MeasureConn(tgtConn, &proxyMetrics.ProxyTarget, &proxyMetrics.TargetProxy) return tgtConn, nil }) - return proxyConnection(ctx, dialer, tgtAddr, innerConn) + return proxyConnection(h.l, ctx, dialer, tgtAddr, innerConn) } // Keep the connection open until we hit the authentication deadline to protect against probing attacks @@ -354,7 +356,7 @@ func (h *streamHandler) absorbProbe(clientConn io.ReadCloser, connMetrics TCPCon // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) - slog.LogAttrs(nil, slog.LevelDebug, "Drain error.", slog.Any("err", drainErr), slog.String("result", drainResult)) + h.l.LogAttrs(nil, slog.LevelDebug, "Drain error.", slog.Any("err", drainErr), slog.String("result", drainResult)) connMetrics.AddProbe(status, drainResult, proxyMetrics.ClientProxy) } diff --git a/service/udp.go b/service/udp.go index 2d08a709..74d8164a 100644 --- a/service/udp.go +++ b/service/udp.go @@ -44,23 +44,23 @@ type UDPMetrics interface { const serverUDPBufferSize = 64 * 1024 // Wrapper for slog.Debug during UDP proxying. -func debugUDP(template string, cipherID string, attr slog.Attr) { +func debugUDP(l logger, template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. - if slog.Default().Enabled(nil, slog.LevelDebug) { - slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("ID", cipherID), attr) + if l.Enabled(nil, slog.LevelDebug) { + l.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("ID", cipherID), attr) } } -func debugUDPAddr(template string, addr net.Addr, attr slog.Attr) { - if slog.Default().Enabled(nil, slog.LevelDebug) { - slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("address", addr.String()), attr) +func debugUDPAddr(l logger, template string, addr net.Addr, attr slog.Attr) { + if l.Enabled(nil, slog.LevelDebug) { + l.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("address", addr.String()), attr) } } // Decrypts src into dst. It tries each cipher until it finds one that authenticates // correctly. dst and src must not overlap. -func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherList) ([]byte, string, *shadowsocks.EncryptionKey, error) { +func findAccessKeyUDP(l logger, clientIP netip.Addr, dst, src []byte, cipherList CipherList) ([]byte, string, *shadowsocks.EncryptionKey, error) { // Try each cipher until we find one that authenticates successfully. This assumes that all ciphers are AEAD. // We snapshot the list because it may be modified while we use it. snapshot := cipherList.SnapshotForClientIP(clientIP) @@ -68,10 +68,10 @@ func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherLis id, cryptoKey := entry.Value.(*CipherEntry).ID, entry.Value.(*CipherEntry).CryptoKey buf, err := shadowsocks.Unpack(dst, src, cryptoKey) if err != nil { - debugUDP("Failed to unpack.", id, slog.Any("err", err)) + debugUDP(l, "Failed to unpack.", id, slog.Any("err", err)) continue } - debugUDP("Found cipher.", id, slog.Int("index", ci)) + debugUDP(l, "Found cipher.", id, slog.Int("index", ci)) // Move the active cipher to the front, so that the search is quicker next time. cipherList.MarkUsedByClientIP(entry, clientIP) return buf, id, cryptoKey, nil @@ -80,6 +80,7 @@ func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherLis } type packetHandler struct { + l logger natTimeout time.Duration ciphers CipherList m UDPMetrics @@ -88,8 +89,8 @@ type packetHandler struct { } // NewPacketHandler creates a UDPService -func NewPacketHandler(natTimeout time.Duration, cipherList CipherList, m UDPMetrics, ssMetrics ShadowsocksConnMetrics) PacketHandler { - return &packetHandler{natTimeout: natTimeout, ciphers: cipherList, m: m, ssm: ssMetrics, targetIPValidator: onet.RequirePublicIP} +func NewPacketHandler(l logger, natTimeout time.Duration, cipherList CipherList, m UDPMetrics, ssMetrics ShadowsocksConnMetrics) PacketHandler { + return &packetHandler{l: l, natTimeout: natTimeout, ciphers: cipherList, m: m, ssm: ssMetrics, targetIPValidator: onet.RequirePublicIP} } // PacketHandler is a running UDP shadowsocks proxy that can be stopped. @@ -109,7 +110,7 @@ func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPVali func (h *packetHandler) Handle(clientConn net.PacketConn) { var running sync.WaitGroup - nm := newNATmap(h.natTimeout, h.m, &running) + nm := newNATmap(h.l, h.natTimeout, h.m, &running) defer nm.Close() cipherBuf := make([]byte, serverUDPBufferSize) textBuf := make([]byte, serverUDPBufferSize) @@ -137,7 +138,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { return onet.NewConnectionError("ERR_READ", "Failed to read from client", err) } defer slog.LogAttrs(nil, slog.LevelDebug, "UDP: Done", slog.String("address", clientAddr.String())) - debugUDPAddr("Outbound packet.", clientAddr, slog.Int("bytes", clientProxyBytes)) + debugUDPAddr(h.l, "Outbound packet.", clientAddr, slog.Int("bytes", clientProxyBytes)) cipherData := cipherBuf[:clientProxyBytes] var payload []byte @@ -148,7 +149,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { var textData []byte var cryptoKey *shadowsocks.EncryptionKey unpackStart := time.Now() - textData, keyID, cryptoKey, err = findAccessKeyUDP(ip, textBuf, cipherData, h.ciphers) + textData, keyID, cryptoKey, err = findAccessKeyUDP(h.l, ip, textBuf, cipherData, h.ciphers) timeToCipher := time.Since(unpackStart) h.ssm.AddCipherSearch(err == nil, timeToCipher) @@ -185,7 +186,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { } } - debugUDPAddr("Proxy exit.", clientAddr, slog.Any("target", targetConn.LocalAddr())) + debugUDPAddr(h.l, "Proxy exit.", clientAddr, slog.Any("target", targetConn.LocalAddr())) proxyTargetBytes, err = targetConn.WriteTo(payload, tgtUDPAddr) // accept only UDPAddr despite the signature if err != nil { return onet.NewConnectionError("ERR_WRITE", "Failed to write to target", err) @@ -294,13 +295,14 @@ func (c *natconn) ReadFrom(buf []byte) (int, net.Addr, error) { type natmap struct { sync.RWMutex keyConn map[string]*natconn + l logger timeout time.Duration metrics UDPMetrics running *sync.WaitGroup } -func newNATmap(timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { - m := &natmap{metrics: sm, running: running} +func newNATmap(l logger, timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { + m := &natmap{l: l, metrics: sm, running: running} m.keyConn = make(map[string]*natconn) m.timeout = timeout return m @@ -346,7 +348,7 @@ func (m *natmap) Add(clientAddr net.Addr, clientConn net.PacketConn, cryptoKey * m.running.Add(1) go func() { - timedCopy(clientAddr, clientConn, entry, keyID) + timedCopy(m.l, clientAddr, clientConn, entry, keyID) connMetrics.RemoveNatEntry() if pc := m.del(clientAddr.String()); pc != nil { pc.Close() @@ -375,7 +377,7 @@ func (m *natmap) Close() error { var maxAddrLen int = len(socks.ParseAddr("[2001:db8::1]:12345")) // copy from target to client until read timeout -func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string) { +func timedCopy(l logger, clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string) { // pkt is used for in-place encryption of downstream UDP packets, with the layout // [padding?][salt][address][body][tag][extra] // Padding is only used if the address is IPv4. @@ -408,7 +410,7 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco return onet.NewConnectionError("ERR_READ", "Failed to read from target", err) } - debugUDPAddr("Got response.", clientAddr, slog.Any("target", raddr)) + debugUDPAddr(l, "Got response.", clientAddr, slog.Any("target", raddr)) srcAddr := socks.ParseAddr(raddr.String()) addrStart := bodyStart - len(srcAddr) // `plainTextBuf` concatenates the SOCKS address and body: From 2073a27e7ae8c3b2f1636a0f09282511c23c0668 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 14:54:42 -0400 Subject: [PATCH 144/182] Refactor metrics to not share with Caddy. --- caddy/app.go | 63 +++++++-------- cmd/outline-ss-server/main.go | 2 +- cmd/outline-ss-server/metrics.go | 73 +++++++++++++++++ cmd/outline-ss-server/metrics_test.go | 110 ++++++++++++++++++++++++++ prometheus/metrics.go | 49 ------------ 5 files changed, 211 insertions(+), 86 deletions(-) create mode 100644 cmd/outline-ss-server/metrics.go create mode 100644 cmd/outline-ss-server/metrics_test.go diff --git a/caddy/app.go b/caddy/app.go index 6ca00335..61cbd8ff 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -18,7 +18,7 @@ package caddy import ( - "errors" + "sync" outline_prometheus "github.com/Jigsaw-Code/outline-ss-server/prometheus" outline "github.com/Jigsaw-Code/outline-ss-server/service" @@ -41,9 +41,11 @@ type OutlineApp struct { Version string `json:"version,omitempty"` ShadowsocksConfig *ShadowsocksConfig `json:"shadowsocks,omitempty"` - Metrics outline.ServiceMetrics ReplayCache outline.ReplayCache logger *zap.Logger + Metrics outline.ServiceMetrics + buildInfo *prometheus.GaugeVec + once sync.Once } func (OutlineApp) CaddyModule() caddy.ModuleInfo { @@ -67,45 +69,34 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { app.ReplayCache = outline.NewReplayCache(app.ShadowsocksConfig.ReplayHistory) } - if err := app.defineMetrics(); err != nil { - app.logger.Error("failed to create Prometheus metrics", zap.Error(err)) - } - - return nil -} - -func (app *OutlineApp) defineMetrics() error { - // TODO: Use `once.Do()` instead of catching already registered collectors? - // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? - r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) - - serverMetrics := outline_prometheus.NewServerMetrics() - registeredServerMetrics := registerCollector(r, serverMetrics) - registeredServerMetrics.SetVersion(app.Version) - // TODO: Call `registeredServerMetrics.SetNumAccessKeys()`. - - // TODO: Allow the configuration of ip2info. - serviceMetrics, err := outline_prometheus.NewServiceMetrics(nil) - if err != nil { - return err - } - registeredServiceMetrics := registerCollector(r, serviceMetrics) - app.Metrics = registeredServiceMetrics + app.defineMetrics() + app.buildInfo.WithLabelValues(app.Version).Set(1) + // TODO: Add replacement metrics for `shadowsocks_keys` and `shadowsocks_ports`. return nil } -func registerCollector[T prometheus.Collector](registerer prometheus.Registerer, coll T) T { - if err := registerer.Register(coll); err != nil { - are := &prometheus.AlreadyRegisteredError{} - if errors.As(err, are) { - // This collector has been registered before. This is expected during a config reload. - coll = are.ExistingCollector.(T) - } else { - panic(err) +func (app *OutlineApp) defineMetrics() { + app.once.Do(func() { + // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? + r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) + + app.buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "build_info", + Help: "Information on the outline-ss-server build", + }, []string{"version"}) + + // TODO: Allow the configuration of ip2info. + metrics, err := outline_prometheus.NewServiceMetrics(nil) + if err != nil { + app.logger.Error("failed to define Prometheus metrics", zap.Error(err)) + return } - } - return coll + app.Metrics = metrics + + r.MustRegister(app.buildInfo) + r.MustRegister(metrics) + }) } // Start starts the App. diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 06241452..7d96e62c 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -396,7 +396,7 @@ func main() { } defer ip2info.Close() - serverMetrics := outline_prometheus.NewServerMetrics() + serverMetrics := newPrometheusServerMetrics() serverMetrics.SetVersion(version) serviceMetrics, err := outline_prometheus.NewServiceMetrics(ip2info) if err != nil { diff --git a/cmd/outline-ss-server/metrics.go b/cmd/outline-ss-server/metrics.go new file mode 100644 index 00000000..32c9b0aa --- /dev/null +++ b/cmd/outline-ss-server/metrics.go @@ -0,0 +1,73 @@ +// Copyright 2023 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// `now` is stubbable for testing. +var now = time.Now + +type serverMetrics struct { + // NOTE: New metrics need to be added to `newPrometheusServerMetrics()`, `Describe()` and `Collect()`. + buildInfo *prometheus.GaugeVec + accessKeys prometheus.Gauge + ports prometheus.Gauge +} + +var _ prometheus.Collector = (*serverMetrics)(nil) + +// newPrometheusServerMetrics constructs a Prometheus metrics collector for server +// related metrics. +func newPrometheusServerMetrics() *serverMetrics { + return &serverMetrics{ + buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "build_info", + Help: "Information on the outline-ss-server build", + }, []string{"version"}), + accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "keys", + Help: "Count of access keys", + }), + ports: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "ports", + Help: "Count of open ports", + }), + } +} + +func (m *serverMetrics) Describe(ch chan<- *prometheus.Desc) { + m.buildInfo.Describe(ch) + m.accessKeys.Describe(ch) + m.ports.Describe(ch) +} + +func (m *serverMetrics) Collect(ch chan<- prometheus.Metric) { + m.buildInfo.Collect(ch) + m.accessKeys.Collect(ch) + m.ports.Collect(ch) +} + +func (m *serverMetrics) SetVersion(version string) { + m.buildInfo.WithLabelValues(version).Set(1) +} + +func (m *serverMetrics) SetNumAccessKeys(numKeys int, ports int) { + m.accessKeys.Set(float64(numKeys)) + m.ports.Set(float64(ports)) +} diff --git a/cmd/outline-ss-server/metrics_test.go b/cmd/outline-ss-server/metrics_test.go new file mode 100644 index 00000000..93cce446 --- /dev/null +++ b/cmd/outline-ss-server/metrics_test.go @@ -0,0 +1,110 @@ +// Copyright 2023 Jigsaw Operations LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net" + "strings" + "testing" + "time" + + "github.com/Jigsaw-Code/outline-ss-server/ipinfo" + "github.com/op/go-logging" + "github.com/prometheus/client_golang/prometheus" + promtest "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +type noopMap struct{} + +func (*noopMap) GetIPInfo(ip net.IP) (ipinfo.IPInfo, error) { + return ipinfo.IPInfo{}, nil +} + +type fakeAddr string + +func (a fakeAddr) String() string { return string(a) } +func (a fakeAddr) Network() string { return "" } + +// Sets the processing clock to be t until changed. +func setNow(t time.Time) { + now = func() time.Time { + return t + } +} + +func init() { + logging.SetLevel(logging.INFO, "") +} + +type fakeConn struct { + net.Conn +} + +func (c *fakeConn) LocalAddr() net.Addr { + return fakeAddr("127.0.0.1:9") +} + +func (c *fakeConn) RemoteAddr() net.Addr { + return fakeAddr("127.0.0.1:10") +} + +func TestMethodsDontPanic(t *testing.T) { + m := newPrometheusServerMetrics() + m.SetVersion("0.0.0-test") + m.SetNumAccessKeys(20, 2) +} + +func TestSetVersion(t *testing.T) { + m := newPrometheusServerMetrics() + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(m) + + m.SetVersion("0.0.0-test") + + err := promtest.GatherAndCompare( + reg, + strings.NewReader(` + # HELP build_info Information on the outline-ss-server build + # TYPE build_info gauge + build_info{version="0.0.0-test"} 1 + `), + "build_info", + ) + require.NoError(t, err, "unexpected metric value found") +} + +func TestSetNumAccessKeys(t *testing.T) { + m := newPrometheusServerMetrics() + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(m) + + m.SetNumAccessKeys(1, 2) + + err := promtest.GatherAndCompare( + reg, + strings.NewReader(` + # HELP keys Count of access keys + # TYPE keys gauge + keys 1 + # HELP ports Count of open ports + # TYPE ports gauge + ports 2 + `), + "keys", + "ports", + ) + require.NoError(t, err, "unexpected metric value found") +} diff --git a/prometheus/metrics.go b/prometheus/metrics.go index 8c05140e..186cba7c 100644 --- a/prometheus/metrics.go +++ b/prometheus/metrics.go @@ -572,52 +572,3 @@ func toIPKey(addr net.Addr, accessKey string) (*IPKey, error) { } return &IPKey{ip, accessKey}, nil } - -type serverMetrics struct { - // NOTE: New metrics need to be added to `NewServerMetrics()`, `Describe()` and `Collect()`. - buildInfo *prometheus.GaugeVec - accessKeys prometheus.Gauge - ports prometheus.Gauge -} - -var _ prometheus.Collector = (*serverMetrics)(nil) - -// NewServerMetrics constructs a Prometheus metrics collector for server -// related metrics. -func NewServerMetrics() *serverMetrics { - return &serverMetrics{ - buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "build_info", - Help: "Information on the outline-ss-server build", - }, []string{"version"}), - accessKeys: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "keys", - Help: "Count of access keys", - }), - ports: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "ports", - Help: "Count of open ports", - }), - } -} - -func (m *serverMetrics) Describe(ch chan<- *prometheus.Desc) { - m.buildInfo.Describe(ch) - m.accessKeys.Describe(ch) - m.ports.Describe(ch) -} - -func (m *serverMetrics) Collect(ch chan<- prometheus.Metric) { - m.buildInfo.Collect(ch) - m.accessKeys.Collect(ch) - m.ports.Collect(ch) -} - -func (m *serverMetrics) SetVersion(version string) { - m.buildInfo.WithLabelValues(version).Set(1) -} - -func (m *serverMetrics) SetNumAccessKeys(numKeys int, ports int) { - m.accessKeys.Set(float64(numKeys)) - m.ports.Set(float64(ports)) -} From 0dfafcb2897985cfeb92adceb5adeb5d78a8b228 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 15:11:59 -0400 Subject: [PATCH 145/182] Set Prometheus metrics handler. --- caddy/README.md | 2 +- caddy/config_example.json | 31 +++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/caddy/README.md b/caddy/README.md index cc2ffb9f..f3c7b67d 100644 --- a/caddy/README.md +++ b/caddy/README.md @@ -22,4 +22,4 @@ In a separate window, confirm you can fetch a page using this server: go run github.com/Jigsaw-Code/outline-sdk/x/examples/fetch -transport "ss://chacha20-ietf-poly1305:Secret1@:9000" http://ipinfo.io ``` -Prometheus metrics are exposed on http://localhost:2019/metrics. +Prometheus metrics are available on http://localhost:9091/metrics. diff --git a/caddy/config_example.json b/caddy/config_example.json index bcbf5345..f3818a5a 100644 --- a/caddy/config_example.json +++ b/caddy/config_example.json @@ -1,4 +1,7 @@ { + "admin": { + "disabled": true + }, "logging": { "logs": { "default": {"level":"DEBUG", "encoder": {"format":"console"}} @@ -6,10 +9,30 @@ }, "apps": { "http": { + "http_port": 9091, "servers": { - "": { - "metrics": {} - } + "": { + "listen": [ + ":9091" + ], + "routes": [ + { + "match": [ + { + "path": [ + "/metrics" + ] + } + ], + "handle": [ + { + "disable_openmetrics": true, + "handler": "metrics" + } + ] + } + ] + } } }, "layer4": { @@ -73,4 +96,4 @@ } } } -} \ No newline at end of file +} From e2cc62fbca7319aa68715caed17489324af4a614 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 15:16:36 -0400 Subject: [PATCH 146/182] Catch already registered collectors instead of using `once.Sync`. --- caddy/app.go | 50 +++++++++++++++++++++++---------------- caddy/config_example.json | 1 - 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index 61cbd8ff..aa4ea907 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -18,7 +18,7 @@ package caddy import ( - "sync" + "errors" outline_prometheus "github.com/Jigsaw-Code/outline-ss-server/prometheus" outline "github.com/Jigsaw-Code/outline-ss-server/service" @@ -45,7 +45,6 @@ type OutlineApp struct { logger *zap.Logger Metrics outline.ServiceMetrics buildInfo *prometheus.GaugeVec - once sync.Once } func (OutlineApp) CaddyModule() caddy.ModuleInfo { @@ -77,26 +76,35 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { } func (app *OutlineApp) defineMetrics() { - app.once.Do(func() { - // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? - r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) - - app.buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "build_info", - Help: "Information on the outline-ss-server build", - }, []string{"version"}) - - // TODO: Allow the configuration of ip2info. - metrics, err := outline_prometheus.NewServiceMetrics(nil) - if err != nil { - app.logger.Error("failed to define Prometheus metrics", zap.Error(err)) - return - } - app.Metrics = metrics + // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? + r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) + + buildInfo := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "build_info", + Help: "Information on the outline-ss-server build", + }, []string{"version"}) + app.buildInfo = registerCollector(r, buildInfo) + + // TODO: Allow the configuration of ip2info. + metrics, err := outline_prometheus.NewServiceMetrics(nil) + if err != nil { + app.logger.Error("failed to define Prometheus metrics", zap.Error(err)) + return + } + app.Metrics = registerCollector(r, metrics) +} - r.MustRegister(app.buildInfo) - r.MustRegister(metrics) - }) +func registerCollector[T prometheus.Collector](registerer prometheus.Registerer, coll T) T { + if err := registerer.Register(coll); err != nil { + are := &prometheus.AlreadyRegisteredError{} + if errors.As(err, are) { + // This collector has been registered before. This is expected during a config reload. + coll = are.ExistingCollector.(T) + } else { + panic(err) + } + } + return coll } // Start starts the App. diff --git a/caddy/config_example.json b/caddy/config_example.json index f3818a5a..764c51f0 100644 --- a/caddy/config_example.json +++ b/caddy/config_example.json @@ -9,7 +9,6 @@ }, "apps": { "http": { - "http_port": 9091, "servers": { "": { "listen": [ From 654e0e912904572d175fe8c0dcd621a02245787e Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 15:23:31 -0400 Subject: [PATCH 147/182] refactor: pass in logger to service so caller can control logs --- cmd/outline-ss-server/main.go | 2 + internal/integration_test/integration_test.go | 25 ++++++--- service/logger.go | 22 +++++++- service/shadowsocks.go | 52 +++++++++++++++---- service/tcp.go | 40 +++++++------- service/tcp_test.go | 20 +++---- service/udp.go | 42 ++++++++------- service/udp_test.go | 14 ++--- 8 files changed, 141 insertions(+), 76 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 543d036d..7d96e62c 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -218,6 +218,7 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { ciphers.Update(cipherList) ssService, err := service.NewShadowsocksService( + service.WithLogger(slog.Default()), service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), @@ -244,6 +245,7 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { return fmt.Errorf("failed to create cipher list from config: %v", err) } ssService, err := service.NewShadowsocksService( + service.WithLogger(slog.Default()), service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index f847f761..9bc3d0d5 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "io" + "log/slog" "net" "net/netip" "sync" @@ -132,7 +133,7 @@ func TestTCPEcho(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(authFunc, testTimeout) + handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -183,6 +184,16 @@ func TestTCPEcho(t *testing.T) { echoRunning.Wait() } +type noopLogger struct { +} + +func (l *noopLogger) Enabled(ctx context.Context, level slog.Level) bool { + return false +} + +func (l *noopLogger) LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { +} + type fakeShadowsocksMetrics struct { } @@ -212,7 +223,7 @@ func TestRestrictedAddresses(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(authFunc, testTimeout) + handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) done := make(chan struct{}) go func() { service.StreamServe( @@ -311,7 +322,7 @@ func TestUDPEcho(t *testing.T) { t.Fatal(err) } testMetrics := &fakeUDPMetrics{} - proxy := service.NewPacketHandler(time.Hour, cipherList, testMetrics, &fakeShadowsocksMetrics{}) + proxy := service.NewPacketHandler(&noopLogger{}, time.Hour, cipherList, testMetrics, &fakeShadowsocksMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { @@ -401,7 +412,7 @@ func BenchmarkTCPThroughput(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(authFunc, testTimeout) + handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -468,7 +479,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(authFunc, testTimeout) + handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -544,7 +555,7 @@ func BenchmarkUDPEcho(b *testing.B) { if err != nil { b.Fatal(err) } - proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) + proxy := service.NewPacketHandler(&noopLogger{}, time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { @@ -588,7 +599,7 @@ func BenchmarkUDPManyKeys(b *testing.B) { if err != nil { b.Fatal(err) } - proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) + proxy := service.NewPacketHandler(&noopLogger{}, time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { diff --git a/service/logger.go b/service/logger.go index 79b8ee3a..2e8cab9e 100644 --- a/service/logger.go +++ b/service/logger.go @@ -14,6 +14,24 @@ package service -import logging "github.com/op/go-logging" +import ( + "context" + "log/slog" +) -var logger = logging.MustGetLogger("shadowsocks") +type logger interface { + Enabled(ctx context.Context, level slog.Level) bool + LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) +} + +type noopLogger struct { +} + +var _ logger = (*noopLogger)(nil) + +func (l *noopLogger) Enabled(ctx context.Context, level slog.Level) bool { + return false +} + +func (l *noopLogger) LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { +} diff --git a/service/shadowsocks.go b/service/shadowsocks.go index 87814df8..fcca785a 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -16,6 +16,7 @@ package service import ( "context" + "fmt" "net" "time" @@ -46,10 +47,11 @@ type Service interface { HandlePacket(conn net.PacketConn) } -// Option is a Shadowsocks service constructor option. -type Option func(s *ssService) +// Option user's option. +type Option func(s *ssService) error type ssService struct { + logger logger m ServiceMetrics ciphers CipherList natTimeout time.Duration @@ -59,12 +61,17 @@ type ssService struct { ph PacketHandler } -// NewShadowsocksService creates a new service func NewShadowsocksService(opts ...Option) (Service, error) { s := &ssService{} for _, opt := range opts { - opt(s) + if err := opt(s); err != nil { + return nil, fmt.Errorf("failed to create new service: %v", err) + } + } + + if s.logger == nil { + s.logger = &noopLogger{} } if s.natTimeout == 0 { @@ -73,40 +80,57 @@ func NewShadowsocksService(opts ...Option) (Service, error) { return s, nil } +// WithLogger can be used to provide a custom log target. +// Defaults to io.Discard. +func WithLogger(l logger) Option { + return func(s *ssService) error { + s.logger = l + return nil + } +} + // WithCiphers option function. func WithCiphers(ciphers CipherList) Option { - return func(s *ssService) { + return func(s *ssService) error { s.ciphers = ciphers + return nil } } // WithMetrics option function. func WithMetrics(metrics ServiceMetrics) Option { - return func(s *ssService) { + return func(s *ssService) error { s.m = metrics + return nil } } // WithReplayCache option function. func WithReplayCache(replayCache *ReplayCache) Option { - return func(s *ssService) { + return func(s *ssService) error { s.replayCache = replayCache + return nil } } // WithNatTimeout option function. func WithNatTimeout(natTimeout time.Duration) Option { - return func(s *ssService) { + return func(s *ssService) error { s.natTimeout = natTimeout + return nil } } // HandleStream handles a Shadowsocks stream-based connection. func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { if s.sh == nil { - authFunc := NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}) + authFunc := NewShadowsocksStreamAuthenticator( + s.ciphers, + s.replayCache, + &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}, + ) // TODO: Register initial data metrics at zero. - s.sh = NewStreamHandler(authFunc, tcpReadTimeout) + s.sh = NewStreamHandler(s.logger, authFunc, tcpReadTimeout) } connMetrics := s.m.AddOpenTCPConnection(conn) s.sh.Handle(ctx, conn, connMetrics) @@ -115,7 +139,13 @@ func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) // HandlePacket handles a Shadowsocks packet connection. func (s *ssService) HandlePacket(conn net.PacketConn) { if s.ph == nil { - s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) + s.ph = NewPacketHandler( + s.logger, + s.natTimeout, + s.ciphers, + s.m, + &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}, + ) } s.ph.Handle(conn) } diff --git a/service/tcp.go b/service/tcp.go index 8637663b..7a490bb5 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -58,11 +58,11 @@ func remoteIP(conn net.Conn) netip.Addr { } // Wrapper for slog.Debug during TCP access key searches. -func debugTCP(template string, cipherID string, attr slog.Attr) { +func debugTCP(l logger, template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. - if slog.Default().Enabled(nil, slog.LevelDebug) { - slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("TCP: %s", template), slog.String("ID", cipherID), attr) + if l.Enabled(nil, slog.LevelDebug) { + l.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("TCP: %s", template), slog.String("ID", cipherID), attr) } } @@ -72,7 +72,7 @@ func debugTCP(template string, cipherID string, attr slog.Attr) { // required = saltSize + 2 + cipher.TagSize, the number of bytes needed to authenticate the connection. const bytesForKeyFinding = 50 -func findAccessKey(clientReader io.Reader, clientIP netip.Addr, cipherList CipherList) (*CipherEntry, io.Reader, []byte, time.Duration, error) { +func findAccessKey(l logger, clientReader io.Reader, clientIP netip.Addr, cipherList CipherList) (*CipherEntry, io.Reader, []byte, time.Duration, error) { // We snapshot the list because it may be modified while we use it. ciphers := cipherList.SnapshotForClientIP(clientIP) firstBytes := make([]byte, bytesForKeyFinding) @@ -81,7 +81,7 @@ func findAccessKey(clientReader io.Reader, clientIP netip.Addr, cipherList Ciphe } findStartTime := time.Now() - entry, elt := findEntry(firstBytes, ciphers) + entry, elt := findEntry(l, firstBytes, ciphers) timeToCipher := time.Since(findStartTime) if entry == nil { // TODO: Ban and log client IPs with too many failures too quick to protect against DoS. @@ -95,7 +95,7 @@ func findAccessKey(clientReader io.Reader, clientIP netip.Addr, cipherList Ciphe } // Implements a trial decryption search. This assumes that all ciphers are AEAD. -func findEntry(firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list.Element) { +func findEntry(l logger, firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list.Element) { // To hold the decrypted chunk length. chunkLenBuf := [2]byte{} for ci, elt := range ciphers { @@ -103,23 +103,23 @@ func findEntry(firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list. cryptoKey := entry.CryptoKey _, err := shadowsocks.Unpack(chunkLenBuf[:0], firstBytes[:cryptoKey.SaltSize()+2+cryptoKey.TagSize()], cryptoKey) if err != nil { - debugTCP("Failed to decrypt length.", entry.ID, slog.Any("err", err)) + debugTCP(l, "Failed to decrypt length.", entry.ID, slog.Any("err", err)) continue } - debugTCP("Found cipher.", entry.ID, slog.Int("index", ci)) + debugTCP(l, "Found cipher.", entry.ID, slog.Int("index", ci)) return entry, elt } return nil, nil } -type StreamAuthenticateFunc func(clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) +type StreamAuthenticateFunc func(l logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) // NewShadowsocksStreamAuthenticator creates a stream authenticator that uses Shadowsocks. // TODO(fortuna): Offer alternative transports. func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCache, metrics ShadowsocksConnMetrics) StreamAuthenticateFunc { - return func(clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { + return func(l logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { // Find the cipher and acess key id. - cipherEntry, clientReader, clientSalt, timeToCipher, keyErr := findAccessKey(clientConn, remoteIP(clientConn), ciphers) + cipherEntry, clientReader, clientSalt, timeToCipher, keyErr := findAccessKey(l, clientConn, remoteIP(clientConn), ciphers) metrics.AddCipherSearch(keyErr == nil, timeToCipher) if keyErr != nil { const status = "ERR_CIPHER" @@ -151,6 +151,7 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa } type streamHandler struct { + l logger listenerId string readTimeout time.Duration authenticate StreamAuthenticateFunc @@ -158,8 +159,9 @@ type streamHandler struct { } // NewStreamHandler creates a StreamHandler -func NewStreamHandler(authenticate StreamAuthenticateFunc, timeout time.Duration) StreamHandler { +func NewStreamHandler(l logger, authenticate StreamAuthenticateFunc, timeout time.Duration) StreamHandler { return &streamHandler{ + l: l, readTimeout: timeout, authenticate: authenticate, dialer: defaultDialer, @@ -251,11 +253,11 @@ func (h *streamHandler) Handle(ctx context.Context, clientConn transport.StreamC status := "OK" if connError != nil { status = connError.Status - slog.LogAttrs(nil, slog.LevelDebug, "TCP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) + h.l.LogAttrs(nil, slog.LevelDebug, "TCP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) } connMetrics.AddClosed(status, proxyMetrics, connDuration) measuredClientConn.Close() // Closing after the metrics are added aids integration testing. - slog.LogAttrs(nil, slog.LevelDebug, "TCP: Done.", slog.String("status", status), slog.Duration("duration", connDuration)) + h.l.LogAttrs(nil, slog.LevelDebug, "TCP: Done.", slog.String("status", status), slog.Duration("duration", connDuration)) } func getProxyRequest(clientConn transport.StreamConn) (string, error) { @@ -270,14 +272,14 @@ func getProxyRequest(clientConn transport.StreamConn) (string, error) { return tgtAddr.String(), nil } -func proxyConnection(ctx context.Context, dialer transport.StreamDialer, tgtAddr string, clientConn transport.StreamConn) *onet.ConnectionError { +func proxyConnection(l logger, ctx context.Context, dialer transport.StreamDialer, tgtAddr string, clientConn transport.StreamConn) *onet.ConnectionError { tgtConn, dialErr := dialer.DialStream(ctx, tgtAddr) if dialErr != nil { // We don't drain so dial errors and invalid addresses are communicated quickly. return ensureConnectionError(dialErr, "ERR_CONNECT", "Failed to connect to target") } defer tgtConn.Close() - slog.LogAttrs(nil, slog.LevelDebug, "Proxy connection.", slog.String("client", clientConn.RemoteAddr().String()), slog.String("target", tgtConn.RemoteAddr().String())) + l.LogAttrs(nil, slog.LevelDebug, "Proxy connection.", slog.String("client", clientConn.RemoteAddr().String()), slog.String("target", tgtConn.RemoteAddr().String())) fromClientErrCh := make(chan error) go func() { @@ -319,7 +321,7 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor } outerConn.SetReadDeadline(readDeadline) - id, innerConn, authErr := h.authenticate(outerConn) + id, innerConn, authErr := h.authenticate(h.l, outerConn) if authErr != nil { // Drain to protect against probing attacks. h.absorbProbe(outerConn, connMetrics, authErr.Status, proxyMetrics) @@ -345,7 +347,7 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor tgtConn = metrics.MeasureConn(tgtConn, &proxyMetrics.ProxyTarget, &proxyMetrics.TargetProxy) return tgtConn, nil }) - return proxyConnection(ctx, dialer, tgtAddr, innerConn) + return proxyConnection(h.l, ctx, dialer, tgtAddr, innerConn) } // Keep the connection open until we hit the authentication deadline to protect against probing attacks @@ -354,7 +356,7 @@ func (h *streamHandler) absorbProbe(clientConn io.ReadCloser, connMetrics TCPCon // This line updates proxyMetrics.ClientProxy before it's used in AddTCPProbe. _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) - slog.LogAttrs(nil, slog.LevelDebug, "Drain error.", slog.Any("err", drainErr), slog.String("result", drainResult)) + h.l.LogAttrs(nil, slog.LevelDebug, "Drain error.", slog.Any("err", drainErr), slog.String("result", drainResult)) connMetrics.AddProbe(status, drainResult, proxyMetrics.ClientProxy) } diff --git a/service/tcp_test.go b/service/tcp_test.go index a4efcb2e..f4d22580 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -102,7 +102,7 @@ func BenchmarkTCPFindCipherFail(b *testing.B) { } clientIP := clientConn.RemoteAddr().(*net.TCPAddr).AddrPort().Addr() b.StartTimer() - findAccessKey(clientConn, clientIP, cipherList) + findAccessKey(&noopLogger{}, clientConn, clientIP, cipherList) b.StopTimer() } } @@ -205,7 +205,7 @@ func BenchmarkTCPFindCipherRepeat(b *testing.B) { cipher := cipherEntries[cipherNumber].CryptoKey go shadowsocks.NewWriter(writer, cipher).Write(makeTestPayload(50)) b.StartTimer() - _, _, _, _, err := findAccessKey(&c, clientIP, cipherList) + _, _, _, _, err := findAccessKey(&noopLogger{}, &c, clientIP, cipherList) b.StopTimer() if err != nil { b.Error(err) @@ -286,7 +286,7 @@ func TestProbeRandom(t *testing.T) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(authFunc, 200*time.Millisecond) + handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe( @@ -366,7 +366,7 @@ func TestProbeClientBytesBasicTruncated(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(authFunc, 200*time.Millisecond) + handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -404,7 +404,7 @@ func TestProbeClientBytesBasicModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(authFunc, 200*time.Millisecond) + handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -443,7 +443,7 @@ func TestProbeClientBytesCoalescedModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(authFunc, 200*time.Millisecond) + handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -489,7 +489,7 @@ func TestProbeServerBytesModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(authFunc, 200*time.Millisecond) + handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe( @@ -523,7 +523,7 @@ func TestReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewStreamHandler(authFunc, testTimeout) + handler := NewStreamHandler(&noopLogger{}, authFunc, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -605,7 +605,7 @@ func TestReverseReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewStreamHandler(authFunc, testTimeout) + handler := NewStreamHandler(&noopLogger{}, authFunc, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -679,7 +679,7 @@ func probeExpectTimeout(t *testing.T, payloadSize int) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(authFunc, testTimeout) + handler := NewStreamHandler(&noopLogger{}, authFunc, testTimeout) done := make(chan struct{}) go func() { diff --git a/service/udp.go b/service/udp.go index 2d08a709..74d8164a 100644 --- a/service/udp.go +++ b/service/udp.go @@ -44,23 +44,23 @@ type UDPMetrics interface { const serverUDPBufferSize = 64 * 1024 // Wrapper for slog.Debug during UDP proxying. -func debugUDP(template string, cipherID string, attr slog.Attr) { +func debugUDP(l logger, template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. - if slog.Default().Enabled(nil, slog.LevelDebug) { - slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("ID", cipherID), attr) + if l.Enabled(nil, slog.LevelDebug) { + l.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("ID", cipherID), attr) } } -func debugUDPAddr(template string, addr net.Addr, attr slog.Attr) { - if slog.Default().Enabled(nil, slog.LevelDebug) { - slog.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("address", addr.String()), attr) +func debugUDPAddr(l logger, template string, addr net.Addr, attr slog.Attr) { + if l.Enabled(nil, slog.LevelDebug) { + l.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("address", addr.String()), attr) } } // Decrypts src into dst. It tries each cipher until it finds one that authenticates // correctly. dst and src must not overlap. -func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherList) ([]byte, string, *shadowsocks.EncryptionKey, error) { +func findAccessKeyUDP(l logger, clientIP netip.Addr, dst, src []byte, cipherList CipherList) ([]byte, string, *shadowsocks.EncryptionKey, error) { // Try each cipher until we find one that authenticates successfully. This assumes that all ciphers are AEAD. // We snapshot the list because it may be modified while we use it. snapshot := cipherList.SnapshotForClientIP(clientIP) @@ -68,10 +68,10 @@ func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherLis id, cryptoKey := entry.Value.(*CipherEntry).ID, entry.Value.(*CipherEntry).CryptoKey buf, err := shadowsocks.Unpack(dst, src, cryptoKey) if err != nil { - debugUDP("Failed to unpack.", id, slog.Any("err", err)) + debugUDP(l, "Failed to unpack.", id, slog.Any("err", err)) continue } - debugUDP("Found cipher.", id, slog.Int("index", ci)) + debugUDP(l, "Found cipher.", id, slog.Int("index", ci)) // Move the active cipher to the front, so that the search is quicker next time. cipherList.MarkUsedByClientIP(entry, clientIP) return buf, id, cryptoKey, nil @@ -80,6 +80,7 @@ func findAccessKeyUDP(clientIP netip.Addr, dst, src []byte, cipherList CipherLis } type packetHandler struct { + l logger natTimeout time.Duration ciphers CipherList m UDPMetrics @@ -88,8 +89,8 @@ type packetHandler struct { } // NewPacketHandler creates a UDPService -func NewPacketHandler(natTimeout time.Duration, cipherList CipherList, m UDPMetrics, ssMetrics ShadowsocksConnMetrics) PacketHandler { - return &packetHandler{natTimeout: natTimeout, ciphers: cipherList, m: m, ssm: ssMetrics, targetIPValidator: onet.RequirePublicIP} +func NewPacketHandler(l logger, natTimeout time.Duration, cipherList CipherList, m UDPMetrics, ssMetrics ShadowsocksConnMetrics) PacketHandler { + return &packetHandler{l: l, natTimeout: natTimeout, ciphers: cipherList, m: m, ssm: ssMetrics, targetIPValidator: onet.RequirePublicIP} } // PacketHandler is a running UDP shadowsocks proxy that can be stopped. @@ -109,7 +110,7 @@ func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPVali func (h *packetHandler) Handle(clientConn net.PacketConn) { var running sync.WaitGroup - nm := newNATmap(h.natTimeout, h.m, &running) + nm := newNATmap(h.l, h.natTimeout, h.m, &running) defer nm.Close() cipherBuf := make([]byte, serverUDPBufferSize) textBuf := make([]byte, serverUDPBufferSize) @@ -137,7 +138,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { return onet.NewConnectionError("ERR_READ", "Failed to read from client", err) } defer slog.LogAttrs(nil, slog.LevelDebug, "UDP: Done", slog.String("address", clientAddr.String())) - debugUDPAddr("Outbound packet.", clientAddr, slog.Int("bytes", clientProxyBytes)) + debugUDPAddr(h.l, "Outbound packet.", clientAddr, slog.Int("bytes", clientProxyBytes)) cipherData := cipherBuf[:clientProxyBytes] var payload []byte @@ -148,7 +149,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { var textData []byte var cryptoKey *shadowsocks.EncryptionKey unpackStart := time.Now() - textData, keyID, cryptoKey, err = findAccessKeyUDP(ip, textBuf, cipherData, h.ciphers) + textData, keyID, cryptoKey, err = findAccessKeyUDP(h.l, ip, textBuf, cipherData, h.ciphers) timeToCipher := time.Since(unpackStart) h.ssm.AddCipherSearch(err == nil, timeToCipher) @@ -185,7 +186,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { } } - debugUDPAddr("Proxy exit.", clientAddr, slog.Any("target", targetConn.LocalAddr())) + debugUDPAddr(h.l, "Proxy exit.", clientAddr, slog.Any("target", targetConn.LocalAddr())) proxyTargetBytes, err = targetConn.WriteTo(payload, tgtUDPAddr) // accept only UDPAddr despite the signature if err != nil { return onet.NewConnectionError("ERR_WRITE", "Failed to write to target", err) @@ -294,13 +295,14 @@ func (c *natconn) ReadFrom(buf []byte) (int, net.Addr, error) { type natmap struct { sync.RWMutex keyConn map[string]*natconn + l logger timeout time.Duration metrics UDPMetrics running *sync.WaitGroup } -func newNATmap(timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { - m := &natmap{metrics: sm, running: running} +func newNATmap(l logger, timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { + m := &natmap{l: l, metrics: sm, running: running} m.keyConn = make(map[string]*natconn) m.timeout = timeout return m @@ -346,7 +348,7 @@ func (m *natmap) Add(clientAddr net.Addr, clientConn net.PacketConn, cryptoKey * m.running.Add(1) go func() { - timedCopy(clientAddr, clientConn, entry, keyID) + timedCopy(m.l, clientAddr, clientConn, entry, keyID) connMetrics.RemoveNatEntry() if pc := m.del(clientAddr.String()); pc != nil { pc.Close() @@ -375,7 +377,7 @@ func (m *natmap) Close() error { var maxAddrLen int = len(socks.ParseAddr("[2001:db8::1]:12345")) // copy from target to client until read timeout -func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string) { +func timedCopy(l logger, clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string) { // pkt is used for in-place encryption of downstream UDP packets, with the layout // [padding?][salt][address][body][tag][extra] // Padding is only used if the address is IPv4. @@ -408,7 +410,7 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco return onet.NewConnectionError("ERR_READ", "Failed to read from target", err) } - debugUDPAddr("Got response.", clientAddr, slog.Any("target", raddr)) + debugUDPAddr(l, "Got response.", clientAddr, slog.Any("target", raddr)) srcAddr := socks.ParseAddr(raddr.String()) addrStart := bodyStart - len(srcAddr) // `plainTextBuf` concatenates the SOCKS address and body: diff --git a/service/udp_test.go b/service/udp_test.go index 8ba00af3..bab91339 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -136,7 +136,7 @@ func sendToDiscard(payloads [][]byte, validator onet.TargetIPValidator) *natTest cipher := ciphers.SnapshotForClientIP(netip.Addr{})[0].Value.(*CipherEntry).CryptoKey clientConn := makePacketConn() metrics := &natTestMetrics{} - handler := NewPacketHandler(timeout, ciphers, metrics, &fakeShadowsocksMetrics{}) + handler := NewPacketHandler(&noopLogger{}, timeout, ciphers, metrics, &fakeShadowsocksMetrics{}) handler.SetTargetIPValidator(validator) done := make(chan struct{}) go func() { @@ -207,14 +207,14 @@ func assertAlmostEqual(t *testing.T, a, b time.Time) { } func TestNATEmpty(t *testing.T) { - nat := newNATmap(timeout, &natTestMetrics{}, &sync.WaitGroup{}) + nat := newNATmap(&noopLogger{}, timeout, &natTestMetrics{}, &sync.WaitGroup{}) if nat.Get("foo") != nil { t.Error("Expected nil value from empty NAT map") } } func setupNAT() (*fakePacketConn, *fakePacketConn, *natconn) { - nat := newNATmap(timeout, &natTestMetrics{}, &sync.WaitGroup{}) + nat := newNATmap(&noopLogger{}, timeout, &natTestMetrics{}, &sync.WaitGroup{}) clientConn := makePacketConn() targetConn := makePacketConn() nat.Add(&clientAddr, clientConn, natCryptoKey, targetConn, "key id") @@ -409,7 +409,7 @@ func BenchmarkUDPUnpackFail(b *testing.B) { testIP := netip.MustParseAddr("192.0.2.1") b.ResetTimer() for n := 0; n < b.N; n++ { - findAccessKeyUDP(testIP, textBuf, testPayload, cipherList) + findAccessKeyUDP(&noopLogger{}, testIP, textBuf, testPayload, cipherList) } } @@ -439,7 +439,7 @@ func BenchmarkUDPUnpackRepeat(b *testing.B) { cipherNumber := n % numCiphers ip := ips[cipherNumber] packet := packets[cipherNumber] - _, _, _, err := findAccessKeyUDP(ip, testBuf, packet, cipherList) + _, _, _, err := findAccessKeyUDP(&noopLogger{}, ip, testBuf, packet, cipherList) if err != nil { b.Error(err) } @@ -468,7 +468,7 @@ func BenchmarkUDPUnpackSharedKey(b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { ip := ips[n%numIPs] - _, _, _, err := findAccessKeyUDP(ip, testBuf, packet, cipherList) + _, _, _, err := findAccessKeyUDP(&noopLogger{}, ip, testBuf, packet, cipherList) if err != nil { b.Error(err) } @@ -482,7 +482,7 @@ func TestUDPEarlyClose(t *testing.T) { } testMetrics := &natTestMetrics{} const testTimeout = 200 * time.Millisecond - s := NewPacketHandler(testTimeout, cipherList, testMetrics, &fakeShadowsocksMetrics{}) + s := NewPacketHandler(&noopLogger{}, testTimeout, cipherList, testMetrics, &fakeShadowsocksMetrics{}) clientConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0}) if err != nil { From be43e847f2c6b293a1cdded948487105152edec5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 16:44:11 -0400 Subject: [PATCH 148/182] Fix test. --- cmd/outline-ss-server/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/outline-ss-server/server_test.go b/cmd/outline-ss-server/server_test.go index 9b8400a0..05999486 100644 --- a/cmd/outline-ss-server/server_test.go +++ b/cmd/outline-ss-server/server_test.go @@ -22,7 +22,7 @@ import ( ) func TestRunSSServer(t *testing.T) { - serverMetrics := prometheus.NewServerMetrics() + serverMetrics := newPrometheusServerMetrics() serviceMetrics, err := prometheus.NewServiceMetrics(nil) if err != nil { t.Fatalf("Failed to create Prometheus service metrics: %v", err) From bca42a0333d45b5fa2de0703fd8cea3b399e21fe Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 16:47:51 -0400 Subject: [PATCH 149/182] Add `--watch` flag to README. --- caddy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caddy/README.md b/caddy/README.md index f3c7b67d..188a9625 100644 --- a/caddy/README.md +++ b/caddy/README.md @@ -13,7 +13,7 @@ Shadowsocks backend. From this directory, build and run a custom binary with `xcaddy`: ```sh -xcaddy run --config config_example.json +xcaddy run --config config_example.json --watch ``` In a separate window, confirm you can fetch a page using this server: From 62206c816d55f04b947694ea0603c3e40820b620 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 9 Sep 2024 17:09:19 -0400 Subject: [PATCH 150/182] Remove changes moved to another PR. --- .github/workflows/go.yml | 2 +- .gitignore | 4 ---- .goreleaser.yml | 1 - caddy/go.mod | 5 +---- caddy/go.sum | 2 -- 5 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index e05d7899..24822c5e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -46,7 +46,7 @@ jobs: git submodule update --init - name: Build - run: GOWORK=off go build -v ./... + run: go build -v ./... - name: Test run: go test -race -benchmem -bench=. ./... -benchtime=100ms diff --git a/.gitignore b/.gitignore index cfb902b3..0f99f759 100644 --- a/.gitignore +++ b/.gitignore @@ -28,9 +28,5 @@ # Go task /.task/ -# Go workspace -go.work -go.work.sum - # Custom caddy binary /caddy/caddy diff --git a/.goreleaser.yml b/.goreleaser.yml index b07a73b4..29842233 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -21,7 +21,6 @@ builds: main: ./cmd/outline-ss-server env: - CGO_ENABLED=0 - - GOWORK=off goos: - darwin - windows diff --git a/caddy/go.mod b/caddy/go.mod index 2f2fa737..c084818c 100644 --- a/caddy/go.mod +++ b/caddy/go.mod @@ -4,15 +4,13 @@ go 1.23 require ( github.com/Jigsaw-Code/outline-sdk v0.0.16 - github.com/Jigsaw-Code/outline-ss-server v1.5.0 + github.com/Jigsaw-Code/outline-ss-server v1.6.0 github.com/caddyserver/caddy/v2 v2.8.4 github.com/mholt/caddy-l4 v0.0.0-20240812213304-afa78d72257b github.com/prometheus/client_golang v1.20.0 go.uber.org/zap v1.27.0 ) -replace github.com/mholt/caddy-l4 v0.0.0-20240812213304-afa78d72257b => ../../caddy-l4 - require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect @@ -71,7 +69,6 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo/v2 v2.15.0 // indirect - github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect github.com/oschwald/geoip2-golang v1.8.0 // indirect github.com/oschwald/maxminddb-golang v1.10.0 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/caddy/go.sum b/caddy/go.sum index 60fce3ee..afabefe1 100644 --- a/caddy/go.sum +++ b/caddy/go.sum @@ -20,8 +20,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Jigsaw-Code/outline-sdk v0.0.16 h1:WbHmv80FKDIpzEmR3GehTbq5CibYTLvcxIIpMMILiEs= github.com/Jigsaw-Code/outline-sdk v0.0.16/go.mod h1:e1oQZbSdLJBBuHgfeQsgEkvkuyIePPwstUeZRGq0KO8= -github.com/Jigsaw-Code/outline-ss-server v1.5.0 h1:Vz+iS0xR7i3PrLD82pzFFwZ9fsh6zrNawMeYERR8VTc= -github.com/Jigsaw-Code/outline-ss-server v1.5.0/go.mod h1:KaebwBiCWDSkgsJrJIbGH0szON8CZq4LgQaFV8v3RM4= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= From d2846128ce2ec526c4c148572145bb4e89b46d44 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 10:52:09 -0400 Subject: [PATCH 151/182] Remove arguments from `Logger()`. --- caddy/app.go | 2 +- caddy/shadowsocks_handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index aa4ea907..aedcb000 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -56,7 +56,7 @@ func (OutlineApp) CaddyModule() caddy.ModuleInfo { // Provision sets up Outline. func (app *OutlineApp) Provision(ctx caddy.Context) error { - app.logger = ctx.Logger(app) + app.logger = ctx.Logger() defer app.logger.Sync() app.logger.Info("provisioning app instance") diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go index 176f33ea..b9ead742 100644 --- a/caddy/shadowsocks_handler.go +++ b/caddy/shadowsocks_handler.go @@ -54,7 +54,7 @@ func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { // Provision implements caddy.Provisioner. func (h *ShadowsocksHandler) Provision(ctx caddy.Context) error { - h.logger = ctx.Logger(h) + h.logger = ctx.Logger() defer h.logger.Sync() ctx.App(moduleName) From 04faca24135a00a9d1a5df4ee61d0df46a65f882 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 11:01:00 -0400 Subject: [PATCH 152/182] Use `slog` instead of `zap`. --- caddy/app.go | 9 +++--- caddy/logger.go | 61 ------------------------------------ caddy/shadowsocks_handler.go | 8 ++--- 3 files changed, 8 insertions(+), 70 deletions(-) delete mode 100644 caddy/logger.go diff --git a/caddy/app.go b/caddy/app.go index aedcb000..b4f8e0b1 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -19,12 +19,12 @@ package caddy import ( "errors" + "log/slog" outline_prometheus "github.com/Jigsaw-Code/outline-ss-server/prometheus" outline "github.com/Jigsaw-Code/outline-ss-server/service" "github.com/caddyserver/caddy/v2" "github.com/prometheus/client_golang/prometheus" - "go.uber.org/zap" ) func init() { @@ -42,7 +42,7 @@ type OutlineApp struct { ShadowsocksConfig *ShadowsocksConfig `json:"shadowsocks,omitempty"` ReplayCache outline.ReplayCache - logger *zap.Logger + logger *slog.Logger Metrics outline.ServiceMetrics buildInfo *prometheus.GaugeVec } @@ -56,8 +56,7 @@ func (OutlineApp) CaddyModule() caddy.ModuleInfo { // Provision sets up Outline. func (app *OutlineApp) Provision(ctx caddy.Context) error { - app.logger = ctx.Logger() - defer app.logger.Sync() + app.logger = ctx.Slogger() app.logger.Info("provisioning app instance") @@ -88,7 +87,7 @@ func (app *OutlineApp) defineMetrics() { // TODO: Allow the configuration of ip2info. metrics, err := outline_prometheus.NewServiceMetrics(nil) if err != nil { - app.logger.Error("failed to define Prometheus metrics", zap.Error(err)) + app.logger.Error("failed to define Prometheus metrics", "err", err) return } app.Metrics = registerCollector(r, metrics) diff --git a/caddy/logger.go b/caddy/logger.go deleted file mode 100644 index 95cc2c7e..00000000 --- a/caddy/logger.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2024 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package caddy - -import ( - "context" - "log/slog" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -type logger struct { - zap *zap.Logger -} - -func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { - return l.zap.Check(toZapLevel(level), "") != nil -} - -func (l *logger) LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) { - fields := toZapFields(attrs) - l.zap.Log(toZapLevel(level), msg, fields...) - -} - -func toZapLevel(level slog.Level) zapcore.Level { - switch level { - case slog.LevelInfo: - return zapcore.InfoLevel - case slog.LevelWarn: - return zapcore.WarnLevel - case slog.LevelError: - return zapcore.ErrorLevel - default: - return zapcore.DebugLevel - } -} - -func toZapFields(attrs []slog.Attr) []zapcore.Field { - fields := make([]zapcore.Field, 0, len(attrs)) - var field zapcore.Field - for _, attr := range attrs { - field = zap.Any(attr.Key, attr.Value) - fields = append(fields, field) - } - - return fields -} diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go index b9ead742..415ad96c 100644 --- a/caddy/shadowsocks_handler.go +++ b/caddy/shadowsocks_handler.go @@ -17,6 +17,7 @@ package caddy import ( "container/list" "fmt" + "log/slog" "net" "github.com/Jigsaw-Code/outline-sdk/transport" @@ -42,7 +43,7 @@ type ShadowsocksHandler struct { NatTimeoutSec int `json:"nat_timeout_sec,omitempty"` service outline.Service - logger *zap.Logger + logger *slog.Logger } func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { @@ -54,8 +55,7 @@ func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { // Provision implements caddy.Provisioner. func (h *ShadowsocksHandler) Provision(ctx caddy.Context) error { - h.logger = ctx.Logger() - defer h.logger.Sync() + h.logger = ctx.Slogger() ctx.App(moduleName) if _, err := ctx.AppIfConfigured(moduleName); err != nil { @@ -94,7 +94,7 @@ func (h *ShadowsocksHandler) Provision(ctx caddy.Context) error { ciphers.Update(cipherList) service, err := outline.NewShadowsocksService( - outline.WithLogger(&logger{app.logger}), + outline.WithLogger(h.logger), outline.WithCiphers(ciphers), outline.WithMetrics(app.Metrics), outline.WithReplayCache(&app.ReplayCache), From 46e2f66bc6d6f89bedde5f7f5375aaf3ce521a49 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 11:06:01 -0400 Subject: [PATCH 153/182] Log error in `Provision()` instead of `defineMetrics()`. --- caddy/app.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index b4f8e0b1..84688b18 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -67,14 +67,16 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { app.ReplayCache = outline.NewReplayCache(app.ShadowsocksConfig.ReplayHistory) } - app.defineMetrics() + if err := app.defineMetrics(); err != nil { + app.logger.Error("failed to define Prometheus metrics", "err", err) + } app.buildInfo.WithLabelValues(app.Version).Set(1) // TODO: Add replacement metrics for `shadowsocks_keys` and `shadowsocks_ports`. return nil } -func (app *OutlineApp) defineMetrics() { +func (app *OutlineApp) defineMetrics() error { // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) @@ -87,10 +89,10 @@ func (app *OutlineApp) defineMetrics() { // TODO: Allow the configuration of ip2info. metrics, err := outline_prometheus.NewServiceMetrics(nil) if err != nil { - app.logger.Error("failed to define Prometheus metrics", "err", err) - return + return err } app.Metrics = registerCollector(r, metrics) + return nil } func registerCollector[T prometheus.Collector](registerer prometheus.Registerer, coll T) T { From a32378ee3a9da4fa653696782394b92034df4b5a Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 11:28:37 -0400 Subject: [PATCH 154/182] Do not panic on bad metrics registrations. --- caddy/app.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index 84688b18..e3b52edb 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -80,32 +80,40 @@ func (app *OutlineApp) defineMetrics() error { // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) + var err error buildInfo := prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "build_info", Help: "Information on the outline-ss-server build", }, []string{"version"}) - app.buildInfo = registerCollector(r, buildInfo) + app.buildInfo, err = registerCollector(r, buildInfo) + if err != nil { + return err + } // TODO: Allow the configuration of ip2info. metrics, err := outline_prometheus.NewServiceMetrics(nil) if err != nil { return err } - app.Metrics = registerCollector(r, metrics) + app.Metrics, err = registerCollector(r, metrics) + if err != nil { + return err + } return nil } -func registerCollector[T prometheus.Collector](registerer prometheus.Registerer, coll T) T { +func registerCollector[T prometheus.Collector](registerer prometheus.Registerer, coll T) (T, error) { if err := registerer.Register(coll); err != nil { are := &prometheus.AlreadyRegisteredError{} - if errors.As(err, are) { + if !errors.As(err, are) { // This collector has been registered before. This is expected during a config reload. coll = are.ExistingCollector.(T) } else { - panic(err) + // Something else went wrong. + return coll, err } } - return coll + return coll, nil } // Start starts the App. From f2b1b6f637b6d8d46b791fd93dbfa40e2211c99f Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 13:03:57 -0400 Subject: [PATCH 155/182] Check if the cast to `OutlineApp` is ok. --- caddy/shadowsocks_handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go index 415ad96c..625fac54 100644 --- a/caddy/shadowsocks_handler.go +++ b/caddy/shadowsocks_handler.go @@ -65,7 +65,10 @@ func (h *ShadowsocksHandler) Provision(ctx caddy.Context) error { if err != nil { return err } - app := mod.(*OutlineApp) + app, ok := mod.(*OutlineApp) + if !ok { + return fmt.Errorf("module `%s` is not an OutlineApp", moduleName) + } if len(h.Keys) == 0 { h.logger.Warn("no keys configured") From ac98ed93b6221226d65ceec5ac4bce33c0e2d26e Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 13:08:13 -0400 Subject: [PATCH 156/182] Remove `version` from the config. --- caddy/app.go | 7 ++----- caddy/config_example.json | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index e3b52edb..81beae35 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -38,7 +38,6 @@ type ShadowsocksConfig struct { } type OutlineApp struct { - Version string `json:"version,omitempty"` ShadowsocksConfig *ShadowsocksConfig `json:"shadowsocks,omitempty"` ReplayCache outline.ReplayCache @@ -60,9 +59,6 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { app.logger.Info("provisioning app instance") - if app.Version == "" { - app.Version = "dev" - } if app.ShadowsocksConfig != nil { app.ReplayCache = outline.NewReplayCache(app.ShadowsocksConfig.ReplayHistory) } @@ -70,7 +66,8 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { if err := app.defineMetrics(); err != nil { app.logger.Error("failed to define Prometheus metrics", "err", err) } - app.buildInfo.WithLabelValues(app.Version).Set(1) + // TODO: Set version at build time. + app.buildInfo.WithLabelValues("dev").Set(1) // TODO: Add replacement metrics for `shadowsocks_keys` and `shadowsocks_ports`. return nil diff --git a/caddy/config_example.json b/caddy/config_example.json index 764c51f0..4f65fc33 100644 --- a/caddy/config_example.json +++ b/caddy/config_example.json @@ -89,7 +89,6 @@ } }, "outline": { - "version": "0.0.0", "shadowsocks": { "replay_history": 10000 } From 40ff316abaad9cb8f92c6b11acf0e17cf186fdcd Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 16:19:50 -0400 Subject: [PATCH 157/182] Use `outline_` prefix for Caddy metrics. --- caddy/app.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index 81beae35..848fb00f 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -74,8 +74,7 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { } func (app *OutlineApp) defineMetrics() error { - // TODO: Decide on what to do about namespace. Can we change to "outline" for Caddy servers? - r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) + r := prometheus.WrapRegistererWithPrefix("outline_", prometheus.DefaultRegisterer) var err error buildInfo := prometheus.NewGaugeVec(prometheus.GaugeOpts{ From f9986123dee36a2aa8bca9cc0ead1f233f293521 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 10 Sep 2024 16:40:10 -0400 Subject: [PATCH 158/182] Remove unused `NatTimeoutSec` config option. --- caddy/shadowsocks_handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go index 625fac54..964f0a12 100644 --- a/caddy/shadowsocks_handler.go +++ b/caddy/shadowsocks_handler.go @@ -40,7 +40,6 @@ type KeyConfig struct { type ShadowsocksHandler struct { Keys []KeyConfig `json:"keys,omitempty"` - NatTimeoutSec int `json:"nat_timeout_sec,omitempty"` service outline.Service logger *slog.Logger From e6686d1aea8be1d4bc017c4423ead7fd11750410 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 11 Sep 2024 11:08:35 -0400 Subject: [PATCH 159/182] Move initialization of handlers to the constructor. --- service/shadowsocks.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/service/shadowsocks.go b/service/shadowsocks.go index 87814df8..cb274600 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -70,6 +70,14 @@ func NewShadowsocksService(opts ...Option) (Service, error) { if s.natTimeout == 0 { s.natTimeout = defaultNatTimeout } + + // TODO: Register initial data metrics at zero. + s.sh = NewStreamHandler( + NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}), + tcpReadTimeout, + ) + s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) + return s, nil } @@ -103,20 +111,12 @@ func WithNatTimeout(natTimeout time.Duration) Option { // HandleStream handles a Shadowsocks stream-based connection. func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { - if s.sh == nil { - authFunc := NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}) - // TODO: Register initial data metrics at zero. - s.sh = NewStreamHandler(authFunc, tcpReadTimeout) - } connMetrics := s.m.AddOpenTCPConnection(conn) s.sh.Handle(ctx, conn, connMetrics) } // HandlePacket handles a Shadowsocks packet connection. func (s *ssService) HandlePacket(conn net.PacketConn) { - if s.ph == nil { - s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) - } s.ph.Handle(conn) } From 1259af8d312fe0676856301c6961b848e96cc967 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 11 Sep 2024 15:25:14 -0400 Subject: [PATCH 160/182] Pass a `list.List` instead of a `CipherList`. --- cmd/outline-ss-server/main.go | 14 ++++---------- service/shadowsocks.go | 9 ++++++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 543d036d..d87a42e6 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -93,12 +93,12 @@ func (s *SSServer) loadConfig(filename string) error { return nil } -func newCipherListFromConfig(config ServiceConfig) (service.CipherList, error) { +func newCipherListFromConfig(config ServiceConfig) (*list.List, error) { type cipherKey struct { cipher string secret string } - cipherList := list.New() + ciphers := list.New() existingCiphers := make(map[cipherKey]bool) for _, keyConfig := range config.Keys { key := cipherKey{keyConfig.Cipher, keyConfig.Secret} @@ -111,12 +111,9 @@ func newCipherListFromConfig(config ServiceConfig) (service.CipherList, error) { return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) } entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - cipherList.PushBack(&entry) + ciphers.PushBack(&entry) existingCiphers[key] = true } - ciphers := service.NewCipherList() - ciphers.Update(cipherList) - return ciphers, nil } @@ -214,11 +211,8 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { for portNum, cipherList := range portCiphers { addr := net.JoinHostPort("::", strconv.Itoa(portNum)) - ciphers := service.NewCipherList() - ciphers.Update(cipherList) - ssService, err := service.NewShadowsocksService( - service.WithCiphers(ciphers), + service.WithCiphers(cipherList), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), service.WithReplayCache(&s.replayCache), diff --git a/service/shadowsocks.go b/service/shadowsocks.go index cb274600..227fc81c 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -15,6 +15,7 @@ package service import ( + "container/list" "context" "net" "time" @@ -61,7 +62,9 @@ type ssService struct { // NewShadowsocksService creates a new service func NewShadowsocksService(opts ...Option) (Service, error) { - s := &ssService{} + s := &ssService{ + ciphers: NewCipherList(), + } for _, opt := range opts { opt(s) @@ -82,9 +85,9 @@ func NewShadowsocksService(opts ...Option) (Service, error) { } // WithCiphers option function. -func WithCiphers(ciphers CipherList) Option { +func WithCiphers(ciphers *list.List) Option { return func(s *ssService) { - s.ciphers = ciphers + s.ciphers.Update(ciphers) } } From 3a64e35b9236ad0564927750253b38a7c1cf7f05 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 11 Sep 2024 15:27:59 -0400 Subject: [PATCH 161/182] Rename `SSServer` to `OutlineServer`. --- cmd/outline-ss-server/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index d87a42e6..76a17051 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -57,7 +57,7 @@ func init() { ) } -type SSServer struct { +type OutlineServer struct { stopConfig func() error lnManager service.ListenerManager natTimeout time.Duration @@ -66,7 +66,7 @@ type SSServer struct { replayCache service.ReplayCache } -func (s *SSServer) loadConfig(filename string) error { +func (s *OutlineServer) loadConfig(filename string) error { configData, err := os.ReadFile(filename) if err != nil { return fmt.Errorf("failed to read config file %s: %w", filename, err) @@ -178,7 +178,7 @@ func (ls *listenerSet) Len() int { return len(ls.listenerCloseFuncs) } -func (s *SSServer) runConfig(config Config) (func() error, error) { +func (s *OutlineServer) runConfig(config Config) (func() error, error) { startErrCh := make(chan error) stopErrCh := make(chan error) stopCh := make(chan struct{}) @@ -290,7 +290,7 @@ func (s *SSServer) runConfig(config Config) (func() error, error) { } // Stop stops serving the current config. -func (s *SSServer) Stop() error { +func (s *OutlineServer) Stop() error { stopFunc := s.stopConfig if stopFunc == nil { return nil @@ -303,9 +303,9 @@ func (s *SSServer) Stop() error { return nil } -// RunSSServer starts a shadowsocks server running, and returns the server or an error. -func RunSSServer(filename string, natTimeout time.Duration, serverMetrics *serverMetrics, serviceMetrics service.ServiceMetrics, replayHistory int) (*SSServer, error) { - server := &SSServer{ +// RunOutlineServer starts an Outline server running, and returns the server or an error. +func RunOutlineServer(filename string, natTimeout time.Duration, serverMetrics *serverMetrics, serviceMetrics service.ServiceMetrics, replayHistory int) (*OutlineServer, error) { + server := &OutlineServer{ lnManager: service.NewListenerManager(), natTimeout: natTimeout, serverMetrics: serverMetrics, @@ -397,7 +397,7 @@ func main() { r := prometheus.WrapRegistererWithPrefix("shadowsocks_", prometheus.DefaultRegisterer) r.MustRegister(serverMetrics, serviceMetrics) - _, err = RunSSServer(flags.ConfigFile, flags.natTimeout, serverMetrics, serviceMetrics, flags.replayHistory) + _, err = RunOutlineServer(flags.ConfigFile, flags.natTimeout, serverMetrics, serviceMetrics, flags.replayHistory) if err != nil { slog.Error("Server failed to start. Aborting.", "err", err) } From 39da61b64587ddab369c572795702984f2531664 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 12 Sep 2024 13:36:00 -0400 Subject: [PATCH 162/182] refactor: make connection metrics optional --- internal/integration_test/integration_test.go | 14 +++++----- service/tcp.go | 23 +++++++--------- service/udp.go | 27 ++++++++++++------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index f847f761..d2626eff 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -130,7 +130,6 @@ func TestTCPEcho(t *testing.T) { } replayCache := service.NewReplayCache(5) const testTimeout = 200 * time.Millisecond - testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -138,7 +137,7 @@ func TestTCPEcho(t *testing.T) { go func() { service.StreamServe( func() (transport.StreamConn, error) { return proxyListener.AcceptTCP() }, - func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, testMetrics) }, + func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, nil) }, ) done <- struct{}{} }() @@ -192,11 +191,14 @@ func (m *fakeShadowsocksMetrics) AddCipherSearch(accessKeyFound bool, timeToCiph } type statusMetrics struct { - service.NoOpTCPConnMetrics sync.Mutex statuses []string } +var _ service.TCPConnMetrics = (*statusMetrics)(nil) + +func (m *statusMetrics) AddAuthenticated(accessKey string) {} +func (m *statusMetrics) AddProbe(status, drainResult string, clientProxyBytes int64) {} func (m *statusMetrics) AddClosed(status string, data metrics.ProxyMetrics, duration time.Duration) { m.Lock() m.statuses = append(m.statuses, status) @@ -399,7 +401,6 @@ func BenchmarkTCPThroughput(b *testing.B) { b.Fatal(err) } const testTimeout = 200 * time.Millisecond - testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -407,7 +408,7 @@ func BenchmarkTCPThroughput(b *testing.B) { go func() { service.StreamServe( service.WrapStreamAcceptFunc(proxyListener.AcceptTCP), - func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, testMetrics) }, + func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, nil) }, ) done <- struct{}{} }() @@ -466,7 +467,6 @@ func BenchmarkTCPMultiplexing(b *testing.B) { } replayCache := service.NewReplayCache(service.MaxCapacity) const testTimeout = 200 * time.Millisecond - testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -474,7 +474,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { go func() { service.StreamServe( service.WrapStreamAcceptFunc(proxyListener.AcceptTCP), - func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, testMetrics) }, + func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, nil) }, ) done <- struct{}{} }() diff --git a/service/tcp.go b/service/tcp.go index 8637663b..5e66261c 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -253,7 +253,9 @@ func (h *streamHandler) Handle(ctx context.Context, clientConn transport.StreamC status = connError.Status slog.LogAttrs(nil, slog.LevelDebug, "TCP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) } - connMetrics.AddClosed(status, proxyMetrics, connDuration) + if connMetrics != nil { + connMetrics.AddClosed(status, proxyMetrics, connDuration) + } measuredClientConn.Close() // Closing after the metrics are added aids integration testing. slog.LogAttrs(nil, slog.LevelDebug, "TCP: Done.", slog.String("status", status), slog.Duration("duration", connDuration)) } @@ -325,7 +327,9 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor h.absorbProbe(outerConn, connMetrics, authErr.Status, proxyMetrics) return authErr } - connMetrics.AddAuthenticated(id) + if connMetrics != nil { + connMetrics.AddAuthenticated(id) + } // Read target address and dial it. tgtAddr, err := getProxyRequest(innerConn) @@ -355,7 +359,9 @@ func (h *streamHandler) absorbProbe(clientConn io.ReadCloser, connMetrics TCPCon _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) slog.LogAttrs(nil, slog.LevelDebug, "Drain error.", slog.Any("err", drainErr), slog.String("result", drainResult)) - connMetrics.AddProbe(status, drainResult, proxyMetrics.ClientProxy) + if connMetrics != nil { + connMetrics.AddProbe(status, drainResult, proxyMetrics.ClientProxy) + } } func drainErrToString(drainErr error) string { @@ -369,14 +375,3 @@ func drainErrToString(drainErr error) string { return "other" } } - -// NoOpTCPConnMetrics is a [TCPConnMetrics] that doesn't do anything. Useful in tests -// or if you don't want to track metrics. -type NoOpTCPConnMetrics struct{} - -var _ TCPConnMetrics = (*NoOpTCPConnMetrics)(nil) - -func (m *NoOpTCPConnMetrics) AddAuthenticated(accessKey string) {} -func (m *NoOpTCPConnMetrics) AddClosed(status string, data metrics.ProxyMetrics, duration time.Duration) { -} -func (m *NoOpTCPConnMetrics) AddProbe(status, drainResult string, clientProxyBytes int64) {} diff --git a/service/udp.go b/service/udp.go index 2d08a709..9cb4a462 100644 --- a/service/udp.go +++ b/service/udp.go @@ -198,7 +198,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { slog.LogAttrs(nil, slog.LevelDebug, "UDP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) status = connError.Status } - if targetConn != nil { + if targetConn != nil && targetConn.metrics != nil { targetConn.metrics.AddPacketFromClient(status, int64(clientProxyBytes), int64(proxyTargetBytes)) } } @@ -299,11 +299,13 @@ type natmap struct { running *sync.WaitGroup } -func newNATmap(timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { - m := &natmap{metrics: sm, running: running} - m.keyConn = make(map[string]*natconn) - m.timeout = timeout - return m +func newNATmap(timeout time.Duration, metrics UDPMetrics, running *sync.WaitGroup) *natmap { + return &natmap{ + metrics: metrics, + running: running, + keyConn: make(map[string]*natconn), + timeout: timeout, + } } func (m *natmap) Get(key string) *natconn { @@ -341,13 +343,18 @@ func (m *natmap) del(key string) net.PacketConn { } func (m *natmap) Add(clientAddr net.Addr, clientConn net.PacketConn, cryptoKey *shadowsocks.EncryptionKey, targetConn net.PacketConn, keyID string) *natconn { - connMetrics := m.metrics.AddUDPNatEntry(clientAddr, keyID) + var connMetrics UDPConnMetrics + if m.metrics != nil { + connMetrics = m.metrics.AddUDPNatEntry(clientAddr, keyID) + } entry := m.set(clientAddr.String(), targetConn, cryptoKey, keyID, connMetrics) m.running.Add(1) go func() { timedCopy(clientAddr, clientConn, entry, keyID) - connMetrics.RemoveNatEntry() + if connMetrics != nil { + connMetrics.RemoveNatEntry() + } if pc := m.del(clientAddr.String()); pc != nil { pc.Close() } @@ -443,7 +450,9 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco if expired { break } - targetConn.metrics.AddPacketFromTarget(status, int64(bodyLen), int64(proxyClientBytes)) + if targetConn.metrics != nil { + targetConn.metrics.AddPacketFromTarget(status, int64(bodyLen), int64(proxyClientBytes)) + } } } From 48796e1a7c1838bed52f95e54945ecb144ce0219 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 16:40:26 -0400 Subject: [PATCH 163/182] Make setting the logger a setter function. --- internal/integration_test/integration_test.go | 14 ++++----- service/logger.go | 4 +-- service/shadowsocks.go | 18 +++++------ service/tcp.go | 24 ++++++++------ service/tcp_test.go | 16 +++++----- service/udp.go | 31 +++++++++++++------ service/udp_test.go | 4 +-- 7 files changed, 65 insertions(+), 46 deletions(-) diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index 9bc3d0d5..d0658353 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -133,7 +133,7 @@ func TestTCPEcho(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) + handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -223,7 +223,7 @@ func TestRestrictedAddresses(t *testing.T) { const testTimeout = 200 * time.Millisecond testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) + handler := service.NewStreamHandler(authFunc, testTimeout) done := make(chan struct{}) go func() { service.StreamServe( @@ -322,7 +322,7 @@ func TestUDPEcho(t *testing.T) { t.Fatal(err) } testMetrics := &fakeUDPMetrics{} - proxy := service.NewPacketHandler(&noopLogger{}, time.Hour, cipherList, testMetrics, &fakeShadowsocksMetrics{}) + proxy := service.NewPacketHandler(time.Hour, cipherList, testMetrics, &fakeShadowsocksMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { @@ -412,7 +412,7 @@ func BenchmarkTCPThroughput(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) + handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -479,7 +479,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { const testTimeout = 200 * time.Millisecond testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) - handler := service.NewStreamHandler(&noopLogger{}, authFunc, testTimeout) + handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) done := make(chan struct{}) go func() { @@ -555,7 +555,7 @@ func BenchmarkUDPEcho(b *testing.B) { if err != nil { b.Fatal(err) } - proxy := service.NewPacketHandler(&noopLogger{}, time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) + proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { @@ -599,7 +599,7 @@ func BenchmarkUDPManyKeys(b *testing.B) { if err != nil { b.Fatal(err) } - proxy := service.NewPacketHandler(&noopLogger{}, time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) + proxy := service.NewPacketHandler(time.Hour, cipherList, &service.NoOpUDPMetrics{}, &fakeShadowsocksMetrics{}) proxy.SetTargetIPValidator(allowAll) done := make(chan struct{}) go func() { diff --git a/service/logger.go b/service/logger.go index 2e8cab9e..b751fb8e 100644 --- a/service/logger.go +++ b/service/logger.go @@ -19,7 +19,7 @@ import ( "log/slog" ) -type logger interface { +type Logger interface { Enabled(ctx context.Context, level slog.Level) bool LogAttrs(ctx context.Context, level slog.Level, msg string, attrs ...slog.Attr) } @@ -27,7 +27,7 @@ type logger interface { type noopLogger struct { } -var _ logger = (*noopLogger)(nil) +var _ Logger = (*noopLogger)(nil) func (l *noopLogger) Enabled(ctx context.Context, level slog.Level) bool { return false diff --git a/service/shadowsocks.go b/service/shadowsocks.go index fcca785a..230bbf14 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -51,7 +51,7 @@ type Service interface { type Option func(s *ssService) error type ssService struct { - logger logger + logger Logger m ServiceMetrics ciphers CipherList natTimeout time.Duration @@ -70,10 +70,6 @@ func NewShadowsocksService(opts ...Option) (Service, error) { } } - if s.logger == nil { - s.logger = &noopLogger{} - } - if s.natTimeout == 0 { s.natTimeout = defaultNatTimeout } @@ -81,8 +77,7 @@ func NewShadowsocksService(opts ...Option) (Service, error) { } // WithLogger can be used to provide a custom log target. -// Defaults to io.Discard. -func WithLogger(l logger) Option { +func WithLogger(l Logger) Option { return func(s *ssService) error { s.logger = l return nil @@ -130,7 +125,10 @@ func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}, ) // TODO: Register initial data metrics at zero. - s.sh = NewStreamHandler(s.logger, authFunc, tcpReadTimeout) + s.sh = NewStreamHandler(authFunc, tcpReadTimeout) + if s.logger != nil { + s.sh.SetLogger(s.logger) + } } connMetrics := s.m.AddOpenTCPConnection(conn) s.sh.Handle(ctx, conn, connMetrics) @@ -140,12 +138,14 @@ func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) func (s *ssService) HandlePacket(conn net.PacketConn) { if s.ph == nil { s.ph = NewPacketHandler( - s.logger, s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}, ) + if s.logger != nil { + s.ph.SetLogger(s.logger) + } } s.ph.Handle(conn) } diff --git a/service/tcp.go b/service/tcp.go index 7a490bb5..68c8a63a 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -58,7 +58,7 @@ func remoteIP(conn net.Conn) netip.Addr { } // Wrapper for slog.Debug during TCP access key searches. -func debugTCP(l logger, template string, cipherID string, attr slog.Attr) { +func debugTCP(l Logger, template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. if l.Enabled(nil, slog.LevelDebug) { @@ -72,7 +72,7 @@ func debugTCP(l logger, template string, cipherID string, attr slog.Attr) { // required = saltSize + 2 + cipher.TagSize, the number of bytes needed to authenticate the connection. const bytesForKeyFinding = 50 -func findAccessKey(l logger, clientReader io.Reader, clientIP netip.Addr, cipherList CipherList) (*CipherEntry, io.Reader, []byte, time.Duration, error) { +func findAccessKey(l Logger, clientReader io.Reader, clientIP netip.Addr, cipherList CipherList) (*CipherEntry, io.Reader, []byte, time.Duration, error) { // We snapshot the list because it may be modified while we use it. ciphers := cipherList.SnapshotForClientIP(clientIP) firstBytes := make([]byte, bytesForKeyFinding) @@ -95,7 +95,7 @@ func findAccessKey(l logger, clientReader io.Reader, clientIP netip.Addr, cipher } // Implements a trial decryption search. This assumes that all ciphers are AEAD. -func findEntry(l logger, firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list.Element) { +func findEntry(l Logger, firstBytes []byte, ciphers []*list.Element) (*CipherEntry, *list.Element) { // To hold the decrypted chunk length. chunkLenBuf := [2]byte{} for ci, elt := range ciphers { @@ -112,12 +112,12 @@ func findEntry(l logger, firstBytes []byte, ciphers []*list.Element) (*CipherEnt return nil, nil } -type StreamAuthenticateFunc func(l logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) +type StreamAuthenticateFunc func(l Logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) // NewShadowsocksStreamAuthenticator creates a stream authenticator that uses Shadowsocks. // TODO(fortuna): Offer alternative transports. func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCache, metrics ShadowsocksConnMetrics) StreamAuthenticateFunc { - return func(l logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { + return func(l Logger, clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { // Find the cipher and acess key id. cipherEntry, clientReader, clientSalt, timeToCipher, keyErr := findAccessKey(l, clientConn, remoteIP(clientConn), ciphers) metrics.AddCipherSearch(keyErr == nil, timeToCipher) @@ -151,7 +151,7 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa } type streamHandler struct { - l logger + l Logger listenerId string readTimeout time.Duration authenticate StreamAuthenticateFunc @@ -159,9 +159,9 @@ type streamHandler struct { } // NewStreamHandler creates a StreamHandler -func NewStreamHandler(l logger, authenticate StreamAuthenticateFunc, timeout time.Duration) StreamHandler { +func NewStreamHandler(authenticate StreamAuthenticateFunc, timeout time.Duration) StreamHandler { return &streamHandler{ - l: l, + l: &noopLogger{}, readTimeout: timeout, authenticate: authenticate, dialer: defaultDialer, @@ -180,10 +180,16 @@ func makeValidatingTCPStreamDialer(targetIPValidator onet.TargetIPValidator) tra // StreamHandler is a handler that handles stream connections. type StreamHandler interface { Handle(ctx context.Context, conn transport.StreamConn, connMetrics TCPConnMetrics) + // SetLogger sets the logger used to log messages. + SetLogger(l Logger) // SetTargetDialer sets the [transport.StreamDialer] to be used to connect to target addresses. SetTargetDialer(dialer transport.StreamDialer) } +func (s *streamHandler) SetLogger(l Logger) { + s.l = l +} + func (s *streamHandler) SetTargetDialer(dialer transport.StreamDialer) { s.dialer = dialer } @@ -272,7 +278,7 @@ func getProxyRequest(clientConn transport.StreamConn) (string, error) { return tgtAddr.String(), nil } -func proxyConnection(l logger, ctx context.Context, dialer transport.StreamDialer, tgtAddr string, clientConn transport.StreamConn) *onet.ConnectionError { +func proxyConnection(l Logger, ctx context.Context, dialer transport.StreamDialer, tgtAddr string, clientConn transport.StreamConn) *onet.ConnectionError { tgtConn, dialErr := dialer.DialStream(ctx, tgtAddr) if dialErr != nil { // We don't drain so dial errors and invalid addresses are communicated quickly. diff --git a/service/tcp_test.go b/service/tcp_test.go index f4d22580..f5f5dece 100644 --- a/service/tcp_test.go +++ b/service/tcp_test.go @@ -286,7 +286,7 @@ func TestProbeRandom(t *testing.T) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe( @@ -366,7 +366,7 @@ func TestProbeClientBytesBasicTruncated(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -404,7 +404,7 @@ func TestProbeClientBytesBasicModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -443,7 +443,7 @@ func TestProbeClientBytesCoalescedModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, 200*time.Millisecond) handler.SetTargetDialer(makeValidatingTCPStreamDialer(allowAll)) done := make(chan struct{}) go func() { @@ -489,7 +489,7 @@ func TestProbeServerBytesModified(t *testing.T) { cipher := firstCipher(cipherList) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(&noopLogger{}, authFunc, 200*time.Millisecond) + handler := NewStreamHandler(authFunc, 200*time.Millisecond) done := make(chan struct{}) go func() { StreamServe( @@ -523,7 +523,7 @@ func TestReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewStreamHandler(&noopLogger{}, authFunc, testTimeout) + handler := NewStreamHandler(authFunc, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -605,7 +605,7 @@ func TestReverseReplayDefense(t *testing.T) { testMetrics := &probeTestMetrics{} const testTimeout = 200 * time.Millisecond authFunc := NewShadowsocksStreamAuthenticator(cipherList, &replayCache, testMetrics) - handler := NewStreamHandler(&noopLogger{}, authFunc, testTimeout) + handler := NewStreamHandler(authFunc, testTimeout) snapshot := cipherList.SnapshotForClientIP(netip.Addr{}) cipherEntry := snapshot[0].Value.(*CipherEntry) cipher := cipherEntry.CryptoKey @@ -679,7 +679,7 @@ func probeExpectTimeout(t *testing.T, payloadSize int) { require.NoError(t, err, "MakeTestCiphers failed: %v", err) testMetrics := &probeTestMetrics{} authFunc := NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) - handler := NewStreamHandler(&noopLogger{}, authFunc, testTimeout) + handler := NewStreamHandler(authFunc, testTimeout) done := make(chan struct{}) go func() { diff --git a/service/udp.go b/service/udp.go index 74d8164a..057229c1 100644 --- a/service/udp.go +++ b/service/udp.go @@ -44,7 +44,7 @@ type UDPMetrics interface { const serverUDPBufferSize = 64 * 1024 // Wrapper for slog.Debug during UDP proxying. -func debugUDP(l logger, template string, cipherID string, attr slog.Attr) { +func debugUDP(l Logger, template string, cipherID string, attr slog.Attr) { // This is an optimization to reduce unnecessary allocations due to an interaction // between Go's inlining/escape analysis and varargs functions like slog.Debug. if l.Enabled(nil, slog.LevelDebug) { @@ -52,7 +52,7 @@ func debugUDP(l logger, template string, cipherID string, attr slog.Attr) { } } -func debugUDPAddr(l logger, template string, addr net.Addr, attr slog.Attr) { +func debugUDPAddr(l Logger, template string, addr net.Addr, attr slog.Attr) { if l.Enabled(nil, slog.LevelDebug) { l.LogAttrs(nil, slog.LevelDebug, fmt.Sprintf("UDP: %s", template), slog.String("address", addr.String()), attr) } @@ -60,7 +60,7 @@ func debugUDPAddr(l logger, template string, addr net.Addr, attr slog.Attr) { // Decrypts src into dst. It tries each cipher until it finds one that authenticates // correctly. dst and src must not overlap. -func findAccessKeyUDP(l logger, clientIP netip.Addr, dst, src []byte, cipherList CipherList) ([]byte, string, *shadowsocks.EncryptionKey, error) { +func findAccessKeyUDP(l Logger, clientIP netip.Addr, dst, src []byte, cipherList CipherList) ([]byte, string, *shadowsocks.EncryptionKey, error) { // Try each cipher until we find one that authenticates successfully. This assumes that all ciphers are AEAD. // We snapshot the list because it may be modified while we use it. snapshot := cipherList.SnapshotForClientIP(clientIP) @@ -80,7 +80,7 @@ func findAccessKeyUDP(l logger, clientIP netip.Addr, dst, src []byte, cipherList } type packetHandler struct { - l logger + l Logger natTimeout time.Duration ciphers CipherList m UDPMetrics @@ -89,18 +89,31 @@ type packetHandler struct { } // NewPacketHandler creates a UDPService -func NewPacketHandler(l logger, natTimeout time.Duration, cipherList CipherList, m UDPMetrics, ssMetrics ShadowsocksConnMetrics) PacketHandler { - return &packetHandler{l: l, natTimeout: natTimeout, ciphers: cipherList, m: m, ssm: ssMetrics, targetIPValidator: onet.RequirePublicIP} +func NewPacketHandler(natTimeout time.Duration, cipherList CipherList, m UDPMetrics, ssMetrics ShadowsocksConnMetrics) PacketHandler { + return &packetHandler{ + l: &noopLogger{}, + natTimeout: natTimeout, + ciphers: cipherList, + m: m, + ssm: ssMetrics, + targetIPValidator: onet.RequirePublicIP, + } } // PacketHandler is a running UDP shadowsocks proxy that can be stopped. type PacketHandler interface { + // SetLogger sets the logger used to log messages. + SetLogger(l Logger) // SetTargetIPValidator sets the function to be used to validate the target IP addresses. SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) // Handle returns after clientConn closes and all the sub goroutines return. Handle(clientConn net.PacketConn) } +func (h *packetHandler) SetLogger(l Logger) { + h.l = l +} + func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPValidator) { h.targetIPValidator = targetIPValidator } @@ -295,13 +308,13 @@ func (c *natconn) ReadFrom(buf []byte) (int, net.Addr, error) { type natmap struct { sync.RWMutex keyConn map[string]*natconn - l logger + l Logger timeout time.Duration metrics UDPMetrics running *sync.WaitGroup } -func newNATmap(l logger, timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { +func newNATmap(l Logger, timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { m := &natmap{l: l, metrics: sm, running: running} m.keyConn = make(map[string]*natconn) m.timeout = timeout @@ -377,7 +390,7 @@ func (m *natmap) Close() error { var maxAddrLen int = len(socks.ParseAddr("[2001:db8::1]:12345")) // copy from target to client until read timeout -func timedCopy(l logger, clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string) { +func timedCopy(l Logger, clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string) { // pkt is used for in-place encryption of downstream UDP packets, with the layout // [padding?][salt][address][body][tag][extra] // Padding is only used if the address is IPv4. diff --git a/service/udp_test.go b/service/udp_test.go index bab91339..33b98531 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -136,7 +136,7 @@ func sendToDiscard(payloads [][]byte, validator onet.TargetIPValidator) *natTest cipher := ciphers.SnapshotForClientIP(netip.Addr{})[0].Value.(*CipherEntry).CryptoKey clientConn := makePacketConn() metrics := &natTestMetrics{} - handler := NewPacketHandler(&noopLogger{}, timeout, ciphers, metrics, &fakeShadowsocksMetrics{}) + handler := NewPacketHandler(timeout, ciphers, metrics, &fakeShadowsocksMetrics{}) handler.SetTargetIPValidator(validator) done := make(chan struct{}) go func() { @@ -482,7 +482,7 @@ func TestUDPEarlyClose(t *testing.T) { } testMetrics := &natTestMetrics{} const testTimeout = 200 * time.Millisecond - s := NewPacketHandler(&noopLogger{}, testTimeout, cipherList, testMetrics, &fakeShadowsocksMetrics{}) + s := NewPacketHandler(testTimeout, cipherList, testMetrics, &fakeShadowsocksMetrics{}) clientConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0}) if err != nil { From 9d126f9b8336ba8736149eea1883dc1205e2a184 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 16:46:46 -0400 Subject: [PATCH 164/182] Revert "Pass a `list.List` instead of a `CipherList`." This reverts commit 1259af8d312fe0676856301c6961b848e96cc967. --- cmd/outline-ss-server/main.go | 14 ++++++++++---- service/shadowsocks.go | 9 +++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 76a17051..bd2aa177 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -93,12 +93,12 @@ func (s *OutlineServer) loadConfig(filename string) error { return nil } -func newCipherListFromConfig(config ServiceConfig) (*list.List, error) { +func newCipherListFromConfig(config ServiceConfig) (service.CipherList, error) { type cipherKey struct { cipher string secret string } - ciphers := list.New() + cipherList := list.New() existingCiphers := make(map[cipherKey]bool) for _, keyConfig := range config.Keys { key := cipherKey{keyConfig.Cipher, keyConfig.Secret} @@ -111,9 +111,12 @@ func newCipherListFromConfig(config ServiceConfig) (*list.List, error) { return nil, fmt.Errorf("failed to create encyption key for key %v: %w", keyConfig.ID, err) } entry := service.MakeCipherEntry(keyConfig.ID, cryptoKey, keyConfig.Secret) - ciphers.PushBack(&entry) + cipherList.PushBack(&entry) existingCiphers[key] = true } + ciphers := service.NewCipherList() + ciphers.Update(cipherList) + return ciphers, nil } @@ -211,8 +214,11 @@ func (s *OutlineServer) runConfig(config Config) (func() error, error) { for portNum, cipherList := range portCiphers { addr := net.JoinHostPort("::", strconv.Itoa(portNum)) + ciphers := service.NewCipherList() + ciphers.Update(cipherList) + ssService, err := service.NewShadowsocksService( - service.WithCiphers(cipherList), + service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), service.WithReplayCache(&s.replayCache), diff --git a/service/shadowsocks.go b/service/shadowsocks.go index 227fc81c..cb274600 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -15,7 +15,6 @@ package service import ( - "container/list" "context" "net" "time" @@ -62,9 +61,7 @@ type ssService struct { // NewShadowsocksService creates a new service func NewShadowsocksService(opts ...Option) (Service, error) { - s := &ssService{ - ciphers: NewCipherList(), - } + s := &ssService{} for _, opt := range opts { opt(s) @@ -85,9 +82,9 @@ func NewShadowsocksService(opts ...Option) (Service, error) { } // WithCiphers option function. -func WithCiphers(ciphers *list.List) Option { +func WithCiphers(ciphers CipherList) Option { return func(s *ssService) { - s.ciphers.Update(ciphers) + s.ciphers = ciphers } } From 213903d58ff806b9bfa8df37d54b443b97841fa6 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 17:01:13 -0400 Subject: [PATCH 165/182] Create noop metrics if nil. --- service/tcp.go | 26 +++++++++++++++++--------- service/udp.go | 26 ++++++++++++++------------ 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/service/tcp.go b/service/tcp.go index 5e66261c..5554454c 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -241,6 +241,9 @@ func StreamServe(accept StreamAcceptFunc, handle StreamHandleFunc) { } func (h *streamHandler) Handle(ctx context.Context, clientConn transport.StreamConn, connMetrics TCPConnMetrics) { + if connMetrics == nil { + connMetrics = &NoOpTCPConnMetrics{} + } var proxyMetrics metrics.ProxyMetrics measuredClientConn := metrics.MeasureConn(clientConn, &proxyMetrics.ProxyClient, &proxyMetrics.ClientProxy) connStart := time.Now() @@ -253,9 +256,7 @@ func (h *streamHandler) Handle(ctx context.Context, clientConn transport.StreamC status = connError.Status slog.LogAttrs(nil, slog.LevelDebug, "TCP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) } - if connMetrics != nil { - connMetrics.AddClosed(status, proxyMetrics, connDuration) - } + connMetrics.AddClosed(status, proxyMetrics, connDuration) measuredClientConn.Close() // Closing after the metrics are added aids integration testing. slog.LogAttrs(nil, slog.LevelDebug, "TCP: Done.", slog.String("status", status), slog.Duration("duration", connDuration)) } @@ -327,9 +328,7 @@ func (h *streamHandler) handleConnection(ctx context.Context, outerConn transpor h.absorbProbe(outerConn, connMetrics, authErr.Status, proxyMetrics) return authErr } - if connMetrics != nil { - connMetrics.AddAuthenticated(id) - } + connMetrics.AddAuthenticated(id) // Read target address and dial it. tgtAddr, err := getProxyRequest(innerConn) @@ -359,9 +358,7 @@ func (h *streamHandler) absorbProbe(clientConn io.ReadCloser, connMetrics TCPCon _, drainErr := io.Copy(io.Discard, clientConn) // drain socket drainResult := drainErrToString(drainErr) slog.LogAttrs(nil, slog.LevelDebug, "Drain error.", slog.Any("err", drainErr), slog.String("result", drainResult)) - if connMetrics != nil { - connMetrics.AddProbe(status, drainResult, proxyMetrics.ClientProxy) - } + connMetrics.AddProbe(status, drainResult, proxyMetrics.ClientProxy) } func drainErrToString(drainErr error) string { @@ -375,3 +372,14 @@ func drainErrToString(drainErr error) string { return "other" } } + +// NoOpTCPConnMetrics is a [TCPConnMetrics] that doesn't do anything. Useful in tests +// or if you don't want to track metrics. +type NoOpTCPConnMetrics struct{} + +var _ TCPConnMetrics = (*NoOpTCPConnMetrics)(nil) + +func (m *NoOpTCPConnMetrics) AddAuthenticated(accessKey string) {} +func (m *NoOpTCPConnMetrics) AddClosed(status string, data metrics.ProxyMetrics, duration time.Duration) { +} +func (m *NoOpTCPConnMetrics) AddProbe(status, drainResult string, clientProxyBytes int64) {} diff --git a/service/udp.go b/service/udp.go index 9cb4a462..444237e3 100644 --- a/service/udp.go +++ b/service/udp.go @@ -89,7 +89,16 @@ type packetHandler struct { // NewPacketHandler creates a UDPService func NewPacketHandler(natTimeout time.Duration, cipherList CipherList, m UDPMetrics, ssMetrics ShadowsocksConnMetrics) PacketHandler { - return &packetHandler{natTimeout: natTimeout, ciphers: cipherList, m: m, ssm: ssMetrics, targetIPValidator: onet.RequirePublicIP} + if m == nil { + m = &NoOpUDPMetrics{} + } + return &packetHandler{ + natTimeout: natTimeout, + ciphers: cipherList, + m: m, + ssm: ssMetrics, + targetIPValidator: onet.RequirePublicIP, + } } // PacketHandler is a running UDP shadowsocks proxy that can be stopped. @@ -198,7 +207,7 @@ func (h *packetHandler) Handle(clientConn net.PacketConn) { slog.LogAttrs(nil, slog.LevelDebug, "UDP: Error", slog.String("msg", connError.Message), slog.Any("cause", connError.Cause)) status = connError.Status } - if targetConn != nil && targetConn.metrics != nil { + if targetConn != nil { targetConn.metrics.AddPacketFromClient(status, int64(clientProxyBytes), int64(proxyTargetBytes)) } } @@ -343,18 +352,13 @@ func (m *natmap) del(key string) net.PacketConn { } func (m *natmap) Add(clientAddr net.Addr, clientConn net.PacketConn, cryptoKey *shadowsocks.EncryptionKey, targetConn net.PacketConn, keyID string) *natconn { - var connMetrics UDPConnMetrics - if m.metrics != nil { - connMetrics = m.metrics.AddUDPNatEntry(clientAddr, keyID) - } + connMetrics := m.metrics.AddUDPNatEntry(clientAddr, keyID) entry := m.set(clientAddr.String(), targetConn, cryptoKey, keyID, connMetrics) m.running.Add(1) go func() { timedCopy(clientAddr, clientConn, entry, keyID) - if connMetrics != nil { - connMetrics.RemoveNatEntry() - } + connMetrics.RemoveNatEntry() if pc := m.del(clientAddr.String()); pc != nil { pc.Close() } @@ -450,9 +454,7 @@ func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natco if expired { break } - if targetConn.metrics != nil { - targetConn.metrics.AddPacketFromTarget(status, int64(bodyLen), int64(proxyClientBytes)) - } + targetConn.metrics.AddPacketFromTarget(status, int64(bodyLen), int64(proxyClientBytes)) } } From e5e8549083751a50177477ac453894deda964641 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 17:02:32 -0400 Subject: [PATCH 166/182] Revert some more changes. --- internal/integration_test/integration_test.go | 14 +++++++------- service/udp.go | 12 +++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/internal/integration_test/integration_test.go b/internal/integration_test/integration_test.go index d2626eff..f847f761 100644 --- a/internal/integration_test/integration_test.go +++ b/internal/integration_test/integration_test.go @@ -130,6 +130,7 @@ func TestTCPEcho(t *testing.T) { } replayCache := service.NewReplayCache(5) const testTimeout = 200 * time.Millisecond + testMetrics := &statusMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -137,7 +138,7 @@ func TestTCPEcho(t *testing.T) { go func() { service.StreamServe( func() (transport.StreamConn, error) { return proxyListener.AcceptTCP() }, - func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, nil) }, + func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, testMetrics) }, ) done <- struct{}{} }() @@ -191,14 +192,11 @@ func (m *fakeShadowsocksMetrics) AddCipherSearch(accessKeyFound bool, timeToCiph } type statusMetrics struct { + service.NoOpTCPConnMetrics sync.Mutex statuses []string } -var _ service.TCPConnMetrics = (*statusMetrics)(nil) - -func (m *statusMetrics) AddAuthenticated(accessKey string) {} -func (m *statusMetrics) AddProbe(status, drainResult string, clientProxyBytes int64) {} func (m *statusMetrics) AddClosed(status string, data metrics.ProxyMetrics, duration time.Duration) { m.Lock() m.statuses = append(m.statuses, status) @@ -401,6 +399,7 @@ func BenchmarkTCPThroughput(b *testing.B) { b.Fatal(err) } const testTimeout = 200 * time.Millisecond + testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, nil, &fakeShadowsocksMetrics{}) handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -408,7 +407,7 @@ func BenchmarkTCPThroughput(b *testing.B) { go func() { service.StreamServe( service.WrapStreamAcceptFunc(proxyListener.AcceptTCP), - func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, nil) }, + func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, testMetrics) }, ) done <- struct{}{} }() @@ -467,6 +466,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { } replayCache := service.NewReplayCache(service.MaxCapacity) const testTimeout = 200 * time.Millisecond + testMetrics := &service.NoOpTCPConnMetrics{} authFunc := service.NewShadowsocksStreamAuthenticator(cipherList, &replayCache, &fakeShadowsocksMetrics{}) handler := service.NewStreamHandler(authFunc, testTimeout) handler.SetTargetDialer(&transport.TCPDialer{}) @@ -474,7 +474,7 @@ func BenchmarkTCPMultiplexing(b *testing.B) { go func() { service.StreamServe( service.WrapStreamAcceptFunc(proxyListener.AcceptTCP), - func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, nil) }, + func(ctx context.Context, conn transport.StreamConn) { handler.Handle(ctx, conn, testMetrics) }, ) done <- struct{}{} }() diff --git a/service/udp.go b/service/udp.go index 444237e3..0215358c 100644 --- a/service/udp.go +++ b/service/udp.go @@ -308,13 +308,11 @@ type natmap struct { running *sync.WaitGroup } -func newNATmap(timeout time.Duration, metrics UDPMetrics, running *sync.WaitGroup) *natmap { - return &natmap{ - metrics: metrics, - running: running, - keyConn: make(map[string]*natconn), - timeout: timeout, - } +func newNATmap(timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { + m := &natmap{metrics: sm, running: running} + m.keyConn = make(map[string]*natconn) + m.timeout = timeout + return m } func (m *natmap) Get(key string) *natconn { From 724260e98b2b2bb90ed35666cf73117a7df334c1 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 17:17:33 -0400 Subject: [PATCH 167/182] Use a noop metrics struct if no metrics provided. --- service/shadowsocks.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/service/shadowsocks.go b/service/shadowsocks.go index cb274600..0f391f26 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -50,7 +50,7 @@ type Service interface { type Option func(s *ssService) type ssService struct { - m ServiceMetrics + metrics ServiceMetrics ciphers CipherList natTimeout time.Duration replayCache *ReplayCache @@ -70,13 +70,16 @@ func NewShadowsocksService(opts ...Option) (Service, error) { if s.natTimeout == 0 { s.natTimeout = defaultNatTimeout } + if s.metrics == nil { + s.metrics = &NoOpShadowsocksMetrics{} + } // TODO: Register initial data metrics at zero. s.sh = NewStreamHandler( - NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.m, proto: "tcp"}), + NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.metrics, proto: "tcp"}), tcpReadTimeout, ) - s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.m, &ssConnMetrics{ServiceMetrics: s.m, proto: "udp"}) + s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.metrics, &ssConnMetrics{ServiceMetrics: s.metrics, proto: "udp"}) return s, nil } @@ -91,7 +94,7 @@ func WithCiphers(ciphers CipherList) Option { // WithMetrics option function. func WithMetrics(metrics ServiceMetrics) Option { return func(s *ssService) { - s.m = metrics + s.metrics = metrics } } @@ -111,7 +114,7 @@ func WithNatTimeout(natTimeout time.Duration) Option { // HandleStream handles a Shadowsocks stream-based connection. func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { - connMetrics := s.m.AddOpenTCPConnection(conn) + connMetrics := s.metrics.AddOpenTCPConnection(conn) s.sh.Handle(ctx, conn, connMetrics) } @@ -130,3 +133,16 @@ var _ ShadowsocksConnMetrics = (*ssConnMetrics)(nil) func (cm *ssConnMetrics) AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { cm.ServiceMetrics.AddCipherSearch(cm.proto, accessKeyFound, timeToCipher) } + +type NoOpShadowsocksMetrics struct { + NoOpUDPMetrics +} + +var _ ServiceMetrics = (*NoOpShadowsocksMetrics)(nil) + +func (m *NoOpShadowsocksMetrics) AddOpenTCPConnection(conn net.Conn) TCPConnMetrics { + return &NoOpTCPConnMetrics{} +} + +func (m *NoOpShadowsocksMetrics) AddCipherSearch(proto string, accessKeyFound bool, timeToCipher time.Duration) { +} From 655c3cce291ae1ff177a555ce81593c906d26509 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 17:24:16 -0400 Subject: [PATCH 168/182] Add noop implementation for `ShadowsocksConnMetrics`. --- service/shadowsocks.go | 9 +++++++++ service/tcp.go | 3 +++ service/udp.go | 11 +++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/service/shadowsocks.go b/service/shadowsocks.go index 97329c3a..72170801 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -20,3 +20,12 @@ import "time" type ShadowsocksConnMetrics interface { AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) } + +// NoOpShadowsocksConnMetrics is a [ShadowsocksConnMetrics] that doesn't do anything. Useful in tests +// or if you don't want to track metrics. +type NoOpShadowsocksConnMetrics struct{} + +var _ ShadowsocksConnMetrics = (*NoOpShadowsocksConnMetrics)(nil) + +func (m *NoOpShadowsocksConnMetrics) AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { +} diff --git a/service/tcp.go b/service/tcp.go index 5554454c..6a12afc0 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -117,6 +117,9 @@ type StreamAuthenticateFunc func(clientConn transport.StreamConn) (string, trans // NewShadowsocksStreamAuthenticator creates a stream authenticator that uses Shadowsocks. // TODO(fortuna): Offer alternative transports. func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCache, metrics ShadowsocksConnMetrics) StreamAuthenticateFunc { + if metrics == nil { + metrics = &NoOpShadowsocksConnMetrics{} + } return func(clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { // Find the cipher and acess key id. cipherEntry, clientReader, clientSalt, timeToCipher, keyErr := findAccessKey(clientConn, remoteIP(clientConn), ciphers) diff --git a/service/udp.go b/service/udp.go index 0215358c..39091239 100644 --- a/service/udp.go +++ b/service/udp.go @@ -92,11 +92,14 @@ func NewPacketHandler(natTimeout time.Duration, cipherList CipherList, m UDPMetr if m == nil { m = &NoOpUDPMetrics{} } + if ssMetrics == nil { + ssMetrics = &NoOpShadowsocksConnMetrics{} + } return &packetHandler{ - natTimeout: natTimeout, - ciphers: cipherList, - m: m, - ssm: ssMetrics, + natTimeout: natTimeout, + ciphers: cipherList, + m: m, + ssm: ssMetrics, targetIPValidator: onet.RequirePublicIP, } } From fd04a2b392a6b0ffa9ce13a78e53dad3cf21de70 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 17:38:53 -0400 Subject: [PATCH 169/182] Move logger arg. --- service/udp.go | 8 ++++---- service/udp_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/service/udp.go b/service/udp.go index dffd9411..341ec2bd 100644 --- a/service/udp.go +++ b/service/udp.go @@ -123,7 +123,7 @@ func (h *packetHandler) SetTargetIPValidator(targetIPValidator onet.TargetIPVali func (h *packetHandler) Handle(clientConn net.PacketConn) { var running sync.WaitGroup - nm := newNATmap(h.l, h.natTimeout, h.m, &running) + nm := newNATmap(h.natTimeout, h.m, &running, h.l) defer nm.Close() cipherBuf := make([]byte, serverUDPBufferSize) textBuf := make([]byte, serverUDPBufferSize) @@ -314,7 +314,7 @@ type natmap struct { running *sync.WaitGroup } -func newNATmap(l Logger, timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup) *natmap { +func newNATmap(timeout time.Duration, sm UDPMetrics, running *sync.WaitGroup, l Logger) *natmap { m := &natmap{l: l, metrics: sm, running: running} m.keyConn = make(map[string]*natconn) m.timeout = timeout @@ -361,7 +361,7 @@ func (m *natmap) Add(clientAddr net.Addr, clientConn net.PacketConn, cryptoKey * m.running.Add(1) go func() { - timedCopy(m.l, clientAddr, clientConn, entry, keyID) + timedCopy(clientAddr, clientConn, entry, keyID, m.l) connMetrics.RemoveNatEntry() if pc := m.del(clientAddr.String()); pc != nil { pc.Close() @@ -390,7 +390,7 @@ func (m *natmap) Close() error { var maxAddrLen int = len(socks.ParseAddr("[2001:db8::1]:12345")) // copy from target to client until read timeout -func timedCopy(l Logger, clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string) { +func timedCopy(clientAddr net.Addr, clientConn net.PacketConn, targetConn *natconn, keyID string, l Logger) { // pkt is used for in-place encryption of downstream UDP packets, with the layout // [padding?][salt][address][body][tag][extra] // Padding is only used if the address is IPv4. diff --git a/service/udp_test.go b/service/udp_test.go index 8e4d2046..1cf020ee 100644 --- a/service/udp_test.go +++ b/service/udp_test.go @@ -207,14 +207,14 @@ func assertAlmostEqual(t *testing.T, a, b time.Time) { } func TestNATEmpty(t *testing.T) { - nat := newNATmap(&noopLogger{}, timeout, &natTestMetrics{}, &sync.WaitGroup{}) + nat := newNATmap(timeout, &natTestMetrics{}, &sync.WaitGroup{}, &noopLogger{}) if nat.Get("foo") != nil { t.Error("Expected nil value from empty NAT map") } } func setupNAT() (*fakePacketConn, *fakePacketConn, *natconn) { - nat := newNATmap(&noopLogger{}, timeout, &natTestMetrics{}, &sync.WaitGroup{}) + nat := newNATmap(timeout, &natTestMetrics{}, &sync.WaitGroup{}, &noopLogger{}) clientConn := makePacketConn() targetConn := makePacketConn() nat.Add(&clientAddr, clientConn, natCryptoKey, targetConn, "key id") From c2bae13ea167e9aa3ab62658c42bfb95956b12f5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 17:45:18 -0400 Subject: [PATCH 170/182] Resolve nil metrics. --- service/shadowsocks.go | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/service/shadowsocks.go b/service/shadowsocks.go index 0f391f26..aacd3152 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -70,9 +70,6 @@ func NewShadowsocksService(opts ...Option) (Service, error) { if s.natTimeout == 0 { s.natTimeout = defaultNatTimeout } - if s.metrics == nil { - s.metrics = &NoOpShadowsocksMetrics{} - } // TODO: Register initial data metrics at zero. s.sh = NewStreamHandler( @@ -114,7 +111,10 @@ func WithNatTimeout(natTimeout time.Duration) Option { // HandleStream handles a Shadowsocks stream-based connection. func (s *ssService) HandleStream(ctx context.Context, conn transport.StreamConn) { - connMetrics := s.metrics.AddOpenTCPConnection(conn) + var connMetrics TCPConnMetrics + if s.metrics != nil { + connMetrics = s.metrics.AddOpenTCPConnection(conn) + } s.sh.Handle(ctx, conn, connMetrics) } @@ -131,18 +131,7 @@ type ssConnMetrics struct { var _ ShadowsocksConnMetrics = (*ssConnMetrics)(nil) func (cm *ssConnMetrics) AddCipherSearch(accessKeyFound bool, timeToCipher time.Duration) { - cm.ServiceMetrics.AddCipherSearch(cm.proto, accessKeyFound, timeToCipher) -} - -type NoOpShadowsocksMetrics struct { - NoOpUDPMetrics -} - -var _ ServiceMetrics = (*NoOpShadowsocksMetrics)(nil) - -func (m *NoOpShadowsocksMetrics) AddOpenTCPConnection(conn net.Conn) TCPConnMetrics { - return &NoOpTCPConnMetrics{} -} - -func (m *NoOpShadowsocksMetrics) AddCipherSearch(proto string, accessKeyFound bool, timeToCipher time.Duration) { + if cm.ServiceMetrics != nil { + cm.ServiceMetrics.AddCipherSearch(cm.proto, accessKeyFound, timeToCipher) + } } From d3e60277d54e9c1c960a2d1c821c4f882281594b Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 16 Sep 2024 19:26:19 -0400 Subject: [PATCH 171/182] Set logger explicitly to `noopLogger` in service creation. --- service/shadowsocks.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/service/shadowsocks.go b/service/shadowsocks.go index ceb3f43b..073f434e 100644 --- a/service/shadowsocks.go +++ b/service/shadowsocks.go @@ -60,6 +60,7 @@ type ssService struct { ph PacketHandler } +// NewShadowsocksService creates a new Shadowsocks service. func NewShadowsocksService(opts ...Option) (Service, error) { s := &ssService{} @@ -67,25 +68,30 @@ func NewShadowsocksService(opts ...Option) (Service, error) { opt(s) } + // If no NAT timeout is provided via options, use the recommended default. if s.natTimeout == 0 { s.natTimeout = defaultNatTimeout } + // If no logger is provided via options, use a noop logger. + if s.logger == nil { + s.logger = &noopLogger{} + } // TODO: Register initial data metrics at zero. s.sh = NewStreamHandler( NewShadowsocksStreamAuthenticator(s.ciphers, s.replayCache, &ssConnMetrics{ServiceMetrics: s.metrics, proto: "tcp"}, s.logger), tcpReadTimeout, ) + s.sh.SetLogger(s.logger) + s.ph = NewPacketHandler(s.natTimeout, s.ciphers, s.metrics, &ssConnMetrics{ServiceMetrics: s.metrics, proto: "udp"}) - if s.logger != nil { - s.sh.SetLogger(s.logger) - s.ph.SetLogger(s.logger) - } + s.ph.SetLogger(s.logger) return s, nil } -// WithLogger can be used to provide a custom log target. +// WithLogger can be used to provide a custom log target. If not provided, +// the service uses a noop logger (i.e., no logging). func WithLogger(l Logger) Option { return func(s *ssService) { s.logger = l From bb19080586f6decef306518b67f6eefa00e78085 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 17 Sep 2024 15:24:32 -0400 Subject: [PATCH 172/182] Address review comments. --- caddy/app.go | 14 +++++++------- caddy/shadowsocks_handler.go | 21 +++++++++------------ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index 848fb00f..f3001233 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -31,7 +31,7 @@ func init() { caddy.RegisterModule(OutlineApp{}) } -const moduleName = "outline" +const outlineModuleName = "outline" type ShadowsocksConfig struct { ReplayHistory int `json:"replay_history,omitempty"` @@ -46,9 +46,14 @@ type OutlineApp struct { buildInfo *prometheus.GaugeVec } +var ( + _ caddy.App = (*OutlineApp)(nil) + _ caddy.Provisioner = (*OutlineApp)(nil) +) + func (OutlineApp) CaddyModule() caddy.ModuleInfo { return caddy.ModuleInfo{ - ID: moduleName, + ID: outlineModuleName, New: func() caddy.Module { return new(OutlineApp) }, } } @@ -123,8 +128,3 @@ func (app *OutlineApp) Stop() error { app.logger.Debug("stopped app instance") return nil } - -var ( - _ caddy.App = (*OutlineApp)(nil) - _ caddy.Provisioner = (*OutlineApp)(nil) -) diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go index 964f0a12..30ab81b9 100644 --- a/caddy/shadowsocks_handler.go +++ b/caddy/shadowsocks_handler.go @@ -45,6 +45,12 @@ type ShadowsocksHandler struct { logger *slog.Logger } +var ( + _ caddy.Provisioner = (*ShadowsocksHandler)(nil) + _ layer4.NextHandler = (*ShadowsocksHandler)(nil) +) + + func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { return caddy.ModuleInfo{ ID: "layer4.handlers.shadowsocks", @@ -56,17 +62,13 @@ func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { func (h *ShadowsocksHandler) Provision(ctx caddy.Context) error { h.logger = ctx.Slogger() - ctx.App(moduleName) - if _, err := ctx.AppIfConfigured(moduleName); err != nil { - return fmt.Errorf("outline app configure error: %w", err) - } - mod, err := ctx.App(moduleName) + mod, err := ctx.AppIfConfigured(outlineModuleName) if err != nil { - return err + return fmt.Errorf("outline app configure error: %w", err) } app, ok := mod.(*OutlineApp) if !ok { - return fmt.Errorf("module `%s` is not an OutlineApp", moduleName) + return fmt.Errorf("module `%s` is of type `%T`, expected `OutlineApp`", outlineModuleName, app) } if len(h.Keys) == 0 { @@ -120,8 +122,3 @@ func (h *ShadowsocksHandler) Handle(cx *layer4.Connection, _ layer4.Handler) err } return nil } - -var ( - _ caddy.Provisioner = (*ShadowsocksHandler)(nil) - _ layer4.NextHandler = (*ShadowsocksHandler)(nil) -) From 7d8892ca94a0e9021cd444996c6bda6c63e17721 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 18 Sep 2024 12:14:38 -0400 Subject: [PATCH 173/182] Set `noopLogger` in `NewShadowsocksStreamAuthenticator()` if nil. --- service/tcp.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/service/tcp.go b/service/tcp.go index 7eb4ae04..926274aa 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -120,6 +120,9 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa if metrics == nil { metrics = &NoOpShadowsocksConnMetrics{} } + if logger == nil { + logger = &noopLogger{} + } return func(clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { // Find the cipher and acess key id. cipherEntry, clientReader, clientSalt, timeToCipher, keyErr := findAccessKey(clientConn, remoteIP(clientConn), ciphers, l) From 53ddc3150f16143bdc01940cdac3723c1850f176 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 18 Sep 2024 15:47:46 -0400 Subject: [PATCH 174/182] Fix logger reference. --- service/tcp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/tcp.go b/service/tcp.go index 926274aa..bb6c4383 100644 --- a/service/tcp.go +++ b/service/tcp.go @@ -120,8 +120,8 @@ func NewShadowsocksStreamAuthenticator(ciphers CipherList, replayCache *ReplayCa if metrics == nil { metrics = &NoOpShadowsocksConnMetrics{} } - if logger == nil { - logger = &noopLogger{} + if l == nil { + l = &noopLogger{} } return func(clientConn transport.StreamConn) (string, transport.StreamConn, *onet.ConnectionError) { // Find the cipher and acess key id. From 64634064bda7b67e232948413c4b9b99831c6876 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 18 Sep 2024 15:52:38 -0400 Subject: [PATCH 175/182] Add TODO comment to persist replay cache. --- caddy/app.go | 1 + 1 file changed, 1 insertion(+) diff --git a/caddy/app.go b/caddy/app.go index f3001233..18439f69 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -65,6 +65,7 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { app.logger.Info("provisioning app instance") if app.ShadowsocksConfig != nil { + // TODO: Persist replay cache across config reloads. app.ReplayCache = outline.NewReplayCache(app.ShadowsocksConfig.ReplayHistory) } From 72c635c733e8b2983b4f597971b065dd66f966fb Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 18 Sep 2024 17:03:49 -0400 Subject: [PATCH 176/182] Remove use of zap. --- caddy/shadowsocks_handler.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go index 30ab81b9..f7f26142 100644 --- a/caddy/shadowsocks_handler.go +++ b/caddy/shadowsocks_handler.go @@ -25,7 +25,6 @@ import ( outline "github.com/Jigsaw-Code/outline-ss-server/service" "github.com/caddyserver/caddy/v2" "github.com/mholt/caddy-l4/layer4" - "go.uber.org/zap" ) func init() { @@ -39,7 +38,7 @@ type KeyConfig struct { } type ShadowsocksHandler struct { - Keys []KeyConfig `json:"keys,omitempty"` + Keys []KeyConfig `json:"keys,omitempty"` service outline.Service logger *slog.Logger @@ -50,7 +49,6 @@ var ( _ layer4.NextHandler = (*ShadowsocksHandler)(nil) ) - func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { return caddy.ModuleInfo{ ID: "layer4.handlers.shadowsocks", @@ -83,7 +81,7 @@ func (h *ShadowsocksHandler) Provision(ctx caddy.Context) error { for _, cfg := range h.Keys { key := cipherKey{cfg.Cipher, cfg.Secret} if _, exists := existingCiphers[key]; exists { - h.logger.Debug("Encryption key already exists. Skipping.", zap.String("id", cfg.ID)) + h.logger.Debug("Encryption key already exists. Skipping.", slog.String("id", cfg.ID)) continue } cryptoKey, err := shadowsocks.NewEncryptionKey(cfg.Cipher, cfg.Secret) From 5321705a0dd2f6762846aac08d6932a751fdf3e2 Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 18 Sep 2024 17:04:34 -0400 Subject: [PATCH 177/182] feat: persist replay cache across config reloads --- caddy/app.go | 7 +++-- service/replay.go | 41 ++++++++++++++++++++++++++++ service/replay_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index 18439f69..a5dd6c0e 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -27,8 +27,11 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +var replayCache outline.ReplayCache + func init() { caddy.RegisterModule(OutlineApp{}) + replayCache = outline.NewReplayCache(0) } const outlineModuleName = "outline" @@ -65,8 +68,8 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { app.logger.Info("provisioning app instance") if app.ShadowsocksConfig != nil { - // TODO: Persist replay cache across config reloads. - app.ReplayCache = outline.NewReplayCache(app.ShadowsocksConfig.ReplayHistory) + replayCache.Resize(app.ShadowsocksConfig.ReplayHistory) + app.ReplayCache = replayCache } if err := app.defineMetrics(); err != nil { diff --git a/service/replay.go b/service/replay.go index 818fde0f..602ed3ba 100644 --- a/service/replay.go +++ b/service/replay.go @@ -100,3 +100,44 @@ func (c *ReplayCache) Add(id string, salt []byte) bool { c.active[hash] = empty{} return !inArchive } + +// Resize adjusts the capacity of the ReplayCache. +// +// If the new capacity is less than the current capacity, and the number of +// active handshakes exceeds the new capacity, then the least recently added +// handshakes are moved to the archive. +func (c *ReplayCache) Resize(capacity int) { + if capacity > MaxCapacity { + panic("ReplayCache capacity would result in too many false positives") + } + c.mutex.Lock() + defer c.mutex.Unlock() + + // Shrink the active handshakes to capacity and move the rest to the archive. + if capacity < c.capacity { + active := make(map[uint32]empty, capacity) + archive := make(map[uint32]empty, 0) + + // Move handshakes up to capacity into a new active, and the remainder into a new archive. + for k, v := range c.active { + if len(active) != capacity { + active[k] = v + } else if len(archive) != capacity { + archive[k] = v + } + } + + // Fill the remainder of the new archive with old archive entries. + for k, v := range c.archive { + if len(archive) != capacity { + archive[k] = v + } + } + + // Use the new active and archive handshakes. + c.active = active + c.archive = archive + } + + c.capacity = capacity +} diff --git a/service/replay_test.go b/service/replay_test.go index c0187c0c..f93b32b6 100644 --- a/service/replay_test.go +++ b/service/replay_test.go @@ -91,6 +91,67 @@ func TestReplayCache_Archive(t *testing.T) { } } +func TestReplayCache_Resize(t *testing.T) { + t.Run("Smaller", func(t *testing.T) { + cache := &ReplayCache{ + capacity: 5, + active: map[uint32]empty{ + 1: {}, 2: {}, 3: {}, 4: {}, 5: {}, + }, + archive: map[uint32]empty{ + 6: {}, 7: {}, 8: {}, 9: {}, 10: {}, + }, + } + + cache.Resize(3) + + if cache.capacity != 3 { + t.Errorf("Expected capacity to be 3, got %d", cache.capacity) + } + if len(cache.active) != 3 { + t.Errorf("Expected active handshakes length to be 3, got %d", len(cache.active)) + } + if len(cache.archive) != 3 { + t.Errorf("Expected archive handshakes length to be 3, got %d", len(cache.active)) + } + }) + + t.Run("Larger", func(t *testing.T) { + cache := &ReplayCache{ + capacity: 5, + active: map[uint32]empty{ + 1: {}, 2: {}, 3: {}, 4: {}, 5: {}, + }, + archive: map[uint32]empty{ + 6: {}, 7: {}, 8: {}, 9: {}, 10: {}, + }, + } + + cache.Resize(10) + + if cache.capacity != 10 { + t.Errorf("Expected capacity to be 10, got %d", cache.capacity) + } + if len(cache.active) != 5 { + t.Errorf("Expected active handshakes length to be 5, got %d", len(cache.active)) + } + if len(cache.archive) != 5 { + t.Errorf("Expected archive handshakes length to be 5, got %d", len(cache.archive)) + } + }) + + t.Run("Exceeding maximum capacity", func(t *testing.T) { + cache := &ReplayCache{} + defer func() { + if r := recover(); r == nil { + t.Errorf("The code did not panic") + } + }() + + cache.Resize(MaxCapacity + 1) + }) +} + // Benchmark to determine the memory usage of ReplayCache. // Note that NewReplayCache only allocates the active set, // so the eventual memory usage will be roughly double. From a13ab28a80f8f696aee81935a1127263621c07ae Mon Sep 17 00:00:00 2001 From: sbruens Date: Wed, 18 Sep 2024 17:09:43 -0400 Subject: [PATCH 178/182] Update comment. --- service/replay.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/replay.go b/service/replay.go index 602ed3ba..4a6c9e81 100644 --- a/service/replay.go +++ b/service/replay.go @@ -104,8 +104,8 @@ func (c *ReplayCache) Add(id string, salt []byte) bool { // Resize adjusts the capacity of the ReplayCache. // // If the new capacity is less than the current capacity, and the number of -// active handshakes exceeds the new capacity, then the least recently added -// handshakes are moved to the archive. +// active handshakes exceeds the new capacity, then we move the excess to the +// archive. func (c *ReplayCache) Resize(capacity int) { if capacity > MaxCapacity { panic("ReplayCache capacity would result in too many false positives") From d435603809cebb3408259d81f518928fabfe7d8d Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 7 Oct 2024 15:07:43 -0400 Subject: [PATCH 179/182] Fix bad merge and don't use a global. --- caddy/app.go | 23 ++++++++++++----------- caddy/config_example.json | 1 - caddy/shadowsocks_handler.go | 12 +++++++----- cmd/outline-ss-server/main.go | 2 -- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index a5dd6c0e..66345e5c 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -27,15 +27,20 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -var replayCache outline.ReplayCache +const outlineModuleName = "outline" func init() { - caddy.RegisterModule(OutlineApp{}) - replayCache = outline.NewReplayCache(0) + replayCache := outline.NewReplayCache(0) + caddy.RegisterModule(ModuleRegistration{ + ID: outlineModuleName, + New: func() caddy.Module { + app := new(OutlineApp) + app.ReplayCache = replayCache + return app + }, + }) } -const outlineModuleName = "outline" - type ShadowsocksConfig struct { ReplayHistory int `json:"replay_history,omitempty"` } @@ -55,10 +60,7 @@ var ( ) func (OutlineApp) CaddyModule() caddy.ModuleInfo { - return caddy.ModuleInfo{ - ID: outlineModuleName, - New: func() caddy.Module { return new(OutlineApp) }, - } + return caddy.ModuleInfo{ID: outlineModuleName} } // Provision sets up Outline. @@ -68,8 +70,7 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { app.logger.Info("provisioning app instance") if app.ShadowsocksConfig != nil { - replayCache.Resize(app.ShadowsocksConfig.ReplayHistory) - app.ReplayCache = replayCache + app.ReplayCache.Resize(app.ShadowsocksConfig.ReplayHistory) } if err := app.defineMetrics(); err != nil { diff --git a/caddy/config_example.json b/caddy/config_example.json index 4f65fc33..2d198890 100644 --- a/caddy/config_example.json +++ b/caddy/config_example.json @@ -36,7 +36,6 @@ }, "layer4": { "servers": { - "1": { "listen": [ "tcp/[::]:9000", diff --git a/caddy/shadowsocks_handler.go b/caddy/shadowsocks_handler.go index f7f26142..a0c48747 100644 --- a/caddy/shadowsocks_handler.go +++ b/caddy/shadowsocks_handler.go @@ -27,8 +27,13 @@ import ( "github.com/mholt/caddy-l4/layer4" ) +const ssModuleName = "layer4.handlers.shadowsocks" + func init() { - caddy.RegisterModule(&ShadowsocksHandler{}) + caddy.RegisterModule(ModuleRegistration{ + ID: ssModuleName, + New: func() caddy.Module { return new(ShadowsocksHandler) }, + }) } type KeyConfig struct { @@ -50,10 +55,7 @@ var ( ) func (*ShadowsocksHandler) CaddyModule() caddy.ModuleInfo { - return caddy.ModuleInfo{ - ID: "layer4.handlers.shadowsocks", - New: func() caddy.Module { return new(ShadowsocksHandler) }, - } + return caddy.ModuleInfo{ID: ssModuleName} } // Provision implements caddy.Provisioner. diff --git a/cmd/outline-ss-server/main.go b/cmd/outline-ss-server/main.go index 09d50003..3a04af0b 100644 --- a/cmd/outline-ss-server/main.go +++ b/cmd/outline-ss-server/main.go @@ -220,7 +220,6 @@ func (s *OutlineServer) runConfig(config Config) (func() error, error) { ciphers.Update(cipherList) ssService, err := service.NewShadowsocksService( - service.WithLogger(slog.Default()), service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), @@ -248,7 +247,6 @@ func (s *OutlineServer) runConfig(config Config) (func() error, error) { return fmt.Errorf("failed to create cipher list from config: %v", err) } ssService, err := service.NewShadowsocksService( - service.WithLogger(slog.Default()), service.WithCiphers(ciphers), service.WithNatTimeout(s.natTimeout), service.WithMetrics(s.serviceMetrics), From ecb636be7c2a035d37d40206c34f72b87bde05e3 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 7 Oct 2024 16:13:28 -0400 Subject: [PATCH 180/182] Address review comments. --- caddy/app.go | 4 +++- service/replay.go | 40 +++++++--------------------------------- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/caddy/app.go b/caddy/app.go index 66345e5c..8c3026ba 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -70,7 +70,9 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { app.logger.Info("provisioning app instance") if app.ShadowsocksConfig != nil { - app.ReplayCache.Resize(app.ShadowsocksConfig.ReplayHistory) + if err := app.ReplayCache.Resize(app.ShadowsocksConfig.ReplayHistory); err != nil { + return err + } } if err := app.defineMetrics(); err != nil { diff --git a/service/replay.go b/service/replay.go index 4a6c9e81..5b493bbb 100644 --- a/service/replay.go +++ b/service/replay.go @@ -16,6 +16,7 @@ package service import ( "encoding/binary" + "errors" "sync" ) @@ -102,42 +103,15 @@ func (c *ReplayCache) Add(id string, salt []byte) bool { } // Resize adjusts the capacity of the ReplayCache. -// -// If the new capacity is less than the current capacity, and the number of -// active handshakes exceeds the new capacity, then we move the excess to the -// archive. -func (c *ReplayCache) Resize(capacity int) { +func (c *ReplayCache) Resize(capacity int) error { if capacity > MaxCapacity { - panic("ReplayCache capacity would result in too many false positives") + return errors.New("ReplayCache capacity would result in too many false positives") } c.mutex.Lock() defer c.mutex.Unlock() - - // Shrink the active handshakes to capacity and move the rest to the archive. - if capacity < c.capacity { - active := make(map[uint32]empty, capacity) - archive := make(map[uint32]empty, 0) - - // Move handshakes up to capacity into a new active, and the remainder into a new archive. - for k, v := range c.active { - if len(active) != capacity { - active[k] = v - } else if len(archive) != capacity { - archive[k] = v - } - } - - // Fill the remainder of the new archive with old archive entries. - for k, v := range c.archive { - if len(archive) != capacity { - archive[k] = v - } - } - - // Use the new active and archive handshakes. - c.active = active - c.archive = archive - } - c.capacity = capacity + // NOTE: The active handshakes and archive lists are not explicitly shrunk. + // Their sizes will naturally adjust as new handshakes are added and the cache + // adheres to the updated capacity. + return nil } From 82090563479634d8817ebd6bea71efec4ac645f2 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 7 Oct 2024 17:03:48 -0400 Subject: [PATCH 181/182] Update tests. --- service/replay.go | 4 +- service/replay_test.go | 95 +++++++++++++++++++++++++----------------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/service/replay.go b/service/replay.go index 5b493bbb..27b3d128 100644 --- a/service/replay.go +++ b/service/replay.go @@ -93,7 +93,7 @@ func (c *ReplayCache) Add(id string, salt []byte) bool { return false } _, inArchive := c.archive[hash] - if len(c.active) == c.capacity { + if len(c.active) >= c.capacity { // Discard the archive and move active to archive. c.archive = c.active c.active = make(map[uint32]empty, c.capacity) @@ -111,7 +111,7 @@ func (c *ReplayCache) Resize(capacity int) error { defer c.mutex.Unlock() c.capacity = capacity // NOTE: The active handshakes and archive lists are not explicitly shrunk. - // Their sizes will naturally adjust as new handshakes are added and the cache + // Their sizes will naturally adjust as new handshakes are added and the cache // adheres to the updated capacity. return nil } diff --git a/service/replay_test.go b/service/replay_test.go index f93b32b6..6cfd98a5 100644 --- a/service/replay_test.go +++ b/service/replay_test.go @@ -17,6 +17,9 @@ package service import ( "encoding/binary" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const keyID = "the key" @@ -92,63 +95,77 @@ func TestReplayCache_Archive(t *testing.T) { } func TestReplayCache_Resize(t *testing.T) { - t.Run("Smaller", func(t *testing.T) { - cache := &ReplayCache{ - capacity: 5, - active: map[uint32]empty{ - 1: {}, 2: {}, 3: {}, 4: {}, 5: {}, - }, - archive: map[uint32]empty{ - 6: {}, 7: {}, 8: {}, 9: {}, 10: {}, - }, + t.Run("Smaller resizes active and archive maps", func(t *testing.T) { + salts := makeSalts(10) + cache := NewReplayCache(5) + for _, s := range salts { + cache.Add(keyID, s) } - cache.Resize(3) + err := cache.Resize(3) - if cache.capacity != 3 { - t.Errorf("Expected capacity to be 3, got %d", cache.capacity) - } - if len(cache.active) != 3 { - t.Errorf("Expected active handshakes length to be 3, got %d", len(cache.active)) + require.NoError(t, err) + assert.Equal(t, cache.capacity, 3, "Expected capacity to be updated") + + // Adding a new salt should trigger a shrinking of the active map as it hits the new + // capacity immediately. + cache.Add(keyID, salts[0]) + assert.Len(t, cache.active, 1, "Expected active handshakes length to have shrunk") + assert.Len(t, cache.archive, 5, "Expected archive handshakes length to not have shrunk") + + // Adding more new salts should eventually trigger a shrinking of the archive map as well, + // when the shrunken active map gets moved to the archive. + for _, s := range salts { + cache.Add(keyID, s) } - if len(cache.archive) != 3 { - t.Errorf("Expected archive handshakes length to be 3, got %d", len(cache.active)) + assert.Len(t, cache.archive, 3, "Expected archive handshakes length to have shrunk") + }) + + t.Run("Larger resizes active and archive maps", func(t *testing.T) { + salts := makeSalts(10) + cache := NewReplayCache(5) + for _, s := range salts { + cache.Add(keyID, s) } + + err := cache.Resize(10) + + require.NoError(t, err) + assert.Equal(t, cache.capacity, 10, "Expected capacity to be updated") + assert.Len(t, cache.active, 5, "Expected active handshakes length not to have changed") + assert.Len(t, cache.archive, 5, "Expected archive handshakes length not to have changed") }) - t.Run("Larger", func(t *testing.T) { - cache := &ReplayCache{ - capacity: 5, - active: map[uint32]empty{ - 1: {}, 2: {}, 3: {}, 4: {}, 5: {}, - }, - archive: map[uint32]empty{ - 6: {}, 7: {}, 8: {}, 9: {}, 10: {}, - }, + t.Run("Still detect salts", func(t *testing.T) { + salts := makeSalts(10) + cache := NewReplayCache(5) + for _, s := range salts { + cache.Add(keyID, s) } cache.Resize(10) - if cache.capacity != 10 { - t.Errorf("Expected capacity to be 10, got %d", cache.capacity) - } - if len(cache.active) != 5 { - t.Errorf("Expected active handshakes length to be 5, got %d", len(cache.active)) + for _, s := range salts { + if cache.Add(keyID, s) { + t.Error("Should still be able to detect the salts after resizing") + } } - if len(cache.archive) != 5 { - t.Errorf("Expected archive handshakes length to be 5, got %d", len(cache.archive)) + + cache.Resize(3) + + for _, s := range salts { + if cache.Add(keyID, s) { + t.Error("Should still be able to detect the salts after resizing") + } } }) t.Run("Exceeding maximum capacity", func(t *testing.T) { cache := &ReplayCache{} - defer func() { - if r := recover(); r == nil { - t.Errorf("The code did not panic") - } - }() - cache.Resize(MaxCapacity + 1) + err := cache.Resize(MaxCapacity + 1) + + require.Error(t, err) }) } From efd2ea37eb6029fc217f1520450ccbac12d033b5 Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 7 Oct 2024 17:26:12 -0400 Subject: [PATCH 182/182] Add more context to error message. --- caddy/app.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/caddy/app.go b/caddy/app.go index 8c3026ba..8f8c1a27 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -19,6 +19,7 @@ package caddy import ( "errors" + "fmt" "log/slog" outline_prometheus "github.com/Jigsaw-Code/outline-ss-server/prometheus" @@ -71,7 +72,7 @@ func (app *OutlineApp) Provision(ctx caddy.Context) error { if app.ShadowsocksConfig != nil { if err := app.ReplayCache.Resize(app.ShadowsocksConfig.ReplayHistory); err != nil { - return err + return fmt.Errorf("failed to configure replay history with capacity %d: %v", app.ShadowsocksConfig.ReplayHistory, err) } }