Add toxiproxy

petergoldstein · Feb 21, 2025 · 4bf48c2 · 4bf48c2
1 parent c626a9b
commit 4bf48c2
Show file tree

Hide file tree

Showing 16 changed files with 132 additions and 100 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -31,6 +31,8 @@ jobs:
       run: |
         chmod +x ./install_memcached.sh
         ./install_memcached.sh
+    - name: Install and start Toxiproxy
+      run: ./bin/start-toxiproxy
     - name: Set up Ruby ${{ matrix.ruby-version }}
       uses: ruby/setup-ruby@v1
       with:

diff --git a/.gitignore b/.gitignore
@@ -35,3 +35,7 @@ gemfiles/*.lock
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+
+# ignore toxiproxy logs/output
+nohup.out
+bin/toxiproxy-server
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -17,3 +17,6 @@ Metrics/BlockLength:
 Style/Documentation:
   Exclude:
     - 'test/**/*'
+
+Metrics/MethodLength:
+  Max: 20
diff --git a/Gemfile b/Gemfile
@@ -15,6 +15,7 @@ group :development, :test do
   gem 'rubocop-performance'
   gem 'rubocop-rake'
   gem 'simplecov'
+  gem 'toxiproxy'
 end
 
 group :test do

diff --git a/bin/start-toxiproxy b/bin/start-toxiproxy
@@ -0,0 +1,16 @@
+#!/bin/bash -e
+
+VERSION='v2.4.0'
+
+if [[ "$OSTYPE" == "linux"* ]]; then
+    DOWNLOAD_TYPE="linux-amd64"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    DOWNLOAD_TYPE="darwin-amd64"
+fi
+
+echo "[dowload toxiproxy for $DOWNLOAD_TYPE]"
+curl --silent -L https://github.com/Shopify/toxiproxy/releases/download/$VERSION/toxiproxy-server-$DOWNLOAD_TYPE -o ./bin/toxiproxy-server
+
+echo "[start toxiproxy]"
+chmod +x ./bin/toxiproxy-server
+nohup bash -c "./bin/toxiproxy-server 2>&1 | sed -e 's/^/[toxiproxy] /' &"
diff --git a/test/benchmark_test.rb b/test/benchmark_test.rb
@@ -37,7 +37,7 @@ def profile(&block)
 
   it 'runs benchmarks' do
     protocol = :binary
-    memcached(protocol, @port) do
+    memcached(protocol, port_or_socket: @port) do
       profile do
         Benchmark.bm(37) do |x|
           n = 2500

diff --git a/test/helper.rb b/test/helper.rb
@@ -12,6 +12,7 @@
 require 'dalli'
 require 'logger'
 require 'securerandom'
+require 'toxiproxy'
 
 Dalli.logger = Logger.new($stdout)
 Dalli.logger.level = Logger::ERROR

diff --git a/test/helpers/memcached.rb b/test/helpers/memcached.rb
@@ -32,31 +32,63 @@ def memcached_mock(prc, meth = :start, meth_args = [])
     #
     # port_or_socket - If numeric or numeric string, treated as a TCP port
     #                  on localhost.  If not, treated as a UNIX domain socket
-    # args - Command line args passed to the memcached invocation
+    # cli_args - Command line args passed to the memcached invocation
     # client_options - Options passed to the Dalli::Client on initialization
     # terminate_process - whether to terminate the memcached process on
     #                     exiting the block
-    def memcached(protocol, port_or_socket, args = '', client_options = {}, terminate_process: true)
-      dc = MemcachedManager.start_and_flush_with_retry(port_or_socket, args, client_options.merge(protocol: protocol))
+    def memcached(protocol, port_or_socket:, cli_args: '', client_options: {}, terminate_process: true)
+      dc = MemcachedManager.start_and_flush_with_retry(port_or_socket, cli_args,
+                                                       client_options.merge(protocol: protocol))
       yield dc, port_or_socket if block_given?
       memcached_kill(port_or_socket) if terminate_process
     end
 
     # Launches a memcached process using the memcached method in this module,
     # but sets terminate_process to false ensuring that the process persists
     # past execution of the block argument.
-    # rubocop:disable Metrics/ParameterLists
-    def memcached_persistent(protocol = :binary, port_or_socket = 21_345, args = '', client_options = {}, &block)
-      memcached(protocol, port_or_socket, args, client_options, terminate_process: false, &block)
+    def memcached_persistent(protocol = :binary, port_or_socket: 21_345, cli_args: '', client_options: {}, &block)
+      memcached(protocol,
+                port_or_socket: port_or_socket,
+                cli_args: cli_args,
+                client_options: client_options,
+                terminate_process: false,
+                &block)
+    end
+
+    ###
+    # Launches a persistent memcached process that is proxied through Toxiproxy
+    # to test network errors.
+    # Uses port 21_345 by default for the Toxiproxy port and the specified
+    # port_or_socket for the memcached process.
+    ###
+    def toxiproxy_memcached_persistent(
+      protocol = :binary,
+      upstream_port: MemcachedManager::TOXIPROXY_UPSTREAM_PORT,
+      listen_port: MemcachedManager::TOXIPROXY_MEMCACHED_PORT,
+      cli_args: '',
+      client_options: {}
+    )
+      raise 'Toxiproxy does not support unix sockets' if listen_port.to_i.zero? || upstream_port.to_i.zero?
+
+      unless @toxy_configured
+        Toxiproxy.populate([{ name: 'memcached', listen: "localhost:#{listen_port}",
+                              upstream: "localhost:#{upstream_port}" }])
+        @toxy_configured = true
+      end
+      memcached_persistent(protocol, port_or_socket: upstream_port, cli_args: cli_args,
+                                     client_options: client_options) do |dc, _|
+        dc.close # We don't need the client to talk directly to memcached
+      end
+      dc = Dalli::Client.new("localhost:#{listen_port}", client_options)
+      yield dc, listen_port
     end
-    # rubocop:enable Metrics/ParameterLists
 
     # Launches a persistent memcached process, configured to use SSL
-    def memcached_ssl_persistent(protocol = :binary, port_or_socket = rand(21_397..21_896), &block)
+    def memcached_ssl_persistent(protocol = :binary, port_or_socket: rand(21_397..21_896), &block)
       memcached_persistent(protocol,
-                           port_or_socket,
-                           CertificateGenerator.ssl_args,
-                           { ssl_context: CertificateGenerator.ssl_context },
+                           port_or_socket: port_or_socket,
+                           args: CertificateGenerator.ssl_args,
+                           client_options: { ssl_context: CertificateGenerator.ssl_context },
                            &block)
     end
 
@@ -68,7 +100,8 @@ def memcached_kill(port_or_socket)
 
     # Launches a persistent memcached process, configured to use SASL authentication
     def memcached_sasl_persistent(port_or_socket = 21_398, &block)
-      memcached_persistent(:binary, port_or_socket, '-S', sasl_credentials, &block)
+      memcached_persistent(:binary, port_or_socket: port_or_socket, cli_args: '-S', client_options: sasl_credentials,
+                           &block)
     end
 
     # The SASL credentials used for the test SASL server

diff --git a/test/integration/test_compressor.rb b/test/integration/test_compressor.rb
@@ -17,22 +17,22 @@ def self.decompress(data)
   MemcachedManager.supported_protocols.each do |p|
     describe "using the #{p} protocol" do
       it 'default to Dalli::Compressor' do
-        memcached(p, 29_199) do |dc|
+        memcached(p, port_or_socket: 29_199) do |dc|
           dc.set 1, 2
 
           assert_equal Dalli::Compressor, dc.instance_variable_get(:@ring).servers.first.compressor
         end
       end
 
       it 'support a custom compressor' do
-        memcached(p, 29_199) do |_dc|
+        memcached(p, port_or_socket: 29_199) do |_dc|
           memcache = Dalli::Client.new('127.0.0.1:29199', { compressor: NoopCompressor })
           memcache.set 1, 2
           begin
             assert_equal NoopCompressor,
                          memcache.instance_variable_get(:@ring).servers.first.compressor
 
-            memcached(p, 19_127) do |newdc|
+            memcached(p, port_or_socket: 19_127) do |newdc|
               assert newdc.set('string-test', 'a test string')
               assert_equal('a test string', newdc.get('string-test'))
             end
@@ -42,7 +42,7 @@ def self.decompress(data)
 
       describe 'GzipCompressor' do
         it 'compress and uncompress data using Zlib::GzipWriter/Reader' do
-          memcached(p, 19_127) do |_dc|
+          memcached(p, port_or_socket: 19_127) do |_dc|
             memcache = Dalli::Client.new('127.0.0.1:19127', { compress: true, compressor: Dalli::GzipCompressor })
             data = (0...1025).map { rand(65..90).chr }.join
 

diff --git a/test/integration/test_failover.rb b/test/integration/test_failover.rb
@@ -34,14 +34,14 @@
       describe 'assuming some bad servers' do
         it 'silently reconnect if server hiccups' do
           server_port = 30_124
-          memcached_persistent(p, server_port) do |dc, port|
+          memcached_persistent(p, port_or_socket: server_port) do |dc, port|
             dc.set 'foo', 'bar'
             foo = dc.get 'foo'
 
             assert_equal('bar', foo)
 
             memcached_kill(port)
-            memcached_persistent(p, port) do
+            memcached_persistent(p, port_or_socket: port) do
               foo = dc.get 'foo'
 
               assert_nil foo
@@ -55,8 +55,8 @@
           port1 = 32_112
           port2 = 37_887
 
-          memcached(p, port1, '-o idle_timeout=1') do |_, first_port|
-            memcached(p, port2, '-o idle_timeout=1') do |_, second_port|
+          memcached(p, port_or_socket: port1, cli_args: '-o idle_timeout=1') do |_, first_port|
+            memcached(p, port_or_socket: port2, cli_args: '-o idle_timeout=1') do |_, second_port|
               dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"]
               dc.set 'foo', 'bar'
               dc.set 'foo2', 'bar2'
@@ -77,8 +77,8 @@
         it 'handle graceful failover' do
           port1 = 31_777
           port2 = 32_113
-          memcached_persistent(p, port1) do |_first_dc, first_port|
-            memcached_persistent(p, port2) do |_second_dc, second_port|
+          memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port|
+            memcached_persistent(p, port_or_socket: port2) do |_second_dc, second_port|
               dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"]
               dc.set 'foo', 'bar'
               foo = dc.get 'foo'
@@ -104,8 +104,8 @@
         it 'handle them gracefully in get_multi' do
           port1 = 32_971
           port2 = 34_312
-          memcached_persistent(p, port1) do |_first_dc, first_port|
-            memcached(p, port2) do |_second_dc, second_port|
+          memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port|
+            memcached(p, port_or_socket: port2) do |_second_dc, second_port|
               dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"]
               dc.set 'a', 'a1'
               result = dc.get_multi ['a']
@@ -124,8 +124,8 @@
         it 'handle graceful failover in get_multi' do
           port1 = 34_541
           port2 = 33_044
-          memcached_persistent(p, port1) do |_first_dc, first_port|
-            memcached_persistent(p, port2) do |_second_dc, second_port|
+          memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port|
+            memcached_persistent(p, port_or_socket: port2) do |_second_dc, second_port|
               dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"]
               dc.set 'foo', 'foo1'
               dc.set 'bar', 'bar1'
@@ -153,8 +153,8 @@
         it 'stats it still properly report' do
           port1 = 34_547
           port2 = 33_219
-          memcached_persistent(p, port1) do |_first_dc, first_port|
-            memcached_persistent(p, port2) do |_second_dc, second_port|
+          memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port|
+            memcached_persistent(p, port_or_socket: port2) do |_second_dc, second_port|
               dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"]
               result = dc.stats