diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bee51d1d..f035fd2d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,8 @@ jobs: run: | chmod +x ./install_memcached.sh ./install_memcached.sh + - name: Install and start Toxiproxy + run: ./bin/start-toxiproxy - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 with: diff --git a/.gitignore b/.gitignore index 7e1a0fcd..4735c974 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,7 @@ gemfiles/*.lock # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: .rvmrc + +# ignore toxiproxy logs/output +nohup.out +bin/toxiproxy-server diff --git a/.rubocop.yml b/.rubocop.yml index 2af66bc5..b84641ba 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -17,3 +17,6 @@ Metrics/BlockLength: Style/Documentation: Exclude: - 'test/**/*' + +Metrics/MethodLength: + Max: 20 diff --git a/Gemfile b/Gemfile index 2591d6fd..ff2c482a 100644 --- a/Gemfile +++ b/Gemfile @@ -15,6 +15,7 @@ group :development, :test do gem 'rubocop-performance' gem 'rubocop-rake' gem 'simplecov' + gem 'toxiproxy' end group :test do diff --git a/bin/start-toxiproxy b/bin/start-toxiproxy new file mode 100755 index 00000000..27a52610 --- /dev/null +++ b/bin/start-toxiproxy @@ -0,0 +1,16 @@ +#!/bin/bash -e + +VERSION='v2.4.0' + +if [[ "$OSTYPE" == "linux"* ]]; then + DOWNLOAD_TYPE="linux-amd64" +elif [[ "$OSTYPE" == "darwin"* ]]; then + DOWNLOAD_TYPE="darwin-amd64" +fi + +echo "[dowload toxiproxy for $DOWNLOAD_TYPE]" +curl --silent -L https://github.com/Shopify/toxiproxy/releases/download/$VERSION/toxiproxy-server-$DOWNLOAD_TYPE -o ./bin/toxiproxy-server + +echo "[start toxiproxy]" +chmod +x ./bin/toxiproxy-server +nohup bash -c "./bin/toxiproxy-server 2>&1 | sed -e 's/^/[toxiproxy] /' &" diff --git a/test/benchmark_test.rb b/test/benchmark_test.rb index c3880343..0d58d3fa 100644 --- a/test/benchmark_test.rb +++ b/test/benchmark_test.rb @@ -37,7 +37,7 @@ def profile(&block) it 'runs benchmarks' do protocol = :binary - memcached(protocol, @port) do + memcached(protocol, port_or_socket: @port) do profile do Benchmark.bm(37) do |x| n = 2500 diff --git a/test/helper.rb b/test/helper.rb index e3c4c8b9..bfbdb58d 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -12,6 +12,7 @@ require 'dalli' require 'logger' require 'securerandom' +require 'toxiproxy' Dalli.logger = Logger.new($stdout) Dalli.logger.level = Logger::ERROR diff --git a/test/helpers/memcached.rb b/test/helpers/memcached.rb index 5923ecee..826da717 100644 --- a/test/helpers/memcached.rb +++ b/test/helpers/memcached.rb @@ -32,12 +32,13 @@ def memcached_mock(prc, meth = :start, meth_args = []) # # port_or_socket - If numeric or numeric string, treated as a TCP port # on localhost. If not, treated as a UNIX domain socket - # args - Command line args passed to the memcached invocation + # cli_args - Command line args passed to the memcached invocation # client_options - Options passed to the Dalli::Client on initialization # terminate_process - whether to terminate the memcached process on # exiting the block - def memcached(protocol, port_or_socket, args = '', client_options = {}, terminate_process: true) - dc = MemcachedManager.start_and_flush_with_retry(port_or_socket, args, client_options.merge(protocol: protocol)) + def memcached(protocol, port_or_socket:, cli_args: '', client_options: {}, terminate_process: true) + dc = MemcachedManager.start_and_flush_with_retry(port_or_socket, cli_args, + client_options.merge(protocol: protocol)) yield dc, port_or_socket if block_given? memcached_kill(port_or_socket) if terminate_process end @@ -45,18 +46,49 @@ def memcached(protocol, port_or_socket, args = '', client_options = {}, terminat # Launches a memcached process using the memcached method in this module, # but sets terminate_process to false ensuring that the process persists # past execution of the block argument. - # rubocop:disable Metrics/ParameterLists - def memcached_persistent(protocol = :binary, port_or_socket = 21_345, args = '', client_options = {}, &block) - memcached(protocol, port_or_socket, args, client_options, terminate_process: false, &block) + def memcached_persistent(protocol = :binary, port_or_socket: 21_345, cli_args: '', client_options: {}, &block) + memcached(protocol, + port_or_socket: port_or_socket, + cli_args: cli_args, + client_options: client_options, + terminate_process: false, + &block) + end + + ### + # Launches a persistent memcached process that is proxied through Toxiproxy + # to test network errors. + # Uses port 21_345 by default for the Toxiproxy port and the specified + # port_or_socket for the memcached process. + ### + def toxiproxy_memcached_persistent( + protocol = :binary, + upstream_port: MemcachedManager::TOXIPROXY_UPSTREAM_PORT, + listen_port: MemcachedManager::TOXIPROXY_MEMCACHED_PORT, + cli_args: '', + client_options: {} + ) + raise 'Toxiproxy does not support unix sockets' if listen_port.to_i.zero? || upstream_port.to_i.zero? + + unless @toxy_configured + Toxiproxy.populate([{ name: 'memcached', listen: "localhost:#{listen_port}", + upstream: "localhost:#{upstream_port}" }]) + @toxy_configured = true + end + memcached_persistent(protocol, port_or_socket: upstream_port, cli_args: cli_args, + client_options: client_options) do |dc, _| + dc.close # We don't need the client to talk directly to memcached + end + dc = Dalli::Client.new("localhost:#{listen_port}", client_options) + yield dc, listen_port end - # rubocop:enable Metrics/ParameterLists # Launches a persistent memcached process, configured to use SSL - def memcached_ssl_persistent(protocol = :binary, port_or_socket = rand(21_397..21_896), &block) + def memcached_ssl_persistent(protocol = :binary, port_or_socket: rand(21_397..21_896), &block) memcached_persistent(protocol, - port_or_socket, - CertificateGenerator.ssl_args, - { ssl_context: CertificateGenerator.ssl_context }, + port_or_socket: port_or_socket, + args: CertificateGenerator.ssl_args, + client_options: { ssl_context: CertificateGenerator.ssl_context }, &block) end @@ -68,7 +100,8 @@ def memcached_kill(port_or_socket) # Launches a persistent memcached process, configured to use SASL authentication def memcached_sasl_persistent(port_or_socket = 21_398, &block) - memcached_persistent(:binary, port_or_socket, '-S', sasl_credentials, &block) + memcached_persistent(:binary, port_or_socket: port_or_socket, cli_args: '-S', client_options: sasl_credentials, + &block) end # The SASL credentials used for the test SASL server diff --git a/test/integration/test_compressor.rb b/test/integration/test_compressor.rb index 26cb378a..c7aac6b3 100644 --- a/test/integration/test_compressor.rb +++ b/test/integration/test_compressor.rb @@ -17,7 +17,7 @@ def self.decompress(data) MemcachedManager.supported_protocols.each do |p| describe "using the #{p} protocol" do it 'default to Dalli::Compressor' do - memcached(p, 29_199) do |dc| + memcached(p, port_or_socket: 29_199) do |dc| dc.set 1, 2 assert_equal Dalli::Compressor, dc.instance_variable_get(:@ring).servers.first.compressor @@ -25,14 +25,14 @@ def self.decompress(data) end it 'support a custom compressor' do - memcached(p, 29_199) do |_dc| + memcached(p, port_or_socket: 29_199) do |_dc| memcache = Dalli::Client.new('127.0.0.1:29199', { compressor: NoopCompressor }) memcache.set 1, 2 begin assert_equal NoopCompressor, memcache.instance_variable_get(:@ring).servers.first.compressor - memcached(p, 19_127) do |newdc| + memcached(p, port_or_socket: 19_127) do |newdc| assert newdc.set('string-test', 'a test string') assert_equal('a test string', newdc.get('string-test')) end @@ -42,7 +42,7 @@ def self.decompress(data) describe 'GzipCompressor' do it 'compress and uncompress data using Zlib::GzipWriter/Reader' do - memcached(p, 19_127) do |_dc| + memcached(p, port_or_socket: 19_127) do |_dc| memcache = Dalli::Client.new('127.0.0.1:19127', { compress: true, compressor: Dalli::GzipCompressor }) data = (0...1025).map { rand(65..90).chr }.join diff --git a/test/integration/test_failover.rb b/test/integration/test_failover.rb index 85de7e7d..05b65030 100644 --- a/test/integration/test_failover.rb +++ b/test/integration/test_failover.rb @@ -34,14 +34,14 @@ describe 'assuming some bad servers' do it 'silently reconnect if server hiccups' do server_port = 30_124 - memcached_persistent(p, server_port) do |dc, port| + memcached_persistent(p, port_or_socket: server_port) do |dc, port| dc.set 'foo', 'bar' foo = dc.get 'foo' assert_equal('bar', foo) memcached_kill(port) - memcached_persistent(p, port) do + memcached_persistent(p, port_or_socket: port) do foo = dc.get 'foo' assert_nil foo @@ -55,8 +55,8 @@ port1 = 32_112 port2 = 37_887 - memcached(p, port1, '-o idle_timeout=1') do |_, first_port| - memcached(p, port2, '-o idle_timeout=1') do |_, second_port| + memcached(p, port_or_socket: port1, cli_args: '-o idle_timeout=1') do |_, first_port| + memcached(p, port_or_socket: port2, cli_args: '-o idle_timeout=1') do |_, second_port| dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] dc.set 'foo', 'bar' dc.set 'foo2', 'bar2' @@ -77,8 +77,8 @@ it 'handle graceful failover' do port1 = 31_777 port2 = 32_113 - memcached_persistent(p, port1) do |_first_dc, first_port| - memcached_persistent(p, port2) do |_second_dc, second_port| + memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port| + memcached_persistent(p, port_or_socket: port2) do |_second_dc, second_port| dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] dc.set 'foo', 'bar' foo = dc.get 'foo' @@ -104,8 +104,8 @@ it 'handle them gracefully in get_multi' do port1 = 32_971 port2 = 34_312 - memcached_persistent(p, port1) do |_first_dc, first_port| - memcached(p, port2) do |_second_dc, second_port| + memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port| + memcached(p, port_or_socket: port2) do |_second_dc, second_port| dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] dc.set 'a', 'a1' result = dc.get_multi ['a'] @@ -124,8 +124,8 @@ it 'handle graceful failover in get_multi' do port1 = 34_541 port2 = 33_044 - memcached_persistent(p, port1) do |_first_dc, first_port| - memcached_persistent(p, port2) do |_second_dc, second_port| + memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port| + memcached_persistent(p, port_or_socket: port2) do |_second_dc, second_port| dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] dc.set 'foo', 'foo1' dc.set 'bar', 'bar1' @@ -153,8 +153,8 @@ it 'stats it still properly report' do port1 = 34_547 port2 = 33_219 - memcached_persistent(p, port1) do |_first_dc, first_port| - memcached_persistent(p, port2) do |_second_dc, second_port| + memcached_persistent(p, port_or_socket: port1) do |_first_dc, first_port| + memcached_persistent(p, port_or_socket: port2) do |_second_dc, second_port| dc = Dalli::Client.new ["localhost:#{first_port}", "localhost:#{second_port}"] result = dc.stats diff --git a/test/integration/test_network.rb b/test/integration/test_network.rb index 73bd7efd..72e38c81 100644 --- a/test/integration/test_network.rb +++ b/test/integration/test_network.rb @@ -15,79 +15,49 @@ describe 'with a fake server' do it 'handle connection reset' do - memcached_mock(lambda(&:close)) do - dc = Dalli::Client.new('localhost:19123') - assert_raises Dalli::RingError, message: 'No server available' do - dc.get('abc') - end - end - end - - it 'handle connection reset with unix socket' do - socket_path = MemcachedMock::UNIX_SOCKET_PATH - memcached_mock(lambda(&:close), :start_unix, socket_path) do - dc = Dalli::Client.new(socket_path) - assert_raises Dalli::RingError, message: 'No server available' do - dc.get('abc') - end - end - end + toxiproxy_memcached_persistent(p) do |dc| + dc.get('abc') - it 'handle malformed response' do - memcached_mock(->(sock) { sock.write('123') }) do - dc = Dalli::Client.new('localhost:19123') - assert_raises Dalli::RingError, message: 'No server available' do - dc.get('abc') + Toxiproxy[/memcached/].down do + assert_raises Dalli::RingError, message: 'No server available' do + dc.get('abc') + end end end end it 'handle socket timeouts' do - dc = Dalli::Client.new('localhost:19123', socket_timeout: 0) - assert_raises Dalli::RingError, message: 'No server available' do + toxiproxy_memcached_persistent(p, client_options: { socket_timeout: 0.1 }) do |dc| dc.get('abc') - end - end - it 'handle connect timeouts' do - memcached_mock(lambda { |sock| - sleep(0.6) - sock.close - }, :delayed_start) do - dc = Dalli::Client.new('localhost:19123') - assert_raises Dalli::RingError, message: 'No server available' do - dc.get('abc') + Toxiproxy[/memcached/].downstream(:latency, 200) do + assert_raises Dalli::RingError, message: 'No server available' do + dc.get('abc') + end end end end - it 'handle read timeouts' do - memcached_mock(lambda { |sock| - sleep(0.6) - sock.write('giraffe') - }) do - dc = Dalli::Client.new('localhost:19123') - assert_raises Dalli::RingError, message: 'No server available' do - dc.get('abc') + it 'handle connect timeouts' do + toxiproxy_memcached_persistent(p, client_options: { socket_timeout: 0.1 }) do |dc| + Toxproxy[/memcached/].downstream(:latency, 200) do + assert_raises Dalli::RingError, message: 'No server available' do + dc.get('abc') + end end end end end - it 'handles operation timeouts' do - next if p == :binary + it 'handles operation timeouts without retries' do + toxiproxy_memcached_persistent(p, client_options: { socket_timeout: 0.1, socket_max_failures: 0, + socket_failure_delay: 0.0, down_retry_delay: 0.0 }) do |dc| + dc.version - memcached_mock(lambda { |sock| - # handle initial version call - sock.gets - sock.write("VERSION 1.6.0\r\n") - - sleep(0.3) - }) do - dc = Dalli::Client.new('localhost:19123', socket_timeout: 0.1, protocol: p, socket_max_failures: 0, - socket_failure_delay: 0.0, down_retry_delay: 0.0) - assert_raises Dalli::RingError, message: 'No server available' do - dc.get('abc') + Toxproxy[/memcached/].downstream(:latency, 200) do + assert_raises Dalli::RingError, message: 'No server available' do + dc.get('abc') + end end end end @@ -140,7 +110,7 @@ it 'handles timeout error during pipelined get' do with_nil_logger do - memcached(p, 19_191) do |dc| + memcached(p, port_or_socket: 19_191) do |dc| dc.send(:ring).server_for_key('abc').sock.stub(:write, proc { raise Timeout::Error }) do assert_empty dc.get_multi(['abc']) end @@ -150,7 +120,7 @@ it 'handles asynchronous Thread#raise' do with_nil_logger do - memcached(p, 19_191) do |dc| + memcached(p, port_or_socket: 19_191) do |dc| 10.times do |i| thread = Thread.new do loop do @@ -174,7 +144,7 @@ it 'handles asynchronous Thread#raise during pipelined get' do with_nil_logger do - memcached(p, 19_191) do |dc| + memcached(p, port_or_socket: 19_191) do |dc| 10.times do |i| expected_response = 100.times.to_h { |x| ["key:#{i}:#{x}", x.to_s] } expected_response.each do |key, val| @@ -203,7 +173,7 @@ it 'handles asynchronous Thread#kill' do with_nil_logger do - memcached(p, 19_191) do |dc| + memcached(p, port_or_socket: 19_191) do |dc| 10.times do |i| thread = Thread.new do loop do @@ -227,7 +197,7 @@ it 'handles asynchronous Thread#kill during pipelined get' do with_nil_logger do - memcached(p, 19_191) do |dc| + memcached(p, port_or_socket: 19_191) do |dc| 10.times do |i| expected_response = 100.times.to_h { |x| ["key:#{i}:#{x}", x.to_s] } expected_response.each do |key, val| @@ -317,7 +287,7 @@ end it 'passes a simple smoke test on unix socket' do - memcached_persistent(:binary, MemcachedMock::UNIX_SOCKET_PATH) do |dc, path| + memcached_persistent(:binary, port_or_socket: MemcachedMock::UNIX_SOCKET_PATH) do |dc, path| resp = dc.flush refute_nil resp diff --git a/test/integration/test_operations.rb b/test/integration/test_operations.rb index 4ea51ce5..6ab6e825 100644 --- a/test/integration/test_operations.rb +++ b/test/integration/test_operations.rb @@ -236,7 +236,7 @@ end it 'supports with nil values when cache_nils: true' do - memcached_persistent(p, 21_345, '', cache_nils: true) do |dc| + memcached_persistent(p, port_or_socket: 21_345, client_options: { cache_nils: true }) do |dc| dc.flush dc.set('fetch_key', nil) @@ -245,7 +245,7 @@ assert_nil res end - memcached_persistent(p, 21_345, '', cache_nils: false) do |dc| + memcached_persistent(p, port_or_socket: 21_345, client_options: { cache_nils: false }) do |dc| dc.flush dc.set('fetch_key', nil) executed = false diff --git a/test/integration/test_serializer.rb b/test/integration/test_serializer.rb index 46dff8ac..b1f142d1 100644 --- a/test/integration/test_serializer.rb +++ b/test/integration/test_serializer.rb @@ -7,7 +7,7 @@ MemcachedManager.supported_protocols.each do |p| describe "using the #{p} protocol" do it 'defaults to Marshal' do - memcached(p, 29_198) do |dc| + memcached(p, port_or_socket: 29_198) do |dc| dc.set 1, 2 assert_equal Marshal, dc.instance_variable_get(:@ring).servers.first.serializer @@ -15,13 +15,13 @@ end it 'supports a custom serializer' do - memcached(p, 29_198) do |_dc, port| + memcached(p, port_or_socket: 29_198) do |_dc, port| memcache = Dalli::Client.new("127.0.0.1:#{port}", serializer: JSON) memcache.set 1, 2 begin assert_equal JSON, memcache.instance_variable_get(:@ring).servers.first.serializer - memcached(p, 21_956) do |newdc| + memcached(p, port_or_socket: 21_956) do |newdc| assert newdc.set('json_test', { 'foo' => 'bar' }) assert_equal({ 'foo' => 'bar' }, newdc.get('json_test')) end diff --git a/test/test_rack_session.rb b/test/test_rack_session.rb index fa4c1fab..3b39612c 100644 --- a/test/test_rack_session.rb +++ b/test/test_rack_session.rb @@ -9,7 +9,7 @@ describe Rack::Session::Dalli do before do @port = 19_129 - memcached_persistent(:binary, @port) + memcached_persistent(:binary, port_or_socket: @port) Rack::Session::Dalli::DEFAULT_DALLI_OPTIONS[:memcache_server] = "localhost:#{@port}" # test memcache connection diff --git a/test/test_ring.rb b/test/test_ring.rb index 41264fe6..4f3c5be1 100644 --- a/test/test_ring.rb +++ b/test/test_ring.rb @@ -45,7 +45,7 @@ def weight it "return the server when it's alive" do servers = ['localhost:19191'] ring = Dalli::Ring.new(servers, Dalli::Protocol::Binary, {}) - memcached(:binary, 19_191) do |mc| + memcached(:binary, port_or_socket: 19_191) do |mc| ring = mc.send(:ring) assert_equal ring.servers.first.port, ring.server_for_key('test').port @@ -65,7 +65,7 @@ def weight it 'return an alive server when at least one is alive' do servers = ['localhost:12346', 'localhost:19191'] ring = Dalli::Ring.new(servers, Dalli::Protocol::Binary, {}) - memcached(:binary, 19_191) do |mc| + memcached(:binary, port_or_socket: 19_191) do |mc| ring = mc.send(:ring) assert_equal ring.servers.first.port, ring.server_for_key('test').port @@ -74,13 +74,13 @@ def weight end it 'detect when a dead server is up again' do - memcached(:binary, 19_997) do + memcached(:binary, port_or_socket: 19_997) do down_retry_delay = 0.5 dc = Dalli::Client.new(['localhost:19997', 'localhost:19998'], down_retry_delay: down_retry_delay) assert_equal 1, dc.stats.values.compact.count - memcached(:binary, 19_998) do + memcached(:binary, port_or_socket: 19_998) do assert_equal 2, dc.stats.values.compact.count end end diff --git a/test/utils/memcached_manager.rb b/test/utils/memcached_manager.rb index e7b778c5..0940f757 100644 --- a/test/utils/memcached_manager.rb +++ b/test/utils/memcached_manager.rb @@ -19,6 +19,8 @@ module MemcachedManager MEMCACHED_VERSION_CMD = "#{MEMCACHED_CMD} -h | head -1" MEMCACHED_VERSION_REGEXP = /^memcached (\d\.\d\.\d+)/.freeze MEMCACHED_MIN_MAJOR_VERSION = ::Dalli::MIN_SUPPORTED_MEMCACHED_VERSION + TOXIPROXY_MEMCACHED_PORT = 21_347 + TOXIPROXY_UPSTREAM_PORT = 21_348 @running_pids = {}