Add retry for resume tests

Gemfile

@@ -7,6 +7,7 @@ group :test do
 
   gem "minitest-hooks"
   gem "minitest-reporters", "~> 1.4"
+  gem "minitest-retry"
   gem "minitest-fail-fast", "~> 0.1.0"
 end
 

Gemfile.lock

@@ -16,6 +16,8 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
+    minitest-retry (0.2.2)
+      minitest (>= 5.0)
     mysql2 (0.5.5)
     pry (0.14.2)
       coderay (~> 1.1)
@@ -35,6 +37,7 @@ DEPENDENCIES
   minitest-fail-fast (~> 0.1.0)
   minitest-hooks
   minitest-reporters (~> 1.4)
+  minitest-retry
   mysql2
   pry-byebug
   tqdm

test/helpers/ghostferry_helper.rb

@@ -15,8 +15,6 @@ def self.remove_all_binaries
     FileUtils.remove_entry(GHOSTFERRY_TEMPDIR) if Dir.exist?(GHOSTFERRY_TEMPDIR)
   end
 
-  class GhostferryExitFailure < StandardError
-  end
 
   class Ghostferry
     # Manages compiling, running, and communicating with Ghostferry.
@@ -33,6 +31,10 @@ class Ghostferry
     # Keep these in sync with integrationferry.go
     ENV_KEY_PORT = "GHOSTFERRY_INTEGRATION_PORT"
 
+    Error = Class.new(StandardError)
+    ExitError = Class.new(Error)
+    TimeoutError = Class.new(Error)
+
     module Status
       # This should be in sync with integrationferry.go
       READY = "READY"
@@ -117,8 +119,8 @@ def run(resuming_state = nil)
     def run_expecting_interrupt(resuming_state = nil)
       @logger.info("[#{@tag}] ghostferry#run_expecting_interrupt(state:#{(!resuming_state.nil?).inspect})")
       run(resuming_state)
-    rescue GhostferryExitFailure
-      @logger.info("[#{@tag}] ghostferry#run_expecting_interrupt: got GhostferryExitFailure")
+    rescue ExitError
+      @logger.info("[#{@tag}] ghostferry#run_expecting_interrupt: got Ghostferry::ExitError")
       dumped_state = @stdout.join("")
       JSON.parse(dumped_state)
     else
@@ -131,8 +133,8 @@ def run_expecting_interrupt(resuming_state = nil)
     def run_expecting_failure(resuming_state = nil)
       @logger.info("[#{@tag}] ghostferry#run_expecting_failure(state:#{(!resuming_state.nil?).inspect})")
       run(resuming_state)
-    rescue GhostferryExitFailure
-      @logger.info("[#{@tag}] ghostferry#run_expecting_failure: got GhostferryExitFailure")
+    rescue ExitError
+      @logger.info("[#{@tag}] ghostferry#run_expecting_failure: got Ghostferry::ExitError")
     else
       raise "[#{@tag}] Ghostferry did not fail"
     end
@@ -337,7 +339,7 @@ def start_ghostferry(resuming_state = nil)
 
         @logger.debug("[#{@tag}] ghostferry test binary exitted: #{@exit_status}")
         if @exit_status.exitstatus != 0
-          raise GhostferryExitFailure, "[#{@tag}] ghostferry test binary returned non-zero status: #{@exit_status}"
+          raise ExitError, "[#{@tag}] ghostferry test binary returned non-zero status: #{@exit_status}"
         end
       end
     end
@@ -351,7 +353,7 @@ def start_server_watchdog
           if (now - @last_message_time) > @message_timeout
             @server.shutdown
             @log_capturer.print_output
-            raise "[#{@tag}] ghostferry did not report to the integration test server for the last #{@message_timeout}s"
+            raise TimeoutError, "[#{@tag}] ghostferry did not report to the integration test server for the last #{@message_timeout}s"
           end
 
           sleep 1
@@ -408,7 +410,7 @@ def kill
 
       begin
         @subprocess_thread.join if @subprocess_thread
-      rescue GhostferryExitFailure
+      rescue ExitError
         # ignore
       end
     end

test/lib/go/integrationferry/ferry.go

@@ -148,14 +148,11 @@ func (f *IntegrationFerry) Main() error {
 		return err
 	}
 
-	f.logger.Debug("flush binlog and stop streaming")
 	// TODO: this method should return errors rather than calling
 	// the error handler to panic directly.
 	f.FlushBinlogAndStopStreaming()
-	f.logger.Debug("wait for Ferry to exit")
 	wg.Wait()
 
-	f.logger.Debug("finished waiting")
 	if f.Verifier != nil {
 		err := f.SendStatusAndWaitUntilContinue(StatusVerifyDuringCutover)
 		if err != nil {

test/main.rb

@@ -15,10 +15,12 @@
 
 require "minitest"
 require "minitest/reporters"
+require "minitest/retry"
 require "minitest/fail_fast"
 require "minitest/hooks/test"
 
 Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
+Minitest::Retry.use!(exceptions_to_retry: [GhostferryHelper::Ghostferry::TimeoutError])
 
 test_files.each do |f|
   require f