Issue #114: doc & tested initialize/reset runners (+fixes)

byexample/executor.py

@@ -20,6 +20,110 @@ def __init__(self, prefix, input, timeout_exc):
         self.prefix, self.input = prefix, input
 
 
+'''
+>>> from byexample.runner import ExampleRunner
+>>> from byexample.executor import FileExecutor
+
+>>> from byexample.log import init_log_system
+>>> init_log_system()
+
+>>> class Buggy(ExampleRunner):
+...     def __init__(self, name, bug_in):
+...         self.language = self.name = name
+...         self.bug_in = bug_in
+...     def run(self, example, options):
+...         print("run()", self.name)
+...         if self.bug_in == 'run':
+...             raise Exception("Faked on %s of %s" % (self.bug_in, self.name))
+...     def initialize(self, options):
+...         print("initialize()", self.name)
+...         if self.bug_in == 'initialize':
+...             raise Exception("Faked on %s of %s" % (self.bug_in, self.name))
+...     def shutdown(self):
+...         print("shutdown()", self.name)
+...         if self.bug_in == 'shutdown':
+...             raise Exception("Faked on %s of %s" % (self.bug_in, self.name))
+...     def reset(self, options):
+...         print("reset()", self.name)
+...         if self.bug_in == 'reset':
+...             raise Exception("Faked on %s of %s" % (self.bug_in, self.name))
+
+>>> fexec = FileExecutor(None, None, None, None, None)
+
+FileExecutor will initialize the runners in order, stopping on the
+first failure and shutting down any previous initialized runner (if any).
+
+In this case 'not-buggy1' should never being called because the first
+runner will fail in the 'initialize' phase.
+
+>>> runners = [Buggy("buggy1", "initialize"), Buggy("not-buggy1", "never")]
+
+>>> fexec.initialize_runners(runners, None)
+initialize() buggy1
+[w] Initialization of Buggy1 Runner failed.
+Traceback (most recent call last):
+<...>
+Exception: Faked on initialize of buggy1
+
+But if we change the order, initialize() of 'not-buggy1' is called
+but on the failure of 'buggy1', the shutdown() is called too.
+
+>>> runners = [Buggy("not-buggy1", "never"), Buggy("buggy1", "initialize")]
+>>> fexec.initialize_runners(runners, None)
+initialize() not-buggy1
+initialize() buggy1
+shutdown() not-buggy1
+[w] Initialization of Buggy1 Runner failed.
+Traceback (most recent call last):
+<...>
+Exception: Faked on initialize of buggy1
+
+If an exception is raised during a shutdown when the initialize() was
+aborted, the call to the shutdown()s is not stopped and only the first
+exception is raised (the other are just logged).
+
+Note the order of the calls to shutdown(): the reverse order of the calls
+to initialize() (like a stack).
+
+>>> runners = [Buggy("not-buggy1", "never"), Buggy("buggy2", "shutdown"), Buggy("buggy1", "initialize")]
+>>> fexec.initialize_runners(runners, None)
+initialize() not-buggy1
+initialize() buggy2
+initialize() buggy1
+shutdown() buggy2
+[w] Shutdown of Buggy2 Runner failed.
+shutdown() not-buggy1
+[w] Initialization of Buggy1 Runner failed.
+Traceback (most recent call last):
+<...>
+Exception: Faked on initialize of buggy1
+
+On reset/shutdown we have a similar situation except that reset_runners
+receives the list of runners and it is caller's responsibility (us) to
+pass a reversed list to have the same stack-like de-initialization order.
+
+>>> runners = [Buggy("not-buggy1", "never"), Buggy("buggy1", "shutdown"), Buggy("buggy2", "shutdown")]
+>>> fexec.initialize_runners(runners, None)
+initialize() not-buggy1
+initialize() buggy1
+initialize() buggy2
+
+>>> fexec.reset_runners(list(reversed(runners)))
+shutdown() buggy2
+shutdown() buggy1
+[w] Shutdown of Buggy1 Runner failed.
+shutdown() not-buggy1
+[w] Shutdown of Buggy2 Runner failed.
+Traceback (most recent call last):
+<...>
+Exception: Faked on shutdown of buggy2
+
+Note: no tests nor documentation about 'reset' is done yet.
+
+>>> fexec.close()   # byexample: -skip
+'''
+
+
 class FileExecutor(object):
     def __init__(
         self, concerns, differ, verbosity, use_colors, options, **unused
@@ -34,7 +138,7 @@ def __init__(
 
     @contextlib.contextmanager
     def on_failure_shutdown_runners(
-        self, should_raise, runners_left, err_args
+        self, should_raise, runners_left, log, err_args
     ):
         try:
             yield
@@ -45,7 +149,6 @@ def on_failure_shutdown_runners(
                 self.reset_runners(
                     runners_left, should_raise=False, force_shutdown=True
                 )
-                assert not runners_left
 
             log.warn(*err_args)
 
@@ -67,7 +170,8 @@ def initialize_runners(self, runners, options):
                 log.info("Initializing %s", str(runner))
                 with self.on_failure_shutdown_runners(
                     should_raise=True,
-                    runners_left=so_far,
+                    runners_left=list(reversed(so_far)),
+                    log=log,
                     err_args=("Initialization of %s failed.", str(runner))
                 ):
                     runner.initialize(options)
@@ -94,6 +198,7 @@ def reset_runners(self, runners, should_raise=True, force_shutdown=True):
                     with self.on_failure_shutdown_runners(
                         should_raise=should_raise,
                         runners_left=left,
+                        log=log,
                         err_args=("Reset of %s failed.", str(runner))
                     ):
                         if runner.reset():
@@ -116,6 +221,7 @@ def reset_runners(self, runners, should_raise=True, force_shutdown=True):
                 with self.on_failure_shutdown_runners(
                     should_raise=should_raise,
                     runners_left=left,
+                    log=log,
                     err_args=("Shutdown of %s failed.", str(runner))
                 ):
                     runner.shutdown()
@@ -124,8 +230,11 @@ def reset_runners(self, runners, should_raise=True, force_shutdown=True):
 
     @log_context('byexample.close')
     def close(self):
+        # no order is guaranteed
         self.reset_runners(
-            self.still_alive_runners, should_raise=True, force_shutdown=True
+            list(self.still_alive_runners),
+            should_raise=True,
+            force_shutdown=True
         )
 
     def __repr__(self):