Separating report_target application in interactive mode

doc/newsfragments/2788_changed.improve_interactive_startup_performance.rst

@@ -0,0 +1 @@
+Internal refactoring to enable faster startup in interactive mode.

testplan/runnable/interactive/base.py

@@ -726,71 +726,6 @@ def all_tests_operation(self, operation, await_results=True):
             else:
                 raise ValueError("Unknown operation: {}".format(operation))
 
-    def create_new_environment(self, env_uid, env_type="local_environment"):
-        """Dynamically create an environment maker object."""
-        if env_uid in self._created_environments:
-            raise RuntimeError(
-                "Environment {} already exists.".format(env_uid)
-            )
-
-        if env_type == "local_environment":
-            from testplan.environment import LocalEnvironment
-
-            env_class = LocalEnvironment
-        else:
-            raise ValueError("Unknown environment type: {}".format(env_type))
-
-        self._created_environments[env_uid] = env_class(env_uid)
-
-    def add_environment_resource(
-        self, env_uid, target_class_name, source_file=None, **kwargs
-    ):
-        """
-        Add a resource to existing environment or to environment maker object.
-        """
-        final_kwargs = {}
-        compiled = re.compile(r"_ctx_(.+)_ctx_(.+)")
-        context_params = {}
-        for key, value in kwargs.items():
-            if key.startswith("_ctx_"):
-                matched = compiled.match(key)
-                if not matched or key.count("_ctx_") != 2:
-                    raise ValueError("Invalid key: {}".format(key))
-                target_key, ctx_key = matched.groups()
-                if target_key not in context_params:
-                    context_params[target_key] = {}
-                context_params[target_key][ctx_key] = value
-            else:
-                final_kwargs[key] = value
-        if context_params:
-            from testplan.common.utils.context import context
-
-            for key in context_params:
-                final_kwargs[key] = context(**context_params[key])
-
-        if source_file is None:  # Invoke class loader
-            resource = self._resource_loader.load(
-                target_class_name, final_kwargs
-            )
-            try:
-                self.get_environment(env_uid).add(resource)
-            except:
-                self._created_environments[env_uid].add_resource(resource)
-        else:
-            raise Exception("Add from source file is not yet supported.")
-
-    def reload_environment_resource(
-        self, env_uid, target_class_name, source_file=None, **kwargs
-    ):
-        # Placeholder for function to delele an existing and registering a new
-        # environment resource with probably altered source code.
-        # This should access the already added Environment to plan.
-        pass
-
-    def add_created_environment(self, env_uid):
-        """Add an environment from the created environment maker instance."""
-        self.target.add_environment(self._created_environments[env_uid])
-
     def reload(self, rebuild_dependencies=False):
         """Reload test suites."""
         if self._reloader is None:
@@ -816,13 +751,13 @@ def reload_report(self):
                                     ].entries[param_index] = case[
                                         param_case.uid
                                     ]
-                                except KeyError:
+                                except (KeyError, IndexError):
                                     continue
                         else:
                             new_report[multitest.uid][suite.uid].entries[
                                 case_index
                             ] = suite[case.uid]
-                    except KeyError:
+                    except (KeyError, IndexError):
                         continue
                 multitest.entries[suite_index] = new_suite
 
@@ -875,32 +810,12 @@ def _initial_report(self):
 
         for test_uid in self.all_tests():
             test = self.test(test_uid)
+            test.reset_context()
             test_report = test.dry_run().report
             report.append(test_report)
 
         return report
 
-    def _run_all_test_operations(self, test_run_generator):
-        """Run all test operations."""
-        return [
-            self._run_test_operation(operation, args, kwargs)
-            for operation, args, kwargs in test_run_generator
-        ]
-
-    def _run_test_operation(self, test_operation, args, kwargs):
-        """Run a test operation and update our report tree with the results."""
-        result = test_operation(*args, **kwargs)
-
-        if isinstance(result, TestGroupReport):
-            self.logger.debug("Merge test result: %s", result)
-            with self.report_mutex:
-                self.report[result.uid].merge(result)
-        elif result is not None:
-            self.logger.debug(
-                "Discarding result from test operation: %s", result
-            )
-        return result
-
     def _auto_start_environment(self, test_uid):
         """Start environment if required."""
         env_status = self.report[test_uid].env_status
@@ -937,14 +852,6 @@ def _set_env_status(self, test_uid, new_status):
             )
             self.report[test_uid].env_status = new_status
 
-    def _log_env_errors(self, test_uid, error_messages):
-        """Log errors during environment start/stop for a given test."""
-        test_report = self.report[test_uid]
-        with self.report_mutex:
-            for errmsg in error_messages:
-                test_report.logger.error(errmsg)
-            test_report.status_override = Status.ERROR
-
     def _clear_env_errors(self, test_uid):
         """Remove error logs about environment start/stop for a given test."""
         test = self.test(test_uid)

testplan/testing/base.py

@@ -116,39 +116,25 @@ class Test(Runnable):
     customize functionality.
 
     :param name: Test instance name, often used as uid of test entity.
-    :type name: ``str``
     :param description: Description of test instance.
-    :type description: ``str``
     :param environment: List of
-        :py:class:`drivers <testplan.tesitng.multitest.driver.base.Driver>` to
+        :py:class:`drivers <testplan.testing.multitest.driver.base.Driver>` to
         be started and made available on tests execution. Can also take a
         callable that returns the list of drivers.
-    :type environment: ``list`` or ``callable``
     :param dependencies: driver start-up dependencies as a directed graph,
         e.g {server1: (client1, client2)} indicates server1 shall start before
         client1 and client2. Can also take a callable that returns a dict.
-    :type dependencies: ``dict`` or ``callable``
     :param initial_context: key: value pairs that will be made available as
         context for drivers in environment. Can also take a callbale that
         returns a dict.
-    :type initial_context: ``dict`` or ``callable``
     :param test_filter: Class with test filtering logic.
-    :type test_filter: :py:class:`~testplan.testing.filtering.BaseFilter`
     :param test_sorter: Class with tests sorting logic.
-    :type test_sorter: :py:class:`~testplan.testing.ordering.BaseSorter`
     :param before_start: Callable to execute before starting the environment.
-    :type before_start: ``callable`` taking an environment argument.
     :param after_start: Callable to execute after starting the environment.
-    :type after_start: ``callable`` taking an environment argument.
     :param before_stop: Callable to execute before stopping the environment.
-    :type before_stop: ``callable`` taking environment and a result arguments.
     :param after_stop: Callable to execute after stopping the environment.
-    :type after_stop: ``callable`` taking environment and a result arguments.
     :param stdout_style: Console output style.
-    :type stdout_style: :py:class:`~testplan.report.testing.styles.Style`
     :param tags: User defined tag value.
-    :type tags: ``string``, ``iterable`` of ``string``, or a ``dict`` with
-        ``string`` keys and ``string`` or ``iterable`` of ``string`` as values.
 
     Also inherits all
     :py:class:`~testplan.common.entity.base.Runnable` options.
@@ -194,10 +180,10 @@ def __init__(
         self._init_test_report()
         self._env_built = False
 
-    def __str__(self):
+    def __str__(self) -> str:
         return "{}[{}]".format(self.__class__.__name__, self.name)
 
-    def _new_test_report(self):
+    def _new_test_report(self) -> TestGroupReport:
         return TestGroupReport(
             name=self.cfg.name,
             description=self.cfg.description,
@@ -206,10 +192,10 @@ def _new_test_report(self):
             env_status=ResourceStatus.STOPPED,
         )
 
-    def _init_test_report(self):
+    def _init_test_report(self) -> TestGroupReport:
         self.result.report = self._new_test_report()
 
-    def get_tags_index(self):
+    def get_tags_index(self) -> Union[str, Iterable[str], Dict]:
         """
         Return the tag index that will be used for filtering.
         By default this is equal to the native tags for this object.
@@ -219,24 +205,24 @@ def get_tags_index(self):
         """
         return self.cfg.tags or {}
 
-    def get_filter_levels(self):
+    def get_filter_levels(self) -> List[filtering.FilterLevel]:
         if not self.filter_levels:
             raise ValueError(
                 "`filter_levels` is not defined by {}".format(self)
             )
         return self.filter_levels
 
     @property
-    def name(self):
+    def name(self) -> str:
         """Instance name."""
         return self.cfg.name
 
     @property
-    def description(self):
+    def description(self) -> str:
         return self.cfg.description
 
     @property
-    def report(self):
+    def report(self) -> TestGroupReport:
         """Shortcut for the test report."""
         return self.result.report
 
@@ -247,17 +233,13 @@ def stdout_style(self):
 
     @property
     def test_context(self):
-        if (
-            getattr(self, "parent", None)
-            and hasattr(self.parent, "cfg")
-            and self.parent.cfg.interactive_port is not None
-        ):
-            return self.get_test_context()
-
         if self._test_context is None:
             self._test_context = self.get_test_context()
         return self._test_context
 
+    def reset_context(self) -> None:
+        self._test_context = None
+
     def get_test_context(self):
         raise NotImplementedError
 

testplan/testing/multitest/base.py

@@ -440,19 +440,6 @@ def get_test_context(self):
             ) < len(sorted_testcases):
                 testcases_to_run = sorted_testcases
 
-            if self.cfg.testcase_report_target:
-                testcases_to_run = [
-                    report_target(
-                        func=testcase,
-                        ref_func=getattr(
-                            suite,
-                            getattr(testcase, "_parametrization_template", ""),
-                            None,
-                        ),
-                    )
-                    for testcase in testcases_to_run
-                ]
-
             if testcases_to_run:
                 if hasattr(self.cfg, "xfail_tests") and self.cfg.xfail_tests:
                     for testcase in testcases_to_run:
@@ -467,7 +454,7 @@ def get_test_context(self):
                             testcase_instance, None
                         )
                         if data is not None:
-                            testcase.__xfail__ = {
+                            testcase.__func__.__xfail__ = {
                                 "reason": data["reason"],
                                 "strict": data["strict"],
                             }
@@ -901,7 +888,7 @@ def _run_serial_testcases(self, testsuite, testcases):
                 break
 
             testcase_report = self._run_testcase(
-                testcase, pre_testcase, post_testcase
+                testcase, testsuite, pre_testcase, post_testcase
             )
 
             param_template = getattr(
@@ -960,7 +947,11 @@ def _run_parallel_testcases(self, testsuite, execution_groups):
             self.logger.debug('Running execution group "%s"', exec_group)
             results = [
                 self._thread_pool.submit(
-                    self._run_testcase, testcase, pre_testcase, post_testcase
+                    self._run_testcase,
+                    testcase,
+                    testsuite,
+                    pre_testcase,
+                    post_testcase,
                 )
                 for testcase in execution_groups[exec_group]
             ]
@@ -1146,6 +1137,7 @@ def _run_case_related(
     def _run_testcase(
         self,
         testcase,
+        testsuite,
         pre_testcase: Callable,
         post_testcase: Callable,
         testcase_report: Optional[TestCaseReport] = None,
@@ -1167,6 +1159,16 @@ def _run_testcase(
             testcase_name=testcase.name, testcase_report=testcase_report
         )
 
+        if self.cfg.testcase_report_target:
+            testcase = report_target(
+                func=testcase,
+                ref_func=getattr(
+                    testsuite,
+                    getattr(testcase, "_parametrization_template", ""),
+                    None,
+                ),
+            )
+
         # specially handle skipped testcases
         if hasattr(testcase, "__should_skip__"):
             with compose_contexts(
@@ -1337,7 +1339,7 @@ def _run_testcases_iter(self, testsuite, testcases):
             ]
 
             testcase_report = self._run_testcase(
-                testcase, pre_testcase, post_testcase
+                testcase, testsuite, pre_testcase, post_testcase
             )
             yield testcase_report, parent_uids