Skip to content

Commit

Permalink
Separating report_target application in interactive mode (#1034)
Browse files Browse the repository at this point in the history 
* Separating report_target application in interactive mode.
* Move report_target decoration to the testcase runner.
* Remove unused functionalities of interactive mode.
* Fix report target side effect inside test context generation.
* Fixed an issue with testcase removal. Unified handling of test context between interactive and batch execution modes.
  • Loading branch information
@M6AI
M6AI authored Jan 26, 2024
1 parent e4cf2dd commit ce027da
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 203 deletions.
1 change: 1 addition & 0 deletions doc/newsfragments/2788_changed.improve_interactive_startup_performance.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Internal refactoring to enable faster startup in interactive mode.
99 changes: 3 additions & 96 deletions testplan/runnable/interactive/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -726,71 +726,6 @@ def all_tests_operation(self, operation, await_results=True):
else:
raise ValueError("Unknown operation: {}".format(operation))

def create_new_environment(self, env_uid, env_type="local_environment"):
"""Dynamically create an environment maker object."""
if env_uid in self._created_environments:
raise RuntimeError(
"Environment {} already exists.".format(env_uid)
)

if env_type == "local_environment":
from testplan.environment import LocalEnvironment

env_class = LocalEnvironment
else:
raise ValueError("Unknown environment type: {}".format(env_type))

self._created_environments[env_uid] = env_class(env_uid)

def add_environment_resource(
self, env_uid, target_class_name, source_file=None, **kwargs
):
"""
Add a resource to existing environment or to environment maker object.
"""
final_kwargs = {}
compiled = re.compile(r"_ctx_(.+)_ctx_(.+)")
context_params = {}
for key, value in kwargs.items():
if key.startswith("_ctx_"):
matched = compiled.match(key)
if not matched or key.count("_ctx_") != 2:
raise ValueError("Invalid key: {}".format(key))
target_key, ctx_key = matched.groups()
if target_key not in context_params:
context_params[target_key] = {}
context_params[target_key][ctx_key] = value
else:
final_kwargs[key] = value
if context_params:
from testplan.common.utils.context import context

for key in context_params:
final_kwargs[key] = context(**context_params[key])

if source_file is None: # Invoke class loader
resource = self._resource_loader.load(
target_class_name, final_kwargs
)
try:
self.get_environment(env_uid).add(resource)
except:
self._created_environments[env_uid].add_resource(resource)
else:
raise Exception("Add from source file is not yet supported.")

def reload_environment_resource(
self, env_uid, target_class_name, source_file=None, **kwargs
):
# Placeholder for function to delele an existing and registering a new
# environment resource with probably altered source code.
# This should access the already added Environment to plan.
pass

def add_created_environment(self, env_uid):
"""Add an environment from the created environment maker instance."""
self.target.add_environment(self._created_environments[env_uid])

def reload(self, rebuild_dependencies=False):
"""Reload test suites."""
if self._reloader is None:
Expand All @@ -816,13 +751,13 @@ def reload_report(self):
].entries[param_index] = case[
param_case.uid
]
except KeyError:
except (KeyError, IndexError):
continue
else:
new_report[multitest.uid][suite.uid].entries[
case_index
] = suite[case.uid]
except KeyError:
except (KeyError, IndexError):
continue
multitest.entries[suite_index] = new_suite

Expand Down Expand Up @@ -875,32 +810,12 @@ def _initial_report(self):

for test_uid in self.all_tests():
test = self.test(test_uid)
test.reset_context()
test_report = test.dry_run().report
report.append(test_report)

return report

def _run_all_test_operations(self, test_run_generator):
"""Run all test operations."""
return [
self._run_test_operation(operation, args, kwargs)
for operation, args, kwargs in test_run_generator
]

def _run_test_operation(self, test_operation, args, kwargs):
"""Run a test operation and update our report tree with the results."""
result = test_operation(*args, **kwargs)

if isinstance(result, TestGroupReport):
self.logger.debug("Merge test result: %s", result)
with self.report_mutex:
self.report[result.uid].merge(result)
elif result is not None:
self.logger.debug(
"Discarding result from test operation: %s", result
)
return result

def _auto_start_environment(self, test_uid):
"""Start environment if required."""
env_status = self.report[test_uid].env_status
Expand Down Expand Up @@ -937,14 +852,6 @@ def _set_env_status(self, test_uid, new_status):
)
self.report[test_uid].env_status = new_status

def _log_env_errors(self, test_uid, error_messages):
"""Log errors during environment start/stop for a given test."""
test_report = self.report[test_uid]
with self.report_mutex:
for errmsg in error_messages:
test_report.logger.error(errmsg)
test_report.status_override = Status.ERROR

def _clear_env_errors(self, test_uid):
"""Remove error logs about environment start/stop for a given test."""
test = self.test(test_uid)
Expand Down
42 changes: 12 additions & 30 deletions testplan/testing/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,39 +116,25 @@ class Test(Runnable):
customize functionality.
:param name: Test instance name, often used as uid of test entity.
:type name: ``str``
:param description: Description of test instance.
:type description: ``str``
:param environment: List of
:py:class:`drivers <testplan.tesitng.multitest.driver.base.Driver>` to
:py:class:`drivers <testplan.testing.multitest.driver.base.Driver>` to
be started and made available on tests execution. Can also take a
callable that returns the list of drivers.
:type environment: ``list`` or ``callable``
:param dependencies: driver start-up dependencies as a directed graph,
e.g {server1: (client1, client2)} indicates server1 shall start before
client1 and client2. Can also take a callable that returns a dict.
:type dependencies: ``dict`` or ``callable``
:param initial_context: key: value pairs that will be made available as
context for drivers in environment. Can also take a callbale that
returns a dict.
:type initial_context: ``dict`` or ``callable``
:param test_filter: Class with test filtering logic.
:type test_filter: :py:class:`~testplan.testing.filtering.BaseFilter`
:param test_sorter: Class with tests sorting logic.
:type test_sorter: :py:class:`~testplan.testing.ordering.BaseSorter`
:param before_start: Callable to execute before starting the environment.
:type before_start: ``callable`` taking an environment argument.
:param after_start: Callable to execute after starting the environment.
:type after_start: ``callable`` taking an environment argument.
:param before_stop: Callable to execute before stopping the environment.
:type before_stop: ``callable`` taking environment and a result arguments.
:param after_stop: Callable to execute after stopping the environment.
:type after_stop: ``callable`` taking environment and a result arguments.
:param stdout_style: Console output style.
:type stdout_style: :py:class:`~testplan.report.testing.styles.Style`
:param tags: User defined tag value.
:type tags: ``string``, ``iterable`` of ``string``, or a ``dict`` with
``string`` keys and ``string`` or ``iterable`` of ``string`` as values.
Also inherits all
:py:class:`~testplan.common.entity.base.Runnable` options.
Expand Down Expand Up @@ -194,10 +180,10 @@ def __init__(
self._init_test_report()
self._env_built = False

def __str__(self):
def __str__(self) -> str:
return "{}[{}]".format(self.__class__.__name__, self.name)

def _new_test_report(self):
def _new_test_report(self) -> TestGroupReport:
return TestGroupReport(
name=self.cfg.name,
description=self.cfg.description,
Expand All @@ -206,10 +192,10 @@ def _new_test_report(self):
env_status=ResourceStatus.STOPPED,
)

def _init_test_report(self):
def _init_test_report(self) -> TestGroupReport:
self.result.report = self._new_test_report()

def get_tags_index(self):
def get_tags_index(self) -> Union[str, Iterable[str], Dict]:
"""
Return the tag index that will be used for filtering.
By default this is equal to the native tags for this object.
Expand All @@ -219,24 +205,24 @@ def get_tags_index(self):
"""
return self.cfg.tags or {}

def get_filter_levels(self):
def get_filter_levels(self) -> List[filtering.FilterLevel]:
if not self.filter_levels:
raise ValueError(
"`filter_levels` is not defined by {}".format(self)
)
return self.filter_levels

@property
def name(self):
def name(self) -> str:
"""Instance name."""
return self.cfg.name

@property
def description(self):
def description(self) -> str:
return self.cfg.description

@property
def report(self):
def report(self) -> TestGroupReport:
"""Shortcut for the test report."""
return self.result.report

Expand All @@ -247,17 +233,13 @@ def stdout_style(self):

@property
def test_context(self):
if (
getattr(self, "parent", None)
and hasattr(self.parent, "cfg")
and self.parent.cfg.interactive_port is not None
):
return self.get_test_context()

if self._test_context is None:
self._test_context = self.get_test_context()
return self._test_context

def reset_context(self) -> None:
self._test_context = None

def get_test_context(self):
raise NotImplementedError

Expand Down
36 changes: 19 additions & 17 deletions testplan/testing/multitest/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,19 +440,6 @@ def get_test_context(self):
) < len(sorted_testcases):
testcases_to_run = sorted_testcases

if self.cfg.testcase_report_target:
testcases_to_run = [
report_target(
func=testcase,
ref_func=getattr(
suite,
getattr(testcase, "_parametrization_template", ""),
None,
),
)
for testcase in testcases_to_run
]

if testcases_to_run:
if hasattr(self.cfg, "xfail_tests") and self.cfg.xfail_tests:
for testcase in testcases_to_run:
Expand All @@ -467,7 +454,7 @@ def get_test_context(self):
testcase_instance, None
)
if data is not None:
testcase.__xfail__ = {
testcase.__func__.__xfail__ = {
"reason": data["reason"],
"strict": data["strict"],
}
Expand Down Expand Up @@ -901,7 +888,7 @@ def _run_serial_testcases(self, testsuite, testcases):
break

testcase_report = self._run_testcase(
testcase, pre_testcase, post_testcase
testcase, testsuite, pre_testcase, post_testcase
)

param_template = getattr(
Expand Down Expand Up @@ -960,7 +947,11 @@ def _run_parallel_testcases(self, testsuite, execution_groups):
self.logger.debug('Running execution group "%s"', exec_group)
results = [
self._thread_pool.submit(
self._run_testcase, testcase, pre_testcase, post_testcase
self._run_testcase,
testcase,
testsuite,
pre_testcase,
post_testcase,
)
for testcase in execution_groups[exec_group]
]
Expand Down Expand Up @@ -1146,6 +1137,7 @@ def _run_case_related(
def _run_testcase(
self,
testcase,
testsuite,
pre_testcase: Callable,
post_testcase: Callable,
testcase_report: Optional[TestCaseReport] = None,
Expand All @@ -1167,6 +1159,16 @@ def _run_testcase(
testcase_name=testcase.name, testcase_report=testcase_report
)

if self.cfg.testcase_report_target:
testcase = report_target(
func=testcase,
ref_func=getattr(
testsuite,
getattr(testcase, "_parametrization_template", ""),
None,
),
)

# specially handle skipped testcases
if hasattr(testcase, "__should_skip__"):
with compose_contexts(
Expand Down Expand Up @@ -1337,7 +1339,7 @@ def _run_testcases_iter(self, testsuite, testcases):
]

testcase_report = self._run_testcase(
testcase, pre_testcase, post_testcase
testcase, testsuite, pre_testcase, post_testcase
)
yield testcase_report, parent_uids

Expand Down
Loading

0 comments on commit ce027da

Please sign in to comment.