From 506225e958d01c155f1dec5ea42c608c9db1d51b Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Tue, 17 Sep 2024 17:52:55 +0200 Subject: [PATCH] Block timeout introduction This commit introduces a new timeout feature, which adds the possibility to set a specific timeout for a block of code with context manager like this: with self.wait_max(3): #code which should take max 3 seconds ... The `wait_max` method will send `SIGALRM` if the code doesn't end within 3 seconds. This signal will be caught by avocado-instrumented runner, which will interrupt the test, the same way as with a regular timeout. Reference: #5994 Signed-off-by: Jan Richter --- avocado/core/test.py | 21 +++++++++++++++ .../plugins/runners/avocado_instrumented.py | 3 ++- .../source/guides/writer/chapters/writing.rst | 16 ++++++++++++ examples/tests/blocktimeouttest.py | 22 ++++++++++++++++ selftests/check.py | 2 +- selftests/functional/basic.py | 26 +++++++++++++++++++ 6 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 examples/tests/blocktimeouttest.py diff --git a/avocado/core/test.py b/avocado/core/test.py index eb79e2960e..710617e05c 100644 --- a/avocado/core/test.py +++ b/avocado/core/test.py @@ -24,11 +24,13 @@ import logging import os import shutil +import signal import sys import tempfile import time import unittest import warnings +from contextlib import contextmanager from avocado.core import exceptions, parameters from avocado.core.settings import settings @@ -514,6 +516,25 @@ def phase(self): """ return self.__phase + @contextmanager + def wait_max(self, timeout): + """ + Context manager for getting block of code with its specific timeout. + + Usage: + with self.wait_max(3): + # code which should take max 3 seconds + ... + + :param timeout: Timeout in seconds for block of code. + :type timeout: int + """ + timeout_factor = float(self.params.get("timeout_factor", default=1.0)) + timeout = round(float(timeout) * timeout_factor) + signal.alarm(timeout) + yield timeout + signal.alarm(0) + def __str__(self): return str(self.name) diff --git a/avocado/plugins/runners/avocado_instrumented.py b/avocado/plugins/runners/avocado_instrumented.py index a08ecceaa2..41f7a4575a 100644 --- a/avocado/plugins/runners/avocado_instrumented.py +++ b/avocado/plugins/runners/avocado_instrumented.py @@ -46,7 +46,7 @@ class and method names should be separated by a ":". One @staticmethod def signal_handler(signum, frame): # pylint: disable=W0613 - if signum == signal.SIGTERM.value: + if signum in [signal.SIGTERM.value, signal.SIGALRM.value]: raise TestInterrupt("Test interrupted: Timeout reached") @staticmethod @@ -77,6 +77,7 @@ def _run_avocado(runnable, queue): # To be defined: if the resolution uri should be composed like # this, or broken down and stored into other data fields signal.signal(signal.SIGTERM, AvocadoInstrumentedTestRunner.signal_handler) + signal.signal(signal.SIGALRM, AvocadoInstrumentedTestRunner.signal_handler) module_path, klass_method = runnable.uri.split(":", 1) klass, method = klass_method.split(".", 1) diff --git a/docs/source/guides/writer/chapters/writing.rst b/docs/source/guides/writer/chapters/writing.rst index ac02fd5983..3c13f58067 100644 --- a/docs/source/guides/writer/chapters/writing.rst +++ b/docs/source/guides/writer/chapters/writing.rst @@ -773,6 +773,19 @@ runner task, making it raise a process is specific to spawner implementation, for more information see :class:`avocado.core.plugin_interfaces.Spawner.terminate_task`. + +Block Timeout +------------- +On more complex (and thus usually) longer tests, there may be multiple +steps to complete. It may be known that some of these steps should not +take more than a small percentage of the overall expected time for the +test as a whole. Therefore, it is not convenient to set the timeout for +the whole test, but it would be better to have timeout for each of those +steps. For such use-case avocado supports `wait_max` context manager, +which let you set specific timeout (in seconds) for a block of code: + +.. literalinclude:: ../../../../../examples/tests/blocktimeouttest.py + Timeout Factor ~~~~~~~~~~~~~~ @@ -810,6 +823,9 @@ test logs. For the previous test execution it shows:: ... [stdlog] 2023-11-29 11:16:23,746 test L0354 DEBUG| actual timeout: 6.0 + +.. note:: Be aweare that timeout factor will also affect timeouts created by `wait_max`` context manager. + Skipping Tests -------------- diff --git a/examples/tests/blocktimeouttest.py b/examples/tests/blocktimeouttest.py new file mode 100644 index 0000000000..eccb4c92c4 --- /dev/null +++ b/examples/tests/blocktimeouttest.py @@ -0,0 +1,22 @@ +import time + +from avocado import Test + + +class TimeoutTest(Test): + """ + Functional test for avocado. Throw a TestTimeoutError. + + :param sleep_time: How long should the test sleep + """ + + def test(self): + """ + This should throw a TestTimeoutError. + """ + with self.wait_max(3): + sleep_time = float(self.params.get("sleep_time", default=5.0)) + self.log.info( + "Sleeping for %.2f seconds (2 more than the timeout)", sleep_time + ) + time.sleep(sleep_time) diff --git a/selftests/check.py b/selftests/check.py index ea81932dfa..a427e7d816 100755 --- a/selftests/check.py +++ b/selftests/check.py @@ -29,7 +29,7 @@ "nrunner-requirement": 28, "unit": 678, "jobs": 11, - "functional-parallel": 307, + "functional-parallel": 308, "functional-serial": 7, "optional-plugins": 0, "optional-plugins-golang": 2, diff --git a/selftests/functional/basic.py b/selftests/functional/basic.py index cf47d72f10..39b75e8825 100644 --- a/selftests/functional/basic.py +++ b/selftests/functional/basic.py @@ -521,6 +521,32 @@ def test_runner_timeout(self): # Ensure no test aborted error messages show up self.assertNotIn(b"TestAbortError: Test aborted unexpectedly", output) + def test_runner_block_timeout(self): + cmd_line = ( + f"{AVOCADO} run --disable-sysinfo --job-results-dir " + f"{self.tmpdir.name} examples/tests/blocktimeouttest.py" + ) + result = process.run(cmd_line, ignore_status=True) + json_path = os.path.join(self.tmpdir.name, "latest", "results.json") + with open(json_path, encoding="utf-8") as json_file: + result_json = json.load(json_file) + output = result.stdout + expected_rc = exit_codes.AVOCADO_JOB_INTERRUPTED + unexpected_rc = exit_codes.AVOCADO_FAIL + self.assertNotEqual( + result.exit_status, + unexpected_rc, + f"Avocado crashed (rc {unexpected_rc}):\n{result}", + ) + self.assertEqual( + result.exit_status, + expected_rc, + f"Avocado did not return rc {expected_rc}:\n{result}", + ) + self.assertIn("Timeout reached", result_json["tests"][0]["fail_reason"]) + # Ensure no test aborted error messages show up + self.assertNotIn(b"TestAbortError: Test aborted unexpectedly", output) + def test_runner_timeout_factor(self): cmd_line = ( f"{AVOCADO} run --disable-sysinfo --job-results-dir "