From abd65550dea4dc72b46fb7225b5e2befd1a06e1b Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Thu, 23 May 2024 10:57:28 +0200 Subject: [PATCH 1/4] Coverage.py simplification This commit simplifies the usage of coverage.py without change of its behaviour. With these changes, it will be easier for users to set up avocado for gathering test coverage data. Reference: #5919 Signed-off-by: Jan Richter --- .../plugins/runners/avocado_instrumented.py | 16 ++++++++-------- .../guides/writer/chapters/integrating.rst | 18 +++++------------- requirements-dev.txt | 2 +- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/avocado/plugins/runners/avocado_instrumented.py b/avocado/plugins/runners/avocado_instrumented.py index d4bec69575..a08ecceaa2 100644 --- a/avocado/plugins/runners/avocado_instrumented.py +++ b/avocado/plugins/runners/avocado_instrumented.py @@ -98,21 +98,21 @@ def _run_avocado(runnable, queue): messages.start_logging(runnable.config, queue) - if "COVERAGE_RUN" in os.environ: - from coverage import Coverage - - coverage = Coverage() - coverage.start() - instance = loader.load_test(test_factory) early_state = instance.get_state() early_state["type"] = "early_state" queue.put(early_state) - instance.run_avocado() + # running the actual test if "COVERAGE_RUN" in os.environ: - coverage.stop() + from coverage import Coverage + + coverage = Coverage(data_suffix=True) + with coverage.collect(): + instance.run_avocado() coverage.save() + else: + instance.run_avocado() state = instance.get_state() fail_reason = state.get("fail_reason") diff --git a/docs/source/guides/writer/chapters/integrating.rst b/docs/source/guides/writer/chapters/integrating.rst index 23150b7a3d..adc169d98d 100644 --- a/docs/source/guides/writer/chapters/integrating.rst +++ b/docs/source/guides/writer/chapters/integrating.rst @@ -9,24 +9,16 @@ like which parts are being exercised by the tests, may help develop new tests. `Coverage.py`_ is a tool designed for measuring code coverage of Python programs. It runs monitoring the program's source, taking notes of which -parts of the code have been executed. - -It is possible to use Coverage.py while running Avocado Instrumented tests. -As Avocado spawn sub-processes to run the tests, the `concurrency` parameter -should be set to `multiprocessing`. +parts of the code have been executed. It is possible to use Coverage.py while +running Avocado Instrumented tests. To make the Coverage.py parameters visible to other processes spawned by -Avocado, create the ``.coveragerc`` file in the project's root folder. +Avocado, create the ``.coveragerc`` file in the project's root folder and set +``source`` parameter to your system under test. Following is an example:: [run] - concurrency = multiprocessing source = foo/bar - parallel = true - -According to the documentation of Coverage.py, when measuring coverage in -a multi-process program, setting the `parallel` parameter will keep the data -separate during the measurement. With the ``.coveragerc`` file set, one possible workflow to use Coverage.py to measure Avocado tests is:: @@ -43,6 +35,6 @@ coverage measurement. For other options related to `Coverage.py`_, visit the software documentation. .. note:: Currently coverage support is limited working only with - `ProcessSpawner` (the default spawner). + `ProcessSpawner` (the default spawner) and Coverage.py>=7.5. .. _Coverage.py: https://coverage.readthedocs.io/ diff --git a/requirements-dev.txt b/requirements-dev.txt index 3faaada25d..ed1b5ee31f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ pylint==2.17.2 autopep8==1.6.0 black==24.3.0 -coverage==5.5 +coverage==7.5 # To run make check psutil==5.9.5 From 02d642ee02fe9fa144688c28b34c5bb1c0c65604 Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Thu, 23 May 2024 14:25:26 +0200 Subject: [PATCH 2/4] coverage.py support for python unittests This commit brings coverage.py feature to python unittest runner. After this change, you can run coverage.py with avocado for avocado-instrumented and python unittest types. Reference: #4957 Signed-off-by: Jan Richter --- avocado/plugins/runners/python_unittest.py | 11 ++++++++++- docs/source/guides/writer/chapters/integrating.rst | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/avocado/plugins/runners/python_unittest.py b/avocado/plugins/runners/python_unittest.py index c9a717efbe..de26d273d1 100644 --- a/avocado/plugins/runners/python_unittest.py +++ b/avocado/plugins/runners/python_unittest.py @@ -109,7 +109,16 @@ def _run_unittest(cls, module_path, module_class_method, queue): return runner = TextTestRunner(stream=stream, verbosity=0) - unittest_result = runner.run(suite) + # running the actual test + if "COVERAGE_RUN" in os.environ: + from coverage import Coverage + + coverage = Coverage(data_suffix=True) + with coverage.collect(): + unittest_result = runner.run(suite) + coverage.save() + else: + unittest_result = runner.run(suite) unittest_result_entries = None if len(unittest_result.errors) > 0: diff --git a/docs/source/guides/writer/chapters/integrating.rst b/docs/source/guides/writer/chapters/integrating.rst index adc169d98d..67c880982b 100644 --- a/docs/source/guides/writer/chapters/integrating.rst +++ b/docs/source/guides/writer/chapters/integrating.rst @@ -10,7 +10,7 @@ like which parts are being exercised by the tests, may help develop new tests. `Coverage.py`_ is a tool designed for measuring code coverage of Python programs. It runs monitoring the program's source, taking notes of which parts of the code have been executed. It is possible to use Coverage.py while -running Avocado Instrumented tests. +running Avocado Instrumented tests or Python unittests. To make the Coverage.py parameters visible to other processes spawned by Avocado, create the ``.coveragerc`` file in the project's root folder and set From fceb0c2c0018de4b4b2fd5c00700f6542db46721 Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Thu, 23 May 2024 15:44:58 +0200 Subject: [PATCH 3/4] check.py for running avocado coverage.py The avocado was using `selftests/run` script for running coverage tests. This script uses python unittests module instead of avocado, which has few disadvantages. The main is that running the `selftests/run` differs from running `check.py`, therefore the coverage report hasn't represented the actual state. After the latest changes in ea0e23dc and 4100cc22 it is possible to run avocado's main selftests entry point `check.py` with the coverage.py. With such change, the running coverage diagnose will use the same process as our selftests, therefore the report will be more precise. Reference: #4992 Signed-off-by: Jan Richter --- .coveragerc | 2 ++ .github/workflows/ci.yml | 6 ++++-- selftests/run_coverage | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..146151a521 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +source = avocado/, optional_plugins/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4981b7c11..5575e05c8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,9 +132,11 @@ jobs: - name: Run pre script run: ./cc-test-reporter before-build - name: Run script - run: make develop && ./selftests/run_coverage + run: | + python setup.py develop --user + ./selftests/run_coverage - name: Run post script - run: ./cc-test-reporter after-build + run: ./cc-test-reporter after-build --debug - run: echo "🥑 This job's status is ${{ job.status }}." diff --git a/selftests/run_coverage b/selftests/run_coverage index 9457131035..22583020b7 100755 --- a/selftests/run_coverage +++ b/selftests/run_coverage @@ -15,8 +15,8 @@ echo "Using coverage utility: $COVERAGE" $COVERAGE erase rm -f .coverage.* -RUNNING_COVERAGE=1 AVOCADO_CHECK_LEVEL=1 UNITTEST_AVOCADO_CMD="$COVERAGE run -p --include 'avocado/*,optional_plugins/*' $PYTHON -m avocado" $COVERAGE run -p --include "avocado/*,optional_plugins/*" ./selftests/run -$COVERAGE combine .coverage* +$COVERAGE run selftests/check.py --skip=static-checks +$COVERAGE combine echo $COVERAGE report -m --include "avocado/core/*" echo From 951de45acfa6fa8e47d65439219f0272528980b3 Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Thu, 23 May 2024 15:59:12 +0200 Subject: [PATCH 4/4] selftests/run removal After the changes in bf58ffc9 the only used selftests entry point is `check.py`, therefore we can remove the `selftests/run` script from avocado repo. Reference: #4992 Signed-off-by: Jan Richter --- selftests/run | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100755 selftests/run diff --git a/selftests/run b/selftests/run deleted file mode 100755 index 44e7905783..0000000000 --- a/selftests/run +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -__author__ = 'Lucas Meneghel Rodrigues ' - -import sys -import unittest - -from selftests.utils import test_suite - -if __name__ == '__main__': - runner = unittest.TextTestRunner(verbosity=1, - resultclass=unittest.TextTestResult) - result = runner.run(test_suite()) - if result.failures or result.errors: - sys.exit(1)