From 6d3c7dedfd868e652fece7d70700fbc42b009af2 Mon Sep 17 00:00:00 2001 From: dzid26 Date: Wed, 10 Jul 2024 16:28:39 +0100 Subject: [PATCH] Enable coverage calculation --- .github/workflows/build.yaml | 15 +++++++++++++-- README.md | 2 ++ pyproject.toml | 1 + tests/conftest.py | 21 ++++++++++++++++++++- tests/gcovr.cfg | 10 ++++++++++ tests/load_c_code.py | 27 +++++++++++++++++++-------- 6 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 tests/gcovr.cfg diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 046c4233..a073b686 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -48,7 +48,7 @@ jobs: run: | pip install -e . rm tests/sim/_tsdz2.cdef # make sure cdef is generated from the source to check testing framework - pytest + pytest --coverage - name: Build run: | cd src @@ -103,6 +103,7 @@ jobs: run: | pip install -e . pip install --upgrade pytest-md-report + pip install gcovr - name: Run tests env: REPORT_OUTPUT: md_report.md @@ -111,7 +112,7 @@ jobs: rm tests/sim/_tsdz2.cdef # make sure cdef is generated from the source to check testing framework echo "REPORT_FILE=${REPORT_OUTPUT}" >> "$GITHUB_ENV" - pytest --md-report --md-report-flavor gfm --md-report-output "$REPORT_OUTPUT" + pytest --coverage --md-report --md-report-flavor gfm --md-report-output "$REPORT_OUTPUT" - name: Output reports to the job summary if: always() shell: bash @@ -131,6 +132,16 @@ jobs: header: test-report recreate: true path: ${{ env.REPORT_FILE }} + - name: Collect coverage data + run: | + echo "### Coverage Report" >> $GITHUB_STEP_SUMMARY + gcovr --html-details -o coverage.html --print-summary >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + - uses: actions/upload-artifact@v4 + with: + name: coverage_report + path: | + coverage.html Compare_builds: needs: [Build_Windows, Build_Linux] diff --git a/README.md b/README.md index f37a3854..d25b3fe8 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,8 @@ Run tests: `pytest` +Calculate coverage and generate html report (probably will not work on Windows): +`pytest --coverage` ### Compile the firmware manually diff --git a/pyproject.toml b/pyproject.toml index e80e7cf7..d94deb8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "pytest", "cffi >=1.16.0", "setuptools", + "gcovr", ] [project.urls] diff --git a/tests/conftest.py b/tests/conftest.py index d9a4be8b..893b2828 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,17 @@ +import sys import subprocess +import importlib from tests.load_c_code import load_code +def pytest_addoption(parser): + parser.addoption("--coverage", action="store_true", help="Enable coverage analysis with gcovr") + def pytest_sessionstart(session): """ Called after the Session object has been created and before performing collection and entering the run test loop. """ - load_code('_tsdz2') + lib, ffi = load_code('_tsdz2', coverage=session.config.option.coverage) def pytest_configure(config): @@ -20,6 +25,20 @@ def pytest_sessionfinish(session, exitstatus): Called after whole test run finished, right before returning the exit status to the system. """ + + if session.config.option.coverage: + module = importlib.import_module("tests.sim._tsdz2") + module.lib.__gcov_flush() + # if "tests.sim._tsdz2" in sys.modules: + try: + # Attempt to call gcovr with the specified arguments + subprocess.call(['gcovr', '-r', 'tests', '--print-summary']) + except FileNotFoundError: + # Handle the case where gcovr is not found (i.e., not installed) + print(" Install Gcovr to generate code coverage report.") + except subprocess.CalledProcessError as e: + # E.g. gcov will fail if cffi compiled code with msvc + print(f"Error running gcovr: {e}") def pytest_unconfigure(config): """ diff --git a/tests/gcovr.cfg b/tests/gcovr.cfg new file mode 100644 index 00000000..85b82abb --- /dev/null +++ b/tests/gcovr.cfg @@ -0,0 +1,10 @@ +# Only show coverage for files in src/, lib/foo, or for main.cpp files. +filter = sim/ +exclude-unreachable-branches=no +exclude-noncode-lines=yes +exclude-function-lines=yes + + +html-details=coverage_report.html +html-self-contained=yes +print-summary=yes \ No newline at end of file diff --git a/tests/load_c_code.py b/tests/load_c_code.py index 896c2e18..3549fd08 100644 --- a/tests/load_c_code.py +++ b/tests/load_c_code.py @@ -149,11 +149,11 @@ def generate_cdef(module_name, src_file): fp.write(cdef) return cdef - -def load_code(module_name, force_recompile=False): +def load_code(module_name, coverage=False, force_recompile=False): # Load previous combined hash hash_file_path = os.path.join(LIB_DIR, f"{module_name}.sha") - with Checksum(hash_file_path, source_dirs, module_name+"".join(define_macros)) as skip: + # Recalculate hash if code or arguments have changed + with Checksum(hash_file_path, source_dirs, module_name+"".join(define_macros)+str(coverage)) as skip: if not skip or force_recompile: print("Collecting source code..") source_content_list: List[str] = [] @@ -166,7 +166,6 @@ def load_code(module_name, force_recompile=False): combined_source = fake_defines + combined_source combined_source = re.sub(r"#\s*include\s*<.*?>", r"//\g<0>", combined_source) # comment out standard includes combined_source_file_path = os.path.join(LIB_DIR, f"{module_name}.i") - with open(combined_source_file_path, "w", encoding="utf8") as fp: fp.write(combined_source) try: @@ -175,7 +174,14 @@ def load_code(module_name, force_recompile=False): print(f"{e}\n\033[93mFailed to generate cdef using your cpp standard headers!!!\nYou may have to edit it manually. Continuing...\033[0m") with open(os.path.join(LIB_DIR, f"{module_name}.cdef"), "r", encoding="utf8") as fp: cdef = fp.read() - + # Coverage + if coverage: # expose gcov api, (again coverage probably will not work on Windows) + combined_source = "// GCOVR_EXCL_STOP\n\n" + combined_source + "\n// GCOVR_EXCL_START" # add inner coverage exclusion markers + combined_source += "\n" + "extern void __gcov_flush(void);" + "\n" + cdef += "\n" + "extern void __gcov_flush(void);" + "\n" + extra_compile_args = compiler_args + ["--coverage"] if coverage else [] + extra_link_args = linker_args + ["--coverage"] if coverage else [] + # Create a CFFI instance ffibuilder = cffi.FFI() print("Processing cdefs...") @@ -183,15 +189,20 @@ def load_code(module_name, force_recompile=False): ffibuilder.set_source(module_name, combined_source, include_dirs=[os.path.abspath(d) for d in include_dirs], define_macros=[(macro, None) for macro in define_macros], - extra_compile_args=compiler_args, - extra_link_args=linker_args + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args ) print("Compiling...") ffibuilder.compile(tmpdir=LIB_DIR) + if coverage: # Add outer coverage exclusion markers after generating cffi apis + with open(os.path.join(LIB_DIR, f"{module_name}.c"), "r+", encoding="utf8") as fp: + c = fp.readlines() + c[0] = c[0].strip() + " // GCOVR_EXCL_START" + c[-1] = c[-1].strip() + " // GCOVR_EXCL_STOP" + fp.seek(0); fp.writelines(c); fp.truncate() else: print("No changes found. Skipping compilation") - module_path = LIB_DIR.replace("/", ".") + module_name print(f"Loading module: {module_path}") module = importlib.import_module(module_path)