Enable coverage calculation

.github/workflows/build.yaml

@@ -48,7 +48,7 @@ jobs:
       run: |
         pip install -e .
         rm tests/sim/_tsdz2.cdef # make sure cdef is generated from the source to check testing framework
-        pytest
+        pytest --coverage
     - name: Build
       run: |
         cd src
@@ -103,6 +103,7 @@ jobs:
         run: |
           pip install -e .
           pip install --upgrade pytest-md-report
+          pip install gcovr
       - name: Run tests
         env:
           REPORT_OUTPUT: md_report.md
@@ -111,7 +112,7 @@ jobs:
           rm tests/sim/_tsdz2.cdef # make sure cdef is generated from the source to check testing framework
 
           echo "REPORT_FILE=${REPORT_OUTPUT}" >> "$GITHUB_ENV"
-          pytest --md-report --md-report-flavor gfm --md-report-output "$REPORT_OUTPUT"
+          pytest --coverage --md-report --md-report-flavor gfm --md-report-output "$REPORT_OUTPUT"
       - name: Output reports to the job summary
         if: always()
         shell: bash
@@ -131,6 +132,16 @@ jobs:
           header: test-report
           recreate: true
           path: ${{ env.REPORT_FILE }}
+      - name: Collect coverage data
+        run: |
+          echo "### Coverage Report" >> $GITHUB_STEP_SUMMARY
+          gcovr --html-details -o coverage.html --print-summary >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage_report
+          path: |
+            coverage.html
 
   Compare_builds:
     needs: [Build_Windows, Build_Linux]

README.md

@@ -73,6 +73,8 @@ Run tests:
 
 `pytest`
 
+Calculate coverage and generate html report (probably will not work on Windows):
+`pytest --coverage`
 
 
 ### Compile the firmware manually

pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
     "pytest",
     "cffi >=1.16.0",
     "setuptools",
+    "gcovr",
 ]
 
 [project.urls]

tests/conftest.py

@@ -1,12 +1,17 @@
+import sys
 import subprocess
+import importlib
 from tests.load_c_code import load_code
 
+def pytest_addoption(parser):
+    parser.addoption("--coverage", action="store_true", help="Enable coverage analysis with gcovr")
+
 def pytest_sessionstart(session):
     """
     Called after the Session object has been created and
     before performing collection and entering the run test loop.
     """
-    load_code('_tsdz2')
+    lib, ffi = load_code('_tsdz2', coverage=session.config.option.coverage)
 
 
 def pytest_configure(config):
@@ -20,6 +25,20 @@ def pytest_sessionfinish(session, exitstatus):
     Called after whole test run finished, right before
     returning the exit status to the system.
     """
+
+    if session.config.option.coverage:
+        module = importlib.import_module("tests.sim._tsdz2")
+        module.lib.__gcov_flush()
+        # if "tests.sim._tsdz2" in sys.modules:
+        try:
+            # Attempt to call gcovr with the specified arguments
+            subprocess.call(['gcovr', '-r', 'tests', '--print-summary'])
+        except FileNotFoundError:
+            # Handle the case where gcovr is not found (i.e., not installed)
+            print(" Install Gcovr to generate code coverage report.")
+        except subprocess.CalledProcessError as e:
+            # E.g. gcov will fail if cffi compiled code with msvc
+            print(f"Error running gcovr: {e}")
 
 def pytest_unconfigure(config):
     """

tests/gcovr.cfg

@@ -0,0 +1,10 @@
+# Only show coverage for files in src/, lib/foo, or for main.cpp files.
+filter = sim/
+exclude-unreachable-branches=no
+exclude-noncode-lines=yes
+exclude-function-lines=yes
+
+
+html-details=coverage_report.html 
+html-self-contained=yes
+print-summary=yes

tests/load_c_code.py

@@ -149,11 +149,11 @@ def generate_cdef(module_name, src_file):
         fp.write(cdef)
     return cdef
 
-
-def load_code(module_name, force_recompile=False):
+def load_code(module_name, coverage=False, force_recompile=False):
     # Load previous combined hash
     hash_file_path = os.path.join(LIB_DIR, f"{module_name}.sha")
-    with Checksum(hash_file_path, source_dirs, module_name+"".join(define_macros)) as skip:
+    # Recalculate hash if code or arguments have changed
+    with Checksum(hash_file_path, source_dirs, module_name+"".join(define_macros)+str(coverage)) as skip:
         if not skip or force_recompile:
             print("Collecting source code..")
             source_content_list: List[str] = []
@@ -166,7 +166,6 @@ def load_code(module_name, force_recompile=False):
             combined_source = fake_defines + combined_source
             combined_source = re.sub(r"#\s*include\s*<.*?>", r"//\g<0>", combined_source) # comment out standard includes
             combined_source_file_path = os.path.join(LIB_DIR, f"{module_name}.i")
-
             with open(combined_source_file_path, "w", encoding="utf8") as fp:
                 fp.write(combined_source)
             try:
@@ -175,23 +174,35 @@ def load_code(module_name, force_recompile=False):
                 print(f"{e}\n\033[93mFailed to generate cdef using your cpp standard headers!!!\nYou may have to edit it manually. Continuing...\033[0m")
                 with open(os.path.join(LIB_DIR, f"{module_name}.cdef"), "r", encoding="utf8") as fp:
                     cdef = fp.read()
-
+            # Coverage
+            if coverage: # expose gcov api, (again coverage probably will not work on Windows)
+                combined_source =  "// GCOVR_EXCL_STOP\n\n" + combined_source + "\n// GCOVR_EXCL_START"  # add inner coverage exclusion markers
+                combined_source += "\n" + "extern void __gcov_flush(void);" + "\n"
+                cdef += "\n" + "extern void __gcov_flush(void);" + "\n"
+            extra_compile_args = compiler_args + ["--coverage"] if coverage else []
+            extra_link_args = linker_args + ["--coverage"] if coverage else []
+
             # Create a CFFI instance
             ffibuilder = cffi.FFI()
             print("Processing cdefs...")
             ffibuilder.cdef(cdef)
             ffibuilder.set_source(module_name, combined_source,
                                 include_dirs=[os.path.abspath(d) for d in include_dirs], 
                                 define_macros=[(macro, None) for macro in define_macros],
-                                extra_compile_args=compiler_args,
-                                extra_link_args=linker_args
+                                extra_compile_args=extra_compile_args,
+                                extra_link_args=extra_link_args
                                 )
             print("Compiling...")
             ffibuilder.compile(tmpdir=LIB_DIR)
+            if coverage: # Add outer coverage exclusion markers after generating cffi apis
+                with open(os.path.join(LIB_DIR, f"{module_name}.c"), "r+", encoding="utf8") as fp:
+                        c = fp.readlines()
+                        c[0] = c[0].strip() + " // GCOVR_EXCL_START"
+                        c[-1] = c[-1].strip() + " // GCOVR_EXCL_STOP"
+                        fp.seek(0); fp.writelines(c); fp.truncate()
         else:
             print("No changes found. Skipping compilation")
 
-
     module_path = LIB_DIR.replace("/", ".") + module_name
     print(f"Loading module: {module_path}")
     module = importlib.import_module(module_path)