Merge pull request PolyJIT#560 from PolyJIT/simbuerg/add-wrapper-to-c…

…overage

feat(coverage): add wrapped binaries to coverage

.coveragerc

@@ -7,6 +7,7 @@ branch = True
 parallel = True
 
 [paths]
+data_file = ${BB_COVERAGE_PATH-default .}/.coverage
 source =
   benchbuild/
   */site-packages/benchbuild/

.github/workflows/ci.yml

@@ -54,6 +54,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.9", "3.10"]
+        db_support: [true, false]
 
     steps:
     - uses: actions/checkout@v2
@@ -76,8 +77,13 @@ jobs:
 
     - name: Run integration tests
       env:
-        BB_CONTAINER_ROOT: '/tmp'
-        BB_CONTAINER_RUNROOT: '/tmp'
+        BB_DB_ENABLED: ${{ matrix.db_support }}
+        BB_DB_CONNECT_STRING: "sqlite://"
+        BB_COVERAGE_COLLECT: true
+        BB_COVERAGE_PATH: ${{ github.workspace }}
+        BB_COVERAGE_CONFIG: ${{ github.workspace }}/.coveragerc
+        BB_CONTAINER_ROOT: ${{ runner.temp }}
+        BB_CONTAINER_RUNROOT: ${{ runner.temp }}
         BB_VERBOSITY: 5
       run: |
         coverage run -p `which benchbuild` bootstrap -s
@@ -93,6 +99,7 @@ jobs:
         coverage run -p `which benchbuild` container bases --import -E raw bzip2/benchbuild
         coverage run -p `which benchbuild` container rmi --with-projects -E raw bzip2/benchbuild
         coverage combine
+        coverage report -m
 
     - uses: actions/upload-artifact@master
       with:

benchbuild/res/wrapping/run_compiler.py.inc

@@ -2,6 +2,11 @@
 #
 import os
 import sys
+
+{% if collect_coverage %}
+import coverage
+{% endif %}
+
 os.environ["OPENBLAS_NUM_THREADS"] = "4"
 
 from plumbum import TEE, local
@@ -57,4 +62,18 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv))
+{% if collect_coverage %}
+    cov = coverage.Coverage(
+      config_file="{{ coverage_config }}",
+      data_file="{{ coverage_path }}/.coverage",
+      data_suffix=True,
+      branch=True
+    )
+    cov.start()
+{% endif %}
+    ret = main(sys.argv)
+{% if collect_coverage %}
+    cov.stop()
+    cov.save()
+{% endif %}
+    sys.exit(ret)

benchbuild/res/wrapping/run_dynamic.py.inc

@@ -6,6 +6,10 @@ import sys
 
 from plumbum import TEE, local
 
+{% if collect_coverage %}
+import coverage
+{% endif %}
+
 # Performance optimization for benchbuild: don't import any experiments or
 # projects. Everything necessary should be imported when loading (unpickling)
 # the project.
@@ -53,4 +57,18 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv))
+{% if collect_coverage %}
+    cov = coverage.Coverage(
+      config_file="{{ coverage_config }}",
+      data_file="{{ coverage_path }}/.coverage",
+      data_suffix=True,
+      branch=True
+    )
+    cov.start()
+{% endif %}
+    ret = main(sys.argv)
+{% if collect_coverage %}
+    cov.stop()
+    cov.save()
+{% endif %}
+    sys.exit(ret)

benchbuild/res/wrapping/run_static.py.inc

@@ -3,6 +3,10 @@
 import os
 import sys
 
+{% if collect_coverage %}
+import coverage
+{% endif %}
+
 from plumbum import TEE, local
 
 # Performance optimization for benchbuild: don't import any experiments or
@@ -38,4 +42,18 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv))
+{% if collect_coverage %}
+    cov = coverage.Coverage(
+      config_file="{{ coverage_config }}",
+      data_file="{{ coverage_path }}/.coverage",
+      data_suffix=True,
+      branch=True
+    )
+    cov.start()
+{% endif %}
+    ret = main(sys.argv)
+{% if collect_coverage %}
+    cov.stop()
+    cov.save()
+{% endif %}
+    sys.exit(ret)

benchbuild/settings.py

@@ -151,7 +151,7 @@
     },
     "connect_string": {
         "desc": "sqlalchemy connect string",
-        "default": ""
+        "default": "sqlite://"
     },
     "rollback": {
         "desc": "Rollback all operations after benchbuild completes.",
@@ -513,5 +513,20 @@
     }
 }
 
+CFG["coverage"] = {
+    "collect": {
+        "desc": "Should benchuild collect coverage inside wrapped binaries.",
+        "default": False
+    },
+    "config": {
+        "desc": "Where is the coverage config?",
+        "default": ".coveragerc"
+    },
+    "path": {
+        "desc": "Where should the coverage files be placed?",
+        "default": None
+    }
+}
+
 s.setup_config(CFG)
 s.update_env(CFG)

benchbuild/utils/wrapping.py

@@ -133,6 +133,10 @@ def wrap(
 
     env = CFG["env"].value
 
+    collect_coverage = bool(CFG["coverage"]["collect"])
+    coverage_config = str(CFG["coverage"]["config"])
+    coverage_path = str(CFG["coverage"]["path"])
+
     bin_path = list_to_path(env.get("PATH", []))
     bin_path = list_to_path([bin_path, os.environ["PATH"]])
 
@@ -149,6 +153,9 @@ def wrap(
                 ld_library_path=str(bin_lib_path),
                 home=str(home),
                 python=python,
+                collect_coverage=collect_coverage,
+                coverage_config=coverage_config,
+                coverage_path=coverage_path
             )
         )
 
@@ -208,6 +215,9 @@ def wrap_dynamic(
     project_file = persist(project, suffix=".project")
 
     cfg_env = CFG["env"].value
+    collect_coverage = bool(CFG["coverage"]["collect"])
+    coverage_config = str(CFG["coverage"]["config"])
+    coverage_path = str(CFG["coverage"]["path"])
 
     bin_path = list_to_path(cfg_env.get("PATH", []))
     bin_path = list_to_path([bin_path, os.environ["PATH"]])
@@ -226,6 +236,9 @@ def wrap_dynamic(
                 home=str(home),
                 python=python,
                 name_filters=name_filters,
+                collect_coverage=collect_coverage,
+                coverage_config=coverage_config,
+                coverage_path=coverage_path
             )
         )
 
@@ -266,13 +279,20 @@ def wrap_cc(
 
     project_file = persist(project, suffix=".project")
 
+    collect_coverage = bool(CFG["coverage"]["collect"])
+    coverage_config = str(CFG["coverage"]["config"])
+    coverage_path = str(CFG["coverage"]["path"])
+
     with open(filepath, "w") as wrapper:
         wrapper.write(
             template.render(
                 cc_f=str(cc_f),
                 project_file=str(project_file),
                 python=python,
                 detect_project=detect_project,
+                collect_coverage=collect_coverage,
+                coverage_config=coverage_config,
+                coverage_path=coverage_path
             )
         )