wip

.github/workflows/buildAndCoverage.yml

@@ -0,0 +1,103 @@
+name: Build and collect test coverage
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [assigned, opened, synchronize, reopened]
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  build-repo:
+    name: Build and Code Coverage
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Get the project repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          submodules: "true"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install Python and other packages
+        run: |
+          pip install cmake numpy psutil pybind11 rich
+
+      - name: Install Ninja
+        run: sudo apt-get install -y ninja-build
+
+      - name: Install llvm-cov
+        run: sudo apt-get install -y clang lld llvm
+
+      - name: Get LLVM
+        id: clone-llvm
+        run: utils/clone-llvm.sh
+
+      - name: Get LLVM commit hash
+        id: get-llvm-commit-hash
+        run: echo "hash=$(cd llvm ; git log -1 --format='%H')" >> $GITHUB_OUTPUT
+
+      - name: Ccache for C++ compilation
+        # https://github.com/hendrikmuhs/ccache-action/releases/tag/v1.2.9
+        uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e
+        with:
+          key: ${{ runner.os }}-${{ matrix.ubuntu_version }}-${{ steps.get-llvm-commit-hash.outputs.hash }}
+          max-size: 1G
+
+      - name: Build and install LLVM
+        run: LLVM_ENABLE_RTTI=ON utils/build-llvm.sh
+
+      - name: Install our python reqs
+        run: pip install -r python/requirements.txt
+
+      - name: Build and generate coverage (Release)
+        run: |
+          mkdir build_release
+          cd build_release
+          cmake .. \
+            -GNinja \
+            -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_PLATFORM_NO_VERSIONED_SONAME=ON \
+            -DCMAKE_VISIBILITY_INLINES_HIDDEN=ON \
+            -DCMAKE_C_VISIBILITY_PRESET=hidden \
+            -DCMAKE_CXX_VISIBILITY_PRESET=hidden \
+            -DAIE_COMPILER=NONE \
+            -DAIE_LINKER=NONE \
+            -DHOST_COMPILER=NONE \
+            -DBUILD_INSTRUMENTED_COVERAGE=ON \
+            -DLLVM_ENABLE_ASSERTIONS=OFF \
+            -DLLVM_ENABLE_RTTI=ON \
+            -DCMAKE_MODULE_PATH=`pwd`/../cmake/modulesXilinx \
+            -DMLIR_DIR=/home/runner/work/mlir-aie/mlir-aie/llvm/install/lib/cmake/mlir \
+            -DLLVM_DIR=/home/runner/work/mlir-aie/mlir-aie/llvm/install/lib/cmake/llvm \
+            -DLLVM_USE_LINKER=lld \
+            -DLLVM_EXTERNAL_LIT=`pwd`/../llvm/build/bin/llvm-lit
+          
+          ninja && ninja generate-aie-coverage-report
+
+      - name: Add PR comment
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: '👋 Thanks for reporting!'
+            })

CMakeLists.txt

@@ -169,6 +169,13 @@ if(AIE_INCLUDE_DOCS)
   add_dependencies(docs mlir-doc)
 endif()
 
+set(LLVM_PROFILE_FILE_PATTERN "code-%p-%100000000m.profraw")
+append_if(BUILD_INSTRUMENTED_COVERAGE "-O0 --coverage -fprofile-arcs -ftest-coverage -fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\" -fcoverage-mapping"
+  CMAKE_CXX_FLAGS
+  CMAKE_C_FLAGS
+  CMAKE_EXE_LINKER_FLAGS
+  CMAKE_SHARED_LINKER_FLAGS)
+
 if(AIE_ENABLE_BINDINGS_PYTHON)
   include(MLIRDetectPythonEnv)
   mlir_configure_python_dev_packages()
@@ -230,3 +237,23 @@ add_subdirectory(reference_designs)
 add_subdirectory(test)
 add_subdirectory(tutorials)
 add_subdirectory(cmake/modules)
+
+get_filename_component(COMPILER_DIRECTORY ${CMAKE_CXX_COMPILER} DIRECTORY)
+find_program(LLVM_COV "llvm-cov" ${COMPILER_DIRECTORY} NO_DEFAULT_PATH)
+find_program(LLVM_PROFDATA "llvm-profdata" ${COMPILER_DIRECTORY} NO_DEFAULT_PATH)
+set(LLVM_PROFILE_DATA_DIR ${PROJECT_BINARY_DIR})
+file(TO_NATIVE_PATH "${CMAKE_BINARY_DIR}/report" REPORT_DIR)
+
+add_custom_target(generate-aie-coverage-report
+  COMMAND ${Python3_EXECUTABLE} ${AIE_SOURCE_DIR}/utils/prepare-code-coverage-artifact.py
+  ${LLVM_PROFDATA} ${LLVM_COV} ${LLVM_PROFILE_DATA_DIR}
+  ${REPORT_DIR}
+    ${CMAKE_BINARY_DIR}/bin/aie-opt
+#    ${CMAKE_BINARY_DIR}/bin/aie-translate
+  --unified-report --restrict
+    "${AIE_SOURCE_DIR}/lib"
+    "${AIE_SOURCE_DIR}/include"
+    "${AIE_SOURCE_DIR}/tools"
+    "${AIE_SOURCE_DIR}/python"
+  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  DEPENDS check-aie) # Run tests

utils/prepare-code-coverage-artifact.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+"""Prepare a code coverage artifact.
+
+- Collate raw profiles into one indexed profile.
+- Generate html reports for the given binaries.
+
+Caution: The positional arguments to this script must be specified before any 
+optional arguments, such as --restrict.
+"""
+
+import argparse
+import glob
+import os
+import subprocess
+import sys
+
+
+def merge_raw_profiles(host_llvm_profdata, profile_data_dir, preserve_profiles):
+    print(":: Merging raw profiles...", end="")
+    sys.stdout.flush()
+    raw_profiles = glob.glob(f"{profile_data_dir}/**/*.profraw", recursive=True)
+    manifest_path = os.path.join(profile_data_dir, "profiles.manifest")
+    profdata_path = os.path.join(profile_data_dir, "Coverage.profdata")
+    with open(manifest_path, "w") as manifest:
+        manifest.write("\n".join(raw_profiles))
+    subprocess.check_call(
+        [
+            host_llvm_profdata,
+            "merge",
+            "-sparse",
+            "-f",
+            manifest_path,
+            "-o",
+            profdata_path,
+        ]
+    )
+    if not preserve_profiles:
+        for raw_profile in raw_profiles:
+            os.remove(raw_profile)
+    # os.remove(manifest_path)
+    print("Done!")
+    return profdata_path
+
+
+def prepare_html_report(
+    host_llvm_cov, profile, report_dir, binaries, restricted_dirs, compilation_dir
+):
+    print(":: Preparing html report for {0}...".format(binaries), end="")
+    sys.stdout.flush()
+    objects = []
+    for i, binary in enumerate(binaries):
+        if i == 0:
+            objects.append(binary)
+        else:
+            objects.extend(("-object", binary))
+    invocation = (
+        [host_llvm_cov, "show"]
+        + objects
+        + [
+            "-format",
+            "html",
+            "-instr-profile",
+            profile,
+            "-o",
+            report_dir,
+            "-show-line-counts-or-regions",
+            # "-show-directory-coverage",
+            "-Xdemangler",
+            "c++filt",
+            "-Xdemangler",
+            "-n",
+        ]
+        + restricted_dirs
+    )
+    if compilation_dir:
+        invocation += ["-compilation-dir=" + compilation_dir]
+    subprocess.check_call(invocation)
+    with open(os.path.join(report_dir, "summary.txt"), "wb") as Summary:
+        subprocess.check_call(
+            [host_llvm_cov, "report"]
+            + objects
+            + ["-instr-profile", profile]
+            + restricted_dirs,
+            stdout=Summary,
+        )
+    print("Done!")
+
+
+def prepare_html_reports(
+    host_llvm_cov,
+    profdata_path,
+    report_dir,
+    binaries,
+    unified_report,
+    restricted_dirs,
+    compilation_dir,
+):
+    if unified_report:
+        prepare_html_report(
+            host_llvm_cov,
+            profdata_path,
+            report_dir,
+            binaries,
+            restricted_dirs,
+            compilation_dir,
+        )
+    else:
+        for binary in binaries:
+            binary_report_dir = os.path.join(report_dir, os.path.basename(binary))
+            prepare_html_report(
+                host_llvm_cov,
+                profdata_path,
+                binary_report_dir,
+                [binary],
+                restricted_dirs,
+                compilation_dir,
+            )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("host_llvm_profdata", help="Path to llvm-profdata")
+    parser.add_argument("host_llvm_cov", help="Path to llvm-cov")
+    parser.add_argument(
+        "profile_data_dir", help="Path to the directory containing the raw profiles"
+    )
+    parser.add_argument(
+        "report_dir", help="Path to the output directory for html reports"
+    )
+    parser.add_argument(
+        "binaries",
+        metavar="B",
+        type=str,
+        nargs="*",
+        help="Path to an instrumented binary",
+    )
+    parser.add_argument(
+        "--only-merge",
+        action="store_true",
+        help="Only merge raw profiles together, skip report " "generation",
+    )
+    parser.add_argument(
+        "--preserve-profiles", help="Do not delete raw profiles", action="store_true"
+    )
+    parser.add_argument(
+        "--use-existing-profdata", help="Specify an existing indexed profile to use"
+    )
+    parser.add_argument(
+        "--unified-report",
+        action="store_true",
+        help="Emit a unified report for all binaries",
+    )
+    parser.add_argument(
+        "--restrict",
+        metavar="R",
+        type=str,
+        nargs="*",
+        default=[],
+        help="Restrict the reporting to the given source paths"
+        " (must be specified after all other positional arguments)",
+    )
+    parser.add_argument(
+        "-C",
+        "--compilation-dir",
+        type=str,
+        default="",
+        help="The compilation directory of the binary",
+    )
+    args = parser.parse_args()
+
+    if args.use_existing_profdata and args.only_merge:
+        print("--use-existing-profdata and --only-merge are incompatible")
+        exit(1)
+
+    if args.use_existing_profdata:
+        profdata_path = args.use_existing_profdata
+    else:
+        profdata_path = merge_raw_profiles(
+            args.host_llvm_profdata, args.profile_data_dir, args.preserve_profiles
+        )
+
+    if not len(args.binaries):
+        print("No binaries specified, no work to do!")
+        exit(1)
+
+    if not args.only_merge:
+        prepare_html_reports(
+            args.host_llvm_cov,
+            profdata_path,
+            args.report_dir,
+            args.binaries,
+            args.unified_report,
+            args.restrict,
+            args.compilation_dir,
+        )