diff --git a/.github/workflows/buildAndCoverage.yml b/.github/workflows/buildAndCoverage.yml new file mode 100644 index 0000000000..e581b823ff --- /dev/null +++ b/.github/workflows/buildAndCoverage.yml @@ -0,0 +1,118 @@ +name: Build and collect test coverage + +on: + push: + branches: + - main + pull_request: + types: [assigned, opened, synchronize, reopened] + workflow_dispatch: + +defaults: + run: + shell: bash + +env: + DEBIAN_FRONTEND: noninteractive + +jobs: + build-repo: + name: Build and Code Coverage + + runs-on: ubuntu-latest + + permissions: + contents: write + id-token: write + + steps: + - name: Get the project repository + uses: actions/checkout@v3 + with: + fetch-depth: 2 + submodules: "true" + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Python and other packages + run: | + pip install cmake numpy psutil pybind11 rich + + - name: Install Ninja + run: sudo apt-get install -y ninja-build + + - name: Install llvm-cov + run: sudo apt-get install -y clang lld llvm + + - name: Get LLVM + id: clone-llvm + run: utils/clone-llvm.sh + + - name: Get LLVM commit hash + id: get-llvm-commit-hash + run: echo "hash=$(cd llvm ; git log -1 --format='%H')" >> $GITHUB_OUTPUT + + - name: Ccache for C++ compilation + # https://github.com/hendrikmuhs/ccache-action/releases/tag/v1.2.9 + uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e + with: + key: ${{ runner.os }}-${{ matrix.ubuntu_version }}-${{ steps.get-llvm-commit-hash.outputs.hash }} + max-size: 1G + + - name: Build and install LLVM + run: LLVM_ENABLE_RTTI=ON utils/build-llvm.sh + + - name: Install our python reqs + run: pip install -r python/requirements.txt + + - name: Build and generate coverage (Release) + run: | + mkdir build_release + cd build_release + cmake .. \ + -GNinja \ + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PLATFORM_NO_VERSIONED_SONAME=ON \ + -DCMAKE_VISIBILITY_INLINES_HIDDEN=ON \ + -DCMAKE_C_VISIBILITY_PRESET=hidden \ + -DCMAKE_CXX_VISIBILITY_PRESET=hidden \ + -DAIE_COMPILER=NONE \ + -DAIE_LINKER=NONE \ + -DHOST_COMPILER=NONE \ + -DBUILD_INSTRUMENTED_COVERAGE=ON \ + -DLLVM_ENABLE_ASSERTIONS=OFF \ + -DLLVM_ENABLE_RTTI=ON \ + -DCMAKE_MODULE_PATH=`pwd`/../cmake/modulesXilinx \ + -DMLIR_DIR=/home/runner/work/mlir-aie/mlir-aie/llvm/install/lib/cmake/mlir \ + -DLLVM_DIR=/home/runner/work/mlir-aie/mlir-aie/llvm/install/lib/cmake/llvm \ + -DLLVM_USE_LINKER=lld \ + -DLLVM_EXTERNAL_LIT=`pwd`/../llvm/build/bin/llvm-lit + + ninja && ninja generate-aie-coverage-report + + - name: Format coverage report + id: format-report + run: | + sed -i.bak "s/class='column-entry-bold'/style='font-weight: bold; text-align: left;'/g" /home/runner/work/mlir-aie/mlir-aie/build_release/report/index.html + sed -i.bak "s/class='column-entry-yellow'/style='text-align: left; background-color: #ffffd0;'/g" /home/runner/work/mlir-aie/mlir-aie/build_release/report/index.html + sed -i.bak "s/class='column-entry-red'/style='text-align: left; background-color: #ffd0d0;'/g" /home/runner/work/mlir-aie/mlir-aie/build_release/report/index.html + sed -i.bak "s/class='column-entry-green'/style='text-align: left; background-color: #d0ffd0;'/g" /home/runner/work/mlir-aie/mlir-aie/build_release/report/index.html + sed -i.bak "s///g" /home/runner/work/mlir-aie/mlir-aie/build_release/report/index.html + + echo "report=$(cat /home/runner/work/mlir-aie/mlir-aie/build_release/report/index.html)" >> $GITHUB_OUTPUT + + + - name: Update PR with coverage results + uses: edumserrano/find-create-or-update-comment@v2 + with: + issue-number: ${{ github.event.pull_request.number }} + body-includes: '' + comment-author: 'github-actions[bot]' + body: | # can be a single value or you can compose text with multi-line values + ${{ steps.format-report.outputs.report }} + edit-mode: replace \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index fe894caa1c..ab1e388952 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,13 @@ if(AIE_INCLUDE_DOCS) add_dependencies(docs mlir-doc) endif() +set(LLVM_PROFILE_FILE_PATTERN "code-%p-%100000000m.profraw") +append_if(BUILD_INSTRUMENTED_COVERAGE "-O0 -fprofile-arcs -ftest-coverage -fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\" -fcoverage-mapping" + CMAKE_CXX_FLAGS + CMAKE_C_FLAGS + CMAKE_EXE_LINKER_FLAGS + CMAKE_SHARED_LINKER_FLAGS) + if(AIE_ENABLE_BINDINGS_PYTHON) include(MLIRDetectPythonEnv) mlir_configure_python_dev_packages() @@ -230,3 +237,23 @@ add_subdirectory(reference_designs) add_subdirectory(test) add_subdirectory(tutorials) add_subdirectory(cmake/modules) + +get_filename_component(COMPILER_DIRECTORY ${CMAKE_CXX_COMPILER} DIRECTORY) +find_program(LLVM_COV "llvm-cov" ${COMPILER_DIRECTORY} NO_DEFAULT_PATH) +find_program(LLVM_PROFDATA "llvm-profdata" ${COMPILER_DIRECTORY} NO_DEFAULT_PATH) +set(LLVM_PROFILE_DATA_DIR ${PROJECT_BINARY_DIR}) +file(TO_NATIVE_PATH "${CMAKE_BINARY_DIR}/report" REPORT_DIR) + +add_custom_target(generate-aie-coverage-report + COMMAND ${Python3_EXECUTABLE} ${AIE_SOURCE_DIR}/utils/prepare-code-coverage-artifact.py + ${LLVM_PROFDATA} ${LLVM_COV} ${LLVM_PROFILE_DATA_DIR} + ${REPORT_DIR} + ${CMAKE_BINARY_DIR}/bin/aie-opt +# ${CMAKE_BINARY_DIR}/bin/aie-translate + --unified-report --restrict + "${AIE_SOURCE_DIR}/lib" + "${AIE_SOURCE_DIR}/include" + "${AIE_SOURCE_DIR}/tools" + "${AIE_SOURCE_DIR}/python" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + DEPENDS check-aie) # Run tests \ No newline at end of file diff --git a/utils/prepare-code-coverage-artifact.py b/utils/prepare-code-coverage-artifact.py new file mode 100755 index 0000000000..d4a47a57a8 --- /dev/null +++ b/utils/prepare-code-coverage-artifact.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python + +from __future__ import print_function + +"""Prepare a code coverage artifact. + +- Collate raw profiles into one indexed profile. +- Generate html reports for the given binaries. + +Caution: The positional arguments to this script must be specified before any +optional arguments, such as --restrict. +""" + +import argparse +import glob +import os +import subprocess +import sys + + +def merge_raw_profiles(host_llvm_profdata, profile_data_dir, preserve_profiles): + print(":: Merging raw profiles...", end="") + sys.stdout.flush() + raw_profiles = glob.glob(f"{profile_data_dir}/**/*.profraw", recursive=True) + manifest_path = os.path.join(profile_data_dir, "profiles.manifest") + profdata_path = os.path.join(profile_data_dir, "Coverage.profdata") + with open(manifest_path, "w") as manifest: + manifest.write("\n".join(raw_profiles)) + subprocess.check_call( + [ + host_llvm_profdata, + "merge", + "-sparse", + "-f", + manifest_path, + "-o", + profdata_path, + ] + ) + if not preserve_profiles: + for raw_profile in raw_profiles: + os.remove(raw_profile) + # os.remove(manifest_path) + print("Done!") + return profdata_path + + +def prepare_html_report( + host_llvm_cov, profile, report_dir, binaries, restricted_dirs, compilation_dir +): + print(":: Preparing html report for {0}...".format(binaries), end="") + sys.stdout.flush() + objects = [] + for i, binary in enumerate(binaries): + if i == 0: + objects.append(binary) + else: + objects.extend(("-object", binary)) + invocation = ( + [host_llvm_cov, "show"] + + objects + + [ + "-format", + "html", + "-instr-profile", + profile, + "-o", + report_dir, + "-show-line-counts-or-regions", + # "-show-directory-coverage", + "-Xdemangler", + "c++filt", + "-Xdemangler", + "-n", + ] + + restricted_dirs + ) + if compilation_dir: + invocation += ["-compilation-dir=" + compilation_dir] + subprocess.check_call(invocation) + with open(os.path.join(report_dir, "summary.txt"), "wb") as Summary: + subprocess.check_call( + [host_llvm_cov, "report"] + + objects + + ["-instr-profile", profile] + + restricted_dirs, + stdout=Summary, + ) + print("Done!") + + +def prepare_html_reports( + host_llvm_cov, + profdata_path, + report_dir, + binaries, + unified_report, + restricted_dirs, + compilation_dir, +): + if unified_report: + prepare_html_report( + host_llvm_cov, + profdata_path, + report_dir, + binaries, + restricted_dirs, + compilation_dir, + ) + else: + for binary in binaries: + binary_report_dir = os.path.join(report_dir, os.path.basename(binary)) + prepare_html_report( + host_llvm_cov, + profdata_path, + binary_report_dir, + [binary], + restricted_dirs, + compilation_dir, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("host_llvm_profdata", help="Path to llvm-profdata") + parser.add_argument("host_llvm_cov", help="Path to llvm-cov") + parser.add_argument( + "profile_data_dir", help="Path to the directory containing the raw profiles" + ) + parser.add_argument( + "report_dir", help="Path to the output directory for html reports" + ) + parser.add_argument( + "binaries", + metavar="B", + type=str, + nargs="*", + help="Path to an instrumented binary", + ) + parser.add_argument( + "--only-merge", + action="store_true", + help="Only merge raw profiles together, skip report " "generation", + ) + parser.add_argument( + "--preserve-profiles", help="Do not delete raw profiles", action="store_true" + ) + parser.add_argument( + "--use-existing-profdata", help="Specify an existing indexed profile to use" + ) + parser.add_argument( + "--unified-report", + action="store_true", + help="Emit a unified report for all binaries", + ) + parser.add_argument( + "--restrict", + metavar="R", + type=str, + nargs="*", + default=[], + help="Restrict the reporting to the given source paths" + " (must be specified after all other positional arguments)", + ) + parser.add_argument( + "-C", + "--compilation-dir", + type=str, + default="", + help="The compilation directory of the binary", + ) + args = parser.parse_args() + + if args.use_existing_profdata and args.only_merge: + print("--use-existing-profdata and --only-merge are incompatible") + exit(1) + + if args.use_existing_profdata: + profdata_path = args.use_existing_profdata + else: + profdata_path = merge_raw_profiles( + args.host_llvm_profdata, args.profile_data_dir, args.preserve_profiles + ) + + if not len(args.binaries): + print("No binaries specified, no work to do!") + exit(1) + + if not args.only_merge: + prepare_html_reports( + args.host_llvm_cov, + profdata_path, + args.report_dir, + args.binaries, + args.unified_report, + args.restrict, + args.compilation_dir, + )