From 88ef82afca716f060a908bcdab67a7ace06ce03e Mon Sep 17 00:00:00 2001 From: Dan MacDonald Date: Tue, 13 Aug 2024 11:08:43 -0400 Subject: [PATCH] [#50] Update Code Coverage Tools & Documentation - remove coverage script that relies on nightly builds - rename script using tarpaulin coverage - add coverage script using llvm-cov - add README for tools Implements [#34] --- .gitignore | 1 + tools/README.md | 34 ++++++++++++++++++++++ tools/coverage.sh | 21 -------------- tools/generate_llvm_coverage.sh | 39 +++++++++++++++++++++++++ tools/generate_tarpaulin_coverage.sh | 40 ++++++++++++++++++++++++++ tools/generate_test_coverage_report.sh | 17 ----------- 6 files changed, 114 insertions(+), 38 deletions(-) create mode 100644 tools/README.md delete mode 100755 tools/coverage.sh create mode 100644 tools/generate_llvm_coverage.sh create mode 100755 tools/generate_tarpaulin_coverage.sh delete mode 100755 tools/generate_test_coverage_report.sh diff --git a/.gitignore b/.gitignore index ef1e7bd7..72c2c178 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ tarpaulin-report.html .idea/ .vscode/launch.json .vscode/settings.json +reports/ diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..a726a78f --- /dev/null +++ b/tools/README.md @@ -0,0 +1,34 @@ +# Tools + +The scripts in this folder help with commands with long arguments + +* generate_llvm_coverage.sh **(RECOMMENDED)** + +Generate coverage using the llvm coverage tool. The output files are generated to reports/llvm/html/index.html, reports/llvm/lcov.info & reports/llvm/lcov/index.html + +``` bash +$ sh tools/generate_llvm_coverage.sh +... +Writing directory view page. +Overall coverage rate: + lines......: 78.1% (890 of 1140 lines) + functions..: 55.1% (201 of 365 functions) +llvm-cov report generated to reports/llvm/html/index.html +lcov report generated to: reports/llvm/lcov/index.html +``` + +* generate_tarpaulin_coverage.sh + +This script generates coverage reports using the tarpaulin interface _and_ lcov. The output files are generated to reports/tarpaulin-report.html, reports/lcov.info & reports/lcov/index.html. + +execute the script from the repo root: +``` bash +$ sh tools/generate_tarpaulin_coverage.sh +... +Overall coverage rate: + lines......: 75.1% (431 of 574 lines) + functions..: 82.3% (51 of 62 functions) +tarpaulin report generated to reports/tarpaulin-report.html +lcov report generated to: reports/lcov/index.html +``` + diff --git a/tools/coverage.sh b/tools/coverage.sh deleted file mode 100755 index a1dfd74b..00000000 --- a/tools/coverage.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# This runs cargo test and creates test coverage data, as well as a test result report, currently this requires the 'nightly' rust toolchain. -# Run this in the project root, and cargo2junit and grcov binaries (do `cargo install cargo2junit grcov`) -# Result files will be placed in ./reports - -PROJECT_NAME_UNDERSCORE="up_streamer_rust" -RUSTFLAGS="--cfg uuid_unstable -Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" -RUSTDOCFLAGS="-Cpanic=abort" -TEMP=`mktemp --directory` - - -mkdir -p $TEMP/reports - -cargo test $CARGO_OPTIONS -- -Z unstable-options --format json | cargo2junit > $TEMP/results.xml -zip -0 $TEMP/ccov.zip `find . \( -name "$PROJECT_NAME_UNDERSCORE*.gc*" \) -print` -grcov $TEMP/ccov.zip -s . -t lcov --llvm --ignore-not-existing --ignore "/*" --ignore "tests/*" --ignore "target/*" -o $TEMP/lcov.info -genhtml $TEMP/lcov.info --output-directory $TEMP/out - -rm $TEMP/ccov.zip -mv -r $TEMP/* ./reports -rm -fr $TEMP diff --git a/tools/generate_llvm_coverage.sh b/tools/generate_llvm_coverage.sh new file mode 100644 index 00000000..b9f8151b --- /dev/null +++ b/tools/generate_llvm_coverage.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +################################################################################ +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + +BLUE='\033[0;34m' +NC='\033[0m' +HAS_LLVMCOV=$(cargo install --list | grep cargo-llvm-cov) +COVERAGE_OUT=reports/llvm + +if [ -z "$HAS_LLVMCOV" ]; then + echo "cargo-llvm-cov not found, please install it with 'cargo install cargo-llvm-cov'" + exit 1 +fi + +mkdir -p $COVERAGE_OUT + +# generate coverage using llvm +# RUST_LOG ensures the debug macros are not flagged by the coverage output +RUST_LOG=trace cargo llvm-cov --html --output-dir $COVERAGE_OUT --ignore-filename-regex 'utils/.*' 2>&1 | grep -Ev 'TRACE|DEBUG|WARN' + +# generate lcov coverage +RUST_LOG=trace cargo llvm-cov --lcov --output-path $COVERAGE_OUT/lcov.info --ignore-filename-regex 'utils/.*' 2>&1 | grep -Ev 'TRACE|DEBUG|WARN' +genhtml -o $COVERAGE_OUT/lcov/ --show-details --highlight --ignore-errors source --legend $COVERAGE_OUT/lcov.info + +printf "${BLUE}" +printf "llvm-cov report generated to \e]8;;file://%s\a%s\e]8;;\a \n" "$PWD/$COVERAGE_OUT/html/index.html" "$COVERAGE_OUT/html/index.html" +printf 'lcov report generated to: \e]8;;file://%s\a%s\e]8;;\a \n' "$PWD/$COVERAGE_OUT/lcov/index.html" "$COVERAGE_OUT/lcov/index.html" +printf "${NC}" diff --git a/tools/generate_tarpaulin_coverage.sh b/tools/generate_tarpaulin_coverage.sh new file mode 100755 index 00000000..c5b57414 --- /dev/null +++ b/tools/generate_tarpaulin_coverage.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +################################################################################ +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + +BLUE='\033[0;34m' +NC='\033[0m' +HAS_TARPAULIN=$(cargo install --list | grep cargo-tarpaulin) +HAS_GENHTML=$(which genhtml) +COVERAGE_OUT=reports + +if [ -z "$HAS_TARPAULIN" ]; then + echo "cargo-tarpaulin not found, please install it with 'cargo install cargo-tarpaulin'" + exit 1 +fi + +if [ -z "$HAS_GENHTML" ]; then + echo "genhtml not found, please install it with 'sudo apt install lcov'" + exit 1 +fi + +# we want both html and lcov output formats +cargo tarpaulin -o lcov -o html --exclude-files 'utils/*' --output-dir $COVERAGE_OUT +# convert the lcov output to html +genhtml -o $COVERAGE_OUT/lcov/ --show-details --highlight --ignore-errors source --legend lcov.info + +printf "${BLUE}" +printf "tarpaulin report generated to \e]8;;file://%s\a%s\e]8;;\a \n" "$PWD/$COVERAGE_OUT/tarpaulin-report.html" "$COVERAGE_OUT/tarpaulin-report.html" +printf 'lcov report generated to: \e]8;;file://%s\a%s\e]8;;\a \n' "$PWD/$COVERAGE_OUT/lcov/index.html" "$COVERAGE_OUT/lcov/index.html" +printf "${NC}" diff --git a/tools/generate_test_coverage_report.sh b/tools/generate_test_coverage_report.sh deleted file mode 100755 index 1a54e482..00000000 --- a/tools/generate_test_coverage_report.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -################################################################################ -# Copyright (c) 2024 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -################################################################################ - -# we want both html and lcov output formats -cargo tarpaulin -o lcov -o html --output-dir target/tarpaulin