Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Report more warnings, summarize job output #1996

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 100 additions & 0 deletions .github/scripts/job_summary.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import argparse
import re

from collections import Counter


def parse_arguments():
na_value = "N/A"
parser = argparse.ArgumentParser(description="Analyze build logs and ctest output, and generate a summary.")
parser.add_argument("--build-log", required=True, help="Path to the build log file.")
parser.add_argument("--ctest-log", required=True, help="Path to the ctest log file.")
parser.add_argument("--output-file", required=True, help="Path to the output file.")
parser.add_argument("--os", default=na_value, help="Operating System.")
parser.add_argument("--compiler-version", default=na_value, help="Version of the compiler.")
parser.add_argument("--cmake-version", default=na_value, help="Version of CMake.")
parser.add_argument("--cpu-model", default=na_value, help="CPU model.")
return parser.parse_args()


def read_file(file_path):
with open(file_path, 'r', encoding="utf8") as f:
return f.read()


def generate_environment_table(os_info, compiler_version, cmake_version, cpu_model):
table = []
table.append( "| Environment Parameter | Value |")
table.append( "|-----------------------|---------------------|")
table.append(f"| OS | {os_info} |")
table.append(f"| Compiler Ver. | {compiler_version} |")
table.append(f"| CMake Ver. | {cmake_version} |")
table.append(f"| CPU Model | {cpu_model} |")
return "\n".join(table)


def generate_warning_table(build_log_content):
warning_regex = re.compile(
r"""
\[(\-W[a-zA-Z0-9\-]+)\] | # GCC/Clang warnings: "[-W<some-flag>]"
((?:STL|C|D|LNK)\d{4}) # MSVC warnings: "<STL|C|D|LNK>xxxx"
""",
re.VERBOSE
)
warnings = tuple(
match.group(1) or match.group(2)
for match in warning_regex.finditer(build_log_content)
)
warning_histogram = Counter(warnings)

# The slowest part: optimize if needed (takes ~3sec for 100K lines)
warning_examples = {
w: re.search(rf".*{w}.*", build_log_content).group(0)
for w in warning_histogram
}

table = []
table.append("| Warning Type | Count | Message example |")
table.append("|----------------|-------|-----------------|")
for warning, count in warning_histogram.items():
example = warning_examples[warning]
table.append(f"| {warning} | {count} | {example} |")
return "\n".join(table)


def generate_ctest_table(ctest_log_content):
# No need to parse the data into Markdown table: it is already in a readable pseudo-table format
result_lines = re.findall(r".*Test\s*#.*sec.*", ctest_log_content)
code_block = ["```"] + result_lines + ["```"]
return "\n".join(code_block)


def extract_ctest_summary(ctest_log_content):
match = re.search(r".*tests passed.*tests failed.*", ctest_log_content)
if match is None:
return ""
else:
return match.group(0)


def combine_tables(environment_table, warning_table, ctest_table, ctest_summary):
# Make the CTest summary collapsible since it can be long
title = f"<summary><b>CTest: {ctest_summary} (expand for details)</b></summary>"
collapsible_ctest_table = f"<details>\n{title}\n\n{ctest_table}\n\n</details>"
# Additional empty line to separate the tables
return "\n\n".join([environment_table, warning_table, collapsible_ctest_table])


if __name__ == "__main__":
args = parse_arguments()
build_log_content = read_file(args.build_log)
ctest_log_content = read_file(args.ctest_log)

environment_table = generate_environment_table(args.os, args.compiler_version, args.cmake_version, args.cpu_model)
warning_table = generate_warning_table(build_log_content)
ctest_table = generate_ctest_table(ctest_log_content)
ctest_summary = extract_ctest_summary(ctest_log_content)
summary = combine_tables(environment_table, warning_table, ctest_table, ctest_summary)

with open(args.output_file, 'w', encoding="utf-8") as f:
f.write(summary)
81 changes: 61 additions & 20 deletions .github/workflows/ci-testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -198,22 +198,33 @@ jobs:
if [[ -f "${LINUX_ONEAPI_PATH}/setvars.sh" ]]; then
source ${LINUX_ONEAPI_PATH}/setvars.sh
fi
echo "::warning::CMake: $(cmake --version)"
echo "::warning::Compiler: $(${{ matrix.cxx_compiler }} --version)"

if [[ "${{ matrix.backend }}" == "dpcpp" ]]; then
make_targets="build-onedpl-sycl_iterator-tests build-onedpl-ranges-tests"
ctest_flags="-R (sycl_iterator_.*)|(std_ranges_.*)\.pass"
echo "::warning::dpcpp backend is set. Compile and run only sycl_iterator tests"
dmitriy-sobolev marked this conversation as resolved.
Show resolved Hide resolved
else
make_targets="build-onedpl-tests"
fi
mkdir build && cd build
lscpu
cmake -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DONEDPL_BACKEND=${{ matrix.backend }} -DONEDPL_DEVICE_TYPE=${{ matrix.device_type }} ..
make VERBOSE=1 -j${BUILD_CONCURRENCY} ${make_targets}
ctest --timeout ${TEST_TIMEOUT} --output-on-failure ${ctest_flags}
-DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DONEDPL_BACKEND=${{ matrix.backend }} -DONEDPL_DEVICE_TYPE=${{ matrix.device_type }} -DCMAKE_CXX_FLAGS="-Wall" ..
make VERBOSE=1 -j${BUILD_CONCURRENCY} ${make_targets} |& tee build.log
ctest --timeout ${TEST_TIMEOUT} --output-on-failure ${ctest_flags} |& tee ctest.log

# Generate a summary
os_name=$(uname -a | head -n 1)
cmake_version=$(cmake --version | head -n 1)
compiler_version=$(${{ matrix.cxx_compiler }} --version | head -n 1)
cpu_model=$(lscpu | grep "Model name")
python ${GITHUB_WORKSPACE}/.github/scripts/job_summary.py --build-log build.log \
--ctest-log ctest.log \
--output-file summary.md \
--os "${os_name}" \
--cmake-version "${cmake_version}" \
--compiler-version "${compiler_version}" \
--cpu-model "${cpu_model}"
cat summary.md > $GITHUB_STEP_SUMMARY

windows-testing:
name: ${{ matrix.device_type }},bknd=${{ matrix.backend }},cmplr=${{ matrix.cxx_compiler }},${{ matrix.os }},std=c++${{ matrix.std }},cfg=${{ matrix.build_type }}
Expand Down Expand Up @@ -267,25 +278,42 @@ jobs:
if "${{ matrix.cxx_compiler }}" == "cl" (
call "C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
)
powershell $output = cmake --version; Write-Host ::warning::CMake: $output
powershell $output = ${{ matrix.cxx_compiler }} --version; Write-Host ::warning::Compiler: $output

if "${{ matrix.backend }}" == "dpcpp" (
set ninja_targets="build-onedpl-sycl_iterator-tests"
set ctest_flags=-R sycl_iterator_.*\.pass
echo ::warning::dpcpp backend is set. Compile and run only sycl_iterator tests
) else (
set ninja_targets=build-onedpl-tests
)
mkdir build && cd build

cmake -G "Ninja" -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DONEDPL_BACKEND=${{ matrix.backend }} -DONEDPL_DEVICE_TYPE=${{ matrix.device_type }} .. || goto :short_circuit_fail
ninja -j 2 -v %ninja_targets% || goto :short_circuit_fail
ctest --timeout %TEST_TIMEOUT% -C ${{ matrix.build_type }} --output-on-failure %ctest_flags% || goto :short_circuit_fail
exit /b 0
:: modify the default behaviour of shell:cmd, which exits with the status of a last command, in order not to unintentially miss an error
:short_circuit_fail
exit /b %errorlevel%
:: Preserve the code of an unsuccessful command if any.
:: By default, CMD shell only reports the error level of the final command.
set exit_code=0

cmake -G "Ninja" -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DONEDPL_BACKEND=${{ matrix.backend }} -DONEDPL_DEVICE_TYPE=${{ matrix.device_type }} -DCMAKE_CXX_FLAGS="/W4" .. || set exit_code=%errorlevel%

ninja -j 2 -v %ninja_targets% > build.log 2>&1 || set exit_code=%errorlevel%
:: Display the results after executing all tests because "tee" alternative is unavailable in CMD.
type build.log

ctest --timeout %TEST_TIMEOUT% -C ${{ matrix.build_type }} --output-on-failure %ctest_flags% > ctest.log 2>&1 || set exit_code=%errorlevel%
type ctest.log

:: Generate a summary
for /f "usebackq tokens=*" %%i in (`powershell -command "(Get-CimInstance -ClassName Win32_OperatingSystem).Caption"`) do set os_name=%%i
for /f "usebackq tokens=*" %%i in (`powershell -command "cmake --version | Select-Object -First 1"`) do set cmake_version=%%i
for /f "usebackq tokens=*" %%i in (`powershell -command "${{ matrix.cxx_compiler }} --version | Select-Object -First 1"`) do set compiler_version=%%i
for /f "usebackq tokens=*" %%i in (`powershell -command "(Get-CimInstance -ClassName Win32_Processor).Name"`) do set cpu_model=%%i
python %GITHUB_WORKSPACE%\.github\scripts\job_summary.py --build-log build.log ^
--ctest-log ctest.log ^
--output-file summary.md ^
--os "%os_name%" ^
--cmake-version "%cmake_version%" ^
--compiler-version "%compiler_version%" ^
--cpu-model "%cpu_model%"
type summary.md > %GITHUB_STEP_SUMMARY%
exit /b %exit_code%

macos-testing:
name: HOST,bknd=${{ matrix.backend }},cmplr=${{ matrix.cxx_compiler }},${{ matrix.os }},std=c++${{ matrix.std }},cfg=${{ matrix.build_type }}
Expand All @@ -309,13 +337,26 @@ jobs:
shell: bash
run: |
set -x
echo "::warning::CMake: $(cmake --version)"
sysctl -a | grep machdep.cpu
# workaround for CMake not being able to find OpenMP: see https://discourse.cmake.org/t/how-to-find-openmp-with-clang-on-macos/8860
# -DCMAKE_POLICY_DEFAULT_CMP0074=NEW below is forced to make sure CMake uses <PackageName>_ROOT variables.
export OpenMP_ROOT=$(brew --prefix)/opt/libomp
mkdir build && cd build
cmake -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DONEDPL_BACKEND=${{ matrix.backend }} -DCMAKE_POLICY_DEFAULT_CMP0074=NEW ..
make VERBOSE=1 build-onedpl-tests -j${MACOS_BUILD_CONCURRENCY}
ctest --timeout ${TEST_TIMEOUT} --output-on-failure -E "${EXCLUDE_FROM_TESTING}"
-DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DONEDPL_BACKEND=${{ matrix.backend }} -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_CXX_FLAGS="-Wall" ..
make VERBOSE=1 build-onedpl-tests -j${MACOS_BUILD_CONCURRENCY} 2>&1 | tee build.log
ctest --timeout ${TEST_TIMEOUT} --output-on-failure -E "${EXCLUDE_FROM_TESTING}" 2>&1 | tee ctest.log

# Generate a summary
os_name=$(uname -a | head -n 1)
cmake_version=$(cmake --version | head -n 1)
compiler_version=$(${{ matrix.cxx_compiler }} --version | head -n 1)
cpu_model=$(sysctl -n machdep.cpu.brand_string)
python ${GITHUB_WORKSPACE}/.github/scripts/job_summary.py --build-log build.log \
--ctest-log ctest.log \
--output-file summary.md \
--os "${os_name}" \
--cmake-version "${cmake_version}" \
--compiler-version "${compiler_version}" \
--cpu-model "${cpu_model}"
cat summary.md > $GITHUB_STEP_SUMMARY