From 9ebda232529af90957f1bfa7297e606da28e620f Mon Sep 17 00:00:00 2001 From: Jean-Roch Coulon Date: Thu, 3 Oct 2024 06:32:32 +0200 Subject: [PATCH] Display report at the end of dhrystone and coremark executions Run benchmarks in single and dual issue Update cache config --- .gitlab-ci.yml | 16 +++++++- .gitlab-ci/scripts/report_benchmark.py | 55 +++++++++++++++----------- verif/regress/coremark.sh | 7 +++- verif/regress/dhrystone.sh | 7 +++- 4 files changed, 59 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d56738b6d1..04d5ebefc7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,9 +181,15 @@ coremark: DASHBOARD_SORT_INDEX: 5 DASHBOARD_JOB_CATEGORY: "Performance" SPIKE_TANDEM: 1 + parallel: + matrix: + - DV_HWCONFIG_OPTS: ["cv32a65x SuperscalarEn=0 IcacheByteSize=16384 IcacheSetAssoc=4 DcacheByteSize=32768 DcacheSetAssoc=8"] + ISSUE: "coremark_single" + - DV_HWCONFIG_OPTS: ["cv32a65x IcacheByteSize=16384 IcacheSetAssoc=4 DcacheByteSize=32768 DcacheSetAssoc=8"] + ISSUE: "coremark_dual" script: - bash verif/regress/coremark.sh --no-print - - python3 .gitlab-ci/scripts/report_benchmark.py --coremark verif/sim/out_*/vcs-uvm_sim/core_main.*.log + - python3 .gitlab-ci/scripts/report_benchmark.py --$ISSUE verif/sim/out_*/vcs-uvm_sim/core_main.*.log hwconfig: extends: @@ -318,9 +324,15 @@ dhrystone: DASHBOARD_SORT_INDEX: 5 DASHBOARD_JOB_CATEGORY: "Performance" SPIKE_TANDEM: 1 + parallel: + matrix: + - DV_HWCONFIG_OPTS: ["cv32a65x SuperscalarEn=0 IcacheByteSize=16384 IcacheSetAssoc=4 DcacheByteSize=32768 DcacheSetAssoc=8"] + ISSUE: "dhrystone_single" + - DV_HWCONFIG_OPTS: ["cv32a65x IcacheByteSize=16384 IcacheSetAssoc=4 DcacheByteSize=32768 DcacheSetAssoc=8"] + ISSUE: "dhrystone_dual" script: - bash verif/regress/dhrystone.sh - - python3 .gitlab-ci/scripts/report_benchmark.py --dhrystone verif/sim/out_*/vcs-uvm_sim/dhrystone_main.*.log + - python3 .gitlab-ci/scripts/report_benchmark.py --$ISSUE verif/sim/out_*/vcs-uvm_sim/dhrystone_main.*.log riscv_arch_test: extends: diff --git a/.gitlab-ci/scripts/report_benchmark.py b/.gitlab-ci/scripts/report_benchmark.py index 85837ccdf2..9f4f74e0d4 100644 --- a/.gitlab-ci/scripts/report_benchmark.py +++ b/.gitlab-ci/scripts/report_benchmark.py @@ -7,7 +7,9 @@ # # Original Author: Côme Allart +import os import sys +import re import report_builder as rb path = None @@ -17,56 +19,65 @@ # Keep it up-to-date with compiler version and core performance improvements # Will fail if the number of cycles is different from this one valid_cycles = { - 'dhrystone': 250453, - 'coremark': 582561, + "dhrystone_dual": 211352, + "dhrystone_single": 254018, + "coremark_dual": 535567, + "coremark_single": 674940, } for arg in sys.argv[1:]: - if arg == '--dhrystone': - mode = 'dhrystone' - # Standard value for Dhrystone - iterations = 500 - elif arg == '--coremark': - mode = 'coremark' - # Defined in verif/regress/coremark.sh - iterations = 2 + if "--dhrystone" in arg or "--coremark" in arg: + if "--dhrystone" in arg: + iterations = 500 + else: + if "--coremark" in arg: + iterations = 2 + mode = arg.replace("-", "") else: path = arg # We do not want to have a report without a check assert mode is not None -with open(path, 'r') as f: +with open(path, "r") as f: log = [l.strip() for l in f.readlines()] stopwatch = [] for index, line in enumerate(log): - if line.split()[-1] == 'mcycle' or line.split()[-2] == 'mcycle,': + if line.split()[-1] == "mcycle" or line.split()[-2] == "mcycle,": stopwatch.append(int(log[index + 1].split()[-1], 16)) # There might be > 2 matches, we use the two at the center N = len(stopwatch) assert N % 2 == 0 -cycles = stopwatch[N//2] - stopwatch[N//2-1] +cycles = stopwatch[N // 2] - stopwatch[N // 2 - 1] -score_metric = rb.TableMetric('Performance results') -score_metric.add_value('cycles', cycles) +score_metric = rb.TableMetric("Performance results") +score_metric.add_value("cycles", cycles) if iterations is not None: ipmhz = iterations * 1000000 / cycles - if mode == 'dhrystone': - score_metric.add_value('Dhrystone/MHz', ipmhz) - score_metric.add_value('DMIPS/MHz', ipmhz / 1757) - if mode == 'coremark': - score_metric.add_value('CoreMark/MHz', ipmhz) + if "dhrystone" in mode: + score_metric.add_value("Dhrystone/MHz", ipmhz) + score_metric.add_value("DMIPS/MHz", ipmhz / 1757) + if "coremark" in mode: + score_metric.add_value("CoreMark/MHz", ipmhz) diff = cycles - valid_cycles[mode] if diff != 0: score_metric.fail() - score_metric.add_value('Cycles diff', diff) + score_metric.add_value("Cycles diff", diff) -report = rb.Report(f'{cycles//1000} kCycles') +report = rb.Report(f"{cycles//1000} kCycles") report.add_metric(score_metric) report.dump() +filename = re.sub(r"[^\w\.\\\/]", "_", os.environ["CI_JOB_NAME"]) +path = "artifacts/reports/" + filename + ".yml" +with open(path, "r") as f: + log = [l.strip() for l in f.readlines()] +for index, line in enumerate(log): + if "MHz" in line: + print(log[index + 1], log[index]) + if report.failed: sys.exit(1) diff --git a/verif/regress/coremark.sh b/verif/regress/coremark.sh index 63e9454398..28f8bddfb4 100644 --- a/verif/regress/coremark.sh +++ b/verif/regress/coremark.sh @@ -30,6 +30,10 @@ if ! [ -n "$DV_SIMULATORS" ]; then DV_SIMULATORS=vcs-uvm fi +if ! [ -n "$DV_HWCONFIG_OPTS" ]; then + DV_HWCONFIG_OPTS="cv32a65x" +fi + if ! [ -n "$UVM_VERBOSITY" ]; then export UVM_VERBOSITY=UVM_NONE fi @@ -82,7 +86,8 @@ isa="rv32imc_zba_zbb_zbc_zbs" set -x python3 cva6.py \ - --target cv32a65x \ + --target hwconfig \ + --hwconfig_opts="$DV_HWCONFIG_OPTS" \ --iss="$DV_SIMULATORS" \ --iss_yaml=cva6.yaml \ --c_tests "$src0" \ diff --git a/verif/regress/dhrystone.sh b/verif/regress/dhrystone.sh index 60db9d84e8..a55602f7b9 100644 --- a/verif/regress/dhrystone.sh +++ b/verif/regress/dhrystone.sh @@ -25,6 +25,10 @@ if ! [ -n "$DV_SIMULATORS" ]; then DV_SIMULATORS=vcs-uvm fi +if ! [ -n "$DV_HWCONFIG_OPTS" ]; then + DV_HWCONFIG_OPTS="cv32a65x" +fi + make clean make -C verif/sim clean_all @@ -55,7 +59,8 @@ cflags=( set -x python3 cva6.py \ - --target cv32a65x \ + --target hwconfig \ + --hwconfig_opts="$DV_HWCONFIG_OPTS" \ --iss="$DV_SIMULATORS" \ --iss_yaml=cva6.yaml \ --c_tests "$src0" \