From ffc5b903c8ccbfd7a52b6e39aa318dd7808bc4be Mon Sep 17 00:00:00 2001 From: Larry Gritz Date: Tue, 4 Feb 2025 10:13:11 -0800 Subject: [PATCH] testing: env variable OSL_TESTSUITE_THRESH_SCALE scales test thresholds (#1934) For any individual test running, the OSL_TESTSUITE_THRESH_SCALE environment variable, if set, will scale the image comparison thresholds used to decide if a test passes. Furthermore, a special OSL_TESTSUITE_OPTIX_THRESH_SCALE environment variable, if set, will cause *all* OptiX related tests to be set up with scaled thresholds. Signed-off-by: Larry Gritz --- src/cmake/testing.cmake | 3 +++ testsuite/runtest.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/cmake/testing.cmake b/src/cmake/testing.cmake index c4780abb5..18b4ec7b6 100644 --- a/src/cmake/testing.cmake +++ b/src/cmake/testing.cmake @@ -90,6 +90,9 @@ macro (add_one_testsuite testname testsrcdir) # Make sure libnvrtc-builtins.so is reachable set_property (TEST ${testname} APPEND PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64:$ENV{LD_LIBRARY_PATH}) endif() + if (DEFINED ENV{OSL_TESTSUITE_OPTIX_THRESH_SCALE}) + set_property (TEST ${testname} APPEND PROPERTY ENVIRONMENT OSL_TESTSUITE_THRESH_SCALE=$ENV{OSL_TESTSUITE_OPTIX_THRESH_SCALE}) + endif () endif () if (${testname} MATCHES "-reg") # These are batched shading regression tests. Some are quite diff --git a/testsuite/runtest.py b/testsuite/runtest.py index 86c1203bf..c3c43d890 100755 --- a/testsuite/runtest.py +++ b/testsuite/runtest.py @@ -406,6 +406,16 @@ def runtest (command, outputs, failureok=0, failthresh=0, failpercent=0, regress hardfail *= 2.0 failpercent *= 2.0 +# Allow an environment variable to scale the testsuite image comparison +# thresholds: +if 'OSL_TESTSUITE_THRESH_SCALE' in os.environ : + thresh_scale = float(os.getenv('OSL_TESTSUITE_THRESH_SCALE', '1.0')) + failthresh *= thresh_scale + hardfail *= thresh_scale + failpercent *= thresh_scale + failrelative *= thresh_scale + allowfailures = int(allowfailures * thresh_scale) + # Force out.txt to be in the outputs ##if "out.txt" not in outputs : ## outputs.append ("out.txt")