testing: env variable OSL_TESTSUITE_THRESH_SCALE scales test threshol…

…ds (#1934) For any individual test running, the OSL_TESTSUITE_THRESH_SCALE environment variable, if set, will scale the image comparison thresholds used to decide if a test passes. Furthermore, a special OSL_TESTSUITE_OPTIX_THRESH_SCALE environment variable, if set, will cause *all* OptiX related tests to be set up with scaled thresholds. Signed-off-by: Larry Gritz <[email protected]>
AcademySoftwareFoundation · Feb 4, 2025 · ffc5b90 · ffc5b90
1 parent ddb959e
commit ffc5b90
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/src/cmake/testing.cmake b/src/cmake/testing.cmake
@@ -90,6 +90,9 @@ macro (add_one_testsuite testname testsrcdir)
             # Make sure libnvrtc-builtins.so is reachable
             set_property (TEST ${testname} APPEND PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64:$ENV{LD_LIBRARY_PATH})
         endif()
+        if (DEFINED ENV{OSL_TESTSUITE_OPTIX_THRESH_SCALE})
+            set_property (TEST ${testname} APPEND PROPERTY ENVIRONMENT OSL_TESTSUITE_THRESH_SCALE=$ENV{OSL_TESTSUITE_OPTIX_THRESH_SCALE})
+        endif ()
     endif ()
     if (${testname} MATCHES "-reg")
         # These are batched shading regression tests. Some are quite

diff --git a/testsuite/runtest.py b/testsuite/runtest.py
@@ -406,6 +406,16 @@ def runtest (command, outputs, failureok=0, failthresh=0, failpercent=0, regress
     hardfail *= 2.0
     failpercent *= 2.0
 
+# Allow an environment variable to scale the testsuite image comparison
+# thresholds:
+if 'OSL_TESTSUITE_THRESH_SCALE' in os.environ :
+    thresh_scale = float(os.getenv('OSL_TESTSUITE_THRESH_SCALE', '1.0'))
+    failthresh *= thresh_scale
+    hardfail *= thresh_scale
+    failpercent *= thresh_scale
+    failrelative *= thresh_scale
+    allowfailures = int(allowfailures * thresh_scale)
+
 # Force out.txt to be in the outputs
 ##if "out.txt" not in outputs :
 ##    outputs.append ("out.txt")