Update nist-validation test to provide more granular results

Scapval tool is failing when we build SCE content by default
in RHEL 9 and RHEL 10 data streams because it doesn't expect
content to use checking systems other than the OVAL and OCIL
(base requirement `SRC-118`). For more details see
ComplianceAsCode/content#12488

We can waive this fail, it shouldn't cause any problems for
3rd party scanners as our content still contains also OVAL
checks. To do so the test has been updated to parse XML results
file generated by the scapval tool.

The test is also updated to work on RHEL 10 where `java-21-openjdk`
is the default as scapval tool has no problem running with this
newer version of java.

conf/waivers/30-permanent

@@ -84,4 +84,11 @@
 /hardening/.*/ospp/configure_crypto_policy
     rhel.is_centos() and rhel == 9
 
+# scapval waivers
+#
+# Caused by SCE content being built by default, enabled
+# in https://github.com/ComplianceAsCode/content/pull/12488
+/static-checks/nist-validation/ssg-rhel9-ds/SRC-118
+    rhel >= 9
+
 # vim: syntax=python

static-checks/nist-validation/main.fmf

@@ -4,14 +4,10 @@ result: custom
 environment+:
     PYTHONPATH: ../..
 duration: 15m
-require+:
-  # we use java-17 specifically here because the NIST tool needs it and does not
-  # work with any newer version
+recommend+:
   - java-17-openjdk
+  - java-21-openjdk
 adjust:
   - when: arch != x86_64
     enabled: false
     because: the test is not architecture-specific, one is enough
-  - when: distro == rhel-10
-    enabled: false
-    because: TODO - RHEL-10 doesn't have Java 17, see requires above

static-checks/nist-validation/test.py

@@ -5,6 +5,7 @@
 import zipfile
 import requests
 import subprocess
+import xml.etree.ElementTree as ET
 
 from lib import util, results
 
@@ -16,6 +17,8 @@
 zip.extractall()
 os.chmod('scapval.sh', 0o755)
 
+ns = {'nist': 'http://csrc.nist.gov/ns/decima/results/1.0'}
+
 for datastream in util.iter_datastreams():
     ds_name = datastream.stem
     report_file = f'{ds_name}.report.html'
@@ -27,12 +30,19 @@
         '-valresultfile', result_file,
         '-file', datastream,
     ]
-    proc = util.subprocess_run(cmd, stdout=subprocess.PIPE, check=True, universal_newlines=True)
-    if 'The target is valid' in proc.stdout:
-        results.report('pass', ds_name)
-    elif 'The target is invalid' in proc.stdout:
-        results.report('fail', ds_name, logs=[report_file, result_file])
-    else:
-        raise RuntimeError("SCAPval out has not been correctly parsed")
+    util.subprocess_run(cmd, stdout=subprocess.DEVNULL, check=True)
+    tree = ET.parse(result_file)
+    root = tree.getroot()
+    for elem in root.findall('./nist:results/nist:base-requirement', ns):
+        name = f'{ds_name}/{elem.attrib["id"]}'
+        status = elem.find('./nist:status', ns).text
+        if status in ['NOT_TESTED', 'NOT_APPLICABLE']:
+            continue
+        elif status in ['PASS', 'WARNING', 'INFORMATIONAL']:
+            results.report('pass', name)
+        elif status == 'FAIL':
+            results.report('fail', name, logs=[report_file, result_file])
+        else:
+            results.report('error', name, logs=[report_file, result_file])
 
 results.report_and_exit()