Add an integration test for the autotailor tool

This commit adds an integration test for autotailor. The goal of the test is to verify if the tailoring produced by autotailor can be loaded and consumed by oscap and if the generated tailoring leads to the intended behavior of the profile evaluation.
OpenSCAP · Oct 3, 2023 · ae87898 · ae87898
1 parent ae3874e
commit ae87898
Show file tree

Hide file tree

Showing 3 changed files with 208 additions and 0 deletions.
diff --git a/tests/utils/CMakeLists.txt b/tests/utils/CMakeLists.txt
@@ -1 +1,2 @@
+add_oscap_test("autotailor_integration_test.sh")
 add_oscap_test("test_utils_args.sh")
diff --git a/tests/utils/autotailor_integration_test.sh b/tests/utils/autotailor_integration_test.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+. $builddir/tests/test_common.sh
+
+set -e -o pipefail
+
+autotailor="$top_srcdir/utils/autotailor"
+tailoring="$(mktemp)"
+ds="$srcdir/data_stream.xml"
+stdout="$(mktemp)"
+original_profile="P1"
+result="$(mktemp)"
+
+# the original profile P1 selects rules R1 and R2
+
+# select additional rule R3
+python3 $autotailor --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# select additional rules R3, R4
+python3 $autotailor --id-namespace "com.example.www" --select R3 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# unselect rule R2
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# unselect rule R2 and select R4
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# select additional rule R3 and change its severity to high
+python3 $autotailor --id-namespace "com.example.www" --select R3 --rule-severity R3=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="unknown"]'
+
+# don't select rules, don't unselect rules, but change severity of all rules to high
+python3 $autotailor --id-namespace "com.example.www" --rule-severity R1=high --rule-severity R2=high --rule-severity R3=high --rule-severity R4=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="high"]'
+
+
+# select additional rule R4 and change its role to "unchecked"
+python3 $autotailor --id-namespace "com.example.www" --select R4 --rule-role R4=unchecked $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notchecked"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @role="unchecked"]'
+
+
+# select additional rule R3; the customized profile will have a special profile ID
+customized_profile="xccdf_com.pink.elephant_profile_pineapple"
+python3 $autotailor --new-profile-id $customized_profile --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile $customized_profile --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult[@id="xccdf_org.open-scap_testresult_'$customized_profile'"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# refine value v1 to 30
+python3 $autotailor --id-namespace "com.example.www" --var-value V1=thirty $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/set-value[@idref="xccdf_com.example.www_value_V1" and text()="thirty"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
diff --git a/tests/utils/data_stream.xml b/tests/utils/data_stream.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="utf-8"?>
+<ds:data-stream-collection xmlns:ds="http://scap.nist.gov/schema/scap/source/1.2" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:cat="urn:oasis:names:tc:entity:xmlns:xml:catalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="scap_org.open-scap_collection_from_xccdf_test_single_rule.xccdf.xml" schematron-version="1.3" xsi:schemaLocation="http://scap.nist.gov/schema/scap/source/1.2 https://scap.nist.gov/schema/scap/1.3/scap-source-data-stream_1.3.xsd">
+  <ds:data-stream id="scap_org.open-scap_datastream_simple" scap-version="1.3" use-case="OTHER">
+    <ds:checklists>
+      <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.xccdf.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.xccdf.xml">
+        <cat:catalog>
+          <cat:uri name="test_single_rule.oval.xml" uri="#scap_org.open-scap_cref_test_single_rule.oval.xml"/>
+        </cat:catalog>
+      </ds:component-ref>
+    </ds:checklists>
+    <ds:checks>
+      <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.oval.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.oval.xml"/>
+    </ds:checks>
+  </ds:data-stream>
+  <ds:component id="scap_org.open-scap_comp_test_single_rule.oval.xml" timestamp="2021-02-01T08:07:06+01:00">
+    <oval_definitions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:ind-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5" xmlns:win-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#windows" xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd    http://oval.mitre.org/XMLSchema/oval-definitions-5#independent independent-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#windows windows-definitions-schema.xsd">
+      <generator>
+        <oval:schema_version>5.11.2</oval:schema_version>
+        <oval:timestamp>2021-02-01T08:07:06+01:00</oval:timestamp>
+      </generator>
+      <definitions>
+        <definition class="compliance" id="oval:x:def:1" version="1">
+          <metadata>
+            <title>PASS</title>
+            <description>pass</description>
+          </metadata>
+          <criteria>
+            <criterion comment="PASS test" test_ref="oval:x:tst:1"/>
+          </criteria>
+        </definition>
+      </definitions>
+      <tests>
+        <variable_test xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:tst:1" check="all" comment="always pass" version="1">
+          <object object_ref="oval:x:obj:1"/>
+        </variable_test>
+      </tests>
+      <objects>
+        <variable_object xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:obj:1" version="1" comment="x">
+          <var_ref>oval:x:var:1</var_ref>
+        </variable_object>
+      </objects>
+      <variables>
+        <constant_variable id="oval:x:var:1" version="1" comment="x" datatype="int">
+          <value>100</value>
+        </constant_variable>
+      </variables>
+    </oval_definitions>
+  </ds:component>
+  <ds:component id="scap_org.open-scap_comp_test_single_rule.xccdf.xml" timestamp="2021-02-01T08:07:06+01:00">
+    <Benchmark xmlns="http://checklists.nist.gov/xccdf/1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="xccdf_com.example.www_benchmark_dummy" xsi:schemaLocation="http://checklists.nist.gov/xccdf/1.1 xccdf-1.1.4.xsd" resolved="false" xml:lang="en-US">
+      <status date="2021-01-21">accepted</status>
+      <title>Test Benchmark</title>
+      <description>Description</description>
+      <version>1.0</version>
+      <metadata>
+        <dc:contributor xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:contributor>
+        <dc:publisher xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:publisher>
+        <dc:creator xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:creator>
+        <dc:source xmlns:dc="http://purl.org/dc/elements/1.1/">http://scap.nist.gov</dc:source>
+      </metadata>
+      <Profile id="xccdf_com.example.www_profile_P1">
+        <title>xccdf_test_profile</title>
+        <description>This profile is for testing.</description>
+        <select idref="xccdf_com.example.www_rule_R1" selected="true"/>
+        <select idref="xccdf_com.example.www_rule_R2" selected="true"/>
+      </Profile>
+      <Value id="xccdf_com.example.www_value_V1" operator="equals" type="number">
+        <title>value</title>
+        <description xml:lang="en">cccc</description>
+        <question xml:lang="en">ssss</question>
+        <value>5</value>
+        <value selector="thirty">30</value>
+      </Value>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R1">
+        <title>Rule R1</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R2">
+        <title>Rule R2</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R3">
+        <title>Rule R3</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R4">
+        <title>Rule R4</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+    </Benchmark>
+  </ds:component>
+</ds:data-stream-collection>