Merge pull request #2039 from jan-cerny/improve_autotailor

Improve the autotailor script
OpenSCAP · Oct 5, 2023 · e0ea126 · e0ea126
2 parents 3a987f3 + e1d1458
commit e0ea126
Show file tree

Hide file tree

Showing 7 changed files with 476 additions and 61 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -34,7 +34,7 @@ jobs:
     - name: Install Deps
       run: |
         sudo apt-get update
-        sudo apt-get -y install lcov swig xsltproc rpm-common lua5.3 libpcre2-dev libyaml-dev libapt-pkg-dev libdbus-1-dev libdbus-glib-1-dev libcurl4-openssl-dev libgcrypt-dev libselinux1-dev libgconf2-dev libacl1-dev libblkid-dev libcap-dev libxml2-dev libxslt1-dev libxml-parser-perl libxml-xpath-perl libperl-dev librpm-dev librtmp-dev libxmlsec1-dev libxmlsec1-openssl python3-dbusmock
+        sudo apt-get -y install lcov swig xsltproc rpm-common lua5.3 libpcre2-dev libyaml-dev libapt-pkg-dev libdbus-1-dev libdbus-glib-1-dev libcurl4-openssl-dev libgcrypt-dev libselinux1-dev libgconf2-dev libacl1-dev libblkid-dev libcap-dev libxml2-dev libxslt1-dev libxml-parser-perl libxml-xpath-perl libperl-dev librpm-dev librtmp-dev libxmlsec1-dev libxmlsec1-openssl python3-dbusmock python3-pytest
         sudo apt-get -y remove rpm
 
     # Runs a set of commands using the runners shell
@@ -57,7 +57,7 @@ jobs:
       image: fedora:latest
     steps:
     - name: Install Deps
-      run: dnf install -y cmake git dbus-devel GConf2-devel libacl-devel libblkid-devel libcap-devel libcurl-devel libgcrypt-devel libselinux-devel libxml2-devel libxslt-devel libattr-devel make openldap-devel pcre2-devel perl-XML-Parser perl-XML-XPath perl-devel python3-devel python3-dbusmock rpm-devel swig bzip2-devel gcc-c++ libyaml-devel xmlsec1-devel xmlsec1-openssl-devel hostname bzip2 lua rpm-build which strace apt-devel
+      run: dnf install -y cmake git dbus-devel GConf2-devel libacl-devel libblkid-devel libcap-devel libcurl-devel libgcrypt-devel libselinux-devel libxml2-devel libxslt-devel libattr-devel make openldap-devel pcre2-devel perl-XML-Parser perl-XML-XPath perl-devel python3-devel python3-dbusmock rpm-devel swig bzip2-devel gcc-c++ libyaml-devel xmlsec1-devel xmlsec1-openssl-devel hostname bzip2 lua rpm-build which strace apt-devel python3-pytest
     - name: Checkout
       uses: actions/checkout@v3
       with:

diff --git a/tests/utils/CMakeLists.txt b/tests/utils/CMakeLists.txt
@@ -1 +1,7 @@
+add_oscap_test("autotailor_integration_test.sh")
 add_oscap_test("test_utils_args.sh")
+
+add_test(
+    NAME "autotailor-unit-tests"
+    COMMAND ${PYTHON_EXECUTABLE} -m pytest -v "${CMAKE_CURRENT_SOURCE_DIR}/test_autotailor.py"
+)
diff --git a/tests/utils/autotailor_integration_test.sh b/tests/utils/autotailor_integration_test.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+. $builddir/tests/test_common.sh
+
+set -e -o pipefail
+
+autotailor="$top_srcdir/utils/autotailor"
+tailoring="$(mktemp)"
+ds="$srcdir/data_stream.xml"
+stdout="$(mktemp)"
+original_profile="P1"
+result="$(mktemp)"
+
+# the original profile P1 selects rules R1 and R2
+
+# select additional rule R3
+python3 $autotailor --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# select additional rules R3, R4
+python3 $autotailor --id-namespace "com.example.www" --select R3 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# unselect rule R2
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# unselect rule R2 and select R4
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# select additional rule R3 and change its severity to high
+python3 $autotailor --id-namespace "com.example.www" --select R3 --rule-severity R3=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="unknown"]'
+
+# don't select rules, don't unselect rules, but change severity of all rules to high
+python3 $autotailor --id-namespace "com.example.www" --rule-severity R1=high --rule-severity R2=high --rule-severity R3=high --rule-severity R4=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="high"]'
+
+
+# select additional rule R4 and change its role to "unchecked"
+python3 $autotailor --id-namespace "com.example.www" --select R4 --rule-role R4=unchecked $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notchecked"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @role="unchecked"]'
+
+
+# select additional rule R3; the customized profile will have a special profile ID
+customized_profile="xccdf_com.pink.elephant_profile_pineapple"
+python3 $autotailor --new-profile-id $customized_profile --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile $customized_profile --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult[@id="xccdf_org.open-scap_testresult_'$customized_profile'"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# refine value v1 to 30
+python3 $autotailor --id-namespace "com.example.www" --var-value V1=thirty $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/set-value[@idref="xccdf_com.example.www_value_V1" and text()="thirty"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
diff --git a/tests/utils/data_stream.xml b/tests/utils/data_stream.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="utf-8"?>
+<ds:data-stream-collection xmlns:ds="http://scap.nist.gov/schema/scap/source/1.2" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:cat="urn:oasis:names:tc:entity:xmlns:xml:catalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="scap_org.open-scap_collection_from_xccdf_test_single_rule.xccdf.xml" schematron-version="1.3" xsi:schemaLocation="http://scap.nist.gov/schema/scap/source/1.2 https://scap.nist.gov/schema/scap/1.3/scap-source-data-stream_1.3.xsd">
+  <ds:data-stream id="scap_org.open-scap_datastream_simple" scap-version="1.3" use-case="OTHER">
+    <ds:checklists>
+      <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.xccdf.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.xccdf.xml">
+        <cat:catalog>
+          <cat:uri name="test_single_rule.oval.xml" uri="#scap_org.open-scap_cref_test_single_rule.oval.xml"/>
+        </cat:catalog>
+      </ds:component-ref>
+    </ds:checklists>
+    <ds:checks>
+      <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.oval.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.oval.xml"/>
+    </ds:checks>
+  </ds:data-stream>
+  <ds:component id="scap_org.open-scap_comp_test_single_rule.oval.xml" timestamp="2021-02-01T08:07:06+01:00">
+    <oval_definitions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:ind-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5" xmlns:win-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#windows" xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd    http://oval.mitre.org/XMLSchema/oval-definitions-5#independent independent-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#windows windows-definitions-schema.xsd">
+      <generator>
+        <oval:schema_version>5.11.2</oval:schema_version>
+        <oval:timestamp>2021-02-01T08:07:06+01:00</oval:timestamp>
+      </generator>
+      <definitions>
+        <definition class="compliance" id="oval:x:def:1" version="1">
+          <metadata>
+            <title>PASS</title>
+            <description>pass</description>
+          </metadata>
+          <criteria>
+            <criterion comment="PASS test" test_ref="oval:x:tst:1"/>
+          </criteria>
+        </definition>
+      </definitions>
+      <tests>
+        <variable_test xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:tst:1" check="all" comment="always pass" version="1">
+          <object object_ref="oval:x:obj:1"/>
+        </variable_test>
+      </tests>
+      <objects>
+        <variable_object xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:obj:1" version="1" comment="x">
+          <var_ref>oval:x:var:1</var_ref>
+        </variable_object>
+      </objects>
+      <variables>
+        <constant_variable id="oval:x:var:1" version="1" comment="x" datatype="int">
+          <value>100</value>
+        </constant_variable>
+      </variables>
+    </oval_definitions>
+  </ds:component>
+  <ds:component id="scap_org.open-scap_comp_test_single_rule.xccdf.xml" timestamp="2021-02-01T08:07:06+01:00">
+    <Benchmark xmlns="http://checklists.nist.gov/xccdf/1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="xccdf_com.example.www_benchmark_dummy" xsi:schemaLocation="http://checklists.nist.gov/xccdf/1.1 xccdf-1.1.4.xsd" resolved="false" xml:lang="en-US">
+      <status date="2021-01-21">accepted</status>
+      <title>Test Benchmark</title>
+      <description>Description</description>
+      <version>1.0</version>
+      <metadata>
+        <dc:contributor xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:contributor>
+        <dc:publisher xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:publisher>
+        <dc:creator xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:creator>
+        <dc:source xmlns:dc="http://purl.org/dc/elements/1.1/">http://scap.nist.gov</dc:source>
+      </metadata>
+      <Profile id="xccdf_com.example.www_profile_P1">
+        <title>xccdf_test_profile</title>
+        <description>This profile is for testing.</description>
+        <select idref="xccdf_com.example.www_rule_R1" selected="true"/>
+        <select idref="xccdf_com.example.www_rule_R2" selected="true"/>
+      </Profile>
+      <Value id="xccdf_com.example.www_value_V1" operator="equals" type="number">
+        <title>value</title>
+        <description xml:lang="en">cccc</description>
+        <question xml:lang="en">ssss</question>
+        <value>5</value>
+        <value selector="thirty">30</value>
+      </Value>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R1">
+        <title>Rule R1</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R2">
+        <title>Rule R2</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R3">
+        <title>Rule R3</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R4">
+        <title>Rule R4</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+    </Benchmark>
+  </ds:component>
+</ds:data-stream-collection>