Add a refine-rule tailoring ability to autotailor

tests/utils/CMakeLists.txt

@@ -1 +1,7 @@
+add_oscap_test("autotailor_integration_test.sh")
 add_oscap_test("test_utils_args.sh")
+
+add_test(
+    NAME "autotailor-unit-tests"
+    COMMAND ${PYTHON_EXECUTABLE} -m pytest -v "${CMAKE_CURRENT_SOURCE_DIR}/test_autotailor.py"
+)

tests/utils/autotailor_integration_test.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+. $builddir/tests/test_common.sh
+
+set -e -o pipefail
+
+autotailor="$top_srcdir/utils/autotailor"
+tailoring="$(mktemp)"
+ds="$srcdir/data_stream.xml"
+stdout="$(mktemp)"
+original_profile="P1"
+result="$(mktemp)"
+
+# the original profile P1 selects rules R1 and R2
+
+# select additional rule R3
+python3 $autotailor --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# select additional rules R3, R4
+python3 $autotailor --id-namespace "com.example.www" --select R3 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# unselect rule R2
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# unselect rule R2 and select R4
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# select additional rule R3 and change its severity to high
+python3 $autotailor --id-namespace "com.example.www" --select R3 --rule-severity R3=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="unknown"]'
+
+# don't select rules, don't unselect rules, but change severity of all rules to high
+python3 $autotailor --id-namespace "com.example.www" --rule-severity R1=high --rule-severity R2=high --rule-severity R3=high --rule-severity R4=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="high"]'
+
+
+# select additional rule R4 and change its role to "unchecked"
+python3 $autotailor --id-namespace "com.example.www" --select R4 --rule-role R4=unchecked $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notchecked"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @role="unchecked"]'
+
+
+# select additional rule R3; the customized profile will have a special profile ID
+customized_profile="xccdf_com.pink.elephant_profile_pineapple"
+python3 $autotailor --new-profile-id $customized_profile --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile $customized_profile --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult[@id="xccdf_org.open-scap_testresult_'$customized_profile'"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# refine value v1 to 30
+python3 $autotailor --id-namespace "com.example.www" --var-value V1=thirty $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/set-value[@idref="xccdf_com.example.www_value_V1" and text()="thirty"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'

tests/utils/data_stream.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="utf-8"?>
+<ds:data-stream-collection xmlns:ds="http://scap.nist.gov/schema/scap/source/1.2" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:cat="urn:oasis:names:tc:entity:xmlns:xml:catalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="scap_org.open-scap_collection_from_xccdf_test_single_rule.xccdf.xml" schematron-version="1.3" xsi:schemaLocation="http://scap.nist.gov/schema/scap/source/1.2 https://scap.nist.gov/schema/scap/1.3/scap-source-data-stream_1.3.xsd">
+  <ds:data-stream id="scap_org.open-scap_datastream_simple" scap-version="1.3" use-case="OTHER">
+    <ds:checklists>
+      <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.xccdf.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.xccdf.xml">
+        <cat:catalog>
+          <cat:uri name="test_single_rule.oval.xml" uri="#scap_org.open-scap_cref_test_single_rule.oval.xml"/>
+        </cat:catalog>
+      </ds:component-ref>
+    </ds:checklists>
+    <ds:checks>
+      <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.oval.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.oval.xml"/>
+    </ds:checks>
+  </ds:data-stream>
+  <ds:component id="scap_org.open-scap_comp_test_single_rule.oval.xml" timestamp="2021-02-01T08:07:06+01:00">
+    <oval_definitions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:ind-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5" xmlns:win-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#windows" xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd    http://oval.mitre.org/XMLSchema/oval-definitions-5#independent independent-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#windows windows-definitions-schema.xsd">
+      <generator>
+        <oval:schema_version>5.11.2</oval:schema_version>
+        <oval:timestamp>2021-02-01T08:07:06+01:00</oval:timestamp>
+      </generator>
+      <definitions>
+        <definition class="compliance" id="oval:x:def:1" version="1">
+          <metadata>
+            <title>PASS</title>
+            <description>pass</description>
+          </metadata>
+          <criteria>
+            <criterion comment="PASS test" test_ref="oval:x:tst:1"/>
+          </criteria>
+        </definition>
+      </definitions>
+      <tests>
+        <variable_test xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:tst:1" check="all" comment="always pass" version="1">
+          <object object_ref="oval:x:obj:1"/>
+        </variable_test>
+      </tests>
+      <objects>
+        <variable_object xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:obj:1" version="1" comment="x">
+          <var_ref>oval:x:var:1</var_ref>
+        </variable_object>
+      </objects>
+      <variables>
+        <constant_variable id="oval:x:var:1" version="1" comment="x" datatype="int">
+          <value>100</value>
+        </constant_variable>
+      </variables>
+    </oval_definitions>
+  </ds:component>
+  <ds:component id="scap_org.open-scap_comp_test_single_rule.xccdf.xml" timestamp="2021-02-01T08:07:06+01:00">
+    <Benchmark xmlns="http://checklists.nist.gov/xccdf/1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="xccdf_com.example.www_benchmark_dummy" xsi:schemaLocation="http://checklists.nist.gov/xccdf/1.1 xccdf-1.1.4.xsd" resolved="false" xml:lang="en-US">
+      <status date="2021-01-21">accepted</status>
+      <title>Test Benchmark</title>
+      <description>Description</description>
+      <version>1.0</version>
+      <metadata>
+        <dc:contributor xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:contributor>
+        <dc:publisher xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:publisher>
+        <dc:creator xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:creator>
+        <dc:source xmlns:dc="http://purl.org/dc/elements/1.1/">http://scap.nist.gov</dc:source>
+      </metadata>
+      <Profile id="xccdf_com.example.www_profile_P1">
+        <title>xccdf_test_profile</title>
+        <description>This profile is for testing.</description>
+        <select idref="xccdf_com.example.www_rule_R1" selected="true"/>
+        <select idref="xccdf_com.example.www_rule_R2" selected="true"/>
+      </Profile>
+      <Value id="xccdf_com.example.www_value_V1" operator="equals" type="number">
+        <title>value</title>
+        <description xml:lang="en">cccc</description>
+        <question xml:lang="en">ssss</question>
+        <value>5</value>
+        <value selector="thirty">30</value>
+      </Value>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R1">
+        <title>Rule R1</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R2">
+        <title>Rule R2</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R3">
+        <title>Rule R3</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+      <Rule selected="false" id="xccdf_com.example.www_rule_R4">
+        <title>Rule R4</title>
+        <description>Description</description>
+        <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+          <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+        </check>
+      </Rule>
+    </Benchmark>
+  </ds:component>
+</ds:data-stream-collection>

tests/utils/test_autotailor.py

@@ -0,0 +1,96 @@
+import importlib
+import pytest
+import xml.etree.ElementTree as ET
+
+NS = "http://checklists.nist.gov/xccdf/1.2"
+
+def import_arbitrary_file_as_module(path, module_name):
+    spec = importlib.util.spec_from_loader(
+        module_name, importlib.machinery.SourceFileLoader(module_name, path))
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+autotailor = import_arbitrary_file_as_module("../../../utils/autotailor", "autotailor")
+
+
+def test_is_valid_xccdf_id():
+    assert autotailor.is_valid_xccdf_id(
+        "xccdf_com.example.www_profile_abcd")
+    assert autotailor.is_valid_xccdf_id(
+        "xccdf_com.example.www_rule_selinux_state")
+    assert not autotailor.is_valid_xccdf_id("")
+    assert not autotailor.is_valid_xccdf_id("foo")
+    assert not autotailor.is_valid_xccdf_id(
+        "xccdf_com_example_www_rule_selinux_state")
+    assert not autotailor.is_valid_xccdf_id("xccdf_rule_selinux_state")
+    assert not autotailor.is_valid_xccdf_id("xccdf_com.example.www_rule_")
+
+
+def test_full_id():
+    t = autotailor.Tailoring()
+    assert t._full_rule_id("accounts_tmout") == \
+        "xccdf_org.ssgproject.content_rule_accounts_tmout"
+    assert t._full_rule_id(
+        "xccdf_org.ssgproject.content_rule_accounts_tmout") == \
+        "xccdf_org.ssgproject.content_rule_accounts_tmout"
+    assert t._full_profile_id("stig") == \
+        "xccdf_org.ssgproject.content_profile_stig"
+    assert t._full_profile_id(
+        "xccdf_org.ssgproject.content_profile_stig") == \
+        "xccdf_org.ssgproject.content_profile_stig"
+    assert t._full_var_id("var_crypto_policy") == \
+        "xccdf_org.ssgproject.content_value_var_crypto_policy"
+    assert t._full_var_id(
+        "xccdf_org.ssgproject.content_value_var_crypto_policy") == \
+        "xccdf_org.ssgproject.content_value_var_crypto_policy"
+
+
+def test_customized_profile_id():
+    t = autotailor.Tailoring()
+    t.extends = "stig"
+    assert t.profile_id == "stig_customized"
+    t.profile_id = "my_cool_profile"
+    assert t.profile_id == "my_cool_profile"
+
+
+def test_refine_rule():
+    t = autotailor.Tailoring()
+    with pytest.raises(ValueError) as e:
+        t.refine_rule("selinux_state", "severity", "high")
+    assert str(e.value) == "Rule id 'selinux_state' is invalid!"
+    with pytest.raises(ValueError) as e:
+        t.refine_rule(
+            "xccdf_org.ssgproject.content_rule_accounts_tmout", "foo", "bar")
+    assert str(e.value) == "Unsupported refine-rule attribute foo"
+    with pytest.raises(ValueError) as e:
+        t.refine_rule(
+            "xccdf_org.ssgproject.content_rule_accounts_tmout",
+            "role", "mnau")
+    assert str(e.value) == (
+        "Can't refine role of rule 'xccdf_org.ssgproject.content_rule_accounts"
+        "_tmout' to 'mnau'. Allowed role values are: \"full\", \"unscored\", "
+        "\"unchecked\".")
+    with pytest.raises(ValueError) as e:
+        t.refine_rule(
+            "xccdf_org.ssgproject.content_rule_accounts_tmout",
+            "severity", "mnau")
+    assert str(e.value) == (
+        "Can't refine severity of rule 'xccdf_org.ssgproject.content_rule_"
+        "accounts_tmout' to 'mnau'. Allowed severity values are: \"unknown\", "
+        "\"info\", \"low\", \"medium\", \"high\".")
+    fav = "xccdf_org.ssgproject.content_rule_accounts_tmout"
+    t.refine_rule(fav, "severity", "high")
+    assert t.rule_refinements(fav, "severity") == "high"
+    t.refine_rule(fav, "role", "full")
+    assert t.rule_refinements(fav, "severity") == "high"
+    assert t.rule_refinements(fav, "role") == "full"
+    with pytest.raises(ValueError) as e:
+        t.refine_rule(fav, "severity", "low")
+    assert str(e.value) == (
+        "Can't refine severity of rule 'xccdf_org.ssgproject.content_rule_"
+        "accounts_tmout' to 'low'. This rule severity is already refined to "
+        "'high'.")
+    assert t.rule_refinements(fav, "severity") == "high"
+    assert t.rule_refinements(fav, "role") == "full"