Merge pull request ComplianceAsCode#11648 from Honny1/strip-cpe

Reduction of CPE content in DS

build-scripts/build_xccdf.py

@@ -136,7 +136,7 @@ def main():
     ssg.xml.ElementTree.ElementTree(xccdftree).write(
         args.xccdf, xml_declaration=True, encoding="utf-8")
 
-    if args.thin_ds_components_dir != "off":
+    if args.thin_ds_components_dir is not None and args.thin_ds_components_dir != "off":
         if not os.path.exists(args.thin_ds_components_dir):
             os.makedirs(args.thin_ds_components_dir)
         store_xccdf_per_profile(loader, oval_linker, args.thin_ds_components_dir)

build-scripts/compose_ds.py

@@ -311,8 +311,11 @@ def _compose_multiple_ds(args):
     for xccdf in glob.glob("{}/xccdf*.xml".format(args.multiple_ds)):
         oval = xccdf.replace("xccdf", "oval")
         ocil = xccdf.replace("xccdf", "ocil")
+        cpe_dict = xccdf.replace("xccdf", "cpe_dict")
+        cpe_oval = xccdf.replace("xccdf", "cpe_oval")
+
         ds = compose_ds(
-            xccdf, oval, ocil, args.cpe_dict, args.cpe_oval, args.enable_sce
+            xccdf, oval, ocil, cpe_dict, cpe_oval, args.enable_sce
         )
         output_13 = _get_thin_ds_output_path(args.output_13, xccdf.replace("xccdf_", ""))
         output_12 = None

build-scripts/cpe_generate.py

@@ -6,23 +6,21 @@
 import os
 import ssg
 import argparse
+import glob
 
 import ssg.build_cpe
 import ssg.id_translate
 import ssg.products
 import ssg.xml
 import ssg.yaml
 import ssg.oval_object_model
-from ssg.constants import XCCDF12_NS
+from ssg.constants import XCCDF12_NS, cpe_language_namespace
 
 # This script requires two arguments: an OVAL file and a CPE dictionary file.
 # It is designed to extract any inventory definitions and the tests, states,
 # objects and variables it references and then write them into a standalone
 # OVAL CPE file, along with a synchronized CPE dictionary file.
 
-oval_ns = "http://oval.mitre.org/XMLSchema/oval-definitions-5"
-cpe_ns = "http://cpe.mitre.org/dictionary/2.0"
-
 
 def parse_args():
     p = argparse.ArgumentParser(
@@ -53,30 +51,20 @@ def parse_args():
         "--cpe-items-dir",
         help="the directory where compiled CPE items are stored"
     )
+    p.add_argument(
+        "--thin-ds-components-dir",
+        help="Directory to store CPE OVAL for thin data stream. (off: to disable)"
+        "e.g.: ~/scap-security-guide/build/rhel7/thin_ds_component/"
+        "Fake profiles are used to create thin DS. Components are generated for each profile."
+        "The minimal cpe will be generated from the minimal XCCDF, "
+        "which is in the same directory.",
+    )
     return p.parse_args()
 
 
-def process_cpe_oval(oval_file_path):
-    oval_document = ssg.oval_object_model.load_oval_document(ssg.xml.parse_file(oval_file_path))
-    oval_document.product_name = os.path.basename(__file__)
-
-    references_to_keep = ssg.oval_object_model.OVALDefinitionReference()
-    for oval_def in oval_document.definitions.values():
-        if oval_def.class_ != "inventory":
-            continue
-        references_to_keep += oval_document.get_all_references_of_definition(oval_def.id_)
-
-    oval_document.keep_referenced_components(references_to_keep)
-
-    translator = ssg.id_translate.IDTranslator("ssg")
-    oval_document = translator.translate_oval_document(oval_document, store_defname=True)
-
-    return oval_document
-
-
-def get_benchmark_cpe_names(xccdf_file):
+def get_benchmark_cpe_names(xccdf_el_root_xml):
     benchmark_cpe_names = set()
-    xccdf_el_root_xml = ssg.xml.parse_file(xccdf_file)
+
     for platform in xccdf_el_root_xml.findall(".//{%s}platform" % XCCDF12_NS):
         cpe_name = platform.get("idref")
         # skip CPE AL platforms (they are handled later)
@@ -85,9 +73,7 @@ def get_benchmark_cpe_names(xccdf_file):
             continue
         benchmark_cpe_names.add(cpe_name)
 
-    for fact_ref in xccdf_el_root_xml.findall(
-        ".//{%s}fact-ref" % ssg.constants.PREFIX_TO_NS["cpe-lang"]
-    ):
+    for fact_ref in xccdf_el_root_xml.findall(".//{%s}fact-ref" % cpe_language_namespace):
         cpe_fact_ref_name = fact_ref.get("name")
         benchmark_cpe_names.add(cpe_fact_ref_name)
     return benchmark_cpe_names
@@ -103,6 +89,51 @@ def load_cpe_dictionary(benchmark_cpe_names, product_yaml, cpe_items_dir):
     return cpe_list
 
 
+def _get_all_check_fact_ref(xccdf_el_root_xml):
+    return xccdf_el_root_xml.findall(".//{%s}check-fact-ref" % cpe_language_namespace)
+
+
+def get_all_cpe_oval_def_ids(xccdf_el_root_xml, cpe_dict, benchmark_cpe_names):
+    out = set()
+
+    for cpe_item in cpe_dict.cpe_items:
+        if cpe_item.name in benchmark_cpe_names:
+            out.add(cpe_item.check_id)
+
+    for check_fact_ref in _get_all_check_fact_ref(xccdf_el_root_xml):
+        out.add(check_fact_ref.get("id-ref"))
+    return out
+
+
+def _save_minimal_cpe_oval(oval_document, path, oval_def_ids):
+    references_to_keep = ssg.oval_object_model.OVALDefinitionReference()
+    for oval_def_id in oval_def_ids:
+        references_to_keep += oval_document.get_all_references_of_definition(oval_def_id)
+
+    oval_document.save_as_xml(path, references_to_keep)
+
+
+def _update_oval_href_in_xccdf(xccdf_el_root_xml, file_name):
+    for check_fact_ref in _get_all_check_fact_ref(xccdf_el_root_xml):
+        check_fact_ref.set("href", file_name)
+
+
+def _generate_cpe_for_thin_xccdf(thin_ds_components_dir, oval_document, cpe_dict):
+    for xccdf_path in glob.glob("{}/xccdf*.xml".format(thin_ds_components_dir)):
+        cpe_oval_path = xccdf_path.replace("xccdf_", "cpe_oval_")
+        cpe_dict_path = xccdf_path.replace("xccdf_", "cpe_dict_")
+
+        xccdf_el_root_xml = ssg.xml.parse_file(xccdf_path)
+        benchmark_cpe_names = get_benchmark_cpe_names(xccdf_el_root_xml)
+        used_cpe_oval_def_ids = get_all_cpe_oval_def_ids(
+            xccdf_el_root_xml, cpe_dict, benchmark_cpe_names
+        )
+        cpe_dict.to_file(cpe_dict_path, os.path.basename(cpe_oval_path), benchmark_cpe_names)
+        _save_minimal_cpe_oval(oval_document, cpe_oval_path, used_cpe_oval_def_ids)
+        _update_oval_href_in_xccdf(xccdf_el_root_xml, os.path.basename(cpe_oval_path))
+        ssg.xml.ElementTree.ElementTree(xccdf_el_root_xml).write(xccdf_path, encoding="utf-8")
+
+
 def main():
     args = parse_args()
 
@@ -116,14 +147,24 @@ def main():
     cpe_dict_filename = "ssg-{}-cpe-dictionary.xml".format(product)
     cpe_dict_path = os.path.join(args.cpeoutdir, cpe_dict_filename)
 
-    oval_document = process_cpe_oval(args.ovalfile)
-    oval_document.save_as_xml(oval_file_path)
+    xccdf_el_root_xml = ssg.xml.parse_file(args.xccdfFile)
 
-    benchmark_cpe_names = get_benchmark_cpe_names(args.xccdfFile)
+    benchmark_cpe_names = get_benchmark_cpe_names(xccdf_el_root_xml)
     cpe_dict = load_cpe_dictionary(benchmark_cpe_names, product_yaml, args.cpe_items_dir)
     cpe_dict.translate_cpe_oval_def_ids()
     cpe_dict.to_file(cpe_dict_path, oval_filename)
 
+    used_cpe_oval_def_ids = get_all_cpe_oval_def_ids(
+        xccdf_el_root_xml, cpe_dict, benchmark_cpe_names
+    )
+    oval_document = ssg.build_cpe.get_linked_cpe_oval_document(args.ovalfile)
+    _save_minimal_cpe_oval(oval_document, oval_file_path, used_cpe_oval_def_ids)
+
+    if args.thin_ds_components_dir is not None and args.thin_ds_components_dir != "off":
+        if not os.path.exists(args.thin_ds_components_dir):
+            os.makedirs(args.thin_ds_components_dir)
+        _generate_cpe_for_thin_xccdf(args.thin_ds_components_dir, oval_document, cpe_dict)
+
     sys.exit(0)
 
 

build-scripts/enable_derivatives.py

@@ -16,8 +16,8 @@
 import sys
 from optparse import OptionParser
 
-import ssg.constants
 import ssg.build_derivatives
+import ssg.constants
 import ssg.xccdf
 import ssg.xml
 
@@ -48,6 +48,12 @@ def parse_args():
     parser.add_option(
         "--cpe-items-dir",
         dest="cpe_items_dir", help="path to the directory where compiled cpe items are stored")
+    parser.add_option(
+        "--unlinked-cpe-oval-path",
+        dest="unlinked_oval_file_path",
+        help="path to the unlinked cpe oval"
+    )
+
     (options, args) = parser.parse_args()
 
     if options.centos and options.sl:
@@ -63,6 +69,12 @@ def parse_args():
     return options, args
 
 
+def store_xml(tree, path):
+    if hasattr(ssg.xml.ElementTree, "indent"):
+        ssg.xml.ElementTree.indent(tree, space="  ", level=0)
+    tree.write(path, encoding="utf-8", xml_declaration=True)
+
+
 def main():
     options, args = parse_args()
 
@@ -112,10 +124,15 @@ def main():
             )
 
     ssg.build_derivatives.replace_platform(root, oval_ns, derivative)
-    ssg.build_derivatives.add_cpe_item_to_dictionary(
-        root, args[0], args[1], options.id_name, options.cpe_items_dir)
+    oval_def_id = ssg.build_derivatives.add_cpe_item_to_dictionary(
+        root, args[0], args[1], options.id_name, options.cpe_items_dir
+    )
+    if oval_def_id is not None:
+        ssg.build_derivatives.add_oval_definition_to_cpe_oval(
+            root, options.unlinked_oval_file_path, oval_def_id
+        )
 
-    tree.write(options.output)
+    store_xml(tree, options.output)
 
 
 if __name__ == "__main__":

cmake/SSGCommon.cmake

@@ -383,7 +383,7 @@ macro(ssg_build_cpe_dictionary PRODUCT)
     add_custom_command(
         OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml"
         OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml"
-        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/cpe_generate.py" --product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
+        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/cpe_generate.py" --product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml" --thin-ds-components-dir "${SSG_THIN_DS_COMPONENTS_DIR}"
         COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml"
         COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml"
         DEPENDS generate-${PRODUCT}-xccdf-oval-ocil "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-oval.xml" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-ocil.xml"
@@ -932,7 +932,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE)
 
     add_custom_command(
         OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml"
-        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items"
+        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
         DEPENDS generate-ssg-${ORIGINAL}-xccdf.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml"
         DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}"
         COMMENT "[${DERIVATIVE}-content] generating ssg-${DERIVATIVE}-xccdf.xml"
@@ -944,7 +944,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE)
 
     add_custom_command(
         OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml"
-        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items"
+        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
         COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml"
         DEPENDS generate-ssg-${ORIGINAL}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml"
         DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}"
@@ -958,7 +958,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE)
     if(SSG_BUILD_SCAP_12_DS)
         add_custom_command(
             OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml"
-            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items"
+            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
             COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml"
             DEPENDS generate-ssg-${ORIGINAL}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml"
             DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}"

ssg/build_cpe.py

@@ -15,6 +15,9 @@
 from .boolean_expression import Algebra, Symbol, Function
 from .entities.common import XCCDFEntity, Templatable
 from .yaml import convert_string_to_bool
+from .oval_object_model import load_oval_document, OVALDefinitionReference
+from .id_translate import IDTranslator
+from .xml import parse_file
 
 
 class CPEDoesNotExist(Exception):
@@ -147,21 +150,35 @@ def __init__(self):
     def add(self, cpe_item):
         self.cpe_items.append(cpe_item)
 
-    def to_xml_element(self, cpe_oval_file):
+    @staticmethod
+    def _create_cpe_list_xml_skeleton():
         cpe_list = ET.Element("{%s}cpe-list" % CPEList.ns)
         cpe_list.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
         cpe_list.set("xsi:schemaLocation",
                      "http://cpe.mitre.org/dictionary/2.0 "
                      "http://cpe.mitre.org/files/cpe-dictionary_2.1.xsd")
+        return cpe_list
 
+    def _add_cpe_items_xml(self, cpe_list, cpe_oval_file, selection_of_cpe_names):
         self.cpe_items.sort(key=lambda cpe: cpe.name)
         for cpe_item in self.cpe_items:
-            cpe_list.append(cpe_item.to_xml_element(cpe_oval_file))
+            if cpe_item.name in selection_of_cpe_names:
+                cpe_list.append(cpe_item.to_xml_element(cpe_oval_file))
+
+    def to_xml_element(self, cpe_oval_file, selection_of_cpe_names=None):
+        cpe_list = self._create_cpe_list_xml_skeleton()
 
+        if selection_of_cpe_names is None:
+            selection_of_cpe_names = [cpe_item.name for cpe_item in self.cpe_items]
+
+        self._add_cpe_items_xml(cpe_list, cpe_oval_file, selection_of_cpe_names)
+
+        if hasattr(ET, "indent"):
+            ET.indent(cpe_list, space="  ", level=0)
         return cpe_list
 
-    def to_file(self, file_name, cpe_oval_file):
-        root = self.to_xml_element(cpe_oval_file)
+    def to_file(self, file_name, cpe_oval_file, selection_of_cpe_names=None):
+        root = self.to_xml_element(cpe_oval_file, selection_of_cpe_names)
         tree = ET.ElementTree(root)
         tree.write(file_name, encoding="utf-8")
 
@@ -439,3 +456,23 @@ def extract_referred_nodes(tree_with_refs, tree_with_ids, attrname):
             elementlist.append(element)
 
     return elementlist
+
+
+def get_linked_cpe_oval_document(unlinked_oval_file_path):
+    oval_document = load_oval_document(parse_file(unlinked_oval_file_path))
+    oval_document.product_name = os.path.basename(__file__)
+
+    references_to_keep = OVALDefinitionReference()
+    for oval_def in oval_document.definitions.values():
+        if oval_def.class_ != "inventory":
+            continue
+        references_to_keep += oval_document.get_all_references_of_definition(
+            oval_def.id_
+        )
+
+    oval_document.keep_referenced_components(references_to_keep)
+
+    translator = IDTranslator("ssg")
+    oval_document = translator.translate_oval_document(oval_document)
+
+    return oval_document