Merge pull request #11618 from Honny1/speedup-thin-ds

Speed up build of thin data streams

build-scripts/build_xccdf.py

@@ -76,6 +76,7 @@ def link_oval(xccdftree, checks, output_file_name, build_ovals_dir):
     oval_linker.link()
     oval_linker.save_linked_tree()
     oval_linker.link_xccdf()
+    return oval_linker
 
 
 def link_ocil(xccdftree, checks, output_file_name, ocil):
@@ -88,39 +89,19 @@ def link_ocil(xccdftree, checks, output_file_name, ocil):
     ocil_linker.link_xccdf()
 
 
-def link_benchmark(loader, xccdftree, args, benchmark=None):
-    checks = xccdftree.findall(".//{%s}check" % ssg.constants.XCCDF12_NS)
-
-    link_oval(xccdftree, checks, args.oval, args.build_ovals_dir)
-
-    ocil = loader.export_ocil_to_xml(benchmark)
-    if ocil is not None:
-        link_ocil(xccdftree, checks, args.ocil, ocil)
-
-    ssg.xml.ElementTree.ElementTree(xccdftree).write(args.xccdf, encoding="utf-8")
-
-
-def get_path(path, file_name):
-    return os.path.join(path, file_name)
-
-
-def link_benchmark_per_profile(loader, args):
-    if not os.path.exists(args.thin_ds_components_dir):
-        os.makedirs(args.thin_ds_components_dir)
+def store_xccdf_per_profile(loader, oval_linker, thin_ds_components_dir):
+    for id_, xccdftree in loader.get_benchmark_xml_by_profile():
+        xccdf_file_name = os.path.join(thin_ds_components_dir, "xccdf_{}.xml".format(id_))
+        oval_file_name = os.path.join(thin_ds_components_dir, "oval_{}.xml".format(id_))
 
-    loader.off_ocil = True
+        checks = xccdftree.findall(".//{%s}check" % ssg.constants.XCCDF12_NS)
+        oval_linker.linked_fname = oval_file_name
+        oval_linker.linked_fname_basename = os.path.basename(oval_file_name)
+        oval_linker.checks_related_to_us = oval_linker.get_related_checks(checks)
 
-    for id_, benchmark in loader.get_benchmark_by_profile():
-        xccdftree = benchmark.to_xml_element(loader.env_yaml)
-        p = Paths_(
-            xccdf=get_path(args.thin_ds_components_dir, "xccdf_{}.xml".format(id_)),
-            oval=get_path(args.thin_ds_components_dir, "oval_{}.xml".format(id_)),
-            ocil=get_path(args.thin_ds_components_dir, "ocil_{}.xml".format(id_)),
-            build_ovals_dir=args.build_ovals_dir
-        )
-        link_benchmark(loader, xccdftree, p, benchmark)
+        oval_linker.link_xccdf()
 
-    loader.off_ocil = False
+        ssg.xml.ElementTree.ElementTree(xccdftree).write(xccdf_file_name, encoding="utf-8")
 
 
 def main():
@@ -143,12 +124,23 @@ def main():
     loader.load_benchmark(benchmark_root)
 
     loader.add_fixes_to_rules()
+    xccdftree = loader.export_benchmark_to_xml()
 
-    if args.thin_ds_components_dir != "off":
-        link_benchmark_per_profile(loader, args)
+    checks = xccdftree.findall(".//{%s}check" % ssg.constants.XCCDF12_NS)
 
-    xccdftree = loader.export_benchmark_to_xml()
-    link_benchmark(loader, xccdftree, args)
+    oval_linker = link_oval(xccdftree, checks, args.oval, args.build_ovals_dir)
+
+    ocil = loader.export_ocil_to_xml()
+    link_ocil(xccdftree, checks, args.ocil, ocil)
+
+    ssg.xml.ElementTree.ElementTree(xccdftree).write(args.xccdf, encoding="utf-8")
+
+    if args.thin_ds_components_dir != "off":
+        if not os.path.exists(args.thin_ds_components_dir):
+            os.makedirs(args.thin_ds_components_dir)
+        store_xccdf_per_profile(loader, oval_linker, args.thin_ds_components_dir)
+        oval_linker.build_ovals_dir = args.thin_ds_components_dir
+        oval_linker.save_oval_document_for_each_xccdf_rule("oval_")
 
 
 if __name__ == "__main__":

ssg/build_renumber.py

@@ -31,14 +31,14 @@ class FileLinker(object):
 
     def __init__(self, translator, xccdftree, checks, output_file_name):
         self.translator = translator
-        self.checks_related_to_us = self._get_related_checks(checks)
+        self.checks_related_to_us = self.get_related_checks(checks)
         self.fname = self._get_input_fname()
         self.tree = None
         self.linked_fname = output_file_name
         self.linked_fname_basename = os.path.basename(self.linked_fname)
         self.xccdftree = xccdftree
 
-    def _get_related_checks(self, checks):
+    def get_related_checks(self, checks):
         """
         Returns a list of checks which have the same check system as this
         class.
@@ -89,6 +89,7 @@ def add_missing_check_exports(self, check, checkcontentref):
         pass
 
     def link_xccdf(self):
+
         for check in self.checks_related_to_us:
             checkcontentref = get_content_ref_if_exists_and_not_remote(check)
             if checkcontentref is None:
@@ -141,16 +142,19 @@ def _get_list_of_names_of_oval_checks(self):
                 checkcontentref = get_content_ref_if_exists_and_not_remote(check)
                 if checkcontentref is None or check.get("system") != oval_cs:
                     continue
-
                 out.append(checkcontentref.get("name"))
         return out
 
-    def _save_oval_document_for_each_xccdf_rule(self):
+    def save_oval_document_for_each_xccdf_rule(self, file_name_prefix=""):
         for name in self._get_list_of_names_of_oval_checks():
+            if name in self.oval_document.definitions:
+                oval_def = self.oval_document.definitions[name]
+                name = oval_def.name
+
             oval_id = self._translate_name_to_oval_definition_id(name)
 
             refs = self.oval_document.get_all_references_of_definition(oval_id)
-            path = self._get_path_for_oval_document(name)
+            path = self._get_path_for_oval_document(file_name_prefix + name)
             with open(path, "wb+") as fd:
                 self.oval_document.save_as_xml(fd, refs)
 
@@ -165,7 +169,7 @@ def save_linked_tree(self):
             self.oval_document.save_as_xml(fd)
 
         if self.build_ovals_dir:
-            self._save_oval_document_for_each_xccdf_rule()
+            self.save_oval_document_for_each_xccdf_rule()
 
     def link(self):
         self.oval_document = load_oval_document(parse_file(self.fname))
@@ -276,7 +280,7 @@ def _ensure_by_xccdf_referenced_oval_no_extra_def_in_oval_file(self):
         # Remove all OVAL checks that are not referenced by XCCDF Rules (checks)
         # or internally via extend-definition
 
-        xccdf_oval_check_refs = [name for name in self._get_list_of_names_of_oval_checks()]
+        xccdf_oval_check_refs = self._get_list_of_names_of_oval_checks()
         document_def_keys = list(self.oval_document.definitions.keys())
 
         references_from_xccdf_to_keep = OVALDefinitionReference()