From 2d3f3e2a881b29b6360c10034aff7881f92ec565 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Fri, 26 Apr 2019 16:11:06 +0200 Subject: [PATCH 01/13] Add script to update SDS to SCAP 1.3 --- build-scripts/update_sds_to_scap_1_3.py | 99 +++++++++++++++++++++++++ cmake/SSGCommon.cmake | 2 + ssg/constants.py | 2 + 3 files changed, 103 insertions(+) create mode 100755 build-scripts/update_sds_to_scap_1_3.py diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py new file mode 100755 index 00000000000..1f78e127d15 --- /dev/null +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -0,0 +1,99 @@ + +from __future__ import print_function + +import sys +from urllib.parse import urlparse + +import ssg.constants +import ssg.xml +ocil_ns = ssg.constants.ocil_namespace + +oval_ns = ssg.constants.oval_namespace +xccdf_ns = ssg.constants.XCCDF12_NS +ds_ns = ssg.constants.datastream_namespace +xlink_ns = ssg.constants.xlink_namespace +cat_ns = ssg.constants.cat_namespace +ns = {'oval': oval_ns, + 'xccdf': xccdf_ns, + 'ds': ds_ns, + 'xlink': xlink_ns, + 'cat': cat_ns} + +component_ref_prefix = "#scap_org.open-scap_cref_" + +# Inspired by openscap ds_sds_mangle_filepath() function +def mangle_path(path): + path = path.replace('/', '-') + path = path.replace('@', '-') + return path + +def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): + ds_checklists = datastreamtree.find(".//ds:checklists", ns) + + for component_ref in ds_checklists: + component_id = component_ref.get('{%s}href' % xlink_ns) + component_id = component_id[1:] + + # Locate the of the with id security_patches_up_to_date + oval_check = datastreamtree.find(".//ds:component[@id='%s']//xccdf:Rule[@id='xccdf_org.ssgproject.content_rule_security_patches_up_to_date']/xccdf:check[@system='%s']" % (component_id, oval_ns), ns ) + # SCAP 1.3 demands multi-check true if the Rules security_patches_up_to_date is + # evaluated by multiple OVAL patch class definitinos. + # See 3.2.4.3, SCAP 1.3 standard (NIST.SP.800-126r3) + if oval_check is None: + continue + oval_check.set('multi-check', 'true') + + check_content_ref = oval_check.find('xccdf:check-content-ref', ns) + href_url = check_content_ref.get('href') + + # Use URL's path to define the component name and URI + component_ref_name = mangle_path(urlparse(href_url).path[1:]) + component_ref_uri = component_ref_prefix + component_ref_name + + # update @href to refer the datastream component name + check_content_ref.set('href', component_ref_name) + + # Add a uri refering the component in Rule's Benchmark component-ref catalog + catalog = component_ref.find('cat:catalog', ns) + uris = catalog.findall("cat:uri[@name='%s']" % component_ref_name, ns) + if not uris: + uri = ssg.xml.ElementTree.Element('{%s}uri' % cat_ns, + attrib = { + 'name' : component_ref_name, + 'uri' : component_ref_uri }) + catalog.append(uri) + + # Add the component-ref to list of datastreams' checks + ds_checks = datastreamtree.find(".//ds:checks", ns) + check_component_ref = ds_checks.findall("ds:component-ref[@id='%s']" % component_ref_uri[1:], ns) + if not check_component_ref: + component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % ds_ns, + attrib = { + 'id' : component_ref_uri[1:], + '{%s}href' % xlink_ns : href_url }) + ds_checks.append(component_ref_feed) + + +def main(): + if len(sys.argv) < 3: + print("This script updates SCAP 1.2 Source DataStream to SCAP 1.3") + sys.exit(1) + + # Input datastream file + indatastreamfile = sys.argv[1] + # Output datastream file + outdatastreamfile = sys.argv[2] + # Datastream element tree + datastreamtree = ssg.xml.ElementTree.parse(indatastreamfile).getroot() + + # Set SCAP version to 1.3 + datastreamtree.set('schematron-version', '1.3') + datastreamtree.find('ds:data-stream', ns).set('scap-version', '1.3') + + # Move reference to remote OVAL content to a source data stream component + move_patches_up_to_date_to_source_data_stream_component(datastreamtree) + + ssg.xml.ElementTree.ElementTree(datastreamtree).write(outdatastreamfile) + +if __name__ == "__main__": + main() diff --git a/cmake/SSGCommon.cmake b/cmake/SSGCommon.cmake index 7e9cd7220af..3bc27d7df06 100644 --- a/cmake/SSGCommon.cmake +++ b/cmake/SSGCommon.cmake @@ -526,6 +526,7 @@ macro(ssg_build_sds PRODUCT) COMMAND "${OPENSCAP_OSCAP_EXECUTABLE}" ds sds-add --skip-valid "ssg-${PRODUCT}-cpe-dictionary.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" COMMAND "${OPENSCAP_OSCAP_EXECUTABLE}" ds sds-add --skip-valid "ssg-${PRODUCT}-pcidss-xccdf-1.2.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/sds_move_ocil_to_checks.py" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" + COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/update_sds_to_scap_1_3.py" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" DEPENDS generate-ssg-${PRODUCT}-xccdf-1.2.xml DEPENDS "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf-1.2.xml" @@ -549,6 +550,7 @@ macro(ssg_build_sds PRODUCT) COMMAND "${SED_EXECUTABLE}" -i 's/schematron-version="[0-9].[0-9]"/schematron-version="1.2"/' "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" COMMAND "${OPENSCAP_OSCAP_EXECUTABLE}" ds sds-add --skip-valid "ssg-${PRODUCT}-cpe-dictionary.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/sds_move_ocil_to_checks.py" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" + COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/update_sds_to_scap_1_3.py" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" DEPENDS generate-ssg-${PRODUCT}-xccdf-1.2.xml DEPENDS "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf-1.2.xml" diff --git a/ssg/constants.py b/ssg/constants.py index 6e4fd3c741b..35983db400d 100644 --- a/ssg/constants.py +++ b/ssg/constants.py @@ -21,6 +21,8 @@ ocil_namespace = "http://scap.nist.gov/schema/ocil/2.0" oval_footer = "" oval_namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" +xlink_namespace = "http://www.w3.org/1999/xlink" +cat_namespace = "urn:oasis:names:tc:entity:xmlns:xml:catalog" ocil_cs = "http://scap.nist.gov/schema/ocil/2" xccdf_header = xml_version + "" xccdf_footer = "" From 73d0fb5e2d230b3e20d37efd6b2a3db62700f8d6 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 11:08:37 +0200 Subject: [PATCH 02/13] Fix import of urlparse for python2 --- build-scripts/update_sds_to_scap_1_3.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 1f78e127d15..87eb9150d0a 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -2,7 +2,10 @@ from __future__ import print_function import sys -from urllib.parse import urlparse +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse import ssg.constants import ssg.xml From 8a97f8fe387df14b8fb25e616609ec91e77134d4 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 11:09:46 +0200 Subject: [PATCH 03/13] Handle tilde during mangling --- build-scripts/update_sds_to_scap_1_3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 87eb9150d0a..9c4ef2dbb69 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -28,6 +28,7 @@ def mangle_path(path): path = path.replace('/', '-') path = path.replace('@', '-') + path = path.replace('~', '-') return path def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): From f0b74944da38dde707b647e1c8c46b61d0c2a4d8 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 13:26:36 +0200 Subject: [PATCH 04/13] Drop use of namespace dictionary It is not supported in python 2.6 --- build-scripts/update_sds_to_scap_1_3.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 9c4ef2dbb69..9653186a9a1 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -16,11 +16,6 @@ ds_ns = ssg.constants.datastream_namespace xlink_ns = ssg.constants.xlink_namespace cat_ns = ssg.constants.cat_namespace -ns = {'oval': oval_ns, - 'xccdf': xccdf_ns, - 'ds': ds_ns, - 'xlink': xlink_ns, - 'cat': cat_ns} component_ref_prefix = "#scap_org.open-scap_cref_" @@ -32,14 +27,14 @@ def mangle_path(path): return path def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): - ds_checklists = datastreamtree.find(".//ds:checklists", ns) + ds_checklists = datastreamtree.find(".//{%s}checklists" % ds_ns) for component_ref in ds_checklists: component_id = component_ref.get('{%s}href' % xlink_ns) component_id = component_id[1:] # Locate the of the with id security_patches_up_to_date - oval_check = datastreamtree.find(".//ds:component[@id='%s']//xccdf:Rule[@id='xccdf_org.ssgproject.content_rule_security_patches_up_to_date']/xccdf:check[@system='%s']" % (component_id, oval_ns), ns ) + oval_check = datastreamtree.find(".//{%s}component[@id='%s']//{%s}Rule[@id='xccdf_org.ssgproject.content_rule_security_patches_up_to_date']/{%s}check[@system='%s']" % (ds_ns, component_id, xccdf_ns, xccdf_ns, oval_ns)) # SCAP 1.3 demands multi-check true if the Rules security_patches_up_to_date is # evaluated by multiple OVAL patch class definitinos. # See 3.2.4.3, SCAP 1.3 standard (NIST.SP.800-126r3) @@ -47,7 +42,7 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): continue oval_check.set('multi-check', 'true') - check_content_ref = oval_check.find('xccdf:check-content-ref', ns) + check_content_ref = oval_check.find('{%s}check-content-ref' % xccdf_ns) href_url = check_content_ref.get('href') # Use URL's path to define the component name and URI @@ -58,8 +53,8 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): check_content_ref.set('href', component_ref_name) # Add a uri refering the component in Rule's Benchmark component-ref catalog - catalog = component_ref.find('cat:catalog', ns) - uris = catalog.findall("cat:uri[@name='%s']" % component_ref_name, ns) + catalog = component_ref.find('{%s}catalog' % cat_ns) + uris = catalog.findall("{%s}uri[@name='%s']" % (cat_ns, component_ref_name)) if not uris: uri = ssg.xml.ElementTree.Element('{%s}uri' % cat_ns, attrib = { @@ -68,8 +63,8 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): catalog.append(uri) # Add the component-ref to list of datastreams' checks - ds_checks = datastreamtree.find(".//ds:checks", ns) - check_component_ref = ds_checks.findall("ds:component-ref[@id='%s']" % component_ref_uri[1:], ns) + ds_checks = datastreamtree.find(".//{%s}checks" % ds_ns) + check_component_ref = ds_checks.findall("{%s}component-ref[@id='%s']" % (ds_ns, component_ref_uri[1:])) if not check_component_ref: component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % ds_ns, attrib = { @@ -92,7 +87,7 @@ def main(): # Set SCAP version to 1.3 datastreamtree.set('schematron-version', '1.3') - datastreamtree.find('ds:data-stream', ns).set('scap-version', '1.3') + datastreamtree.find('{%s}data-stream' % ds_ns).set('scap-version', '1.3') # Move reference to remote OVAL content to a source data stream component move_patches_up_to_date_to_source_data_stream_component(datastreamtree) From 2f0e4bd00a0f4c042825285ef1701fc0158a785d Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 13:45:07 +0200 Subject: [PATCH 05/13] Separate element creation and attribute setting Use of 'attrib' parameter is not supported in python 2.7. --- build-scripts/update_sds_to_scap_1_3.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 9653186a9a1..96fb54160f5 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -56,20 +56,18 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): catalog = component_ref.find('{%s}catalog' % cat_ns) uris = catalog.findall("{%s}uri[@name='%s']" % (cat_ns, component_ref_name)) if not uris: - uri = ssg.xml.ElementTree.Element('{%s}uri' % cat_ns, - attrib = { - 'name' : component_ref_name, - 'uri' : component_ref_uri }) + uri = ssg.xml.ElementTree.Element('{%s}uri' % cat_ns) + uri.set('name', component_ref_name) + uri.set('uri', component_ref_uri) catalog.append(uri) # Add the component-ref to list of datastreams' checks ds_checks = datastreamtree.find(".//{%s}checks" % ds_ns) check_component_ref = ds_checks.findall("{%s}component-ref[@id='%s']" % (ds_ns, component_ref_uri[1:])) if not check_component_ref: - component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % ds_ns, - attrib = { - 'id' : component_ref_uri[1:], - '{%s}href' % xlink_ns : href_url }) + component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % ds_ns) + component_ref_feed.set('id', component_ref_uri[1:]) + component_ref_feed.set('{%s}href' % xlink_ns, href_url) ds_checks.append(component_ref_feed) From a519fa07c17f46ff68b44e53a5864fcc8f2a5f77 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 13:53:58 +0200 Subject: [PATCH 06/13] Explicitly define feed component-ref ID --- build-scripts/update_sds_to_scap_1_3.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 96fb54160f5..c7533e69f88 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -62,11 +62,15 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): catalog.append(uri) # Add the component-ref to list of datastreams' checks + # The component-ref ID is the catalog uri without leading '#' + component_ref_feed_id = component_ref_uri[1:] ds_checks = datastreamtree.find(".//{%s}checks" % ds_ns) - check_component_ref = ds_checks.findall("{%s}component-ref[@id='%s']" % (ds_ns, component_ref_uri[1:])) + check_component_ref = ds_checks.findall("{%s}component-ref[@id='%s']" % + (ds_ns, component_ref_feed_id)) if not check_component_ref: - component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % ds_ns) - component_ref_feed.set('id', component_ref_uri[1:]) + component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % + ds_ns) + component_ref_feed.set('id', component_ref_feed_id) component_ref_feed.set('{%s}href' % xlink_ns, href_url) ds_checks.append(component_ref_feed) From bbcd27188e60b0af24f315ad9205c09b46f2aa7e Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 15:04:49 +0200 Subject: [PATCH 07/13] Filter the component and rules manually Python 2.6 doesn't support attribute filtering in XPath expressions. For every findall, run through the elements lookcing for the one which interests us. --- build-scripts/update_sds_to_scap_1_3.py | 37 +++++++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index c7533e69f88..aaf102ac633 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -19,6 +19,7 @@ component_ref_prefix = "#scap_org.open-scap_cref_" + # Inspired by openscap ds_sds_mangle_filepath() function def mangle_path(path): path = path.replace('/', '-') @@ -26,20 +27,44 @@ def mangle_path(path): path = path.replace('~', '-') return path + def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): ds_checklists = datastreamtree.find(".//{%s}checklists" % ds_ns) for component_ref in ds_checklists: - component_id = component_ref.get('{%s}href' % xlink_ns) - component_id = component_id[1:] + component_ref_id = component_ref.get('id') + component_ref_href = component_ref.get('{%s}href' % xlink_ns) + component_id = component_ref_href[1:] + + # Locate the element of an with id security_patches_up_to_date + component = None + oval_check = None + components = datastreamtree.findall(".//{%s}component" % ds_ns) + for component in components: + if component.get('id') == component_id: + component = component + if component is None: + # Something strange happened + sys.stderr.write("Couldn't find %s referenced by %s" % + (component_id, component_ref_id)) + sys.exit(1) + + rules = component.findall(".//{%s}Rule" % (xccdf_ns)) + for rule in rules: + if rule.get('id').endswith('rule_security_patches_up_to_date'): + rule_checks = rule.findall("{%s}check" % xccdf_ns) + for check in rule_checks: + if check.get('system') == oval_ns: + oval_check = check + break + + if oval_check is None: + # The component doesn't have a security patches up to date rule with an OVAL check + continue - # Locate the of the with id security_patches_up_to_date - oval_check = datastreamtree.find(".//{%s}component[@id='%s']//{%s}Rule[@id='xccdf_org.ssgproject.content_rule_security_patches_up_to_date']/{%s}check[@system='%s']" % (ds_ns, component_id, xccdf_ns, xccdf_ns, oval_ns)) # SCAP 1.3 demands multi-check true if the Rules security_patches_up_to_date is # evaluated by multiple OVAL patch class definitinos. # See 3.2.4.3, SCAP 1.3 standard (NIST.SP.800-126r3) - if oval_check is None: - continue oval_check.set('multi-check', 'true') check_content_ref = oval_check.find('{%s}check-content-ref' % xccdf_ns) From 6786ff54d320dd4f7dd4c4afcde88d329279c112 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 15:36:18 +0200 Subject: [PATCH 08/13] Filter existence of uri and component-ref manually --- build-scripts/update_sds_to_scap_1_3.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index aaf102ac633..098892d49b3 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -78,21 +78,31 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): check_content_ref.set('href', component_ref_name) # Add a uri refering the component in Rule's Benchmark component-ref catalog + uri_exists = False catalog = component_ref.find('{%s}catalog' % cat_ns) - uris = catalog.findall("{%s}uri[@name='%s']" % (cat_ns, component_ref_name)) - if not uris: + uris = catalog.findall("{%s}uri" % (cat_ns)) + for uri in uris: + if uri.get('name') == component_ref_name: + uri_exists = True + break + if not uri_exists: uri = ssg.xml.ElementTree.Element('{%s}uri' % cat_ns) uri.set('name', component_ref_name) uri.set('uri', component_ref_uri) catalog.append(uri) - # Add the component-ref to list of datastreams' checks # The component-ref ID is the catalog uri without leading '#' component_ref_feed_id = component_ref_uri[1:] + + # Add the component-ref to list of datastreams' checks + check_component_ref_exists = False ds_checks = datastreamtree.find(".//{%s}checks" % ds_ns) - check_component_ref = ds_checks.findall("{%s}component-ref[@id='%s']" % - (ds_ns, component_ref_feed_id)) - if not check_component_ref: + check_component_refs = ds_checks.findall("{%s}component-ref" % ds_ns) + for check_component_ref in check_component_refs: + if check_component_ref.get('id') == component_ref_feed_id: + check_component_ref_exists = True + break + if not check_component_ref_exists: component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % ds_ns) component_ref_feed.set('id', component_ref_feed_id) From 57dadd375de118537c2c38565f3778893a3611e7 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 15:37:05 +0200 Subject: [PATCH 09/13] Comment why we strip some leading chars --- build-scripts/update_sds_to_scap_1_3.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 098892d49b3..11d63c5624d 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -34,6 +34,7 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): for component_ref in ds_checklists: component_ref_id = component_ref.get('id') component_ref_href = component_ref.get('{%s}href' % xlink_ns) + # The component ID is the component-ref href without leading '#' component_id = component_ref_href[1:] # Locate the element of an with id security_patches_up_to_date @@ -71,6 +72,9 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): href_url = check_content_ref.get('href') # Use URL's path to define the component name and URI + # Path attribute returned from urlparse contains a leading '/', when mangling it + # it will get replaced by '-'. + # Let's strip the '/' to avoid a sequence of "_-" in the component-ref ID. component_ref_name = mangle_path(urlparse(href_url).path[1:]) component_ref_uri = component_ref_prefix + component_ref_name From ce7ec593fd2db4f6850b21b8fad331c142130f1f Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 16:19:19 +0200 Subject: [PATCH 10/13] Fix bug in component filtering --- build-scripts/update_sds_to_scap_1_3.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 11d63c5624d..cf25c99c907 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -38,19 +38,19 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): component_id = component_ref_href[1:] # Locate the element of an with id security_patches_up_to_date - component = None + checklist_component = None oval_check = None components = datastreamtree.findall(".//{%s}component" % ds_ns) for component in components: if component.get('id') == component_id: - component = component - if component is None: + checklist_component = component + if checklist_component is None: # Something strange happened sys.stderr.write("Couldn't find %s referenced by %s" % (component_id, component_ref_id)) sys.exit(1) - rules = component.findall(".//{%s}Rule" % (xccdf_ns)) + rules = checklist_component.findall(".//{%s}Rule" % (xccdf_ns)) for rule in rules: if rule.get('id').endswith('rule_security_patches_up_to_date'): rule_checks = rule.findall("{%s}check" % xccdf_ns) @@ -83,7 +83,7 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): # Add a uri refering the component in Rule's Benchmark component-ref catalog uri_exists = False - catalog = component_ref.find('{%s}catalog' % cat_ns) + catalog = checklist_component_ref.find('{%s}catalog' % cat_ns) uris = catalog.findall("{%s}uri" % (cat_ns)) for uri in uris: if uri.get('name') == component_ref_name: From 15c70d0012f84ca5c0b868202ea8ba288c93871a Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 16:24:26 +0200 Subject: [PATCH 11/13] Add ds_ prefix to component variables related to DS --- build-scripts/update_sds_to_scap_1_3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index cf25c99c907..87cf1d476c6 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -40,10 +40,10 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): # Locate the element of an with id security_patches_up_to_date checklist_component = None oval_check = None - components = datastreamtree.findall(".//{%s}component" % ds_ns) - for component in components: - if component.get('id') == component_id: - checklist_component = component + ds_components = datastreamtree.findall(".//{%s}component" % ds_ns) + for ds_component in ds_components: + if ds_component.get('id') == component_id: + checklist_component = ds_component if checklist_component is None: # Something strange happened sys.stderr.write("Couldn't find %s referenced by %s" % From 304270f872a51bc5ee4121ae2aeb859c10890f04 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 16:25:31 +0200 Subject: [PATCH 12/13] Add checklists_ prefix to components related to checklist components --- build-scripts/update_sds_to_scap_1_3.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 87cf1d476c6..71d7edfc0ea 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -31,23 +31,22 @@ def mangle_path(path): def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): ds_checklists = datastreamtree.find(".//{%s}checklists" % ds_ns) - for component_ref in ds_checklists: - component_ref_id = component_ref.get('id') - component_ref_href = component_ref.get('{%s}href' % xlink_ns) + for checklists_component_ref in ds_checklists: + checklists_component_ref_id = checklists_component_ref.get('id') # The component ID is the component-ref href without leading '#' - component_id = component_ref_href[1:] + checklists_component_id = checklists_component_ref.get('{%s}href' % xlink_ns)[1:] # Locate the element of an with id security_patches_up_to_date checklist_component = None oval_check = None ds_components = datastreamtree.findall(".//{%s}component" % ds_ns) for ds_component in ds_components: - if ds_component.get('id') == component_id: + if ds_component.get('id') == checklists_component_id: checklist_component = ds_component if checklist_component is None: # Something strange happened sys.stderr.write("Couldn't find %s referenced by %s" % - (component_id, component_ref_id)) + (checklists_component_id, checklists_component_ref_id)) sys.exit(1) rules = checklist_component.findall(".//{%s}Rule" % (xccdf_ns)) @@ -83,7 +82,7 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): # Add a uri refering the component in Rule's Benchmark component-ref catalog uri_exists = False - catalog = checklist_component_ref.find('{%s}catalog' % cat_ns) + catalog = checklists_component_ref.find('{%s}catalog' % cat_ns) uris = catalog.findall("{%s}uri" % (cat_ns)) for uri in uris: if uri.get('name') == component_ref_name: From 3af98cc1d2687ec792ea4266c85ea0883e3955d9 Mon Sep 17 00:00:00 2001 From: Watson Sato Date: Mon, 29 Apr 2019 16:40:38 +0200 Subject: [PATCH 13/13] Reuse namespace definitions from ssg.constants --- build-scripts/update_sds_to_scap_1_3.py | 40 +++++++++++-------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/build-scripts/update_sds_to_scap_1_3.py b/build-scripts/update_sds_to_scap_1_3.py index 71d7edfc0ea..f1c9a29702d 100755 --- a/build-scripts/update_sds_to_scap_1_3.py +++ b/build-scripts/update_sds_to_scap_1_3.py @@ -7,15 +7,10 @@ except ImportError: from urlparse import urlparse -import ssg.constants import ssg.xml -ocil_ns = ssg.constants.ocil_namespace +from ssg.constants import (oval_namespace, XCCDF12_NS, cat_namespace, + datastream_namespace, xlink_namespace) -oval_ns = ssg.constants.oval_namespace -xccdf_ns = ssg.constants.XCCDF12_NS -ds_ns = ssg.constants.datastream_namespace -xlink_ns = ssg.constants.xlink_namespace -cat_ns = ssg.constants.cat_namespace component_ref_prefix = "#scap_org.open-scap_cref_" @@ -29,17 +24,17 @@ def mangle_path(path): def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): - ds_checklists = datastreamtree.find(".//{%s}checklists" % ds_ns) + ds_checklists = datastreamtree.find(".//{%s}checklists" % datastream_namespace) for checklists_component_ref in ds_checklists: checklists_component_ref_id = checklists_component_ref.get('id') # The component ID is the component-ref href without leading '#' - checklists_component_id = checklists_component_ref.get('{%s}href' % xlink_ns)[1:] + checklists_component_id = checklists_component_ref.get('{%s}href' % xlink_namespace)[1:] # Locate the element of an with id security_patches_up_to_date checklist_component = None oval_check = None - ds_components = datastreamtree.findall(".//{%s}component" % ds_ns) + ds_components = datastreamtree.findall(".//{%s}component" % datastream_namespace) for ds_component in ds_components: if ds_component.get('id') == checklists_component_id: checklist_component = ds_component @@ -49,12 +44,12 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): (checklists_component_id, checklists_component_ref_id)) sys.exit(1) - rules = checklist_component.findall(".//{%s}Rule" % (xccdf_ns)) + rules = checklist_component.findall(".//{%s}Rule" % XCCDF12_NS) for rule in rules: if rule.get('id').endswith('rule_security_patches_up_to_date'): - rule_checks = rule.findall("{%s}check" % xccdf_ns) + rule_checks = rule.findall("{%s}check" % XCCDF12_NS) for check in rule_checks: - if check.get('system') == oval_ns: + if check.get('system') == oval_namespace: oval_check = check break @@ -67,7 +62,7 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): # See 3.2.4.3, SCAP 1.3 standard (NIST.SP.800-126r3) oval_check.set('multi-check', 'true') - check_content_ref = oval_check.find('{%s}check-content-ref' % xccdf_ns) + check_content_ref = oval_check.find('{%s}check-content-ref' % XCCDF12_NS) href_url = check_content_ref.get('href') # Use URL's path to define the component name and URI @@ -82,14 +77,14 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): # Add a uri refering the component in Rule's Benchmark component-ref catalog uri_exists = False - catalog = checklists_component_ref.find('{%s}catalog' % cat_ns) - uris = catalog.findall("{%s}uri" % (cat_ns)) + catalog = checklists_component_ref.find('{%s}catalog' % cat_namespace) + uris = catalog.findall("{%s}uri" % cat_namespace) for uri in uris: if uri.get('name') == component_ref_name: uri_exists = True break if not uri_exists: - uri = ssg.xml.ElementTree.Element('{%s}uri' % cat_ns) + uri = ssg.xml.ElementTree.Element('{%s}uri' % cat_namespace) uri.set('name', component_ref_name) uri.set('uri', component_ref_uri) catalog.append(uri) @@ -99,17 +94,17 @@ def move_patches_up_to_date_to_source_data_stream_component(datastreamtree): # Add the component-ref to list of datastreams' checks check_component_ref_exists = False - ds_checks = datastreamtree.find(".//{%s}checks" % ds_ns) - check_component_refs = ds_checks.findall("{%s}component-ref" % ds_ns) + ds_checks = datastreamtree.find(".//{%s}checks" % datastream_namespace) + check_component_refs = ds_checks.findall("{%s}component-ref" % datastream_namespace) for check_component_ref in check_component_refs: if check_component_ref.get('id') == component_ref_feed_id: check_component_ref_exists = True break if not check_component_ref_exists: component_ref_feed = ssg.xml.ElementTree.Element('{%s}component-ref' % - ds_ns) + datastream_namespace) component_ref_feed.set('id', component_ref_feed_id) - component_ref_feed.set('{%s}href' % xlink_ns, href_url) + component_ref_feed.set('{%s}href' % xlink_namespace, href_url) ds_checks.append(component_ref_feed) @@ -127,12 +122,13 @@ def main(): # Set SCAP version to 1.3 datastreamtree.set('schematron-version', '1.3') - datastreamtree.find('{%s}data-stream' % ds_ns).set('scap-version', '1.3') + datastreamtree.find('{%s}data-stream' % datastream_namespace).set('scap-version', '1.3') # Move reference to remote OVAL content to a source data stream component move_patches_up_to_date_to_source_data_stream_component(datastreamtree) ssg.xml.ElementTree.ElementTree(datastreamtree).write(outdatastreamfile) + if __name__ == "__main__": main()