From 2d842980d0547179289d8c49de70e3ac766b5f1e Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Mon, 8 Jan 2024 15:09:17 +0100 Subject: [PATCH 01/10] Separate subparsers to functions --- build-scripts/profile_tool.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index b6c61cc08c9..6fddd73fc48 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -23,14 +23,12 @@ exit(1) -def parse_args(): +def parse_stats(subparsers): script_desc = \ "Obtains and displays XCCDF profile statistics. Namely number " + \ "of rules in the profile, how many of these rules have their OVAL " + \ "check implemented, how many have a remediation available, ..." - parser = argparse.ArgumentParser(description="Profile statistics and utilities tool") - subparsers = parser.add_subparsers(title='subcommands', dest="subcommand") parser_stats = subparsers.add_parser("stats", description=script_desc, help=("Show profile statistics")) parser_stats.add_argument("--profile", "-p", @@ -111,6 +109,8 @@ def parse_args(): parser_stats.add_argument("--output", help="If defined, statistics will be stored under this directory.") + +def parse_sub(subparsers): subtracted_profile_desc = \ "Subtract rules and variable selections from profile1 based on rules present in " + \ "profile2. As a result, a new profile is generated. It doesn't support profile " + \ @@ -141,11 +141,14 @@ def parse_args(): parser_sub.add_argument('--profile2', type=str, dest="profile2", required=True, help='YAML profile') - args = parser.parse_args() - if not args.subcommand: - parser.print_help() - exit(0) +def parse_args(): + parser = argparse.ArgumentParser(description="Profile statistics and utilities tool") + subparsers = parser.add_subparsers(title='subcommands', dest="subcommand", required=True) + parse_stats(subparsers) + parse_sub(subparsers) + + args = parser.parse_args() if args.subcommand == "stats": if args.all: From c752f001bc43f0ef1325f185219a13ef1d729022 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Mon, 8 Jan 2024 15:15:36 +0100 Subject: [PATCH 02/10] Separate commands to functions --- build-scripts/profile_tool.py | 85 +++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index 6fddd73fc48..869b761fac6 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -176,47 +176,43 @@ def parse_args(): return args -def main(): - args = parse_args() +def sub(args): + product_yaml = os.path.join(args.ssg_root, "products", args.product, "product.yml") + env_yaml = ssg.environment.open_environment(args.build_config_yaml, product_yaml) + try: + profile1 = ssg.build_yaml.Profile.from_yaml(args.profile1, env_yaml) + profile2 = ssg.build_yaml.Profile.from_yaml(args.profile2, env_yaml) + except jinja2.exceptions.TemplateNotFound as e: + print("Error: Profile {} could not be found.".format(str(e))) + exit(1) + + subtracted_profile = profile1 - profile2 + + exclusive_rules = len(subtracted_profile.get_rule_selectors()) + exclusive_vars = len(subtracted_profile.get_variable_selectors()) + if exclusive_rules > 0: + print("{} rules were left after subtraction.".format(exclusive_rules)) + if exclusive_vars > 0: + print("{} variables were left after subtraction.".format(exclusive_vars)) + + if exclusive_rules > 0 or exclusive_vars > 0: + profile1_basename = os.path.splitext( + os.path.basename(args.profile1))[0] + profile2_basename = os.path.splitext( + os.path.basename(args.profile2))[0] + subtracted_profile_filename = "{}_sub_{}.profile".format( + profile1_basename, profile2_basename) + print("Creating a new profile containing the exclusive selections: {}".format( + subtracted_profile_filename)) + subtracted_profile.title = profile1.title + " subtracted by " + profile2.title + subtracted_profile.dump_yaml(subtracted_profile_filename) + print("Profile {} was created successfully".format( + subtracted_profile_filename)) + else: + print("Subtraction would produce an empty profile. No new profile was generated") - if args.subcommand == "sub": - product_yaml = os.path.join(args.ssg_root, "products", args.product, "product.yml") - env_yaml = ssg.environment.open_environment(args.build_config_yaml, product_yaml) - try: - profile1 = ssg.build_yaml.Profile.from_yaml(args.profile1, env_yaml) - profile2 = ssg.build_yaml.Profile.from_yaml(args.profile2, env_yaml) - except jinja2.exceptions.TemplateNotFound as e: - print("Error: Profile {} could not be found.".format(str(e))) - exit(1) - - subtracted_profile = profile1 - profile2 - - exclusive_rules = len(subtracted_profile.get_rule_selectors()) - exclusive_vars = len(subtracted_profile.get_variable_selectors()) - if exclusive_rules > 0: - print("{} rules were left after subtraction.".format(exclusive_rules)) - if exclusive_vars > 0: - print("{} variables were left after subtraction.".format(exclusive_vars)) - - if exclusive_rules > 0 or exclusive_vars > 0: - profile1_basename = os.path.splitext( - os.path.basename(args.profile1))[0] - profile2_basename = os.path.splitext( - os.path.basename(args.profile2))[0] - - subtracted_profile_filename = "{}_sub_{}.profile".format( - profile1_basename, profile2_basename) - print("Creating a new profile containing the exclusive selections: {}".format( - subtracted_profile_filename)) - - subtracted_profile.title = profile1.title + " subtracted by " + profile2.title - subtracted_profile.dump_yaml(subtracted_profile_filename) - print("Profile {} was created successfully".format( - subtracted_profile_filename)) - else: - print("Subtraction would produce an empty profile. No new profile was generated") - exit(0) +def stats(args): benchmark = ssg.build_profile.XCCDFBenchmark(args.benchmark, args.product) ret = [] if args.profile: @@ -293,5 +289,16 @@ def main(): print(",".join([str(value) for value in line.values()])) +SUBCMDS = dict( + stats=stats, + sub=sub +) + + +def main(): + args = parse_args() + SUBCMDS[args.subcommand](args) + + if __name__ == '__main__': main() From a6784e0b1d554952b04b27fe88fc776b44aefd53 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Wed, 10 Jan 2024 12:01:08 +0100 Subject: [PATCH 03/10] Decompose statistics generation in html --- build-scripts/profile_tool.py | 122 +++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index 869b761fac6..29f22a67152 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -212,28 +212,25 @@ def sub(args): print("Subtraction would produce an empty profile. No new profile was generated") -def stats(args): - benchmark = ssg.build_profile.XCCDFBenchmark(args.benchmark, args.product) - ret = [] - if args.profile: - ret.append(benchmark.show_profile_stats(args.profile, args)) - else: - ret.extend(benchmark.show_all_profile_stats(args)) - - if args.format == "json": - print(json.dumps(ret, indent=4)) - if args.format == "html": - from json2html import json2html - filtered_output = [] - output_path = "./" - if args.output: - output_path = args.output - mkdir_p(output_path) - - content_path = os.path.join(output_path, "content") - mkdir_p(content_path) - - content_list = [ +def _process_stats_content(profile, bash_fixes_count, content, content_filepath): + link = """
{}
""" + count = len(profile[content]) + if content == "ansible_parity": + # custom text link for ansible parity + count = link.format( + content_filepath, + "{} out of {} ({}%)".format( + bash_fixes_count - count, + bash_fixes_count, + int(((bash_fixes_count-count)/bash_fixes_count)*100) + ) + ) + count_href_element = link.format(content_filepath, count) + profile['{}_count'.format(content)] = count_href_element + + +def _filter_profile_for_html_stats(profile, filtered_output, content_path): + content_list = [ 'rules', 'missing_stig_ids', 'missing_cis_refs', @@ -256,36 +253,65 @@ def stats(args): 'missing_checks', 'missing_fixes' ] - link = """
{}
""" - - for profile in ret: - bash_fixes_count = profile['rules_count'] - profile['missing_bash_fixes_count'] - for content in content_list: - content_file = "{}_{}.txt".format(profile['profile_id'], content) - content_filepath = os.path.join("content", content_file) - count = len(profile[content]) - if count > 0: - if content == "ansible_parity": - #custom text link for ansible parity - count = link.format(content_filepath, "{} out of {} ({}%)".format(bash_fixes_count-count, bash_fixes_count, int(((bash_fixes_count-count)/bash_fixes_count)*100))) - count_href_element = link.format(content_filepath, count) - profile['{}_count'.format(content)] = count_href_element - with open(os.path.join(content_path, content_file), 'w+') as f: - f.write('\n'.join(profile[content])) - else: - profile['{}_count'.format(content)] = count - - del profile[content] - filtered_output.append(profile) - - with open(os.path.join(output_path, "statistics.html"), 'w+') as f: - f.write(json2html.convert(json=json.dumps(filtered_output), escape=False)) + + bash_fixes_count = profile['rules_count'] - profile['missing_bash_fixes_count'] + + for content in content_list: + content_file = "{}_{}.txt".format(profile['profile_id'], content) + content_filepath = os.path.join("content", content_file) + count = len(profile[content]) + if count > 0: + _process_stats_content(profile, bash_fixes_count, content, content_filepath) + with open(os.path.join(content_path, content_file), 'w+') as f: + f.write('\n'.join(profile[content])) + else: + profile['{}_count'.format(content)] = count + del profile[content] + filtered_output.append(profile) + + +def _get_profiles(args): + benchmark = ssg.build_profile.XCCDFBenchmark(args.benchmark, args.product) + ret = [] + if args.profile: + ret.append(benchmark.show_profile_stats(args.profile, args)) + else: + ret.extend(benchmark.show_all_profile_stats(args)) + return ret + + +def _generate_html_stats(args, profiles): + from json2html import json2html + filtered_output = [] + output_path = "./" + if args.output: + output_path = args.output + mkdir_p(output_path) + + content_path = os.path.join(output_path, "content") + mkdir_p(content_path) + + for profile in profiles: + _filter_profile_for_html_stats(profile, filtered_output, content_path) + + with open(os.path.join(output_path, "statistics.html"), 'w+') as f: + f.write(json2html.convert(json=json.dumps(filtered_output), escape=False)) + + +def stats(args): + profiles = _get_profiles(args) + + if args.format == "json": + print(json.dumps(profiles, indent=4)) + + elif args.format == "html": + _generate_html_stats(args, profiles) elif args.format == "csv": # we can assume ret has at least one element # CSV header - print(",".join(ret[0].keys())) - for line in ret: + print(",".join(profiles[0].keys())) + for line in profiles: print(",".join([str(value) for value in line.values()])) From f07eb011579b51fd0921a0d4a4ce921ffdc6dd0f Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Wed, 10 Jan 2024 13:53:38 +0100 Subject: [PATCH 04/10] Move command processing to separate files --- build-scripts/profile_tool.py | 154 +-------------------------------- utils/profile_tool/__init__.py | 2 + utils/profile_tool/stats.py | 109 +++++++++++++++++++++++ utils/profile_tool/sub.py | 41 +++++++++ 4 files changed, 155 insertions(+), 151 deletions(-) create mode 100644 utils/profile_tool/__init__.py create mode 100644 utils/profile_tool/stats.py create mode 100644 utils/profile_tool/sub.py diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index 29f22a67152..3353531387b 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -2,19 +2,10 @@ from __future__ import print_function -import json import argparse -import jinja2 -import os -import os.path try: - import ssg.build_profile - import ssg.constants - import ssg.environment - import ssg.xml - import ssg.build_yaml - from ssg.utils import mkdir_p + from utils.profile_tool import command_stats, command_sub except ImportError: print("The ssg module could not be found.") print("Run .pyenv.sh available in the project root diretory," @@ -176,148 +167,9 @@ def parse_args(): return args -def sub(args): - product_yaml = os.path.join(args.ssg_root, "products", args.product, "product.yml") - env_yaml = ssg.environment.open_environment(args.build_config_yaml, product_yaml) - try: - profile1 = ssg.build_yaml.Profile.from_yaml(args.profile1, env_yaml) - profile2 = ssg.build_yaml.Profile.from_yaml(args.profile2, env_yaml) - except jinja2.exceptions.TemplateNotFound as e: - print("Error: Profile {} could not be found.".format(str(e))) - exit(1) - - subtracted_profile = profile1 - profile2 - - exclusive_rules = len(subtracted_profile.get_rule_selectors()) - exclusive_vars = len(subtracted_profile.get_variable_selectors()) - if exclusive_rules > 0: - print("{} rules were left after subtraction.".format(exclusive_rules)) - if exclusive_vars > 0: - print("{} variables were left after subtraction.".format(exclusive_vars)) - - if exclusive_rules > 0 or exclusive_vars > 0: - profile1_basename = os.path.splitext( - os.path.basename(args.profile1))[0] - profile2_basename = os.path.splitext( - os.path.basename(args.profile2))[0] - subtracted_profile_filename = "{}_sub_{}.profile".format( - profile1_basename, profile2_basename) - print("Creating a new profile containing the exclusive selections: {}".format( - subtracted_profile_filename)) - subtracted_profile.title = profile1.title + " subtracted by " + profile2.title - subtracted_profile.dump_yaml(subtracted_profile_filename) - print("Profile {} was created successfully".format( - subtracted_profile_filename)) - else: - print("Subtraction would produce an empty profile. No new profile was generated") - - -def _process_stats_content(profile, bash_fixes_count, content, content_filepath): - link = """
{}
""" - count = len(profile[content]) - if content == "ansible_parity": - # custom text link for ansible parity - count = link.format( - content_filepath, - "{} out of {} ({}%)".format( - bash_fixes_count - count, - bash_fixes_count, - int(((bash_fixes_count-count)/bash_fixes_count)*100) - ) - ) - count_href_element = link.format(content_filepath, count) - profile['{}_count'.format(content)] = count_href_element - - -def _filter_profile_for_html_stats(profile, filtered_output, content_path): - content_list = [ - 'rules', - 'missing_stig_ids', - 'missing_cis_refs', - 'missing_hipaa_refs', - 'missing_anssi_refs', - 'missing_ospp_refs', - 'missing_cui_refs', - 'missing_ovals', - 'missing_sces', - 'missing_bash_fixes', - 'missing_ansible_fixes', - 'missing_ignition_fixes', - 'missing_kubernetes_fixes', - 'missing_puppet_fixes', - 'missing_anaconda_fixes', - 'missing_cces', - 'ansible_parity', - 'implemented_checks', - 'implemented_fixes', - 'missing_checks', - 'missing_fixes' - ] - - bash_fixes_count = profile['rules_count'] - profile['missing_bash_fixes_count'] - - for content in content_list: - content_file = "{}_{}.txt".format(profile['profile_id'], content) - content_filepath = os.path.join("content", content_file) - count = len(profile[content]) - if count > 0: - _process_stats_content(profile, bash_fixes_count, content, content_filepath) - with open(os.path.join(content_path, content_file), 'w+') as f: - f.write('\n'.join(profile[content])) - else: - profile['{}_count'.format(content)] = count - del profile[content] - filtered_output.append(profile) - - -def _get_profiles(args): - benchmark = ssg.build_profile.XCCDFBenchmark(args.benchmark, args.product) - ret = [] - if args.profile: - ret.append(benchmark.show_profile_stats(args.profile, args)) - else: - ret.extend(benchmark.show_all_profile_stats(args)) - return ret - - -def _generate_html_stats(args, profiles): - from json2html import json2html - filtered_output = [] - output_path = "./" - if args.output: - output_path = args.output - mkdir_p(output_path) - - content_path = os.path.join(output_path, "content") - mkdir_p(content_path) - - for profile in profiles: - _filter_profile_for_html_stats(profile, filtered_output, content_path) - - with open(os.path.join(output_path, "statistics.html"), 'w+') as f: - f.write(json2html.convert(json=json.dumps(filtered_output), escape=False)) - - -def stats(args): - profiles = _get_profiles(args) - - if args.format == "json": - print(json.dumps(profiles, indent=4)) - - elif args.format == "html": - _generate_html_stats(args, profiles) - - elif args.format == "csv": - # we can assume ret has at least one element - # CSV header - print(",".join(profiles[0].keys())) - for line in profiles: - print(",".join([str(value) for value in line.values()])) - - SUBCMDS = dict( - stats=stats, - sub=sub + stats=command_stats, + sub=command_sub ) diff --git a/utils/profile_tool/__init__.py b/utils/profile_tool/__init__.py new file mode 100644 index 00000000000..016d023e6b1 --- /dev/null +++ b/utils/profile_tool/__init__.py @@ -0,0 +1,2 @@ +from .sub import command_sub +from .stats import command_stats diff --git a/utils/profile_tool/stats.py b/utils/profile_tool/stats.py new file mode 100644 index 00000000000..fa6aa9c9f22 --- /dev/null +++ b/utils/profile_tool/stats.py @@ -0,0 +1,109 @@ +import os +import json + +from ssg.build_profile import XCCDFBenchmark +from ssg.utils import mkdir_p + + +def _process_stats_content(profile, bash_fixes_count, content, content_filepath): + link = """
{}
""" + count = len(profile[content]) + if content == "ansible_parity": + # custom text link for ansible parity + count = link.format( + content_filepath, + "{} out of {} ({}%)".format( + bash_fixes_count - count, + bash_fixes_count, + int(((bash_fixes_count - count) / bash_fixes_count) * 100), + ), + ) + count_href_element = link.format(content_filepath, count) + profile["{}_count".format(content)] = count_href_element + + +def _filter_profile_for_html_stats(profile, filtered_output, content_path): + content_list = [ + "rules", + "missing_stig_ids", + "missing_cis_refs", + "missing_hipaa_refs", + "missing_anssi_refs", + "missing_ospp_refs", + "missing_cui_refs", + "missing_ovals", + "missing_sces", + "missing_bash_fixes", + "missing_ansible_fixes", + "missing_ignition_fixes", + "missing_kubernetes_fixes", + "missing_puppet_fixes", + "missing_anaconda_fixes", + "missing_cces", + "ansible_parity", + "implemented_checks", + "implemented_fixes", + "missing_checks", + "missing_fixes", + ] + + bash_fixes_count = profile["rules_count"] - profile["missing_bash_fixes_count"] + + for content in content_list: + content_file = "{}_{}.txt".format(profile["profile_id"], content) + content_filepath = os.path.join("content", content_file) + count = len(profile[content]) + if count > 0: + _process_stats_content(profile, bash_fixes_count, content, content_filepath) + with open(os.path.join(content_path, content_file), "w+") as f: + f.write("\n".join(profile[content])) + else: + profile["{}_count".format(content)] = count + del profile[content] + filtered_output.append(profile) + + +def _get_profiles(args): + benchmark = XCCDFBenchmark(args.benchmark, args.product) + ret = [] + if args.profile: + ret.append(benchmark.show_profile_stats(args.profile, args)) + else: + ret.extend(benchmark.show_all_profile_stats(args)) + return ret + + +def _generate_html_stats(args, profiles): + from json2html import json2html + + filtered_output = [] + output_path = "./" + if args.output: + output_path = args.output + mkdir_p(output_path) + + content_path = os.path.join(output_path, "content") + mkdir_p(content_path) + + for profile in profiles: + _filter_profile_for_html_stats(profile, filtered_output, content_path) + + with open(os.path.join(output_path, "statistics.html"), "w+") as f: + f.write(json2html.convert(json=json.dumps(filtered_output), escape=False)) + + +def command_stats(args): + profiles = _get_profiles(args) + + if args.format == "json": + print(json.dumps(profiles, indent=4)) + + elif args.format == "html": + _generate_html_stats(args, profiles) + + elif args.format == "csv": + # we can assume ret has at least one element + # CSV header + print(",".join(profiles[0].keys())) + for line in profiles: + print(",".join([str(value) for value in line.values()])) diff --git a/utils/profile_tool/sub.py b/utils/profile_tool/sub.py new file mode 100644 index 00000000000..c93c4677fcc --- /dev/null +++ b/utils/profile_tool/sub.py @@ -0,0 +1,41 @@ +import os +import jinja2 +from ssg.build_yaml import Profile +from ssg.environment import open_environment + + +def command_sub(args): + product_yaml = os.path.join(args.ssg_root, "products", args.product, "product.yml") + env_yaml = open_environment(args.build_config_yaml, product_yaml) + try: + profile1 = Profile.from_yaml(args.profile1, env_yaml) + profile2 = Profile.from_yaml(args.profile2, env_yaml) + except jinja2.exceptions.TemplateNotFound as e: + print("Error: Profile {} could not be found.".format(str(e))) + exit(1) + + subtracted_profile = profile1 - profile2 + + exclusive_rules = len(subtracted_profile.get_rule_selectors()) + exclusive_vars = len(subtracted_profile.get_variable_selectors()) + if exclusive_rules > 0: + print("{} rules were left after subtraction.".format(exclusive_rules)) + if exclusive_vars > 0: + print("{} variables were left after subtraction.".format(exclusive_vars)) + + if exclusive_rules > 0 or exclusive_vars > 0: + profile1_basename = os.path.splitext(os.path.basename(args.profile1))[0] + profile2_basename = os.path.splitext(os.path.basename(args.profile2))[0] + subtracted_profile_filename = "{}_sub_{}.profile".format( + profile1_basename, profile2_basename + ) + print( + "Creating a new profile containing the exclusive selections: {}".format( + subtracted_profile_filename + ) + ) + subtracted_profile.title = profile1.title + " subtracted by " + profile2.title + subtracted_profile.dump_yaml(subtracted_profile_filename) + print("Profile {} was created successfully".format(subtracted_profile_filename)) + else: + print("Subtraction would produce an empty profile. No new profile was generated") From 473b2269934a6686a5dfe450cf73461dd656b3bb Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Wed, 10 Jan 2024 13:56:59 +0100 Subject: [PATCH 05/10] Format via black tool --- build-scripts/profile_tool.py | 319 ++++++++++++++++++++++------------ 1 file changed, 206 insertions(+), 113 deletions(-) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index 3353531387b..4f8e4de6111 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -8,134 +8,230 @@ from utils.profile_tool import command_stats, command_sub except ImportError: print("The ssg module could not be found.") - print("Run .pyenv.sh available in the project root diretory," - " or add it to PYTHONPATH manually.") + print( + "Run .pyenv.sh available in the project root diretory," + " or add it to PYTHONPATH manually." + ) print("$ source .pyenv.sh") exit(1) def parse_stats(subparsers): - script_desc = \ - "Obtains and displays XCCDF profile statistics. Namely number " + \ - "of rules in the profile, how many of these rules have their OVAL " + \ - "check implemented, how many have a remediation available, ..." - - parser_stats = subparsers.add_parser("stats", description=script_desc, - help=("Show profile statistics")) - parser_stats.add_argument("--profile", "-p", - action="store", - help="Show statistics for this XCCDF Profile only. If " - "not provided the script will show stats for all " - "available profiles.") - parser_stats.add_argument( - "--benchmark", "-b", required=True, action="store", - help="Specify XCCDF file or a SCAP source data stream file to act on.") - parser_stats.add_argument("--implemented-ovals", default=False, - action="store_true", dest="implemented_ovals", - help="Show IDs of implemented OVAL checks.") - parser_stats.add_argument("--implemented-sces", default=False, - action="store_true", dest="implemented_sces", - help="Show IDs of implemented SCE checks.") - parser_stats.add_argument("--missing-stig-ids", default=False, - action="store_true", dest="missing_stig_ids", - help="Show rules in STIG profiles that don't have STIG IDs.") - parser_stats.add_argument("--missing-cis-refs", default=False, - action="store_true", dest="missing_cis_refs", - help="Show rules in CIS profiles that don't have CIS references.") - parser_stats.add_argument("--missing-hipaa-refs", default=False, - action="store_true", dest="missing_hipaa_refs", - help="Show rules in HIPAA profiles that don't have HIPAA references.") - parser_stats.add_argument("--missing-anssi-refs", default=False, - action="store_true", dest="missing_anssi_refs", - help="Show rules in ANSSI profiles that don't have ANSSI references.") - parser_stats.add_argument("--missing-ospp-refs", default=False, - action="store_true", dest="missing_ospp_refs", - help="Show rules in OSPP profiles that don't have OSPP references.") - parser_stats.add_argument("--missing-cui-refs", default=False, - action="store_true", dest="missing_cui_refs", - help="Show rules in CUI profiles that don't have CUI references.") - parser_stats.add_argument("--missing-ovals", default=False, - action="store_true", dest="missing_ovals", - help="Show IDs of unimplemented OVAL checks.") - parser_stats.add_argument("--missing-sces", default=False, - action="store_true", dest="missing_sces", - help="Show IDs of unimplemented SCE checks.") - parser_stats.add_argument("--implemented-fixes", default=False, - action="store_true", dest="implemented_fixes", - help="Show IDs of implemented remediations.") - parser_stats.add_argument("--missing-fixes", default=False, - action="store_true", dest="missing_fixes", - help="Show IDs of unimplemented remediations.") - parser_stats.add_argument("--assigned-cces", default=False, - action="store_true", dest="assigned_cces", - help="Show IDs of rules having CCE assigned.") - parser_stats.add_argument("--missing-cces", default=False, - action="store_true", dest="missing_cces", - help="Show IDs of rules missing CCE element.") - parser_stats.add_argument("--implemented", default=False, - action="store_true", - help="Equivalent of --implemented-ovals, " - "--implemented_fixes and --assigned-cves " - "all being set.") - parser_stats.add_argument("--missing", default=False, - action="store_true", - help="Equivalent of --missing-ovals, --missing-fixes" - " and --missing-cces all being set.") - parser_stats.add_argument("--ansible-parity", - action="store_true", - help="Show IDs of rules with Bash fix which miss Ansible fix." - " Rules missing both Bash and Ansible are not shown.") - parser_stats.add_argument("--all", default=False, - action="store_true", dest="all", - help="Show all available statistics.") - parser_stats.add_argument("--product", action="store", dest="product", - help="Product directory to evaluate XCCDF under " - "(e.g., ~/scap-security-guide/rhel8)") - parser_stats.add_argument("--skip-stats", default=False, - action="store_true", dest="skip_overall_stats", - help="Do not show overall statistics.") - parser_stats.add_argument("--format", default="plain", - choices=["plain", "json", "csv", "html"], - help="Which format to use for output.") - parser_stats.add_argument("--output", - help="If defined, statistics will be stored under this directory.") + parser_stats = subparsers.add_parser( + "stats", + description=( + "Obtains and displays XCCDF profile statistics. Namely number of rules in the profile," + " how many of these rules have their OVAL check implemented, how many have " + "a remediation available, ..." + ), + help=("Show profile statistics"), + ) + parser_stats.add_argument( + "--profile", + "-p", + action="store", + help=( + "Show statistics for this XCCDF Profile only. If not provided the script will show " + "stats for all available profiles." + ), + ) + parser_stats.add_argument( + "--benchmark", + "-b", + required=True, + action="store", + help="Specify XCCDF file or a SCAP source data stream file to act on.", + ) + parser_stats.add_argument( + "--implemented-ovals", + default=False, + action="store_true", + dest="implemented_ovals", + help="Show IDs of implemented OVAL checks.", + ) + parser_stats.add_argument( + "--implemented-sces", + default=False, + action="store_true", + dest="implemented_sces", + help="Show IDs of implemented SCE checks.", + ) + parser_stats.add_argument( + "--missing-stig-ids", + default=False, + action="store_true", + dest="missing_stig_ids", + help="Show rules in STIG profiles that don't have STIG IDs.", + ) + parser_stats.add_argument( + "--missing-cis-refs", + default=False, + action="store_true", + dest="missing_cis_refs", + help="Show rules in CIS profiles that don't have CIS references.", + ) + parser_stats.add_argument( + "--missing-hipaa-refs", + default=False, + action="store_true", + dest="missing_hipaa_refs", + help="Show rules in HIPAA profiles that don't have HIPAA references.", + ) + parser_stats.add_argument( + "--missing-anssi-refs", + default=False, + action="store_true", + dest="missing_anssi_refs", + help="Show rules in ANSSI profiles that don't have ANSSI references.", + ) + parser_stats.add_argument( + "--missing-ospp-refs", + default=False, + action="store_true", + dest="missing_ospp_refs", + help="Show rules in OSPP profiles that don't have OSPP references.", + ) + parser_stats.add_argument( + "--missing-cui-refs", + default=False, + action="store_true", + dest="missing_cui_refs", + help="Show rules in CUI profiles that don't have CUI references.", + ) + parser_stats.add_argument( + "--missing-ovals", + default=False, + action="store_true", + dest="missing_ovals", + help="Show IDs of unimplemented OVAL checks.", + ) + parser_stats.add_argument( + "--missing-sces", + default=False, + action="store_true", + dest="missing_sces", + help="Show IDs of unimplemented SCE checks.", + ) + parser_stats.add_argument( + "--implemented-fixes", + default=False, + action="store_true", + dest="implemented_fixes", + help="Show IDs of implemented remediations.", + ) + parser_stats.add_argument( + "--missing-fixes", + default=False, + action="store_true", + dest="missing_fixes", + help="Show IDs of unimplemented remediations.", + ) + parser_stats.add_argument( + "--assigned-cces", + default=False, + action="store_true", + dest="assigned_cces", + help="Show IDs of rules having CCE assigned.", + ) + parser_stats.add_argument( + "--missing-cces", + default=False, + action="store_true", + dest="missing_cces", + help="Show IDs of rules missing CCE element.", + ) + parser_stats.add_argument( + "--implemented", + default=False, + action="store_true", + help="Equivalent of --implemented-ovals, --implemented_fixes and --assigned-cves " + "all being set.", + ) + parser_stats.add_argument( + "--missing", + default=False, + action="store_true", + help="Equivalent of --missing-ovals, --missing-fixes and --missing-cces all being set.", + ) + parser_stats.add_argument( + "--ansible-parity", + action="store_true", + help="Show IDs of rules with Bash fix which miss Ansible fix." + " Rules missing both Bash and Ansible are not shown.", + ) + parser_stats.add_argument( + "--all", + default=False, + action="store_true", + dest="all", + help="Show all available statistics.", + ) + parser_stats.add_argument( + "--product", + action="store", + dest="product", + help="Product directory to evaluate XCCDF under (e.g., ~/scap-security-guide/rhel8)", + ) + parser_stats.add_argument( + "--skip-stats", + default=False, + action="store_true", + dest="skip_overall_stats", + help="Do not show overall statistics.", + ) + parser_stats.add_argument( + "--format", + default="plain", + choices=["plain", "json", "csv", "html"], + help="Which format to use for output.", + ) + parser_stats.add_argument( + "--output", help="If defined, statistics will be stored under this directory." + ) def parse_sub(subparsers): - subtracted_profile_desc = \ - "Subtract rules and variable selections from profile1 based on rules present in " + \ - "profile2. As a result, a new profile is generated. It doesn't support profile " + \ - "inheritance, this means that only rules explicitly " + \ - "listed in the profiles will be taken in account." - - parser_sub = subparsers.add_parser("sub", description=subtracted_profile_desc, - help=("Subtract rules and variables from profile1 " - "based on selections present in profile2.")) + parser_sub = subparsers.add_parser( + "sub", + description=( + "Subtract rules and variable selections from profile1 based on rules present in " + "profile2. As a result, a new profile is generated. It doesn't support profile " + "inheritance, this means that only rules explicitly " + "listed in the profiles will be taken in account." + ), + help=( + "Subtract rules and variables from profile1 " + "based on selections present in profile2." + ), + ) parser_sub.add_argument( - "--build-config-yaml", required=True, + "--build-config-yaml", + required=True, help="YAML file with information about the build configuration. " "e.g.: ~/scap-security-guide/build/build_config.yml " - "needed for autodetection of profile root" + "needed for autodetection of profile root", + ) + parser_sub.add_argument( + "--ssg-root", + required=True, + help="Directory containing the source tree. e.g. ~/scap-security-guide/", + ) + parser_sub.add_argument( + "--product", + required=True, + help="ID of the product for which we are building Playbooks. e.g.: 'fedora'", ) parser_sub.add_argument( - "--ssg-root", required=True, - help="Directory containing the source tree. " - "e.g. ~/scap-security-guide/" + "--profile1", type=str, dest="profile1", required=True, help="YAML profile" ) parser_sub.add_argument( - "--product", required=True, - help="ID of the product for which we are building Playbooks. " - "e.g.: 'fedora'" + "--profile2", type=str, dest="profile2", required=True, help="YAML profile" ) - parser_sub.add_argument('--profile1', type=str, dest="profile1", - required=True, help='YAML profile') - parser_sub.add_argument('--profile2', type=str, dest="profile2", - required=True, help='YAML profile') def parse_args(): parser = argparse.ArgumentParser(description="Profile statistics and utilities tool") - subparsers = parser.add_subparsers(title='subcommands', dest="subcommand", required=True) + subparsers = parser.add_subparsers(title="subcommands", dest="subcommand", required=True) parse_stats(subparsers) parse_sub(subparsers) @@ -167,10 +263,7 @@ def parse_args(): return args -SUBCMDS = dict( - stats=command_stats, - sub=command_sub -) +SUBCMDS = dict(stats=command_stats, sub=command_sub) def main(): @@ -178,5 +271,5 @@ def main(): SUBCMDS[args.subcommand](args) -if __name__ == '__main__': +if __name__ == "__main__": main() From 9bd9461470ec13e61431a9983dab92f0d479aae9 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Fri, 12 Jan 2024 15:47:13 +0100 Subject: [PATCH 06/10] Rename functions --- build-scripts/profile_tool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index 4f8e4de6111..a5b310962cb 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -16,7 +16,7 @@ exit(1) -def parse_stats(subparsers): +def parse_stats_subcommand(subparsers): parser_stats = subparsers.add_parser( "stats", description=( @@ -190,7 +190,7 @@ def parse_stats(subparsers): ) -def parse_sub(subparsers): +def parse_sub_subcommand(subparsers): parser_sub = subparsers.add_parser( "sub", description=( @@ -232,8 +232,8 @@ def parse_sub(subparsers): def parse_args(): parser = argparse.ArgumentParser(description="Profile statistics and utilities tool") subparsers = parser.add_subparsers(title="subcommands", dest="subcommand", required=True) - parse_stats(subparsers) - parse_sub(subparsers) + parse_stats_subcommand(subparsers) + parse_sub_subcommand(subparsers) args = parser.parse_args() From 59a88ea0e12d741642162164c6ba199c4b82c4b1 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Fri, 12 Jan 2024 15:47:25 +0100 Subject: [PATCH 07/10] Fix typo --- build-scripts/profile_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/profile_tool.py b/build-scripts/profile_tool.py index a5b310962cb..d316290889f 100755 --- a/build-scripts/profile_tool.py +++ b/build-scripts/profile_tool.py @@ -9,7 +9,7 @@ except ImportError: print("The ssg module could not be found.") print( - "Run .pyenv.sh available in the project root diretory," + "Run .pyenv.sh available in the project root directory," " or add it to PYTHONPATH manually." ) print("$ source .pyenv.sh") From c32928774d46a0df987a6b988e375254d338f583 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Fri, 12 Jan 2024 15:49:33 +0100 Subject: [PATCH 08/10] Move import to top of file --- utils/profile_tool/stats.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/profile_tool/stats.py b/utils/profile_tool/stats.py index fa6aa9c9f22..a3ff5a026e6 100644 --- a/utils/profile_tool/stats.py +++ b/utils/profile_tool/stats.py @@ -1,5 +1,6 @@ import os import json +from json2html import json2html from ssg.build_profile import XCCDFBenchmark from ssg.utils import mkdir_p @@ -74,8 +75,6 @@ def _get_profiles(args): def _generate_html_stats(args, profiles): - from json2html import json2html - filtered_output = [] output_path = "./" if args.output: From fd0ce5dd36fe7f3700a9788c35b67078d350075b Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Fri, 12 Jan 2024 16:01:03 +0100 Subject: [PATCH 09/10] Remove unnessary conditionals --- utils/profile_tool/sub.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/utils/profile_tool/sub.py b/utils/profile_tool/sub.py index c93c4677fcc..005dd9e1417 100644 --- a/utils/profile_tool/sub.py +++ b/utils/profile_tool/sub.py @@ -18,12 +18,11 @@ def command_sub(args): exclusive_rules = len(subtracted_profile.get_rule_selectors()) exclusive_vars = len(subtracted_profile.get_variable_selectors()) - if exclusive_rules > 0: - print("{} rules were left after subtraction.".format(exclusive_rules)) - if exclusive_vars > 0: - print("{} variables were left after subtraction.".format(exclusive_vars)) - if exclusive_rules > 0 or exclusive_vars > 0: + print( + "{} rules were left after subtraction.\n" + "{} variables were left after subtraction.".format(exclusive_rules, exclusive_vars) + ) profile1_basename = os.path.splitext(os.path.basename(args.profile1))[0] profile2_basename = os.path.splitext(os.path.basename(args.profile2))[0] subtracted_profile_filename = "{}_sub_{}.profile".format( From 36fc0e07da40eae8694f1f661fe6d18f15d1c583 Mon Sep 17 00:00:00 2001 From: Jan Rodak Date: Mon, 15 Jan 2024 13:41:55 +0100 Subject: [PATCH 10/10] Fix import problem No module named 'json2html' --- utils/profile_tool/stats.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/utils/profile_tool/stats.py b/utils/profile_tool/stats.py index a3ff5a026e6..55ce6493f3e 100644 --- a/utils/profile_tool/stats.py +++ b/utils/profile_tool/stats.py @@ -1,11 +1,18 @@ import os import json -from json2html import json2html from ssg.build_profile import XCCDFBenchmark from ssg.utils import mkdir_p +OFF_JSON_TO_HTML = False + +try: + from json2html import json2html +except ImportError: + OFF_JSON_TO_HTML = True + + def _process_stats_content(profile, bash_fixes_count, content, content_filepath): link = """
{}
""" count = len(profile[content]) @@ -98,6 +105,9 @@ def command_stats(args): print(json.dumps(profiles, indent=4)) elif args.format == "html": + if OFF_JSON_TO_HTML: + print("No module named 'json2html'. Please install module to enable this function.") + return _generate_html_stats(args, profiles) elif args.format == "csv":