From 56473e26c0046cfdf2c06f3f7422187724054bf7 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 17 Feb 2022 01:08:26 +0100 Subject: [PATCH 01/27] replaced pv2.yml generation scripts with build.yml generator --- PantheonCMD/buildyml-generator.sh | 19 ++++++++++++++ PantheonCMD/generate-build-yml.sh | 25 ++++++++++++++++++ PantheonCMD/generate-pv2-yml.sh | 42 ------------------------------- PantheonCMD/pcmd.py | 8 +++--- PantheonCMD/pv2yml-generator.sh | 38 ---------------------------- 5 files changed, 48 insertions(+), 84 deletions(-) create mode 100644 PantheonCMD/buildyml-generator.sh create mode 100644 PantheonCMD/generate-build-yml.sh delete mode 100644 PantheonCMD/generate-pv2-yml.sh delete mode 100644 PantheonCMD/pv2yml-generator.sh diff --git a/PantheonCMD/buildyml-generator.sh b/PantheonCMD/buildyml-generator.sh new file mode 100644 index 0000000..6e31b83 --- /dev/null +++ b/PantheonCMD/buildyml-generator.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +ROOT_REPO=$(git rev-parse --show-toplevel) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +cd $ROOT_REPO +if [ ! -f "build.yml" ]; then + sh $SCRIPT_DIR/generate-build-yml.sh > build.yml + echo "build.yml succsessfully generated"; + else + read -p "build.yml already exists. Do you want to overwrite it? [y/N] " yn + case $yn in + [Yy]*) + sh $SCRIPT_DIR/generate-build-yml.sh > build.yml && echo "build.yml succsessfully generated"; exit;; + [Nn]*) + echo "exiting..."; exit;; + *) + esac +fi diff --git a/PantheonCMD/generate-build-yml.sh b/PantheonCMD/generate-build-yml.sh new file mode 100644 index 0000000..c1b6656 --- /dev/null +++ b/PantheonCMD/generate-build-yml.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +#define parameters +repo_name=$(basename $PWD) + +#define the template +cat << EOF +# Your repository name goes here +repository: $repo_name +variants: + # Your variant name goes here + - name: PLACEHOLDER + # Path to your attributes file goes here + attributes: + - PATH/TO/_attributes.adoc + nav: PATH/TO/nav.yml + build: true + files: + # Path to your assemblies, modules, and images go here + included: + - PATH/TO/ASSEMBLIES/*.adoc + - PATH/TO/MODULES/**/*.adoc + - PATH/TO/images/*.png + +EOF diff --git a/PantheonCMD/generate-pv2-yml.sh b/PantheonCMD/generate-pv2-yml.sh deleted file mode 100644 index e0b4d77..0000000 --- a/PantheonCMD/generate-pv2-yml.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -#define parameters -repo_name=$(basename $PWD) - -find_images_dir=$(find . -type d -name "images") - -#if images dir is detected record, otherwise create a placeholder -if [ ! -z "$find_images_dir" ]; then - images_dir=$(for i in $find_images_dir; do echo " - ${i/\.\/}/*.png" && echo " - ${i/\.\/}/*.svg"; done) -else - images_dir=$(printf ' - PATH/TO/YOUR/IMAGES/DIRECTORY/*.png\n - PATH/TO/YOUR/IMAGES/DIRECTORY/*.svg') -fi - -#define the template -cat << EOF -server: https://pantheon.corp.redhat.com -# Your repository name goes here -repository: $repo_name -variants: - # Your chosen name goes here - - name: PLACEHOLDER - # Path to your attributes file goes here - path: PATH/TO/attributes.adoc - canonical: true - -assemblies: - # Your assemblies go here - - PATH/TO/ASSEMBLIES/assembly_TEMPLATE-ASSEMBLY.adoc - - -modules: - # Your modules go here - - PATH/TO/MODULES/con_TEMPLATE_CONCEPT.adoc - - PATH/TO/MODULES/proc_TEMPLATE_PROCEDURE.adoc - - PATH/TO/MODULES/ref_TEMPLATE_REFERENCE.adoc - - -resources: - # Path to your images directory goes here -$images_dir -EOF diff --git a/PantheonCMD/pcmd.py b/PantheonCMD/pcmd.py index 35dda17..b114513 100644 --- a/PantheonCMD/pcmd.py +++ b/PantheonCMD/pcmd.py @@ -49,7 +49,7 @@ def parse_args(): parser_d.add_argument('--mr', action='store_true', help='Validate files commited on a merge request.') # 'Generate' command - parser_e = subparsers.add_parser('generate', help='Generate pantheon2.yml file from a template.') + parser_e = subparsers.add_parser('generate', help='Generate build.yml file from a template.') return parser.parse_args() @@ -68,7 +68,7 @@ def parse_args(): # Action - generate a pantheon2.yml file if args.command == 'generate': path_to_script = os.path.dirname(os.path.realpath(__file__)) - call("sh " + path_to_script + "/pv2yml-generator.sh", shell=True) + call("sh " + path_to_script + "/buildyml-generator.sh", shell=True) sys.exit(0) # Action - validate yaml syntax, validate yaml keys and values @@ -156,13 +156,13 @@ def parse_args(): # Validate the pantheon2.yml file yaml_validation(pantheon_repo.yaml_file_location) - + # Set the output format if args.format == 'pdf': output_format = 'pdf' else: output_format = 'html' - + # Did a user specify a set of files? If so, only build those. if args.files: # Handle different interpretations of directories diff --git a/PantheonCMD/pv2yml-generator.sh b/PantheonCMD/pv2yml-generator.sh deleted file mode 100644 index 0b94021..0000000 --- a/PantheonCMD/pv2yml-generator.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -# a simple yay/nay prompt for users -while true; do - read -p "Are you at the root of your repository where you want to create a pantheo2.yml? [y/N] " yn - case $yn in - [Yy]*) - read -p "Do you want to create the pantheon2.yml in $PWD directory? [y/N] " yn - case $yn in - [Yy]*) - if [ ! -f "pantheon2.yml" ]; then - sh $SCRIPT_DIR/generate-pv2-yml.sh > pantheon2.yml - echo "pantheon2.yml succsessfully generated"; - else - read -p "pantheon2.yml already exists. Do you want to overwrite it? [y/N] " yn - case $yn in - [Yy]*) - sh $SCRIPT_DIR/generate-pv2-yml.sh > pantheon2.yml && echo "pantheon2.yml succsessfully generated"; exit;; - [Nn]*) - echo "exiting..."; exit;; - *) - echo "Please answer yes or no.";; - esac - fi - exit;; - [Nn]*) - echo "exiting..."; exit;; - *) - echo "Please answer yes or no.";; - esac;; - [Nn]*) - echo "Please navigate to the root of the repository where you want to create the pantheo2.yml."; exit;; - *) - echo "Please answer yes or no.";; - esac -done From 24466ce47d432675a70ec63cfeb2f5a29036b3cc Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 17 Feb 2022 01:16:49 +0100 Subject: [PATCH 02/27] s/pantheon2.yml/build.yml/g --- PantheonCMD/pcmd.py | 30 +++++++++++++----------------- PantheonCMD/pcutil.py | 16 ++++++++-------- PantheonCMD/pcyamlchecks.py | 6 +++--- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/PantheonCMD/pcmd.py b/PantheonCMD/pcmd.py index b114513..c4cc440 100644 --- a/PantheonCMD/pcmd.py +++ b/PantheonCMD/pcmd.py @@ -42,10 +42,10 @@ def parse_args(): parser_b = subparsers.add_parser('clean', help='Clean the build directory.') # 'Duplicates' command - parser_c = subparsers.add_parser('duplicates', help='Enumerate duplicate entries in your pantheon2.yml file.') + parser_c = subparsers.add_parser('duplicates', help='Enumerate duplicate entries in your build.yml file.') # 'Validate' command - parser_d = subparsers.add_parser('validate', help='Validate entries in your pantheon2.yml file.') + parser_d = subparsers.add_parser('validate', help='Validate entries in your build.yml file.') parser_d.add_argument('--mr', action='store_true', help='Validate files commited on a merge request.') # 'Generate' command @@ -65,10 +65,10 @@ def parse_args(): repo_location = is_pantheon_repo() - # Action - generate a pantheon2.yml file + # Action - generate a build.yml file if args.command == 'generate': path_to_script = os.path.dirname(os.path.realpath(__file__)) - call("sh " + path_to_script + "/buildyml-generator.sh", shell=True) + call("sh " + path_to_script + "/pv2yml-generator.sh", shell=True) sys.exit(0) # Action - validate yaml syntax, validate yaml keys and values @@ -107,16 +107,16 @@ def parse_args(): pantheon_repo = PantheonRepo(repo_location) - if os.path.exists('pantheon2.yml'): + if os.path.exists('build.yml'): # call yaml file validation + attribute file validation - yaml_validation('pantheon2.yml') + yaml_validation('build.yml') exists = get_not_exist(pantheon_repo.get_content()) if exists: - print("\nYour pantheon2.yml contains the following files that do not exist in your repository:\n") + print("\nYour build.yml contains the following files that do not exist in your repository:\n") for exist in exists: @@ -131,7 +131,7 @@ def parse_args(): validate = validation(files_found, modules_found, assemblies_found) if validate.count != 0: - print("\nYour pantheon2.yml contains the following files that did not pass validation:\n") + print("\nYour build.yml contains the following files that did not pass validation:\n") validate.print_report() sys.exit(2) else: @@ -139,7 +139,7 @@ def parse_args(): else: - print("ERROR: You must run this command from the same directory as the pantheon2.yml file.\n") + print("ERROR: You must run this command from the same directory as the build.yml file.\n") sys.exit(1) # Exit if not a Pantheon V2 repository @@ -154,10 +154,6 @@ def parse_args(): # Action - preview if args.command == 'preview': - # Validate the pantheon2.yml file - yaml_validation(pantheon_repo.yaml_file_location) - - # Set the output format if args.format == 'pdf': output_format = 'pdf' else: @@ -183,9 +179,9 @@ def parse_args(): pcbuild.prepare_build_directory() pcbuild.copy_resources(pantheon_repo.get_existing_content("resources")) pcbuild.build_content(content_subset, args.lang, output_format, pantheon_repo.repo_location, pantheon_repo.yaml_file_location) - # Otherwise, attempt to build all files in the pantheon2.yml file. + # Otherwise, attempt to build all files in the build.yml file. else: - if os.path.exists('pantheon2.yml'): + if os.path.exists('build.yml'): content_types = ['assemblies','modules'] continue_run = True @@ -197,7 +193,7 @@ def parse_args(): print("Building %s...\n" % content_type) continue_run = pcbuild.build_content(pantheon_repo.get_existing_content(content_type), args.lang, output_format, pantheon_repo.repo_location, pantheon_repo.yaml_file_location) else: - print("ERROR: You must run this command from the same directory as the pantheon2.yml file.\n") + print("ERROR: You must run this command from the same directory as the build.yml file.\n") sys.exit(1) # Action - clean @@ -216,7 +212,7 @@ def parse_args(): if duplicates: - print("Your pantheon2.yml contains the following duplicate entries:\n") + print("Your build.yml contains the following duplicate entries:\n") for duplicate in duplicates: print(duplicate) diff --git a/PantheonCMD/pcutil.py b/PantheonCMD/pcutil.py index a91274b..783f972 100644 --- a/PantheonCMD/pcutil.py +++ b/PantheonCMD/pcutil.py @@ -15,16 +15,16 @@ def __init__(self, repo_location): """Default constructor; accepts repo location and initializes YAML file location.""" self.repo_location = repo_location - self.yaml_file_location = repo_location + "pantheon2.yml" + self.yaml_file_location = repo_location + "build.yml" def count_content(self): - """Counts the number of assemblies and modules in a pantheon2.yml file.""" + """Counts the number of assemblies and modules in a build.yml file.""" # Initialize dictionary content_counts = {'assemblies': 0, 'modules': 0} # Parse the main YAML file - with open(self.yaml_file_location + 'pantheon2.yml', 'r') as f: + with open(self.yaml_file_location + 'build.yml', 'r') as f: yaml_file = yaml.safe_load(f) # Count assemblies @@ -39,7 +39,7 @@ def count_content(self): def get_content(self): - """Returns a sorted list of the modules and assemblies specified in a pantheon2.yml file.""" + """Returns a sorted list of the modules and assemblies specified in a build.yml file.""" with open(self.yaml_file_location, 'r') as f: yaml_file = yaml.safe_load(f) content_list, content_duplicates = self.get_files(yaml_file, "assemblies", "modules") @@ -48,7 +48,7 @@ def get_content(self): def get_duplicates(self): - """Returns duplicate entries of modules and assemblies found in a pantheon2.yml file.""" + """Returns duplicate entries of modules and assemblies found in a build.yml file.""" with open(self.yaml_file_location, 'r') as f: yaml_file = yaml.safe_load(f) content_list, content_duplicates = self.get_files(yaml_file, "assemblies", "modules") @@ -57,7 +57,7 @@ def get_duplicates(self): def get_existing_content(self, content_type): - """Returns content found in a pantheon2.yml file that exist as files.""" + """Returns content found in a build.yml file that exist as files.""" content_found = [] with open(self.yaml_file_location, 'r') as f: @@ -73,7 +73,7 @@ def get_existing_content(self, content_type): def get_files(self, main_yaml_file, *arguments): - """Returns a sorted list of the modules and assemblies specified in a pantheon2.yml file.""" + """Returns a sorted list of the modules and assemblies specified in a build.yml file.""" content_files = [] content_list = [] content_duplicates = [] @@ -166,7 +166,7 @@ def is_pantheon_repo(): repo_location = None while path_components: - if os.path.exists(os.sep.join(path_components) + os.sep + 'pantheon2.yml'): + if os.path.exists(os.sep.join(path_components) + os.sep + 'build.yml'): repo_location = os.sep.join(path_components) + os.sep break path_components.pop() diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 4df0687..f919c6d 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -21,7 +21,7 @@ class CustomErrorHandler(BasicErrorHandler): def get_yaml_size(yaml_file): """Test if pv2.yml is empty.""" if os.path.getsize(yaml_file) == 0: - print("\nYour pantheon2.yml file is empty; exiting...") + print("\nYour build.yml file is empty; exiting...") sys.exit(2) @@ -31,7 +31,7 @@ def load_doc(yaml_file): try: return yaml.safe_load(file) except yaml.YAMLError: - print("There's a syntax error in your pantheon2.yml file. Please fix it and try again.\nTo detect an error try running yaml lint on your pantheo2.yml file.") + print("There's a syntax error in your build.yml file. Please fix it and try again.\nTo detect an error try running yaml lint on your pantheo2.yml file.") sys.exit(2) @@ -80,7 +80,7 @@ def get_yaml_errors(yaml_schema, yaml_doc): path_exists.append(variant['path']) if path_does_not_exist: - print('FAIL: Your pantheon2.yml contains the following files or directories that do not exist in your repository:\n') + print('FAIL: Your build.yml contains the following files or directories that do not exist in your repository:\n') for path in path_does_not_exist: print('\t', path) sys.exit(2) From 13bbde5df4f115eb3b3e17a2fdd85a1320e71087 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 17 Feb 2022 15:50:05 +0100 Subject: [PATCH 03/27] updated schema to validate build.yml --- PantheonCMD/schema.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/PantheonCMD/schema.py b/PantheonCMD/schema.py index 2ac401a..61a72a6 100644 --- a/PantheonCMD/schema.py +++ b/PantheonCMD/schema.py @@ -1,8 +1,4 @@ { - 'server': { - 'required': True, - 'type': 'string' - }, 'repository': { 'required': True, 'type': 'string' @@ -17,27 +13,33 @@ 'required': True, 'type': 'string' }, - 'path': { + 'attributes': { 'required': True, + 'type': 'list' + }, + 'nav': { + 'required': False, 'type': 'string' }, - 'canonical': { + 'build': { 'required': True, 'allowed': [True, False] + }, + 'files': { + 'required': True, + 'type': 'dict', + 'schema': { + 'included': { + 'required': True, + 'type': 'list' + }, + 'excluded': { + 'required': False, + 'type': 'list' + } + } } } } - }, - 'assemblies': { - 'required': True, - 'type': 'list' - }, - 'modules': { - 'required': True, - 'type': 'list' - }, - 'resources': { - 'required': True, - 'type': 'list' } } From 16f7b73f334bb4a60e4e5a353fe5db44079dbfe6 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 17 Feb 2022 16:23:44 +0100 Subject: [PATCH 04/27] added check for a defined nspb attribute within the attributes file --- PantheonCMD/pcchecks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/PantheonCMD/pcchecks.py b/PantheonCMD/pcchecks.py index 207fd6b..c522a24 100644 --- a/PantheonCMD/pcchecks.py +++ b/PantheonCMD/pcchecks.py @@ -9,6 +9,8 @@ class Tags: ABSTRACT = '[role="_abstract"]' ADD_RES = '[role="_additional-resources"]' EXPERIMENTAL = ':experimental:' + NBSP_ATT = ':nbsp:  ' + NBSP_VAR = '{nbsp}' LVLOFFSET = ':leveloffset:' ICONS = ':icons:' TOC = ':toc:' @@ -62,6 +64,13 @@ def toc_check(report, stripped_file, file_path): report.create_report('toc attribute', file_path) +def nbsp_check(report, stripped_file, file_path): + if re.findall(Tags.NBSP_ATT, stripped_file): + return + elif re.findall(Tags.NBSP_VAR, stripped_file): + report.create_report('`{nsbp}` attribute is used but not defined. `:nbsp:  ` attribute is not', file_path) + + def vanilla_xref_check(stripped_file): """Check if the file contains vanilla xrefs.""" if re.findall(Regex.VANILLA_XREF, stripped_file): From 0179d20b12407e647c5d813a0dd911eff3108398 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 17 Feb 2022 16:25:18 +0100 Subject: [PATCH 05/27] mods how attribute file is sourced from the build.yml --- PantheonCMD/pcyamlchecks.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index f919c6d..1cf1114 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -5,7 +5,7 @@ import yaml from cerberus import Validator, errors from cerberus.errors import BasicErrorHandler -from pcchecks import Regex, icons_check, toc_check +from pcchecks import Regex, icons_check, toc_check, nbsp_check from pcvalidator import Report import glob @@ -31,7 +31,7 @@ def load_doc(yaml_file): try: return yaml.safe_load(file) except yaml.YAMLError: - print("There's a syntax error in your build.yml file. Please fix it and try again.\nTo detect an error try running yaml lint on your pantheo2.yml file.") + print("There's a syntax error in your build.yml file. Please fix it and try again.\nTo detect an error try running yaml lint on your build.yml file.") sys.exit(2) @@ -47,6 +47,7 @@ def get_attribute_file_validation_results(attribute_file): icons_check(report, stripped, path) toc_check(report, stripped, path) + nbsp_check(report, stripped, path) return report @@ -68,16 +69,12 @@ def get_yaml_errors(yaml_schema, yaml_doc): path_does_not_exist = [] path_exists = [] - for item in yaml_doc['resources']: - path_to_images_dir = os.path.split(item)[0] - if not glob.glob(path_to_images_dir): - path_does_not_exist.append(item) - for variant in yaml_doc['variants']: - if not os.path.exists(variant['path']): - path_does_not_exist.append(variant['path']) + for item in variant['attributes']: + if not os.path.exists(item): + path_does_not_exist.append(item) else: - path_exists.append(variant['path']) + path_exists.append(item) if path_does_not_exist: print('FAIL: Your build.yml contains the following files or directories that do not exist in your repository:\n') From 00d48b565573b02cd26594747e4fab609facc3a2 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 24 Feb 2022 20:03:57 +0100 Subject: [PATCH 06/27] collecting files --- PantheonCMD/pcyamlchecks.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 1cf1114..d4d8637 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -8,6 +8,7 @@ from pcchecks import Regex, icons_check, toc_check, nbsp_check from pcvalidator import Report import glob +import re class CustomErrorHandler(BasicErrorHandler): @@ -88,6 +89,30 @@ def get_yaml_errors(yaml_schema, yaml_doc): attribute_file_validation.print_report() +def get_files(yaml_doc): + included_files = [] + file_list = [] + wildcards = re.compile(r'[*?\[\]]') + + for yaml_dict in yaml_doc['variants']: + for include in yaml_dict['files']['included']: + if wildcards.search(include): + expanded_items = glob.glob(include) + if expanded_items: + for expanded_item in expanded_items: + included_files.append(expanded_item) + else: + continue + else: + included_files.append(include) + + for file in included_files: + if file not in file_list: + file_list.append(file) + + return included_files + + def yaml_validation(yaml_file): """Validate pv2.yml; get path to attributes while we're at it.""" # define path to script @@ -99,3 +124,4 @@ def yaml_validation(yaml_file): get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) + print(get_files(loaded_yaml)) From d4ea04d944de687ef62e04fe452b598f59c65bcc Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 24 Feb 2022 20:46:32 +0100 Subject: [PATCH 07/27] mods how files are sourced --- PantheonCMD/pcyamlchecks.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index d4d8637..8da2cfe 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -89,13 +89,22 @@ def get_yaml_errors(yaml_schema, yaml_doc): attribute_file_validation.print_report() -def get_files(yaml_doc): - included_files = [] +def remove_duplicates(files): file_list = [] + + for file in files: + if file not in file_list: + file_list.append(file) + + return file_list + + +def get_files(yaml_doc, var): + included_files = [] wildcards = re.compile(r'[*?\[\]]') for yaml_dict in yaml_doc['variants']: - for include in yaml_dict['files']['included']: + for include in yaml_dict['files'][var]: if wildcards.search(include): expanded_items = glob.glob(include) if expanded_items: @@ -106,13 +115,20 @@ def get_files(yaml_doc): else: included_files.append(include) - for file in included_files: - if file not in file_list: - file_list.append(file) - return included_files +def get_content_list(yaml_doc): + + included_files = get_files(yaml_doc, 'included') + #excluded_files = get_files(yaml_doc, 'excluded') + + includes = remove_duplicates(included_files) + #excludes = remove_duplicates(excluded_files) + + return includes + + def yaml_validation(yaml_file): """Validate pv2.yml; get path to attributes while we're at it.""" # define path to script @@ -124,4 +140,4 @@ def yaml_validation(yaml_file): get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) - print(get_files(loaded_yaml)) + print(get_content_list(loaded_yaml)) From 301f133e24d94e4e6f2965de85e4b373b0670876 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 24 Feb 2022 20:50:10 +0100 Subject: [PATCH 08/27] removing excludes from content list --- PantheonCMD/pcyamlchecks.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 8da2cfe..2df97ee 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -104,16 +104,17 @@ def get_files(yaml_doc, var): wildcards = re.compile(r'[*?\[\]]') for yaml_dict in yaml_doc['variants']: - for include in yaml_dict['files'][var]: - if wildcards.search(include): - expanded_items = glob.glob(include) - if expanded_items: - for expanded_item in expanded_items: - included_files.append(expanded_item) + if var in yaml_dict['files']: + for include in yaml_dict['files'][var]: + if wildcards.search(include): + expanded_items = glob.glob(include) + if expanded_items: + for expanded_item in expanded_items: + included_files.append(expanded_item) + else: + continue else: - continue - else: - included_files.append(include) + included_files.append(include) return included_files @@ -121,12 +122,14 @@ def get_files(yaml_doc, var): def get_content_list(yaml_doc): included_files = get_files(yaml_doc, 'included') - #excluded_files = get_files(yaml_doc, 'excluded') + excluded_files = get_files(yaml_doc, 'excluded') includes = remove_duplicates(included_files) - #excludes = remove_duplicates(excluded_files) + excludes = remove_duplicates(excluded_files) + + content_list = [x for x in includes if x not in excludes] - return includes + return content_list def yaml_validation(yaml_file): From 1ea9481e50ca0b559bb889044abeb1030f568eb3 Mon Sep 17 00:00:00 2001 From: Levi V Date: Fri, 25 Feb 2022 00:03:28 +0100 Subject: [PATCH 09/27] removes excludes from includes --- PantheonCMD/pcyamlchecks.py | 52 +++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 2df97ee..1818af7 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -36,23 +36,6 @@ def load_doc(yaml_file): sys.exit(2) -def get_attribute_file_validation_results(attribute_file): - """Validate attributes file.""" - report = Report() - - for path in attribute_file: - with open(path, 'r') as file: - original = file.read() - stripped = Regex.MULTI_LINE_COMMENT.sub('', original) - stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - - icons_check(report, stripped, path) - toc_check(report, stripped, path) - nbsp_check(report, stripped, path) - - return report - - def get_yaml_errors(yaml_schema, yaml_doc): # load validator with custom error handler v = Validator(yaml_schema, error_handler=CustomErrorHandler()) @@ -89,14 +72,21 @@ def get_yaml_errors(yaml_schema, yaml_doc): attribute_file_validation.print_report() -def remove_duplicates(files): - file_list = [] +def get_attribute_file_validation_results(attribute_file): + """Validate attributes file.""" + report = Report() - for file in files: - if file not in file_list: - file_list.append(file) + for path in attribute_file: + with open(path, 'r') as file: + original = file.read() + stripped = Regex.MULTI_LINE_COMMENT.sub('', original) + stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - return file_list + icons_check(report, stripped, path) + toc_check(report, stripped, path) + nbsp_check(report, stripped, path) + + return report def get_files(yaml_doc, var): @@ -119,6 +109,16 @@ def get_files(yaml_doc, var): return included_files +def remove_duplicates(files): + file_list = [] + + for file in files: + if file not in file_list: + file_list.append(file) + + return file_list + + def get_content_list(yaml_doc): included_files = get_files(yaml_doc, 'included') @@ -127,9 +127,11 @@ def get_content_list(yaml_doc): includes = remove_duplicates(included_files) excludes = remove_duplicates(excluded_files) - content_list = [x for x in includes if x not in excludes] + for i in excludes: + if i in includes: + includes.remove(i) - return content_list + return includes def yaml_validation(yaml_file): From fafcc622d90e1f3eb361831c60a371bc7adbafb7 Mon Sep 17 00:00:00 2001 From: Levi V Date: Tue, 1 Mar 2022 21:25:55 +0100 Subject: [PATCH 10/27] new way to get unique file list from build.yml + reorg --- PantheonCMD/pcmd.py | 29 ----- PantheonCMD/pcyamlchecks.py | 223 ++++++++++++++++++++++++++---------- 2 files changed, 164 insertions(+), 88 deletions(-) diff --git a/PantheonCMD/pcmd.py b/PantheonCMD/pcmd.py index c4cc440..a48b48a 100644 --- a/PantheonCMD/pcmd.py +++ b/PantheonCMD/pcmd.py @@ -112,35 +112,6 @@ def parse_args(): # call yaml file validation + attribute file validation yaml_validation('build.yml') - exists = get_not_exist(pantheon_repo.get_content()) - - if exists: - - print("\nYour build.yml contains the following files that do not exist in your repository:\n") - - for exist in exists: - - print('\t' + exist) - - print("\nTotal: ", str(len(exists))) - - files_found = get_exist(pantheon_repo.get_content()) - modules_found = pantheon_repo.get_existing_content("modules") - assemblies_found = pantheon_repo.get_existing_content("assemblies") - - validate = validation(files_found, modules_found, assemblies_found) - - if validate.count != 0: - print("\nYour build.yml contains the following files that did not pass validation:\n") - validate.print_report() - sys.exit(2) - else: - print("All files passed validation.") - - else: - - print("ERROR: You must run this command from the same directory as the build.yml file.\n") - sys.exit(1) # Exit if not a Pantheon V2 repository if repo_location: diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 1818af7..0b32e70 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -9,6 +9,7 @@ from pcvalidator import Report import glob import re +import subprocess class CustomErrorHandler(BasicErrorHandler): @@ -19,15 +20,22 @@ class CustomErrorHandler(BasicErrorHandler): messages[errors.NOT_NULLABLE.code] = "value can't be empty" +def printing_build_yml_error(msg, *files): + print('\nERROR: Your build.yml contains the following files or directories that do not {}:\n'.format(msg)) + for file in files: + if file: + print('\t', file) + + def get_yaml_size(yaml_file): - """Test if pv2.yml is empty.""" + """Test if build.yml is empty.""" if os.path.getsize(yaml_file) == 0: print("\nYour build.yml file is empty; exiting...") sys.exit(2) def load_doc(yaml_file): - """Load pv2.yml and test for syntax errors.""" + """Load build.yml and test for syntax errors.""" with open(yaml_file, 'r') as file: try: return yaml.safe_load(file) @@ -39,44 +47,113 @@ def load_doc(yaml_file): def get_yaml_errors(yaml_schema, yaml_doc): # load validator with custom error handler v = Validator(yaml_schema, error_handler=CustomErrorHandler()) - # validate the pv2.yml with schema + # validate the build.yml with schema v.validate(yaml_doc, yaml_schema) if v.errors: - print("FAIL: there is an error in your yaml file:") + print("ERROR: there is an error in your yaml file:") for key in v.errors.keys(): print("\n\t'{}' {}".format(key, ', '.join(str(item) for item in v.errors[key]))) sys.exit(2) - else: - path_does_not_exist = [] - path_exists = [] +def get_attribute_files(yaml_doc): + attribute_files = [] - for variant in yaml_doc['variants']: - for item in variant['attributes']: - if not os.path.exists(item): - path_does_not_exist.append(item) - else: - path_exists.append(item) + for variant in yaml_doc['variants']: + for item in variant['attributes']: + attribute_files.append(item) - if path_does_not_exist: - print('FAIL: Your build.yml contains the following files or directories that do not exist in your repository:\n') - for path in path_does_not_exist: - print('\t', path) - sys.exit(2) - else: - attribute_file_validation = get_attribute_file_validation_results(path_exists) - if attribute_file_validation.count != 0: - print("Your attributes file has the following errors:\n") - attribute_file_validation.print_report() + return attribute_files + + +def get_existence(files): + files_found = [] + files_not_found = [] + + for item in files: + if os.path.exists(item): + files_found.append(item) + else: + files_not_found.append(item) + + return files_found, files_not_found + + +def get_not_exist(content_list): + files_found, files_not_found = get_existence(content_list) + + return files_not_found + + +def get_exist(content_list): + files_found, files_not_found = get_existence(content_list) + + return files_found + + +def get_files(yaml_doc, var): + content_list = [] + missing_files = [] + + for yaml_dict in yaml_doc['variants']: + for include in yaml_dict['files'][var]: + content = get_files_bash(include) + if content: + if '' in content: + missing_files.append(include) + else: + for i in content: + if i not in content_list: + content_list.append(i) + + return content_list, missing_files + + +def get_missing_files(yaml_doc, var): + content_list, missing_files = get_files(yaml_doc, var) + return missing_files + + +def get_exitsing_files(yaml_doc, var): + content_list, missing_files = get_files(yaml_doc, var) + return content_list -def get_attribute_file_validation_results(attribute_file): +def get_files_bash(file_path): + command = ("find " + file_path + " -type f 2>/dev/null") + process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout + files = process.strip().decode('utf-8').split('\n') + + return files + + +def get_md5sum(file_path): + command = ("md5sum " + file_path + " | cut -d\ -f1") + process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout + md5sum = process.strip().decode('utf-8').split('\n') + + return md5sum + + +def get_unique_files(files): + md5sum_list = [] + unique_files = [] + + for file in files: + md5sum = get_md5sum(file) + if md5sum not in md5sum_list: + md5sum_list.append(md5sum) + unique_files.append(file) + + return unique_files + + +def validate_attribute_files(attribute_files): """Validate attributes file.""" report = Report() - for path in attribute_file: + for path in attribute_files: with open(path, 'r') as file: original = file.read() stripped = Regex.MULTI_LINE_COMMENT.sub('', original) @@ -89,60 +166,88 @@ def get_attribute_file_validation_results(attribute_file): return report -def get_files(yaml_doc, var): - included_files = [] - wildcards = re.compile(r'[*?\[\]]') +def get_attribute_file_errors(yaml_doc): + attribute_files = get_attribute_files(yaml_doc) + missing_attribute_files = get_not_exist(attribute_files) + exiting_attribute_files = get_exist(attribute_files) - for yaml_dict in yaml_doc['variants']: - if var in yaml_dict['files']: - for include in yaml_dict['files'][var]: - if wildcards.search(include): - expanded_items = glob.glob(include) - if expanded_items: - for expanded_item in expanded_items: - included_files.append(expanded_item) - else: - continue - else: - included_files.append(include) + if missing_attribute_files: + printing_build_yml_error("exist in your repository", missing_attribute_files) - return included_files + if exiting_attribute_files: + for item in attribute_files: + file_name = os.path.basename(item) + file_path = os.path.dirname(item) + if not file_path.startswith("_"): + if "/_" not in file_path: + if not file_name.startswith("_"): + printing_build_yml_error("follow the attribute naming conventions. Attribute files or directory they are stored in should start with an undescore", item) + attribute_validation = validate_attribute_files(exiting_attribute_files) -def remove_duplicates(files): - file_list = [] + if attribute_validation.count != 0: + attribute_validation.print_report() - for file in files: - if file not in file_list: - file_list.append(file) - return file_list +def get_content_list(yaml_doc): + missing_includes = get_missing_files(yaml_doc, 'included') + missing_excludes = get_missing_files(yaml_doc, 'excluded') + + if missing_includes or missing_excludes: + printing_build_yml_error("exist in your repository", missing_includes, missing_excludes) + included = get_exitsing_files(yaml_doc, 'included') + excluded = get_exitsing_files(yaml_doc, 'excluded') -def get_content_list(yaml_doc): + unique_includes = get_unique_files(included) + unique_excludes = get_unique_files(excluded) + + md5sum_exc_list = [] + unique_files = [] - included_files = get_files(yaml_doc, 'included') - excluded_files = get_files(yaml_doc, 'excluded') + for item in unique_excludes: + md5sum_exc = get_md5sum(item) + md5sum_exc_list.append(md5sum_exc) - includes = remove_duplicates(included_files) - excludes = remove_duplicates(excluded_files) + for item in unique_includes: + md5sum_inc = get_md5sum(item) + if md5sum_inc not in md5sum_exc_list: + unique_files.append(item) - for i in excludes: - if i in includes: - includes.remove(i) + return unique_files - return includes + +def sort_content_list(yaml_doc): + prefix_assembly = [] + prefix_modules = [] + unidentifiyed_content = [] + + content_list = get_content_list(yaml_doc) + + for item in content_list: + if item.endswith('.adoc'): + file_name = os.path.basename(item) + if file_name.startswith('assembly'): + prefix_assembly.append(item) + elif file_name.startswith(("proc_", "con_", "ref_", "proc-", "con-", "ref-")): + prefix_modules.append(item) + else: + unidentifiyed_content.append(item) + + return prefix_assembly, prefix_modules, unidentifiyed_content def yaml_validation(yaml_file): - """Validate pv2.yml; get path to attributes while we're at it.""" + """Validate build.yml; get path to attributes while we're at it.""" # define path to script path_to_script = os.path.dirname(os.path.realpath(__file__)) # load schema schema = eval(open(path_to_script + '/schema.py', 'r').read()) - # load pv2.yml + # load build.yml loaded_yaml = load_doc(yaml_file) get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) - print(get_content_list(loaded_yaml)) + get_attribute_file_errors(loaded_yaml) + + sort_content_list(loaded_yaml) From d1e7544c40f2194b45fd5f530d4a97b87286a144 Mon Sep 17 00:00:00 2001 From: Levi V Date: Tue, 1 Mar 2022 23:03:39 +0100 Subject: [PATCH 11/27] changes md5sum to realpath --- PantheonCMD/pcchecks.py | 3 +- PantheonCMD/pcyamlchecks.py | 105 ++++++++++++++++++++++-------------- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/PantheonCMD/pcchecks.py b/PantheonCMD/pcchecks.py index c522a24..258d238 100644 --- a/PantheonCMD/pcchecks.py +++ b/PantheonCMD/pcchecks.py @@ -20,7 +20,8 @@ class Regex: """Define regular expresiions for the checks.""" INCLUDE = re.compile(r'include::.*\]\n') - MODULE_TYPE = re.compile(r':_module-type: (PROCEDURE|CONCEPT|REFERENCE)') + MODULE_TYPE = re.compile(r':_content-type: (PROCEDURE|CONCEPT|REFERENCE)') + ASSEMBLY_TYPE = re.compile(r':_content-type: ASSEMBLY') PREFIX_ASSEMBLIES = re.compile(r'.*\/assembly.*\.adoc') PREFIX_MODULES = re.compile(r'.*\/con.*\.adoc|.*\/proc.*\.adoc|.*\/ref.*\.adoc') # should exclude pseudo vanilla like <> diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 0b32e70..a00208a 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -128,27 +128,6 @@ def get_files_bash(file_path): return files -def get_md5sum(file_path): - command = ("md5sum " + file_path + " | cut -d\ -f1") - process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout - md5sum = process.strip().decode('utf-8').split('\n') - - return md5sum - - -def get_unique_files(files): - md5sum_list = [] - unique_files = [] - - for file in files: - md5sum = get_md5sum(file) - if md5sum not in md5sum_list: - md5sum_list.append(md5sum) - unique_files.append(file) - - return unique_files - - def validate_attribute_files(attribute_files): """Validate attributes file.""" report = Report() @@ -189,6 +168,24 @@ def get_attribute_file_errors(yaml_doc): attribute_validation.print_report() +def get_realpath(files): + unique_files = [] + + for file in files: + real_path = os.path.realpath(file) + if real_path not in unique_files: + unique_files.append(real_path) + + files = [] + + pwd = os.getcwd() + for i in unique_files: + relative_path = os.path.relpath(i, pwd) + files.append(relative_path) + + return files + + def get_content_list(yaml_doc): missing_includes = get_missing_files(yaml_doc, 'included') missing_excludes = get_missing_files(yaml_doc, 'excluded') @@ -199,28 +196,20 @@ def get_content_list(yaml_doc): included = get_exitsing_files(yaml_doc, 'included') excluded = get_exitsing_files(yaml_doc, 'excluded') - unique_includes = get_unique_files(included) - unique_excludes = get_unique_files(excluded) - - md5sum_exc_list = [] - unique_files = [] - - for item in unique_excludes: - md5sum_exc = get_md5sum(item) - md5sum_exc_list.append(md5sum_exc) + unique_includes = get_realpath(included) + unique_excludes = get_realpath(excluded) for item in unique_includes: - md5sum_inc = get_md5sum(item) - if md5sum_inc not in md5sum_exc_list: - unique_files.append(item) + if item in unique_excludes: + unique_includes.remove(item) - return unique_files + return unique_includes -def sort_content_list(yaml_doc): +def sort_prefix_files(yaml_doc): prefix_assembly = [] prefix_modules = [] - unidentifiyed_content = [] + undefined_content = [] content_list = get_content_list(yaml_doc) @@ -232,13 +221,48 @@ def sort_content_list(yaml_doc): elif file_name.startswith(("proc_", "con_", "ref_", "proc-", "con-", "ref-")): prefix_modules.append(item) else: - unidentifiyed_content.append(item) + undefined_content.append(item) + + return prefix_assembly, prefix_modules, undefined_content + + +def get_prefix_assemblies(yaml_doc): + prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_doc) + + return prefix_assembly + + +def get_prefix_modules(yaml_doc): + prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_doc) + + return prefix_modules + + +def get_undefined_content(yaml_doc): + prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_doc) - return prefix_assembly, prefix_modules, unidentifiyed_content + return undefined_content + + +def sort_no_prefix_files(yaml_doc): + undefined_content = get_undefined_content(yaml_doc) + + for path in undefined_content: + with open(path, 'r') as file: + original = file.read() + stripped = Regex.MULTI_LINE_COMMENT.sub('', original) + stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) + + if re.findall(Regex.MODULE_TYPE, stripped): + print('module') + elif re.findall(Regex.ASSEMBLY_TYPE, stripped): + print('assembly') + else: + print('no clue') + print(path) def yaml_validation(yaml_file): - """Validate build.yml; get path to attributes while we're at it.""" # define path to script path_to_script = os.path.dirname(os.path.realpath(__file__)) # load schema @@ -249,5 +273,4 @@ def yaml_validation(yaml_file): get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) get_attribute_file_errors(loaded_yaml) - - sort_content_list(loaded_yaml) + sort_no_prefix_files(loaded_yaml) From 3ba811157d9ad68bcf6bbee1b8b08cd83e3979ef Mon Sep 17 00:00:00 2001 From: Levi V Date: Tue, 1 Mar 2022 23:24:02 +0100 Subject: [PATCH 12/27] hooked up validation routine for files with no prefix --- PantheonCMD/pcyamlchecks.py | 68 +++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index a00208a..10a40e7 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -6,10 +6,11 @@ from cerberus import Validator, errors from cerberus.errors import BasicErrorHandler from pcchecks import Regex, icons_check, toc_check, nbsp_check -from pcvalidator import Report +#from pcvalidator import Report import glob import re import subprocess +from pcchecks import Regex, checks, nesting_in_modules_check, nesting_in_assemblies_check, add_res_section_module_check, add_res_section_assembly_check, icons_check, toc_check class CustomErrorHandler(BasicErrorHandler): @@ -21,12 +22,37 @@ class CustomErrorHandler(BasicErrorHandler): def printing_build_yml_error(msg, *files): - print('\nERROR: Your build.yml contains the following files or directories that do not {}:\n'.format(msg)) + print('\nERROR: Your build.yml contains the following {}:\n'.format(msg)) for file in files: if file: print('\t', file) +class Report(): + """Create and print report. thank u J.""" + + def __init__(self): + """Create placeholder for problem description.""" + self.report = {} + self.count = 0 + + def create_report(self, category, file_path): + """Generate report.""" + self.count += 1 + if not category in self.report: + self.report[category] = [] + self.report[category].append(file_path) + + def print_report(self): + + """Print report.""" + separator = "\n\t" + + for category, files in self.report.items(): + print("\nERROR: {} found in the following files:".format(category)) + print('\t' + separator.join(files)) + + def get_yaml_size(yaml_file): """Test if build.yml is empty.""" if os.path.getsize(yaml_file) == 0: @@ -151,7 +177,7 @@ def get_attribute_file_errors(yaml_doc): exiting_attribute_files = get_exist(attribute_files) if missing_attribute_files: - printing_build_yml_error("exist in your repository", missing_attribute_files) + printing_build_yml_error("files or directories that do not exist in your repository", missing_attribute_files) if exiting_attribute_files: for item in attribute_files: @@ -160,7 +186,7 @@ def get_attribute_file_errors(yaml_doc): if not file_path.startswith("_"): if "/_" not in file_path: if not file_name.startswith("_"): - printing_build_yml_error("follow the attribute naming conventions. Attribute files or directory they are stored in should start with an undescore", item) + printing_build_yml_error("files or directories that do not follow the attribute naming conventions. Attribute files or directory they are stored in should start with an undescore", item) attribute_validation = validate_attribute_files(exiting_attribute_files) @@ -191,7 +217,7 @@ def get_content_list(yaml_doc): missing_excludes = get_missing_files(yaml_doc, 'excluded') if missing_includes or missing_excludes: - printing_build_yml_error("exist in your repository", missing_includes, missing_excludes) + printing_build_yml_error("files or directories that do not exist in your repository", missing_includes, missing_excludes) included = get_exitsing_files(yaml_doc, 'included') excluded = get_exitsing_files(yaml_doc, 'excluded') @@ -246,20 +272,39 @@ def get_undefined_content(yaml_doc): def sort_no_prefix_files(yaml_doc): undefined_content = get_undefined_content(yaml_doc) + undetermined_file_type = [] + report = Report() for path in undefined_content: with open(path, 'r') as file: original = file.read() stripped = Regex.MULTI_LINE_COMMENT.sub('', original) stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) + stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) + stripped = Regex.INTERNAL_IFDEF.sub('', stripped) if re.findall(Regex.MODULE_TYPE, stripped): - print('module') + checks(report, stripped, original, path) + icons_check(report, stripped, path) + toc_check(report, stripped, path) + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + elif re.findall(Regex.ASSEMBLY_TYPE, stripped): - print('assembly') + checks(report, stripped, original, path) + icons_check(report, stripped, path) + toc_check(report, stripped, path) + nesting_in_assemblies_check(report, stripped, path) + add_res_section_assembly_check(report, stripped, path) + else: - print('no clue') - print(path) + undetermined_file_type.append(path) + + if undetermined_file_type: + printing_build_yml_error('files that can not be classifiyed as modules or assemblies', undetermined_file_type) + + return report def yaml_validation(yaml_file): @@ -273,4 +318,7 @@ def yaml_validation(yaml_file): get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) get_attribute_file_errors(loaded_yaml) - sort_no_prefix_files(loaded_yaml) + no_prefix_files_validation = sort_no_prefix_files(loaded_yaml) + + if no_prefix_files_validation.count != 0: + no_prefix_files_validation.print_report() From 67b9ecb34cbcaea73fd2dc66f529a60f0e035afc Mon Sep 17 00:00:00 2001 From: Levi V Date: Tue, 1 Mar 2022 23:52:35 +0100 Subject: [PATCH 13/27] adds validation for files with prefix + catches and validates files with identity crisis --- PantheonCMD/pcyamlchecks.py | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 10a40e7..cea401d 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -301,6 +301,54 @@ def sort_no_prefix_files(yaml_doc): else: undetermined_file_type.append(path) + confused_files = [] + + prefix_assemblies = get_prefix_assemblies(yaml_doc) + + for path in prefix_assemblies: + with open(path, "r") as file: + original = file.read() + stripped = Regex.MULTI_LINE_COMMENT.sub('', original) + stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) + stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) + stripped = Regex.INTERNAL_IFDEF.sub('', stripped) + checks(report, stripped, original, path) + icons_check(report, stripped, path) + toc_check(report, stripped, path) + + if re.findall(Regex.MODULE_TYPE, stripped): + confused_files.append(path) + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + else: + nesting_in_assemblies_check(report, stripped, path) + add_res_section_assembly_check(report, stripped, path) + + prefix_modules = get_prefix_modules(yaml_doc) + for path in prefix_modules: + with open(path, "r") as file: + original = file.read() + stripped = Regex.MULTI_LINE_COMMENT.sub('', original) + stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) + stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) + stripped = Regex.INTERNAL_IFDEF.sub('', stripped) + checks(report, stripped, original, path) + icons_check(report, stripped, path) + toc_check(report, stripped, path) + + if re.findall(Regex.ASSEMBLY_TYPE, stripped): + confused_files.append(path) + nesting_in_assemblies_check(report, stripped, path) + add_res_section_assembly_check(report, stripped, path) + else: + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + + if confused_files: + printing_build_yml_error("files that have missmathed name prefix and content type tag. Content type tag takes precident. The files were checked according to the tag", confused_files) + if undetermined_file_type: printing_build_yml_error('files that can not be classifiyed as modules or assemblies', undetermined_file_type) From a232aada111de9fa75905663de8149e52c49e4ad Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 00:01:24 +0100 Subject: [PATCH 14/27] fixes false positives in -- code blokes --- PantheonCMD/pcchecks.py | 1 + PantheonCMD/pcyamlchecks.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/PantheonCMD/pcchecks.py b/PantheonCMD/pcchecks.py index 258d238..a586763 100644 --- a/PantheonCMD/pcchecks.py +++ b/PantheonCMD/pcchecks.py @@ -39,6 +39,7 @@ class Regex: INTERNAL_IFDEF = re.compile(r'(ifdef::internal\[\])(.*\n)*?(endif::\[\])') CODE_BLOCK_DASHES = re.compile(r'(-{4,})(.*\n)*?(-{4,})') CODE_BLOCK_DOTS = re.compile(r'(\.{4,})(.*\n)*?(\.{4,})') + CODE_BLOCK_TWO_DASHES = re.compile(r'(-{2,})(.*\n)*?(-{2,})') HUMAN_READABLE_LABEL_XREF = re.compile(r'xref:.*\[]') HUMAN_READABLE_LABEL_LINK = re.compile(r'\b(?:https?|file|ftp|irc):\/\/[^\s\[\]<]*\[\]') NESTED_ASSEMBLY = re.compile(r'include.*assembly([a-z|0-9|A-Z|\-|_]+)\.adoc(\[.*\])') diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index cea401d..0b29ee4 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -270,7 +270,7 @@ def get_undefined_content(yaml_doc): return undefined_content -def sort_no_prefix_files(yaml_doc): +def file_validation(yaml_doc): undefined_content = get_undefined_content(yaml_doc) undetermined_file_type = [] report = Report() @@ -281,6 +281,7 @@ def sort_no_prefix_files(yaml_doc): stripped = Regex.MULTI_LINE_COMMENT.sub('', original) stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_TWO_DASHES.sub('', stripped) stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) stripped = Regex.INTERNAL_IFDEF.sub('', stripped) @@ -311,6 +312,7 @@ def sort_no_prefix_files(yaml_doc): stripped = Regex.MULTI_LINE_COMMENT.sub('', original) stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_TWO_DASHES.sub('', stripped) stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) stripped = Regex.INTERNAL_IFDEF.sub('', stripped) checks(report, stripped, original, path) @@ -328,10 +330,12 @@ def sort_no_prefix_files(yaml_doc): prefix_modules = get_prefix_modules(yaml_doc) for path in prefix_modules: with open(path, "r") as file: + original = file.read() original = file.read() stripped = Regex.MULTI_LINE_COMMENT.sub('', original) stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_TWO_DASHES.sub('', stripped) stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) stripped = Regex.INTERNAL_IFDEF.sub('', stripped) checks(report, stripped, original, path) @@ -366,7 +370,7 @@ def yaml_validation(yaml_file): get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) get_attribute_file_errors(loaded_yaml) - no_prefix_files_validation = sort_no_prefix_files(loaded_yaml) + validation = file_validation(loaded_yaml) - if no_prefix_files_validation.count != 0: - no_prefix_files_validation.print_report() + if validation.count != 0: + validation.print_report() From a2df6425fd329d7e0ea3ff4d7dd24eacc5c3c7ed Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 00:08:32 +0100 Subject: [PATCH 15/27] fixes the imports --- PantheonCMD/pcyamlchecks.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 0b29ee4..4afa33c 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -5,12 +5,9 @@ import yaml from cerberus import Validator, errors from cerberus.errors import BasicErrorHandler -from pcchecks import Regex, icons_check, toc_check, nbsp_check -#from pcvalidator import Report -import glob +from pcchecks import Regex, icons_check, toc_check, nbsp_check, checks, nesting_in_modules_check, nesting_in_assemblies_check, add_res_section_module_check, add_res_section_assembly_check import re import subprocess -from pcchecks import Regex, checks, nesting_in_modules_check, nesting_in_assemblies_check, add_res_section_module_check, add_res_section_assembly_check, icons_check, toc_check class CustomErrorHandler(BasicErrorHandler): From 48072cb272f123ab9a8655dad37b0450cab8ba66 Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 00:23:06 +0100 Subject: [PATCH 16/27] split function into two to avoid printing output multiple times --- PantheonCMD/pcyamlchecks.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 4afa33c..09882e8 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -210,12 +210,6 @@ def get_realpath(files): def get_content_list(yaml_doc): - missing_includes = get_missing_files(yaml_doc, 'included') - missing_excludes = get_missing_files(yaml_doc, 'excluded') - - if missing_includes or missing_excludes: - printing_build_yml_error("files or directories that do not exist in your repository", missing_includes, missing_excludes) - included = get_exitsing_files(yaml_doc, 'included') excluded = get_exitsing_files(yaml_doc, 'excluded') @@ -229,6 +223,15 @@ def get_content_list(yaml_doc): return unique_includes +def get_fake_path_files(yaml_doc): + missing_includes = get_missing_files(yaml_doc, 'included') + missing_excludes = get_missing_files(yaml_doc, 'excluded') + + missing_files = missing_excludes + missing_includes + if missing_files: + printing_build_yml_error("files or directories that do not exist in your repository", missing_files) + + def sort_prefix_files(yaml_doc): prefix_assembly = [] prefix_modules = [] @@ -367,6 +370,7 @@ def yaml_validation(yaml_file): get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) get_attribute_file_errors(loaded_yaml) + get_fake_path_files(loaded_yaml) validation = file_validation(loaded_yaml) if validation.count != 0: From f024446949d6642fca9024490e337ca9f0248cb0 Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 00:31:20 +0100 Subject: [PATCH 17/27] msg change --- PantheonCMD/pcyamlchecks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 09882e8..4e170fd 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -174,7 +174,7 @@ def get_attribute_file_errors(yaml_doc): exiting_attribute_files = get_exist(attribute_files) if missing_attribute_files: - printing_build_yml_error("files or directories that do not exist in your repository", missing_attribute_files) + printing_build_yml_error("attribute files that do not exist in your repository", missing_attribute_files) if exiting_attribute_files: for item in attribute_files: From 051c18af4041b8e21fc8113b1162238812cb54f7 Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 18:46:21 +0100 Subject: [PATCH 18/27] removes outdated checks --- PantheonCMD/pcchecks.py | 28 ++++------------------------ PantheonCMD/pcvalidator.py | 3 +-- PantheonCMD/pcyamlchecks.py | 5 +---- 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/PantheonCMD/pcchecks.py b/PantheonCMD/pcchecks.py index a586763..e9b93a5 100644 --- a/PantheonCMD/pcchecks.py +++ b/PantheonCMD/pcchecks.py @@ -85,12 +85,6 @@ def inline_anchor_check(stripped_file): return True -def var_in_title_check(stripped_file): - """Check if the file contains a variable in the level 1 heading.""" - if re.findall(Regex.VAR_IN_TITLE, stripped_file): - return True - - def experimental_tag_check(stripped_file): """Check if the experimental tag is set.""" if stripped_file.count(Tags.EXPERIMENTAL) > 0: @@ -132,14 +126,6 @@ def add_res_section_module_check(report, stripped_file, file_path): if not re.findall(Regex.ADD_RES_MODULE, stripped_file): report.create_report("Additional resources section for modules should be `.Additional resources`. Wrong section name was", file_path) - -# Standalone check on assemblies_found -def nesting_in_assemblies_check(report, stripped_file, file_path): - """Check if file contains nested assemblies.""" - if re.findall(Regex.NESTED_ASSEMBLY, stripped_file): - return report.create_report('nesting in assemblies. nesting', file_path) - - # Standalone check on assemblies_found def add_res_section_assembly_check(report, stripped_file, file_path): if re.findall(Regex.ADDITIONAL_RES, stripped_file): @@ -153,9 +139,9 @@ def lvloffset_check(stripped_file): return True -def abstarct_tag_none_or_multiple_check(stripped_file): +def abstarct_tag_multiple_check(stripped_file): """Checks if the abstract tag is not set or set more than once.""" - if stripped_file.count(Tags.ABSTRACT) != 1: + if stripped_file.count(Tags.ABSTRACT) > 1: return True @@ -235,9 +221,6 @@ def checks(report, stripped_file, original_file, file_path): if inline_anchor_check(stripped_file): report.create_report('in-line anchors', file_path) - if var_in_title_check(stripped_file): - report.create_report('variable in the level 1 heading', file_path) - if experimental_tag_check(stripped_file): report.create_report('files contain UI macros but the :experimental: tag not', file_path) @@ -253,11 +236,8 @@ def checks(report, stripped_file, original_file, file_path): if lvloffset_check(stripped_file): report.create_report('unsupported use of :leveloffset:. unsupported includes', file_path) - if abstarct_tag_none_or_multiple_check(stripped_file): - if stripped_file.count(Tags.ABSTRACT) == 0: - report.create_report('abstract tag not', file_path) - else: - report.create_report('multiple abstract tags', file_path) + if abstarct_tag_multiple_check(stripped_file): + report.create_report('multiple abstract tags', file_path) if abstract_tag_check(original_file): if re.findall(Regex.FIRST_PARA, original_file): diff --git a/PantheonCMD/pcvalidator.py b/PantheonCMD/pcvalidator.py index 31f32b4..9a10787 100644 --- a/PantheonCMD/pcvalidator.py +++ b/PantheonCMD/pcvalidator.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -from pcchecks import Regex, checks, nesting_in_modules_check, nesting_in_assemblies_check, add_res_section_module_check, add_res_section_assembly_check, icons_check, toc_check +from pcchecks import Regex, checks, nesting_in_modules_check, add_res_section_module_check, add_res_section_assembly_check, icons_check, toc_check import sys @@ -64,7 +64,6 @@ def validation(files_found, modules_found, assemblies_found): stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) stripped = Regex.INTERNAL_IFDEF.sub('', stripped) - nesting_in_assemblies_check(report, stripped, path) add_res_section_assembly_check(report, stripped, path) return report diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 4e170fd..751b708 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -5,7 +5,7 @@ import yaml from cerberus import Validator, errors from cerberus.errors import BasicErrorHandler -from pcchecks import Regex, icons_check, toc_check, nbsp_check, checks, nesting_in_modules_check, nesting_in_assemblies_check, add_res_section_module_check, add_res_section_assembly_check +from pcchecks import Regex, icons_check, toc_check, nbsp_check, checks, nesting_in_modules_check, add_res_section_module_check, add_res_section_assembly_check import re import subprocess @@ -296,7 +296,6 @@ def file_validation(yaml_doc): checks(report, stripped, original, path) icons_check(report, stripped, path) toc_check(report, stripped, path) - nesting_in_assemblies_check(report, stripped, path) add_res_section_assembly_check(report, stripped, path) else: @@ -324,7 +323,6 @@ def file_validation(yaml_doc): nesting_in_modules_check(report, stripped, path) add_res_section_module_check(report, stripped, path) else: - nesting_in_assemblies_check(report, stripped, path) add_res_section_assembly_check(report, stripped, path) prefix_modules = get_prefix_modules(yaml_doc) @@ -344,7 +342,6 @@ def file_validation(yaml_doc): if re.findall(Regex.ASSEMBLY_TYPE, stripped): confused_files.append(path) - nesting_in_assemblies_check(report, stripped, path) add_res_section_assembly_check(report, stripped, path) else: nesting_in_modules_check(report, stripped, path) From c5b6668f2b4534e591354a0ad11657485100768b Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 19:20:34 +0100 Subject: [PATCH 19/27] weird way to deal with optional exclude key --- PantheonCMD/pcyamlchecks.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 751b708..639addf 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -120,15 +120,17 @@ def get_files(yaml_doc, var): missing_files = [] for yaml_dict in yaml_doc['variants']: - for include in yaml_dict['files'][var]: - content = get_files_bash(include) - if content: - if '' in content: - missing_files.append(include) - else: - for i in content: - if i not in content_list: - content_list.append(i) + for subkey in yaml_dict['files'] == var: + if subkey == var: + for include in yaml_dict['files'][var]: + content = get_files_bash(include) + if content: + if '' in content: + missing_files.append(include) + else: + for i in content: + if i not in content_list: + content_list.append(i) return content_list, missing_files From 36496f2eaf77c91b3f3a82f6bf0560cf73766e76 Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 19:21:20 +0100 Subject: [PATCH 20/27] fix --- PantheonCMD/pcyamlchecks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 639addf..0f9418e 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -120,7 +120,7 @@ def get_files(yaml_doc, var): missing_files = [] for yaml_dict in yaml_doc['variants']: - for subkey in yaml_dict['files'] == var: + for subkey in yaml_dict['files']: if subkey == var: for include in yaml_dict['files'][var]: content = get_files_bash(include) From bf63dc202b5e2ce60dd3134a5e2992208e52d106 Mon Sep 17 00:00:00 2001 From: Levi V Date: Wed, 2 Mar 2022 19:28:55 +0100 Subject: [PATCH 21/27] removes attribute files from file pull --- PantheonCMD/pcyamlchecks.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 0f9418e..0110ccb 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -179,7 +179,7 @@ def get_attribute_file_errors(yaml_doc): printing_build_yml_error("attribute files that do not exist in your repository", missing_attribute_files) if exiting_attribute_files: - for item in attribute_files: + for item in exiting_attribute_files: file_name = os.path.basename(item) file_path = os.path.dirname(item) if not file_path.startswith("_"): @@ -208,6 +208,16 @@ def get_realpath(files): relative_path = os.path.relpath(i, pwd) files.append(relative_path) + for item in files: + file_name = os.path.basename(item) + file_path = os.path.dirname(item) + if file_path.startswith("_"): + files.remove(item) + elif "/_" in file_path: + files.remove(item) + elif file_name.startswith("_"): + files.remove(item) + return files From 56786815748c639e0d7a027e846ab241f7084ca9 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 3 Mar 2022 18:37:05 +0100 Subject: [PATCH 22/27] implemented Jaromir's feedback on build.yml generation script --- PantheonCMD/buildyml-generator.sh | 44 +++++++++++++++++++++---------- PantheonCMD/generate-build-yml.sh | 25 ------------------ PantheonCMD/pcmd.py | 2 +- 3 files changed, 31 insertions(+), 40 deletions(-) delete mode 100644 PantheonCMD/generate-build-yml.sh diff --git a/PantheonCMD/buildyml-generator.sh b/PantheonCMD/buildyml-generator.sh index 6e31b83..4bddec2 100644 --- a/PantheonCMD/buildyml-generator.sh +++ b/PantheonCMD/buildyml-generator.sh @@ -1,19 +1,35 @@ #!/bin/bash ROOT_REPO=$(git rev-parse --show-toplevel) -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +SCRIPT_DIR=$(realpath $(dirname "$0")) +REPO_NAME=$(basename $ROOT_REPO) -cd $ROOT_REPO -if [ ! -f "build.yml" ]; then - sh $SCRIPT_DIR/generate-build-yml.sh > build.yml - echo "build.yml succsessfully generated"; - else - read -p "build.yml already exists. Do you want to overwrite it? [y/N] " yn - case $yn in - [Yy]*) - sh $SCRIPT_DIR/generate-build-yml.sh > build.yml && echo "build.yml succsessfully generated"; exit;; - [Nn]*) - echo "exiting..."; exit;; - *) - esac +if [[ -f "$ROOT_REPO/build.yml" ]]; then + read -p "build.yml already exists. Do you want to overwrite it? [y/N] " -n 1 + echo + if [[ ! "$REPLY" =~ ^[yY]$ ]]; then + exit + fi fi + +cat >$ROOT_REPO/build.yml << EOF +# Your repository name goes here +repository: $REPO_NAME +variants: + # Your variant name goes here + - name: PLACEHOLDER + # Path to your attributes file goes here + attributes: + - PATH/TO/_attributes.adoc + nav: PATH/TO/nav.yml + build: true + files: + # Path to your assemblies, modules, and images go here + included: + - PATH/TO/ASSEMBLIES/*.adoc + - PATH/TO/MODULES/**/*.adoc + - PATH/TO/images/*.png + +EOF + +echo "build.yml successfully generated" diff --git a/PantheonCMD/generate-build-yml.sh b/PantheonCMD/generate-build-yml.sh deleted file mode 100644 index c1b6656..0000000 --- a/PantheonCMD/generate-build-yml.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -#define parameters -repo_name=$(basename $PWD) - -#define the template -cat << EOF -# Your repository name goes here -repository: $repo_name -variants: - # Your variant name goes here - - name: PLACEHOLDER - # Path to your attributes file goes here - attributes: - - PATH/TO/_attributes.adoc - nav: PATH/TO/nav.yml - build: true - files: - # Path to your assemblies, modules, and images go here - included: - - PATH/TO/ASSEMBLIES/*.adoc - - PATH/TO/MODULES/**/*.adoc - - PATH/TO/images/*.png - -EOF diff --git a/PantheonCMD/pcmd.py b/PantheonCMD/pcmd.py index a48b48a..3a1a818 100644 --- a/PantheonCMD/pcmd.py +++ b/PantheonCMD/pcmd.py @@ -68,7 +68,7 @@ def parse_args(): # Action - generate a build.yml file if args.command == 'generate': path_to_script = os.path.dirname(os.path.realpath(__file__)) - call("sh " + path_to_script + "/pv2yml-generator.sh", shell=True) + call("bash " + path_to_script + "/buildyml-generator.sh", shell=True) sys.exit(0) # Action - validate yaml syntax, validate yaml keys and values From 2dd92c1fff81a1c73a11592bcb9f4226553d7668 Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 3 Mar 2022 20:27:31 +0100 Subject: [PATCH 23/27] adds docstring; removes some functions; simplifies validation routine --- PantheonCMD/pcyamlchecks.py | 212 +++++++++++++++--------------------- 1 file changed, 86 insertions(+), 126 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 0110ccb..494ecee 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -12,6 +12,7 @@ class CustomErrorHandler(BasicErrorHandler): """Custom error messages.""" + messages = errors.BasicErrorHandler.messages.copy() messages[errors.REQUIRED_FIELD.code] = "key is missing" messages[errors.UNKNOWN_FIELD.code] = "unsupported key" @@ -19,6 +20,7 @@ class CustomErrorHandler(BasicErrorHandler): def printing_build_yml_error(msg, *files): + """Print error message.""" print('\nERROR: Your build.yml contains the following {}:\n'.format(msg)) for file in files: if file: @@ -68,6 +70,7 @@ def load_doc(yaml_file): def get_yaml_errors(yaml_schema, yaml_doc): + """Validate build.yml against a schema abd report errors.""" # load validator with custom error handler v = Validator(yaml_schema, error_handler=CustomErrorHandler()) # validate the build.yml with schema @@ -81,6 +84,7 @@ def get_yaml_errors(yaml_schema, yaml_doc): def get_attribute_files(yaml_doc): + """Get attribute files specifiyed in build.yml.""" attribute_files = [] for variant in yaml_doc['variants']: @@ -91,6 +95,7 @@ def get_attribute_files(yaml_doc): def get_existence(files): + """Return a list of found files and a list of not found files.""" files_found = [] files_not_found = [] @@ -103,54 +108,39 @@ def get_existence(files): return files_found, files_not_found -def get_not_exist(content_list): - files_found, files_not_found = get_existence(content_list) - - return files_not_found - - -def get_exist(content_list): - files_found, files_not_found = get_existence(content_list) +def get_files_bash(file_path): + """Expand filepaths.""" + command = ("find " + file_path + " -type f 2>/dev/null") + process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout + files = process.strip().decode('utf-8').split('\n') - return files_found + return files def get_files(yaml_doc, var): + """Get files listed in the build.yml.""" content_list = [] missing_files = [] for yaml_dict in yaml_doc['variants']: for subkey in yaml_dict['files']: - if subkey == var: - for include in yaml_dict['files'][var]: - content = get_files_bash(include) - if content: - if '' in content: - missing_files.append(include) - else: - for i in content: - if i not in content_list: - content_list.append(i) - - return content_list, missing_files - - -def get_missing_files(yaml_doc, var): - content_list, missing_files = get_files(yaml_doc, var) - return missing_files + if subkey != var: + continue + for include in yaml_dict['files'][var]: + content = get_files_bash(include) + if not content: + continue -def get_exitsing_files(yaml_doc, var): - content_list, missing_files = get_files(yaml_doc, var) - return content_list + if '' in content: + missing_files.append(include) + continue + for i in content: + if i not in content_list: + content_list.append(i) -def get_files_bash(file_path): - command = ("find " + file_path + " -type f 2>/dev/null") - process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout - files = process.strip().decode('utf-8').split('\n') - - return files + return content_list, missing_files def validate_attribute_files(attribute_files): @@ -171,9 +161,9 @@ def validate_attribute_files(attribute_files): def get_attribute_file_errors(yaml_doc): + """Report errors found with attribute files.""" attribute_files = get_attribute_files(yaml_doc) - missing_attribute_files = get_not_exist(attribute_files) - exiting_attribute_files = get_exist(attribute_files) + missing_attribute_files, exiting_attribute_files = get_existence(attribute_files) if missing_attribute_files: printing_build_yml_error("attribute files that do not exist in your repository", missing_attribute_files) @@ -194,36 +184,46 @@ def get_attribute_file_errors(yaml_doc): def get_realpath(files): + """Get unique file list of content excluding attributes""" + # get unique file list through realpath unique_files = [] for file in files: - real_path = os.path.realpath(file) - if real_path not in unique_files: - unique_files.append(real_path) + if file.endswith('adoc'): + real_path = os.path.realpath(file) + if real_path not in unique_files: + unique_files.append(real_path) - files = [] + # convert realpath back to relative path + relative_path_files = [] pwd = os.getcwd() for i in unique_files: relative_path = os.path.relpath(i, pwd) - files.append(relative_path) + relative_path_files.append(relative_path) - for item in files: + # remove attribute files from list + files = [] + + for item in relative_path_files: file_name = os.path.basename(item) file_path = os.path.dirname(item) if file_path.startswith("_"): - files.remove(item) + continue elif "/_" in file_path: - files.remove(item) + continue elif file_name.startswith("_"): - files.remove(item) + continue + else: + files.append(item) return files def get_content_list(yaml_doc): - included = get_exitsing_files(yaml_doc, 'included') - excluded = get_exitsing_files(yaml_doc, 'excluded') + """Get a unique list of included files with removed excludes.""" + included, fake_path_includes = get_files(yaml_doc, 'included') + excluded, fake_path_excludes = get_files(yaml_doc, 'excluded') unique_includes = get_realpath(included) unique_excludes = get_realpath(excluded) @@ -236,15 +236,17 @@ def get_content_list(yaml_doc): def get_fake_path_files(yaml_doc): - missing_includes = get_missing_files(yaml_doc, 'included') - missing_excludes = get_missing_files(yaml_doc, 'excluded') + """Error out on fake filepaths in build.yml""" + included, fake_path_includes = get_files(yaml_doc, 'included') + excluded, fake_path_excludes = get_files(yaml_doc, 'excluded') - missing_files = missing_excludes + missing_includes + missing_files = fake_path_excludes + fake_path_includes if missing_files: printing_build_yml_error("files or directories that do not exist in your repository", missing_files) def sort_prefix_files(yaml_doc): + """Get a list of assemblies, modulesa, and unidentifiyed files.""" prefix_assembly = [] prefix_modules = [] undefined_content = [] @@ -264,61 +266,19 @@ def sort_prefix_files(yaml_doc): return prefix_assembly, prefix_modules, undefined_content -def get_prefix_assemblies(yaml_doc): - prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_doc) - - return prefix_assembly - - -def get_prefix_modules(yaml_doc): - prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_doc) - - return prefix_modules - - -def get_undefined_content(yaml_doc): +def file_validation(yaml_doc): + """Validate all files.""" prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_doc) - return undefined_content + all_files = prefix_assembly + prefix_modules + undefined_content - -def file_validation(yaml_doc): - undefined_content = get_undefined_content(yaml_doc) undetermined_file_type = [] - report = Report() - - for path in undefined_content: - with open(path, 'r') as file: - original = file.read() - stripped = Regex.MULTI_LINE_COMMENT.sub('', original) - stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) - stripped = Regex.CODE_BLOCK_TWO_DASHES.sub('', stripped) - stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) - stripped = Regex.INTERNAL_IFDEF.sub('', stripped) - - if re.findall(Regex.MODULE_TYPE, stripped): - checks(report, stripped, original, path) - icons_check(report, stripped, path) - toc_check(report, stripped, path) - nesting_in_modules_check(report, stripped, path) - add_res_section_module_check(report, stripped, path) - - elif re.findall(Regex.ASSEMBLY_TYPE, stripped): - checks(report, stripped, original, path) - icons_check(report, stripped, path) - toc_check(report, stripped, path) - add_res_section_assembly_check(report, stripped, path) - - else: - undetermined_file_type.append(path) - confused_files = [] - prefix_assemblies = get_prefix_assemblies(yaml_doc) + report = Report() - for path in prefix_assemblies: - with open(path, "r") as file: + for path in all_files: + with open(path, 'r') as file: original = file.read() stripped = Regex.MULTI_LINE_COMMENT.sub('', original) stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) @@ -326,38 +286,37 @@ def file_validation(yaml_doc): stripped = Regex.CODE_BLOCK_TWO_DASHES.sub('', stripped) stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) stripped = Regex.INTERNAL_IFDEF.sub('', stripped) - checks(report, stripped, original, path) - icons_check(report, stripped, path) - toc_check(report, stripped, path) - - if re.findall(Regex.MODULE_TYPE, stripped): - confused_files.append(path) - nesting_in_modules_check(report, stripped, path) - add_res_section_module_check(report, stripped, path) - else: - add_res_section_assembly_check(report, stripped, path) - prefix_modules = get_prefix_modules(yaml_doc) - for path in prefix_modules: - with open(path, "r") as file: - original = file.read() - original = file.read() - stripped = Regex.MULTI_LINE_COMMENT.sub('', original) - stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) - stripped = Regex.CODE_BLOCK_TWO_DASHES.sub('', stripped) - stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) - stripped = Regex.INTERNAL_IFDEF.sub('', stripped) checks(report, stripped, original, path) icons_check(report, stripped, path) toc_check(report, stripped, path) - if re.findall(Regex.ASSEMBLY_TYPE, stripped): - confused_files.append(path) - add_res_section_assembly_check(report, stripped, path) - else: - nesting_in_modules_check(report, stripped, path) - add_res_section_module_check(report, stripped, path) + if path in undefined_content: + if re.findall(Regex.MODULE_TYPE, stripped): + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + + elif re.findall(Regex.ASSEMBLY_TYPE, stripped): + add_res_section_assembly_check(report, stripped, path) + + else: + undetermined_file_type.append(path) + + if path in prefix_assembly: + if re.findall(Regex.MODULE_TYPE, stripped): + confused_files.append(path) + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + else: + add_res_section_assembly_check(report, stripped, path) + + if path in prefix_modules: + if re.findall(Regex.ASSEMBLY_TYPE, stripped): + confused_files.append(path) + add_res_section_assembly_check(report, stripped, path) + else: + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) if confused_files: printing_build_yml_error("files that have missmathed name prefix and content type tag. Content type tag takes precident. The files were checked according to the tag", confused_files) @@ -369,6 +328,7 @@ def file_validation(yaml_doc): def yaml_validation(yaml_file): + """Execute yml and general validation and report errors.""" # define path to script path_to_script = os.path.dirname(os.path.realpath(__file__)) # load schema From 52ac8ab52584b65132f2b3d5474a632bfb23e47b Mon Sep 17 00:00:00 2001 From: Levi V Date: Thu, 3 Mar 2022 20:55:07 +0100 Subject: [PATCH 24/27] fixes typos --- PantheonCMD/pcyamlchecks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index 494ecee..d3709a7 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -175,7 +175,7 @@ def get_attribute_file_errors(yaml_doc): if not file_path.startswith("_"): if "/_" not in file_path: if not file_name.startswith("_"): - printing_build_yml_error("files or directories that do not follow the attribute naming conventions. Attribute files or directory they are stored in should start with an undescore", item) + printing_build_yml_error("files or directories that do not follow the attribute naming conventions. Attribute files or directory they are stored in should start with an underscore", item) attribute_validation = validate_attribute_files(exiting_attribute_files) @@ -319,10 +319,10 @@ def file_validation(yaml_doc): add_res_section_module_check(report, stripped, path) if confused_files: - printing_build_yml_error("files that have missmathed name prefix and content type tag. Content type tag takes precident. The files were checked according to the tag", confused_files) + printing_build_yml_error("files that have mismatched name prefix and content type tag. Content type tag takes precedence. The files were checked according to the tag", confused_files) if undetermined_file_type: - printing_build_yml_error('files that can not be classifiyed as modules or assemblies', undetermined_file_type) + printing_build_yml_error('files that can not be classified as modules or assemblies', undetermined_file_type) return report From c9457438ded4845d3da1bbe8f997e19329b2cb13 Mon Sep 17 00:00:00 2001 From: Levi V Date: Fri, 4 Mar 2022 00:47:51 +0100 Subject: [PATCH 25/27] main validation routine now expects user input --- PantheonCMD/pcyamlchecks.py | 72 +++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/pcyamlchecks.py index d3709a7..9365b21 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/pcyamlchecks.py @@ -94,30 +94,31 @@ def get_attribute_files(yaml_doc): return attribute_files -def get_existence(files): +def get_existence(path, files): """Return a list of found files and a list of not found files.""" - files_found = [] - files_not_found = [] + missing_files = [] + exiting_files = [] for item in files: - if os.path.exists(item): - files_found.append(item) + file = path + '/' + item + if os.path.exists(file): + exiting_files.append(item) else: - files_not_found.append(item) + missing_files.append(item) - return files_found, files_not_found + return exiting_files, missing_files -def get_files_bash(file_path): +def get_files_bash(yaml_path, file_path): """Expand filepaths.""" - command = ("find " + file_path + " -type f 2>/dev/null") + command = ("find "+ yaml_path + '/' + file_path + " -type f 2>/dev/null") process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout files = process.strip().decode('utf-8').split('\n') return files -def get_files(yaml_doc, var): +def get_files(yaml_path, yaml_doc, var): """Get files listed in the build.yml.""" content_list = [] missing_files = [] @@ -128,7 +129,7 @@ def get_files(yaml_doc, var): continue for include in yaml_dict['files'][var]: - content = get_files_bash(include) + content = get_files_bash(yaml_path, include) if not content: continue @@ -143,27 +144,28 @@ def get_files(yaml_doc, var): return content_list, missing_files -def validate_attribute_files(attribute_files): +def validate_attribute_files(path, attribute_files): """Validate attributes file.""" report = Report() - for path in attribute_files: - with open(path, 'r') as file: - original = file.read() + for item in attribute_files: + file = path + '/' + item + with open(file, 'r') as f: + original = f.read() stripped = Regex.MULTI_LINE_COMMENT.sub('', original) stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - icons_check(report, stripped, path) - toc_check(report, stripped, path) - nbsp_check(report, stripped, path) + icons_check(report, stripped, item) + toc_check(report, stripped, item) + nbsp_check(report, stripped, item) return report -def get_attribute_file_errors(yaml_doc): +def get_attribute_file_errors(yaml_doc, path): """Report errors found with attribute files.""" attribute_files = get_attribute_files(yaml_doc) - missing_attribute_files, exiting_attribute_files = get_existence(attribute_files) + exiting_attribute_files, missing_attribute_files = get_existence(path, attribute_files) if missing_attribute_files: printing_build_yml_error("attribute files that do not exist in your repository", missing_attribute_files) @@ -177,7 +179,7 @@ def get_attribute_file_errors(yaml_doc): if not file_name.startswith("_"): printing_build_yml_error("files or directories that do not follow the attribute naming conventions. Attribute files or directory they are stored in should start with an underscore", item) - attribute_validation = validate_attribute_files(exiting_attribute_files) + attribute_validation = validate_attribute_files(path, exiting_attribute_files) if attribute_validation.count != 0: attribute_validation.print_report() @@ -220,10 +222,10 @@ def get_realpath(files): return files -def get_content_list(yaml_doc): +def get_content_list(yaml_path, yaml_doc): """Get a unique list of included files with removed excludes.""" - included, fake_path_includes = get_files(yaml_doc, 'included') - excluded, fake_path_excludes = get_files(yaml_doc, 'excluded') + included, fake_path_includes = get_files(yaml_path, yaml_doc, 'included') + excluded, fake_path_excludes = get_files(yaml_path, yaml_doc, 'excluded') unique_includes = get_realpath(included) unique_excludes = get_realpath(excluded) @@ -235,23 +237,23 @@ def get_content_list(yaml_doc): return unique_includes -def get_fake_path_files(yaml_doc): +def get_fake_path_files(yaml_path, yaml_doc): """Error out on fake filepaths in build.yml""" - included, fake_path_includes = get_files(yaml_doc, 'included') - excluded, fake_path_excludes = get_files(yaml_doc, 'excluded') + included, fake_path_includes = get_files(yaml_path, yaml_doc, 'included') + excluded, fake_path_excludes = get_files(yaml_path, yaml_doc, 'excluded') missing_files = fake_path_excludes + fake_path_includes if missing_files: printing_build_yml_error("files or directories that do not exist in your repository", missing_files) -def sort_prefix_files(yaml_doc): +def sort_prefix_files(yaml_path, yaml_doc): """Get a list of assemblies, modulesa, and unidentifiyed files.""" prefix_assembly = [] prefix_modules = [] undefined_content = [] - content_list = get_content_list(yaml_doc) + content_list = get_content_list(yaml_path, yaml_doc) for item in content_list: if item.endswith('.adoc'): @@ -266,9 +268,9 @@ def sort_prefix_files(yaml_doc): return prefix_assembly, prefix_modules, undefined_content -def file_validation(yaml_doc): +def file_validation(yaml_path, yaml_doc): """Validate all files.""" - prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_doc) + prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(yaml_path, yaml_doc) all_files = prefix_assembly + prefix_modules + undefined_content @@ -327,7 +329,7 @@ def file_validation(yaml_doc): return report -def yaml_validation(yaml_file): +def yaml_validation(yaml_file, path_to_yaml): """Execute yml and general validation and report errors.""" # define path to script path_to_script = os.path.dirname(os.path.realpath(__file__)) @@ -338,9 +340,9 @@ def yaml_validation(yaml_file): get_yaml_size(yaml_file) get_yaml_errors(schema, loaded_yaml) - get_attribute_file_errors(loaded_yaml) - get_fake_path_files(loaded_yaml) - validation = file_validation(loaded_yaml) + get_attribute_file_errors(loaded_yaml, path_to_yaml) + get_fake_path_files(path_to_yaml, loaded_yaml) + validation = file_validation(path_to_yaml, loaded_yaml) if validation.count != 0: validation.print_report() From 10799082db4ea90719e83ea58c077ccdc982e661 Mon Sep 17 00:00:00 2001 From: Levi V Date: Fri, 4 Mar 2022 02:30:54 +0100 Subject: [PATCH 26/27] raname; new parser; validation throug build/dir/single file --- PantheonCMD/enki.py | 31 +++++ PantheonCMD/{pcchecks.py => enki_checks.py} | 0 PantheonCMD/enki_files_valiadtor.py | 126 ++++++++++++++++++ ...pcyamlchecks.py => enki_yaml_valiadtor.py} | 7 +- PantheonCMD/pcmd.py | 2 +- PantheonCMD/pcprvalidator.py | 2 +- PantheonCMD/pcvalidator.py | 69 ---------- 7 files changed, 164 insertions(+), 73 deletions(-) create mode 100644 PantheonCMD/enki.py rename PantheonCMD/{pcchecks.py => enki_checks.py} (100%) create mode 100644 PantheonCMD/enki_files_valiadtor.py rename PantheonCMD/{pcyamlchecks.py => enki_yaml_valiadtor.py} (97%) delete mode 100644 PantheonCMD/pcvalidator.py diff --git a/PantheonCMD/enki.py b/PantheonCMD/enki.py new file mode 100644 index 0000000..27f7fa1 --- /dev/null +++ b/PantheonCMD/enki.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 + +import argparse +from pathlib import Path +import os +from enki_yaml_valiadtor import yaml_validation +from enki_files_valiadtor import multi_file_validation, single_file_validation + +parser = argparse.ArgumentParser() +parser.add_argument("path", type=Path) + +p = parser.parse_args() + + +user_input = p.path + +if user_input.is_file(): + file_name = os.path.basename(user_input) + file_path = os.path.dirname(user_input) + if file_name == 'build.yml': + file_name = os.path.basename(user_input) + yaml_validation(user_input, file_path) + else: + str = str(user_input) + list = str.split() + single_file_validation(list) + +elif user_input.is_dir(): + multi_file_validation(user_input) +else: + print("ERROR: Provided path doesn't exist.") diff --git a/PantheonCMD/pcchecks.py b/PantheonCMD/enki_checks.py similarity index 100% rename from PantheonCMD/pcchecks.py rename to PantheonCMD/enki_checks.py diff --git a/PantheonCMD/enki_files_valiadtor.py b/PantheonCMD/enki_files_valiadtor.py new file mode 100644 index 0000000..9d06a96 --- /dev/null +++ b/PantheonCMD/enki_files_valiadtor.py @@ -0,0 +1,126 @@ +#!/usr/bin/python3 +import os +import sys +from enki_checks import Regex, icons_check, toc_check, nbsp_check, checks, nesting_in_modules_check, add_res_section_module_check, add_res_section_assembly_check +import re +import subprocess +from enki_yaml_valiadtor import Report, printing_build_yml_error + + +def get_files_bash(file_path): + """Expand filepaths.""" + command = "find " + str(file_path) + " -type f -name \*adoc 2>/dev/null" + process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout + files = process.strip().decode('utf-8').split('\n') + + return files + + +def sort_prefix_files(files): + """Get a list of assemblies, modulesa, and unidentifiyed files.""" + prefix_assembly = [] + prefix_modules = [] + undefined_content = [] + attribute_file = [] + + for item in files: + file_name = os.path.basename(item) + file_path = os.path.basename(item) + + if file_path.startswith("_"): + attribute_file.append(item) + elif "/_" in file_path: + attribute_file.append(item) + + if file_name.startswith('assembly'): + prefix_assembly.append(item) + elif file_name.startswith(("proc_", "con_", "ref_", "proc-", "con-", "ref-")): + prefix_modules.append(item) + elif file_name.startswith("_"): + attribute_file.append(item) + elif file_name.startswith(("snip_", "snip-")): + continue + elif file_name == 'master.adoc': + continue + else: + undefined_content.append(item) + + return attribute_file, prefix_assembly, prefix_modules, undefined_content + + +def file_validation(files): + """Validate all files.""" + attribute_file, prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(files) + + all_files = prefix_assembly + prefix_modules + undefined_content + + undetermined_file_type = [] + confused_files = [] + + report = Report() + + for path in all_files: + with open(path, 'r') as file: + original = file.read() + stripped = Regex.MULTI_LINE_COMMENT.sub('', original) + stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) + stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_TWO_DASHES.sub('', stripped) + stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) + stripped = Regex.INTERNAL_IFDEF.sub('', stripped) + + checks(report, stripped, original, path) + icons_check(report, stripped, path) + toc_check(report, stripped, path) + + if path in undefined_content: + if re.findall(Regex.MODULE_TYPE, stripped): + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + + elif re.findall(Regex.ASSEMBLY_TYPE, stripped): + add_res_section_assembly_check(report, stripped, path) + + else: + undetermined_file_type.append(path) + + if path in prefix_assembly: + if re.findall(Regex.MODULE_TYPE, stripped): + confused_files.append(path) + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + else: + add_res_section_assembly_check(report, stripped, path) + + if path in prefix_modules: + if re.findall(Regex.ASSEMBLY_TYPE, stripped): + confused_files.append(path) + add_res_section_assembly_check(report, stripped, path) + else: + nesting_in_modules_check(report, stripped, path) + add_res_section_module_check(report, stripped, path) + + if confused_files: + printing_build_yml_error("files that have mismatched name prefix and content type tag. Content type tag takes precedence. The files were checked according to the tag", confused_files) + + if undetermined_file_type: + printing_build_yml_error('files that can not be classified as modules or assemblies', undetermined_file_type) + + return report + + +def multi_file_validation(file_path): + files = get_files_bash(file_path) + validation = file_validation(files) + + if validation.count != 0: + validation.print_report() + sys.exit(2) + + +def single_file_validation(files): + validation = file_validation(files) + + if validation.count != 0: + validation.print_report() + sys.exit(2) diff --git a/PantheonCMD/pcyamlchecks.py b/PantheonCMD/enki_yaml_valiadtor.py similarity index 97% rename from PantheonCMD/pcyamlchecks.py rename to PantheonCMD/enki_yaml_valiadtor.py index 9365b21..77be410 100644 --- a/PantheonCMD/pcyamlchecks.py +++ b/PantheonCMD/enki_yaml_valiadtor.py @@ -5,7 +5,7 @@ import yaml from cerberus import Validator, errors from cerberus.errors import BasicErrorHandler -from pcchecks import Regex, icons_check, toc_check, nbsp_check, checks, nesting_in_modules_check, add_res_section_module_check, add_res_section_assembly_check +from enki_checks import Regex, icons_check, toc_check, nbsp_check, checks, nesting_in_modules_check, add_res_section_module_check, add_res_section_assembly_check import re import subprocess @@ -111,7 +111,7 @@ def get_existence(path, files): def get_files_bash(yaml_path, file_path): """Expand filepaths.""" - command = ("find "+ yaml_path + '/' + file_path + " -type f 2>/dev/null") + command = ("find " + yaml_path + '/' + file_path + " -type f 2>/dev/null") process = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout files = process.strip().decode('utf-8').split('\n') @@ -262,6 +262,8 @@ def sort_prefix_files(yaml_path, yaml_doc): prefix_assembly.append(item) elif file_name.startswith(("proc_", "con_", "ref_", "proc-", "con-", "ref-")): prefix_modules.append(item) + elif file_name.startswith(("snip_", "snip-")): + continue else: undefined_content.append(item) @@ -346,3 +348,4 @@ def yaml_validation(yaml_file, path_to_yaml): if validation.count != 0: validation.print_report() + sys.exit(2) diff --git a/PantheonCMD/pcmd.py b/PantheonCMD/pcmd.py index 3a1a818..e218dde 100644 --- a/PantheonCMD/pcmd.py +++ b/PantheonCMD/pcmd.py @@ -9,7 +9,7 @@ from pcutil import PantheonRepo, get_not_exist, get_exist, is_pantheon_repo from pcvalidator import validation -from pcyamlchecks import yaml_validation +from enki_yaml_valiadtor import yaml_validation from subprocess import call from pcprvalidator import get_changed_files, get_all_modules, get_all_assemblies, get_undetermined_files, get_no_prefix_files diff --git a/PantheonCMD/pcprvalidator.py b/PantheonCMD/pcprvalidator.py index adc7873..a546081 100644 --- a/PantheonCMD/pcprvalidator.py +++ b/PantheonCMD/pcprvalidator.py @@ -6,7 +6,7 @@ import sys import subprocess import re -from pcchecks import Regex +from enki_checks import Regex if subprocess.call(["git", "branch"], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w')) != 0: diff --git a/PantheonCMD/pcvalidator.py b/PantheonCMD/pcvalidator.py deleted file mode 100644 index 9a10787..0000000 --- a/PantheonCMD/pcvalidator.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python3 - -from pcchecks import Regex, checks, nesting_in_modules_check, add_res_section_module_check, add_res_section_assembly_check, icons_check, toc_check -import sys - - -class Report(): - """Create and print report. thank u J.""" - - def __init__(self): - """Create placeholder for problem description.""" - self.report = {} - self.count = 0 - - def create_report(self, category, file_path): - """Generate report.""" - self.count += 1 - if not category in self.report: - self.report[category] = [] - self.report[category].append(file_path) - - def print_report(self): - - """Print report.""" - separator = "\n\t" - - for category, files in self.report.items(): - print("\nERROR: {} found in the following files:".format(category)) - print('\t' + separator.join(files)) - - -def validation(files_found, modules_found, assemblies_found): - """Validate files.""" - report = Report() - - for path in files_found: - with open(path, "r") as file: - original = file.read() - stripped = Regex.MULTI_LINE_COMMENT.sub('', original) - stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) - stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) - stripped = Regex.INTERNAL_IFDEF.sub('', stripped) - checks(report, stripped, original, path) - icons_check(report, stripped, path) - toc_check(report, stripped, path) - - for path in modules_found: - with open(path, "r") as file: - original = file.read() - stripped = Regex.MULTI_LINE_COMMENT.sub('', original) - stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) - stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) - stripped = Regex.INTERNAL_IFDEF.sub('', stripped) - nesting_in_modules_check(report, stripped, path) - add_res_section_module_check(report, stripped, path) - - for path in assemblies_found: - with open(path, "r") as file: - original = file.read() - stripped = Regex.MULTI_LINE_COMMENT.sub('', original) - stripped = Regex.SINGLE_LINE_COMMENT.sub('', stripped) - stripped = Regex.CODE_BLOCK_DASHES.sub('', stripped) - stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) - stripped = Regex.INTERNAL_IFDEF.sub('', stripped) - add_res_section_assembly_check(report, stripped, path) - - return report From bbd504d337a46e5d6599b8028362f6bc1af6212b Mon Sep 17 00:00:00 2001 From: Levi V Date: Fri, 4 Mar 2022 02:39:22 +0100 Subject: [PATCH 27/27] adds attribute check to adoc files validation --- PantheonCMD/enki_files_valiadtor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/PantheonCMD/enki_files_valiadtor.py b/PantheonCMD/enki_files_valiadtor.py index 9d06a96..547e40c 100644 --- a/PantheonCMD/enki_files_valiadtor.py +++ b/PantheonCMD/enki_files_valiadtor.py @@ -52,7 +52,7 @@ def file_validation(files): """Validate all files.""" attribute_file, prefix_assembly, prefix_modules, undefined_content = sort_prefix_files(files) - all_files = prefix_assembly + prefix_modules + undefined_content + all_files = attribute_file + prefix_assembly + prefix_modules + undefined_content undetermined_file_type = [] confused_files = [] @@ -69,10 +69,14 @@ def file_validation(files): stripped = Regex.CODE_BLOCK_DOTS.sub('', stripped) stripped = Regex.INTERNAL_IFDEF.sub('', stripped) - checks(report, stripped, original, path) icons_check(report, stripped, path) toc_check(report, stripped, path) + if path in attribute_file: + nbsp_check(report, stripped, path) + else: + checks(report, stripped, original, path) + if path in undefined_content: if re.findall(Regex.MODULE_TYPE, stripped): nesting_in_modules_check(report, stripped, path)