diff --git a/.travis.yml b/.travis.yml index f08e7af8fe..ad653178a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -151,7 +151,8 @@ script: BUILD_DIR=$TRAVIS_BUILD_DIR/build make -C analyzer \ test_unit \ test_functional \ - test_tu_collector && + test_tu_collector \ + test_merge_clang_extdef_mappings && if [[ "$TRAVIS_OS_NAME" = "linux" ]]; then make -C analyzer test_build_logger fi diff --git a/Makefile b/Makefile index b8c3d55969..425f194826 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,17 @@ build_report_hash: package_report_hash: build_report_hash package_dir_structure cp -r $(CC_TOOLS)/codechecker_report_hash/build/codechecker_report_hash/codechecker_report_hash $(CC_BUILD_LIB_DIR) -package: package_dir_structure set_git_commit_template package_plist_to_html package_tu_collector package_report_converter package_report_hash +build_merge_clang_extdef_mappings: + $(MAKE) -C $(CC_ANALYZER)/tools/merge_clang_extdef_mappings build + +package_merge_clang_extdef_mappings: build_merge_clang_extdef_mappings package_dir_structure + # Copy merge-clang-extdef-mappings files. + cp -r $(CC_ANALYZER)/tools/merge_clang_extdef_mappings/build/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings $(CC_BUILD_LIB_DIR) && \ + chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_merge_clang_extdef_mappings/cli.py && \ + cd $(CC_BUILD_DIR) && \ + ln -sf ../lib/python3/codechecker_merge_clang_extdef_mappings/cli.py bin/merge-clang-extdef-mappings + +package: package_dir_structure set_git_commit_template package_plist_to_html package_tu_collector package_report_converter package_report_hash package_merge_clang_extdef_mappings BUILD_DIR=$(BUILD_DIR) BUILD_LOGGER_64_BIT_ONLY=$(BUILD_LOGGER_64_BIT_ONLY) $(MAKE) -C $(CC_ANALYZER) package_analyzer BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_WEB) package_web diff --git a/analyzer/Makefile b/analyzer/Makefile index f608677826..b0aa463f0b 100644 --- a/analyzer/Makefile +++ b/analyzer/Makefile @@ -54,12 +54,22 @@ package_tu_collector: build_tu_collector package_dir_structure cd $(CC_BUILD_DIR) && \ ln -sf ../lib/python3/tu_collector/tu_collector.py bin/tu_collector +build_merge_clang_extdef_mappings: + $(MAKE) -C tools/merge_clang_extdef_mappings build + +package_merge_clang_extdef_mappings: build_merge_clang_extdef_mappings package_dir_structure + # Copy plist-to-html files. + cp -r tools/merge_clang_extdef_mappings/build/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings $(CC_BUILD_LIB_DIR) && \ + chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_merge_clang_extdef_mappings/cli.py && \ + cd $(CC_BUILD_DIR) && \ + ln -sf ../lib/python3/codechecker_merge_clang_extdef_mappings/cli.py bin/merge-clang-extdef-mappings + # This target should be used from the top level Makefile to build the package # together with the web part. This way we will not build plist-to-html # multiple times. package_analyzer: package_dir_structure -package: package_plist_to_html package_tu_collector package_analyzer +package: package_plist_to_html package_tu_collector package_analyzer package_merge_clang_extdef_mappings # Copy libraries. cp -r $(ROOT)/codechecker_common $(CC_BUILD_LIB_DIR) && \ cp -r $(CURRENT_DIR)/codechecker_analyzer $(CC_BUILD_LIB_DIR) diff --git a/analyzer/codechecker_analyzer/analyzers/clangsa/ctu_manager.py b/analyzer/codechecker_analyzer/analyzers/clangsa/ctu_manager.py index e4ac2d5bbd..1b480f0307 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangsa/ctu_manager.py +++ b/analyzer/codechecker_analyzer/analyzers/clangsa/ctu_manager.py @@ -17,82 +17,33 @@ from codechecker_common.logger import get_logger +from merge_clang_extdef_mappings.merge import merge + from .. import analyzer_base from . import ctu_triple_arch LOG = get_logger('analyzer') -def generate_func_map_lines(fnmap_dir): - """ Iterate over all lines of input files in random order. """ - - files = glob.glob(os.path.join(fnmap_dir, '*')) - for filename in files: - with open(filename, 'r', encoding='utf-8', - errors="ignore") as in_file: - for line in in_file: - yield line - - -def create_global_ctu_function_map(func_map_lines): - """ Takes iterator of individual function maps and creates a global map - keeping only unique names. We leave conflicting names out of CTU. - A function map contains the id of a function (mangled name) and the - originating source (the corresponding AST file) name.""" - - mangled_to_asts = {} - - for line in func_map_lines: - mangled_name, ast_file = line.strip().split(' ', 1) - # We collect all occurences of a function name into a list - if mangled_name not in mangled_to_asts: - mangled_to_asts[mangled_name] = {ast_file} - else: - mangled_to_asts[mangled_name].add(ast_file) - - mangled_ast_pairs = [] - - for mangled_name, ast_files in mangled_to_asts.items(): - if len(ast_files) == 1: - mangled_ast_pairs.append((mangled_name, ast_files.pop())) - - return mangled_ast_pairs - - -def write_global_map(ctu_dir, arch, ctu_func_map_file, mangled_ast_pairs): - """ Write (mangled function name, ast file) pairs into final file. """ - - extern_fns_map_file = os.path.join(ctu_dir, arch, ctu_func_map_file) - with open(extern_fns_map_file, 'w', - encoding='utf-8', errors='ignore') as out_file: - for mangled_name, ast_file in mangled_ast_pairs: - out_file.write('%s %s\n' % (mangled_name, ast_file)) +def merge_clang_extdef_mappings(ctu_dir, ctu_func_map_file, + ctu_temp_fnmap_folder): + """ Merge individual function maps into a global one.""" + triple_arches = glob.glob(os.path.join(ctu_dir, '*')) + for triple_path in triple_arches: + if not os.path.isdir(triple_path): + continue -def merge_ctu_func_maps(ctu_dir, ctu_func_map_file, ctu_temp_fnmap_folder): - """ Merge individual function maps into a global one. + triple_arch = os.path.basename(triple_path) + fnmap_dir = os.path.join(ctu_dir, triple_arch, + ctu_temp_fnmap_folder) - As the collect phase runs parallel on multiple threads, all compilation - units are separately mapped into a temporary file in ctu_temp_fnmap_folder. - These function maps contain the mangled names of functions and the source - (AST generated from the source) which had them. - These files should be merged at the end into a global map file: - ctu_func_map_file.""" + merged_fn_map = os.path.join(ctu_dir, triple_arch, + ctu_func_map_file) + merge(fnmap_dir, merged_fn_map) - triple_arches = glob.glob(os.path.join(ctu_dir, '*')) - for triple_path in triple_arches: - if os.path.isdir(triple_path): - triple_arch = os.path.basename(triple_path) - fnmap_dir = os.path.join(ctu_dir, triple_arch, - ctu_temp_fnmap_folder) - - func_map_lines = generate_func_map_lines(fnmap_dir) - mangled_ast_pairs = create_global_ctu_function_map(func_map_lines) - write_global_map(ctu_dir, triple_arch, ctu_func_map_file, - mangled_ast_pairs) - - # Remove all temporary files - shutil.rmtree(fnmap_dir, ignore_errors=True) + # Remove all temporary files. + shutil.rmtree(fnmap_dir, ignore_errors=True) def generate_ast(triple_arch, action, source, config, env): diff --git a/analyzer/codechecker_analyzer/pre_analysis_manager.py b/analyzer/codechecker_analyzer/pre_analysis_manager.py index 5d07b46ad7..52881b6033 100644 --- a/analyzer/codechecker_analyzer/pre_analysis_manager.py +++ b/analyzer/codechecker_analyzer/pre_analysis_manager.py @@ -202,7 +202,7 @@ def signal_handler(signum, frame): # Postprocessing the pre analysis results. if ctu_data: - ctu_manager.merge_ctu_func_maps( + ctu_manager.merge_clang_extdef_mappings( ctu_data.get('ctu_dir'), ctu_data.get('ctu_func_map_file'), ctu_data.get('ctu_temp_fnmap_folder')) diff --git a/analyzer/tests/Makefile b/analyzer/tests/Makefile index b099c90794..ff2667179e 100644 --- a/analyzer/tests/Makefile +++ b/analyzer/tests/Makefile @@ -20,8 +20,8 @@ pycodestyle: pycodestyle_in_env: venv_dev $(ACTIVATE_DEV_VENV) && $(PYCODESTYLE_TEST_CMD) -PYLINT_TEST_CMD = PYLINTRC=$(ROOT)/.pylintrc \ - pylint ./bin/** ./codechecker_analyzer ./tests/** +PYLINT_TEST_CMD = $(MAKE) -C $(CURRENT_DIR)/tools/merge_clang_extdef_mappings pylint && \ + PYLINTRC=$(ROOT)/.pylintrc pylint ./bin/** ./codechecker_analyzer ./tests/** pylint: $(PYLINT_TEST_CMD) @@ -65,3 +65,10 @@ test_tu_collector: test_tu_collector_in_env: $(ACTIVATE_DEV_VENV) && \ $(REPO_ROOT) make -C $(ROOT)/tools/tu_collector test + +test_merge_clang_extdef_mappings: + make -C tools/merge_clang_extdef_mappings test + +test_merge_clang_extdef_mappings_in_env: + $(ACTIVATE_DEV_VENV) && \ + make -C tools/merge_clang_extdef_mappings test diff --git a/analyzer/tools/merge_clang_extdef_mappings/.gitignore b/analyzer/tools/merge_clang_extdef_mappings/.gitignore new file mode 100644 index 0000000000..e195e845f1 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/.gitignore @@ -0,0 +1,3 @@ +build/ +dist/ +merge_clang_extdef_mappings.egg-info diff --git a/analyzer/tools/merge_clang_extdef_mappings/.noserc b/analyzer/tools/merge_clang_extdef_mappings/.noserc new file mode 100644 index 0000000000..512f4e1a08 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/.noserc @@ -0,0 +1,13 @@ +[nosetests] + +# increase verbosity level +verbosity=3 + +# more detailed error messages on failed asserts +detailed-errors=1 + +# stop running tests on first error +stop=1 + +# do not capture stdout +#nocapture=1 diff --git a/analyzer/tools/merge_clang_extdef_mappings/.pylintrc b/analyzer/tools/merge_clang_extdef_mappings/.pylintrc new file mode 100644 index 0000000000..4822958e6c --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/.pylintrc @@ -0,0 +1,377 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=all + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +enable=logging-format-interpolation,old-style-class,unused-import,unused-variable,len-as-condition,bad-indentation,unpacking-in-except,import-star-module-level,parameter-unpacking,long-suffix,old-octal-literal,old-ne-operator,backtick,old-raise-syntax,print-statement,unpacking-in-except,import-star-module-level,parameter-unpacking,long-suffix,old-octal-literal,old-ne-operator,backtick,old-raise-syntax,print-statement,not-in-loop,function-redefined,continue-in-finally,abstract-class-instantiated,sstar-needs-assignment-target,duplicate-argument-name,too-many-star-expressions,nonlocal-and-global,return-outside-function,return-arg-in-generator,invalid-star-assignment-target,bad-reversed-sequence,nonexistent-operator,yield-outside-function,init-is-generator,nonlocal-without-binding,invalid-unary-operand-type,unsupported-binary-operation,no-member,not-callable,redundant-keyword-arg,assignment-from-no-return,assignment-from-none,not-context-manager,repeated-keyword,missing-kwoa,no-value-for-parameter,invalid-sequence-index,invalid-slice-index,too-many-function-args,unexpected-keyword-arg,unsupported-membership-test,unsubscriptable-object,unpacking-non-sequence,invalid-all-object,no-name-in-module,unbalanced-tuple-unpacking,undefined-variable,undefined-all-variable,used-before-assignment,format-needs-mapping,truncated-format-string,missing-format-string-key,mixed-format-string,too-few-format-args,bad-str-strip-call,too-many-format-args,bad-format-character,access-member-before-definition,method-hidden,assigning-non-slot,duplicate-bases,inconsistent-mro,inherit-non-class,invalid-slots,invalid-slots-object,no-method-argument,no-self-argument,unexpected-special-method-signature,non-iterator-returned,invalid-length-returned + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +msg-template=[{msg_id}] {path}:{line:3d}:{column}: {msg} + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=2000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +#notes=FIXME,XXX,TODO + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions= + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=50 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +ignored-classes= + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/analyzer/tools/merge_clang_extdef_mappings/.pypirc b/analyzer/tools/merge_clang_extdef_mappings/.pypirc new file mode 100644 index 0000000000..52d57ec25f --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/.pypirc @@ -0,0 +1,10 @@ +[distutils] +index-servers = + pypi + testpypi + +[pypi] +repository: https://upload.pypi.org/legacy/ + +[testpypi] +repository: https://test.pypi.org/legacy/ diff --git a/analyzer/tools/merge_clang_extdef_mappings/LICENSE.txt b/analyzer/tools/merge_clang_extdef_mappings/LICENSE.txt new file mode 100644 index 0000000000..bd8b243dfa --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/LICENSE.txt @@ -0,0 +1,218 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. diff --git a/analyzer/tools/merge_clang_extdef_mappings/MANIFEST.in b/analyzer/tools/merge_clang_extdef_mappings/MANIFEST.in new file mode 100644 index 0000000000..c1ebcaeab4 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include *.txt diff --git a/analyzer/tools/merge_clang_extdef_mappings/Makefile b/analyzer/tools/merge_clang_extdef_mappings/Makefile new file mode 100644 index 0000000000..b6633c2a8f --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/Makefile @@ -0,0 +1,64 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- + +CURRENT_DIR = $(shell pwd) +ROOT = $(CURRENT_DIR) + +BUILD_DIR = $(CURRENT_DIR)/build +MERGE_CLANG_EXTDEF_MAPS_DIR = $(BUILD_DIR)/merge_clang_extdef_mappings + +ACTIVATE_DEV_VENV ?= . venv_dev/bin/activate +ACTIVATE_RUNTIME_VENV ?= . venv/bin/activate + +VENV_DEV_REQ_FILE ?= requirements_py/dev/requirements.txt + +default: all + +all: package + +venv: + # Create a virtual environment which can be used to run the build package. + virtualenv -p python3 venv && $(ACTIVATE_RUNTIME_VENV) + +venv_dev: + # Create a virtual environment for development. + virtualenv -p python3 venv_dev && \ + $(ACTIVATE_DEV_VENV) && pip install -r $(VENV_DEV_REQ_FILE) + +clean_venv_dev: + rm -rf venv_dev + +include tests/Makefile + +package: + # Install package in 'development mode'. + python setup.py develop + +build: + python setup.py build --build-purelib $(MERGE_CLANG_EXTDEF_MAPS_DIR) + +dist: + # Create a source distribution. + python setup.py sdist + +upload_test: dist + # Upload package to the TestPyPI repository. + $(eval PKG_NAME := $(shell python setup.py --name)) + $(eval PKG_VERSION := $(shell python setup.py --version)) + twine upload -r testpypi dist/$(PKG_NAME)-$(PKG_VERSION).tar.gz + +upload: dist + # Upload package to the PyPI repository. + $(eval PKG_NAME := $(shell python setup.py --name)) + $(eval PKG_VERSION := $(shell python setup.py --version)) + twine upload -r pypi dist/$(PKG_NAME)-$(PKG_VERSION).tar.gz + +clean: + rm -rf $(BUILD_DIR) + rm -rf dist + rm -rf merge_clang_extdef_mappings.egg-info diff --git a/analyzer/tools/merge_clang_extdef_mappings/README.md b/analyzer/tools/merge_clang_extdef_mappings/README.md new file mode 100644 index 0000000000..e3fe815fb0 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/README.md @@ -0,0 +1,46 @@ +# merge-clang-extdef-mappings +As the collect phase runs parallel on multiple threads, all compilation units +are separately mapped into a temporary file in a temporary folder. These +function maps contain the mangled names of functions and the source (AST +generated from the source) which had them. These files should be merged at +the end into a global map file. + +`merge-clang-extdef-mappings` is a python tool which can be used to merge +individual function maps created by the +[clang-extdef-mapping](https://github.com/llvm/llvm-project/blob/master/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp) +tool into a global one. + + +## Install guide +```sh +# Create a Python virtualenv and set it as your environment. +make venv +source $PWD/venv/bin/activate + +# Build and install plist-to-html package. +make package +``` + +## Usage +```sh +usage: merge-clang-extdef-mappings [-h] -i input -o output + +Merge individual clang extdef mapping files into one mapping file. + +optional arguments: + -h, --help show this help message and exit + -i input, --input input + Folder which contains multiple output of the 'clang- + extdef-mapping' tool. + -o output, --output output + Output file where the merged function maps will be + stored into. + +Example: + merge-ctu-func-maps -i /path/to/fn_map_folder /path/to/externalDefMap.txt +``` + +## License + +The project is licensed under University of Illinois/NCSA Open Source License. +See LICENSE.TXT for details. \ No newline at end of file diff --git a/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/__init__.py b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/__init__.py new file mode 100644 index 0000000000..4259749345 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/__init__.py @@ -0,0 +1,7 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- diff --git a/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/cli.py b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/cli.py new file mode 100644 index 0000000000..cf3189d21a --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/cli.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- + + +import argparse +import logging +import os +import sys + +# If we run this script in an environment where +# 'codechecker_merge_clang_extdef_mappings' module is not available we should +# add the grandparent directory of this file to the system path. +# TODO: This section will not be needed when CodeChecker will be delivered as +# a python package and will be installed in a virtual environment with all the +# dependencies. +if __name__ == '__main__': + current_dir = os.path.dirname(os.path.realpath(__file__)) + os.sys.path.append(os.path.dirname(current_dir)) + +from codechecker_merge_clang_extdef_mappings import \ + merge_clang_extdef_mappings # noqa + + +LOG = logging.getLogger('MergeClangExtdefMappings') + +msg_formatter = logging.Formatter('[%(levelname)s] - %(message)s') +log_handler = logging.StreamHandler(sys.stdout) +log_handler.setFormatter(msg_formatter) +LOG.setLevel(logging.INFO) +LOG.addHandler(log_handler) + + +def __add_arguments_to_parser(parser): + """ Add arguments to the the given parser. """ + parser.add_argument('-i', '--input', + type=str, + metavar='input', + required=True, + help="Folder which contains multiple output of the " + "'clang-extdef-mapping' tool.") + + parser.add_argument('-o', '--output', + type=str, + metavar='output', + required=True, + help="Output file where the merged function maps will " + "be stored into.") + + +def main(): + """ Merge CTU funcs maps main command line. """ + parser = argparse.ArgumentParser( + prog="merge-clang-extdef-mappings", + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Merge individual clang extdef mapping files into one " + "mapping file.", + epilog="""Example: + merge-clang-extdef-mappings -i /path/to/fn_map_folder + /path/to/externalDefMap.txt""") + + __add_arguments_to_parser(parser) + + args = parser.parse_args() + + merge_clang_extdef_mappings.merge(args.input, args.output) + + +if __name__ == "__main__": + main() diff --git a/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/merge_clang_extdef_mappings.py b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/merge_clang_extdef_mappings.py new file mode 100755 index 0000000000..482f050596 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings/merge_clang_extdef_mappings.py @@ -0,0 +1,66 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- + +import glob +import os + + +def _generate_func_map_lines(func_map_dir): + """ Iterate over all lines of input files in random order. """ + files = glob.glob(os.path.join(func_map_dir, '*')) + for func_map_file in files: + with open(func_map_file, 'r', + encoding='utf-8', errors="ignore") as func_map: + for line in func_map: + yield line + + +def _create_global_ctu_function_map(func_map_lines): + """ Takes iterator of individual function maps and creates a global map. + + It will keeping only unique names. We leave conflicting names out of CTU. + A function map contains the id of a function (mangled name) and the + originating source (the corresponding AST file) name. + """ + mangled_to_asts = {} + + # We collect all occurences of a function name into a set. + for line in func_map_lines: + mangled_name, ast_file = line.strip().split(' ', 1) + if mangled_name not in mangled_to_asts: + mangled_to_asts[mangled_name] = {ast_file} + else: + mangled_to_asts[mangled_name].add(ast_file) + + mangled_ast_pairs = [] + + for mangled_name, ast_files in mangled_to_asts.items(): + if len(ast_files) == 1: + mangled_ast_pairs.append((mangled_name, ast_files.pop())) + + return mangled_ast_pairs + + +def merge(func_map_dir, output_file): + """ Merge individual function maps into a global one. + + As the collect phase runs parallel on multiple threads, all compilation + units are separately mapped into a temporary file in ctu_temp_fnmap_folder. + These function maps contain the mangled names of functions and the source + (AST generated from the source) which had them. + These files should be merged at the end into a global map file: + ctu_func_map_file. + """ + func_map_lines = _generate_func_map_lines(func_map_dir) + mangled_ast_pairs = _create_global_ctu_function_map(func_map_lines) + + # Write (mangled function name, ast file) pairs into final file. + with open(output_file, 'w', + encoding='utf-8', errors='ignore') as out_file: + for mangled_name, ast_file in mangled_ast_pairs: + out_file.write('%s %s\n' % (mangled_name, ast_file)) diff --git a/analyzer/tools/merge_clang_extdef_mappings/requirements_py/dev/requirements.txt b/analyzer/tools/merge_clang_extdef_mappings/requirements_py/dev/requirements.txt new file mode 100644 index 0000000000..bf35683890 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/requirements_py/dev/requirements.txt @@ -0,0 +1,3 @@ +nose==1.3.7 +pycodestyle==2.4.0 +pylint==1.9.4 diff --git a/analyzer/tools/merge_clang_extdef_mappings/setup.py b/analyzer/tools/merge_clang_extdef_mappings/setup.py new file mode 100644 index 0000000000..a61ab76b3e --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/setup.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="merge-clang-extdef-mappings", + version="0.1.0", + author='CodeChecker Team (Ericsson)', + description="Merge individual function maps into a global one.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/Ericsson/CodeChecker", + keywords=['clang', 'ctu', 'merge', 'func-map', 'static-analysis', + 'analysis'], + license='LICENSE.txt', + packages=setuptools.find_packages(), + include_package_data=True, + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: POSIX", + "Programming Language :: Python :: 3" + ], + entry_points={ + 'console_scripts': [ + 'merge-clang-extdef-mappings = codechecker_merge_clang_extdef_mappings.cli:main' + ] + }, +) diff --git a/analyzer/tools/merge_clang_extdef_mappings/tests/Makefile b/analyzer/tools/merge_clang_extdef_mappings/tests/Makefile new file mode 100644 index 0000000000..02e9452bde --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/tests/Makefile @@ -0,0 +1,36 @@ +# Environment variables to run tests. + +REPO_ROOT ?= REPO_ROOT=$(ROOT) + +# Nose test runner configuration options. +NOSECFG = --config .noserc + +test: pycodestyle pylint test_unit + +test_in_env: pycodestyle_in_env pylint_in_env test_unit_in_env + +PYCODESTYLE_TEST_CMD = pycodestyle codechecker_merge_clang_extdef_mappings tests + +pycodestyle: + $(PYCODESTYLE_TEST_CMD) + +pycodestyle_in_env: venv_dev + $(ACTIVATE_DEV_VENV) && $(PYCODESTYLE_TEST_CMD) + +PYLINT_TEST_CMD = PYLINTRC=$(ROOT)/.pylintrc \ + pylint ./codechecker_merge_clang_extdef_mappings ./tests/** + +pylint: + $(PYLINT_TEST_CMD) + +pylint_in_env: venv + $(ACTIVATE_DEV_VENV) && $(PYLINT_TEST_CMD) + +UNIT_TEST_CMD = $(REPO_ROOT) \ + nosetests $(NOSECFG) tests/unit + +test_unit: + $(UNIT_TEST_CMD) + +test_unit_in_env: venv_dev + $(ACTIVATE_DEV_VENV) && $(UNIT_TEST_CMD) diff --git a/analyzer/tools/merge_clang_extdef_mappings/tests/unit/__init__.py b/analyzer/tools/merge_clang_extdef_mappings/tests/unit/__init__.py new file mode 100644 index 0000000000..4259749345 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/tests/unit/__init__.py @@ -0,0 +1,7 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- diff --git a/analyzer/tools/merge_clang_extdef_mappings/tests/unit/merge_clang_extdef_mappings/__init__.py b/analyzer/tools/merge_clang_extdef_mappings/tests/unit/merge_clang_extdef_mappings/__init__.py new file mode 100644 index 0000000000..b3a2aa1f9d --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/tests/unit/merge_clang_extdef_mappings/__init__.py @@ -0,0 +1,54 @@ +# coding=utf-8 +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- +""" Setup for the test package analyze. """ + +import os +import shutil +import tempfile + + +def get_workspace(test_id='test'): + """ Return a temporary workspace for the tests. """ + workspace_root = os.environ.get("MERGE_CTU_FUNC_MAPS_TEST_WORKSPACE_ROOT") + if not workspace_root: + # if no external workspace is set create under the build dir + workspace_root = os.path.join(os.environ['REPO_ROOT'], 'build', + 'workspace') + + if not os.path.exists(workspace_root): + os.makedirs(workspace_root) + + if test_id: + return tempfile.mkdtemp(prefix=test_id + "-", dir=workspace_root) + else: + return workspace_root + + +# Test workspace should be initialized in this module. +TEST_WORKSPACE = None + + +def setup_package(): + """ Setup the environment for the tests. """ + + global TEST_WORKSPACE + TEST_WORKSPACE = get_workspace('merge_clang_extdef_mappings') + + os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE + + +def teardown_package(): + """ Delete the workspace associated with this test. """ + + # TODO: If environment variable is set keep the workspace + # and print out the path. + global TEST_WORKSPACE + + print("Removing: " + TEST_WORKSPACE) + shutil.rmtree(TEST_WORKSPACE) diff --git a/analyzer/tools/merge_clang_extdef_mappings/tests/unit/merge_clang_extdef_mappings/merge_clang_extdef_mappings.py b/analyzer/tools/merge_clang_extdef_mappings/tests/unit/merge_clang_extdef_mappings/merge_clang_extdef_mappings.py new file mode 100644 index 0000000000..dd0fc9ddd2 --- /dev/null +++ b/analyzer/tools/merge_clang_extdef_mappings/tests/unit/merge_clang_extdef_mappings/merge_clang_extdef_mappings.py @@ -0,0 +1,62 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- + +import os +import unittest + +from codechecker_merge_clang_extdef_mappings import merge_clang_extdef_mappings + + +class MergeClangExtdefMappingsTest(unittest.TestCase): + @classmethod + def setUpClass(self): + """ Initialize test files. """ + self.test_workspace = os.environ['TEST_WORKSPACE'] + + self.extdef_maps_dir = os.path.join(self.test_workspace, + "extdef_maps") + + if not os.path.exists(self.extdef_maps_dir): + os.makedirs(self.extdef_maps_dir) + + self.extdef_map_1_lines = ["c:@F@f# path/to/file.cpp.ast", + "c:@F@g# path/to/file.cpp.ast", + "c:@F@both# path/to/file.cpp.ast"] + + extdef_map_file_1 = os.path.join(self.extdef_maps_dir, + 'externalDefMap1.txt') + with open(extdef_map_file_1, 'w', + encoding='utf-8', errors='ignore') as map_f: + map_f.write('\n'.join(self.extdef_map_1_lines)) + + self.extdef_map_2_lines = ["c:@F@main# path/to/file2.cpp.ast", + "c:@F@h# path/to/file2.cpp.ast", + "c:@F@both# path/to/file2.cpp.ast"] + + extdef_map_file_2 = os.path.join(self.extdef_maps_dir, + 'externalDefMap2.txt') + with open(extdef_map_file_2, 'w', + encoding='utf-8', errors='ignore') as map_f: + map_f.write('\n'.join(self.extdef_map_2_lines)) + + def test_merge_clang_extdef_mappings(self): + """ Test merging multiple func map files. """ + + output_file = os.path.join(self.test_workspace, 'externalDefMap.txt') + merge_clang_extdef_mappings.merge(self.extdef_maps_dir, output_file) + + with open(output_file, 'r', + encoding='utf-8', errors='ignore') as o_file: + lines = o_file.read().split('\n') + + expected_lines = ["c:@F@f# path/to/file.cpp.ast", + "c:@F@g# path/to/file.cpp.ast", + "c:@F@main# path/to/file2.cpp.ast", + "c:@F@h# path/to/file2.cpp.ast"] + for expected_line in expected_lines: + self.assertTrue(expected_line in lines)