[analyzer] Create separate tool to merge clang CTU extdef maps

- Create a separate analyzer tool which can be used to merge individual
function maps into a global one.
- Add unit tests.
- Use this new tool in the `ctu_manager.py`.

.travis.yml

@@ -151,7 +151,8 @@ script:
         BUILD_DIR=$TRAVIS_BUILD_DIR/build make -C analyzer \
           test_unit \
           test_functional \
-          test_tu_collector &&
+          test_tu_collector \
+          test_merge_clang_extdef_mappings &&
         if [[ "$TRAVIS_OS_NAME" = "linux" ]]; then
           make -C analyzer test_build_logger
         fi

Makefile

@@ -69,7 +69,17 @@ build_report_hash:
 package_report_hash: build_report_hash package_dir_structure
 	cp -r $(CC_TOOLS)/codechecker_report_hash/build/codechecker_report_hash/codechecker_report_hash $(CC_BUILD_LIB_DIR)
 
-package: package_dir_structure set_git_commit_template package_plist_to_html package_tu_collector package_report_converter package_report_hash
+build_merge_clang_extdef_mappings:
+	$(MAKE) -C $(CC_ANALYZER)/tools/merge_clang_extdef_mappings build
+
+package_merge_clang_extdef_mappings: build_merge_clang_extdef_mappings package_dir_structure
+	# Copy merge-clang-extdef-mappings files.
+	cp -r $(CC_ANALYZER)/tools/merge_clang_extdef_mappings/build/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings $(CC_BUILD_LIB_DIR) && \
+	chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_merge_clang_extdef_mappings/cli.py && \
+	cd $(CC_BUILD_DIR) && \
+	ln -sf ../lib/python3/codechecker_merge_clang_extdef_mappings/cli.py bin/merge-clang-extdef-mappings
+
+package: package_dir_structure set_git_commit_template package_plist_to_html package_tu_collector package_report_converter package_report_hash package_merge_clang_extdef_mappings
 	BUILD_DIR=$(BUILD_DIR) BUILD_LOGGER_64_BIT_ONLY=$(BUILD_LOGGER_64_BIT_ONLY) $(MAKE) -C $(CC_ANALYZER) package_analyzer
 	BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_WEB) package_web
 

analyzer/Makefile

@@ -54,12 +54,22 @@ package_tu_collector: build_tu_collector package_dir_structure
 	cd $(CC_BUILD_DIR) && \
 	ln -sf ../lib/python3/tu_collector/tu_collector.py bin/tu_collector
 
+build_merge_clang_extdef_mappings:
+	$(MAKE) -C tools/merge_clang_extdef_mappings build
+
+package_merge_clang_extdef_mappings: build_merge_clang_extdef_mappings package_dir_structure
+	# Copy plist-to-html files.
+	cp -r tools/merge_clang_extdef_mappings/build/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings $(CC_BUILD_LIB_DIR) && \
+	chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_merge_clang_extdef_mappings/cli.py && \
+	cd $(CC_BUILD_DIR) && \
+	ln -sf ../lib/python3/codechecker_merge_clang_extdef_mappings/cli.py bin/merge-clang-extdef-mappings
+
 # This target should be used from the top level Makefile to build the package
 # together with the web part. This way we will not build plist-to-html
 # multiple times.
 package_analyzer: package_dir_structure
 
-package: package_plist_to_html package_tu_collector package_analyzer
+package: package_plist_to_html package_tu_collector package_analyzer package_merge_clang_extdef_mappings
 	# Copy libraries.
 	cp -r $(ROOT)/codechecker_common $(CC_BUILD_LIB_DIR) && \
 	cp -r $(CURRENT_DIR)/codechecker_analyzer $(CC_BUILD_LIB_DIR)

analyzer/codechecker_analyzer/analyzers/clangsa/ctu_manager.py

@@ -17,82 +17,33 @@
 
 from codechecker_common.logger import get_logger
 
+from merge_clang_extdef_mappings.merge import merge
+
 from .. import analyzer_base
 from . import ctu_triple_arch
 
 LOG = get_logger('analyzer')
 
 
-def generate_func_map_lines(fnmap_dir):
-    """ Iterate over all lines of input files in random order. """
-
-    files = glob.glob(os.path.join(fnmap_dir, '*'))
-    for filename in files:
-        with open(filename, 'r', encoding='utf-8',
-                  errors="ignore") as in_file:
-            for line in in_file:
-                yield line
-
-
-def create_global_ctu_function_map(func_map_lines):
-    """ Takes iterator of individual function maps and creates a global map
-    keeping only unique names. We leave conflicting names out of CTU.
-    A function map contains the id of a function (mangled name) and the
-    originating source (the corresponding AST file) name."""
-
-    mangled_to_asts = {}
-
-    for line in func_map_lines:
-        mangled_name, ast_file = line.strip().split(' ', 1)
-        # We collect all occurences of a function name into a list
-        if mangled_name not in mangled_to_asts:
-            mangled_to_asts[mangled_name] = {ast_file}
-        else:
-            mangled_to_asts[mangled_name].add(ast_file)
-
-    mangled_ast_pairs = []
-
-    for mangled_name, ast_files in mangled_to_asts.items():
-        if len(ast_files) == 1:
-            mangled_ast_pairs.append((mangled_name, ast_files.pop()))
-
-    return mangled_ast_pairs
-
-
-def write_global_map(ctu_dir, arch, ctu_func_map_file, mangled_ast_pairs):
-    """ Write (mangled function name, ast file) pairs into final file. """
-
-    extern_fns_map_file = os.path.join(ctu_dir, arch, ctu_func_map_file)
-    with open(extern_fns_map_file, 'w',
-              encoding='utf-8', errors='ignore') as out_file:
-        for mangled_name, ast_file in mangled_ast_pairs:
-            out_file.write('%s %s\n' % (mangled_name, ast_file))
+def merge_clang_extdef_mappings(ctu_dir, ctu_func_map_file,
+                                ctu_temp_fnmap_folder):
+    """ Merge individual function maps into a global one."""
 
+    triple_arches = glob.glob(os.path.join(ctu_dir, '*'))
+    for triple_path in triple_arches:
+        if not os.path.isdir(triple_path):
+            continue
 
-def merge_ctu_func_maps(ctu_dir, ctu_func_map_file, ctu_temp_fnmap_folder):
-    """ Merge individual function maps into a global one.
+        triple_arch = os.path.basename(triple_path)
+        fnmap_dir = os.path.join(ctu_dir, triple_arch,
+                                 ctu_temp_fnmap_folder)
 
-    As the collect phase runs parallel on multiple threads, all compilation
-    units are separately mapped into a temporary file in ctu_temp_fnmap_folder.
-    These function maps contain the mangled names of functions and the source
-    (AST generated from the source) which had them.
-    These files should be merged at the end into a global map file:
-    ctu_func_map_file."""
+        merged_fn_map = os.path.join(ctu_dir, triple_arch,
+                                     ctu_func_map_file)
+        merge(fnmap_dir, merged_fn_map)
 
-    triple_arches = glob.glob(os.path.join(ctu_dir, '*'))
-    for triple_path in triple_arches:
-        if os.path.isdir(triple_path):
-            triple_arch = os.path.basename(triple_path)
-            fnmap_dir = os.path.join(ctu_dir, triple_arch,
-                                     ctu_temp_fnmap_folder)
-
-            func_map_lines = generate_func_map_lines(fnmap_dir)
-            mangled_ast_pairs = create_global_ctu_function_map(func_map_lines)
-            write_global_map(ctu_dir, triple_arch, ctu_func_map_file,
-                             mangled_ast_pairs)
-
-            # Remove all temporary files
-            shutil.rmtree(fnmap_dir, ignore_errors=True)
+        # Remove all temporary files.
+        shutil.rmtree(fnmap_dir, ignore_errors=True)
 
 
 def generate_ast(triple_arch, action, source, config, env):

analyzer/codechecker_analyzer/pre_analysis_manager.py

@@ -202,7 +202,7 @@ def signal_handler(signum, frame):
 
     # Postprocessing the pre analysis results.
     if ctu_data:
-        ctu_manager.merge_ctu_func_maps(
+        ctu_manager.merge_clang_extdef_mappings(
                 ctu_data.get('ctu_dir'),
                 ctu_data.get('ctu_func_map_file'),
                 ctu_data.get('ctu_temp_fnmap_folder'))

analyzer/tests/Makefile

@@ -20,8 +20,8 @@ pycodestyle:
 pycodestyle_in_env: venv_dev
 	$(ACTIVATE_DEV_VENV) && $(PYCODESTYLE_TEST_CMD)
 
-PYLINT_TEST_CMD = PYLINTRC=$(ROOT)/.pylintrc \
-	pylint ./bin/** ./codechecker_analyzer ./tests/**
+PYLINT_TEST_CMD = $(MAKE) -C $(CURRENT_DIR)/tools/merge_clang_extdef_mappings pylint && \
+	PYLINTRC=$(ROOT)/.pylintrc pylint ./bin/** ./codechecker_analyzer ./tests/**
 
 pylint:
 	$(PYLINT_TEST_CMD)
@@ -65,3 +65,10 @@ test_tu_collector:
 test_tu_collector_in_env:
 	$(ACTIVATE_DEV_VENV) && \
 	$(REPO_ROOT) make -C $(ROOT)/tools/tu_collector test
+
+test_merge_clang_extdef_mappings:
+	make -C tools/merge_clang_extdef_mappings test
+
+test_merge_clang_extdef_mappings_in_env:
+	$(ACTIVATE_DEV_VENV) && \
+	make -C tools/merge_clang_extdef_mappings test

analyzer/tools/merge_clang_extdef_mappings/.gitignore

@@ -0,0 +1,3 @@
+build/
+dist/
+merge_clang_extdef_mappings.egg-info

analyzer/tools/merge_clang_extdef_mappings/.noserc

@@ -0,0 +1,13 @@
+[nosetests]
+
+# increase verbosity level
+verbosity=3
+
+# more detailed error messages on failed asserts
+detailed-errors=1
+
+# stop running tests on first error
+stop=1
+
+# do not capture stdout
+#nocapture=1