Skip to content

Commit

Permalink
Merge pull request Ericsson#2465 from csordasmarton/analyze_makefile_…
Browse files Browse the repository at this point in the history 
…output

[analyzer] Analyze makefile output
  • Loading branch information
Gyorgy Orban authored Jul 7, 2020
2 parents b35d25e + 02d18c6 commit d589767
Show file tree
Hide file tree
Showing 58 changed files with 3,111 additions and 528 deletions.
4 changes: 3 additions & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,9 @@ script:
BUILD_DIR=$TRAVIS_BUILD_DIR/build make -C analyzer \
test_unit \
test_functional \
test_tu_collector &&
test_tu_collector \
test_merge_clang_extdef_mappings \
test_statistics_collector &&
if [[ "$TRAVIS_OS_NAME" = "linux" ]]; then
make -C analyzer test_build_logger
fi
Expand Down
28 changes: 26 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ CC_CLIENT = $(CC_WEB)/client/
CC_ANALYZER = $(CURRENT_DIR)/analyzer

CC_TOOLS = $(CURRENT_DIR)/tools
CC_ANALYZER_TOOLS = $(CC_ANALYZER)/tools

# Root of the repository.
ROOT = $(CURRENT_DIR)
Expand Down Expand Up @@ -69,7 +70,27 @@ build_report_hash:
package_report_hash: build_report_hash package_dir_structure
cp -r $(CC_TOOLS)/codechecker_report_hash/build/codechecker_report_hash/codechecker_report_hash $(CC_BUILD_LIB_DIR)

package: package_dir_structure set_git_commit_template package_plist_to_html package_tu_collector package_report_converter package_report_hash
build_merge_clang_extdef_mappings:
$(MAKE) -C $(CC_ANALYZER)/tools/merge_clang_extdef_mappings build

package_merge_clang_extdef_mappings: build_merge_clang_extdef_mappings package_dir_structure
# Copy merge-clang-extdef-mappings files.
cp -r $(CC_ANALYZER)/tools/merge_clang_extdef_mappings/build/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings $(CC_BUILD_LIB_DIR) && \
chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_merge_clang_extdef_mappings/cli.py && \
cd $(CC_BUILD_DIR) && \
ln -sf ../lib/python3/codechecker_merge_clang_extdef_mappings/cli.py bin/merge-clang-extdef-mappings

build_statistics_collector:
$(MAKE) -C $(CC_ANALYZER_TOOLS)/statistics_collector build

package_statistics_collector: build_statistics_collector package_dir_structure
# Copy statistics-collector files.
cp -r $(CC_ANALYZER_TOOLS)/statistics_collector/build/statistics_collector/codechecker_statistics_collector $(CC_BUILD_LIB_DIR) && \
chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_statistics_collector/cli.py && \
cd $(CC_BUILD_DIR) && \
ln -sf ../lib/python3/codechecker_statistics_collector/cli.py bin/post-process-stats

package: package_dir_structure set_git_commit_template package_plist_to_html package_tu_collector package_report_converter package_report_hash package_merge_clang_extdef_mappings package_statistics_collector
BUILD_DIR=$(BUILD_DIR) BUILD_LOGGER_64_BIT_ONLY=$(BUILD_LOGGER_64_BIT_ONLY) $(MAKE) -C $(CC_ANALYZER) package_analyzer
BUILD_DIR=$(BUILD_DIR) $(MAKE) -C $(CC_WEB) package_web

Expand Down Expand Up @@ -154,7 +175,7 @@ clean_venv_dev:
clean: clean_package
$(MAKE) -C $(CC_WEB) clean

clean_package: clean_plist_to_html clean_tu_collector clean_report_converter clean_report_hash
clean_package: clean_plist_to_html clean_tu_collector clean_report_converter clean_report_hash clean_statistics_collector
rm -rf $(BUILD_DIR)
find . -name "*.pyc" -delete

Expand All @@ -170,6 +191,9 @@ clean_report_converter:
clean_report_hash:
$(MAKE) -C $(CC_TOOLS)/codechecker_report_hash clean

clean_statistics_collector:
$(MAKE) -C $(CC_ANALYZER_TOOLS)/statistics_collector clean

clean_travis:
# Clean CodeChecker config files stored in the users home directory.
rm -rf ~/.codechecker*
Expand Down
22 changes: 21 additions & 1 deletion analyzer/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,32 @@ package_tu_collector: build_tu_collector package_dir_structure
cd $(CC_BUILD_DIR) && \
ln -sf ../lib/python3/tu_collector/tu_collector.py bin/tu_collector

build_merge_clang_extdef_mappings:
$(MAKE) -C tools/merge_clang_extdef_mappings build

package_merge_clang_extdef_mappings: build_merge_clang_extdef_mappings package_dir_structure
# Copy plist-to-html files.
cp -r tools/merge_clang_extdef_mappings/build/merge_clang_extdef_mappings/codechecker_merge_clang_extdef_mappings $(CC_BUILD_LIB_DIR) && \
chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_merge_clang_extdef_mappings/cli.py && \
cd $(CC_BUILD_DIR) && \
ln -sf ../lib/python3/codechecker_merge_clang_extdef_mappings/cli.py bin/merge-clang-extdef-mappings

build_statistics_collector:
$(MAKE) -C $(CC_ANALYZER_TOOLS)/statistics_collector build

package_statistics_collector: build_statistics_collector package_dir_structure
# Copy statistics-collector files.
cp -r tools/statistics_collector/build/statistics_collector/codechecker_statistics_collector $(CC_BUILD_LIB_DIR) && \
chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_statistics_collector/cli.py && \
cd $(CC_BUILD_DIR) && \
ln -sf ../lib/python3/codechecker_statistics_collector/cli.py bin/post-process-stats

# This target should be used from the top level Makefile to build the package
# together with the web part. This way we will not build plist-to-html
# multiple times.
package_analyzer: package_dir_structure

package: package_plist_to_html package_tu_collector package_analyzer
package: package_plist_to_html package_tu_collector package_analyzer package_merge_clang_extdef_mappings package_statistics_collector
# Copy libraries.
cp -r $(ROOT)/codechecker_common $(CC_BUILD_LIB_DIR) && \
cp -r $(CURRENT_DIR)/codechecker_analyzer $(CC_BUILD_LIB_DIR)
Expand Down
58 changes: 28 additions & 30 deletions analyzer/codechecker_analyzer/analysis_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@
from codechecker_common import plist_parser
from codechecker_common.logger import get_logger

from codechecker_statistics_collector.collectors.special_return_value import \
SpecialReturnValueCollector

from . import gcc_toolchain

from .analyzers import analyzer_types
from .analyzers.config_handler import CheckerState
from .analyzers.clangsa.analyzer import ClangSA
from .analyzers.clangsa.statistics_collector import SpecialReturnValueCollector

LOG = get_logger('analyzer')

Expand Down Expand Up @@ -166,15 +168,9 @@ def is_ctu_active(source_analyzer):
source_analyzer.is_ctu_enabled()


def prepare_check(action, analyzer_config, output_dir,
severity_map, skip_handler, statistics_data,
disable_ctu=False):
"""
Construct the source analyzer build the analysis command
and result handler for the analysis.
"""
reanalyzed = False

def prepare_check(action, analyzer_config, output_dir, severity_map,
skip_handler, statistics_data, disable_ctu=False):
""" Construct the source analyzer and result handler. """
# Create a source analyzer.
source_analyzer = \
analyzer_types.construct_analyzer(action,
Expand All @@ -198,7 +194,11 @@ def prepare_check(action, analyzer_config, output_dir,
stats_cfg = \
SpecialReturnValueCollector.checker_analyze_cfg(stats_dir)

source_analyzer.add_checker_config(stats_cfg)
if os.path.exists(SpecialReturnValueCollector.stats_file(stats_dir)):
source_analyzer.add_checker_config(stats_cfg)
else:
LOG.debug('No checker statistics file was found for %s',
SpecialReturnValueCollector.checker_analyze)

# Source is the currently analyzed source file
# there can be more in one buildaction.
Expand All @@ -217,14 +217,7 @@ def prepare_check(action, analyzer_config, output_dir,
# The analyzer output file is based on the currently
# analyzed source.
rh.analyzed_source_file = action.source

if os.path.exists(rh.analyzer_result_file):
reanalyzed = True

# Construct the analyzer cmd.
analyzer_cmd = source_analyzer.construct_analyzer_cmd(rh)

return source_analyzer, analyzer_cmd, rh, reanalyzed
return source_analyzer, rh


def handle_success(rh, result_file, result_base, skip_handler,
Expand Down Expand Up @@ -477,10 +470,14 @@ def check(check_data):
if analyzer_config is None:
raise Exception("Analyzer configuration is missing.")

source_analyzer, analyzer_cmd, rh, reanalyzed = \
prepare_check(action, analyzer_config,
output_dir, context.severity_map,
skip_handler, statistics_data)
source_analyzer, rh = prepare_check(action, analyzer_config,
output_dir, context.severity_map,
skip_handler, statistics_data)

reanalyzed = os.path.exists(rh.analyzer_result_file)

# Construct the analyzer cmd.
analyzer_cmd = source_analyzer.construct_analyzer_cmd(rh)

# The analyzer invocation calls __create_timeout as a callback
# when the analyzer starts. This callback creates the timeout
Expand Down Expand Up @@ -594,14 +591,15 @@ def __create_timeout(analyzer_process):
if ctu_active and ctu_reanalyze_on_failure:
LOG.error("Try to reanalyze without CTU")
# Try to reanalyze with CTU disabled.
source_analyzer, analyzer_cmd, rh, reanalyzed = \
prepare_check(action,
analyzer_config,
output_dir,
context.severity_map,
skip_handler,
statistics_data,
source_analyzer, rh = \
prepare_check(action, analyzer_config,
output_dir, context.severity_map,
skip_handler, statistics_data,
True)
reanalyzed = os.path.exists(rh.analyzer_result_file)

# Construct the analyzer cmd.
analyzer_cmd = source_analyzer.construct_analyzer_cmd(rh)

# Fills up the result handler with
# the analyzer information.
Expand Down
59 changes: 42 additions & 17 deletions analyzer/codechecker_analyzer/analyzer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,17 @@

from codechecker_common.logger import get_logger

from codechecker_statistics_collector.collectors.special_return_value import \
SpecialReturnValueCollector
from codechecker_statistics_collector.collectors.return_value import \
ReturnValueCollector

from . import analysis_manager, pre_analysis_manager, env, checkers
from .analyzers import analyzer_types
from .analyzers.config_handler import CheckerState
from .analyzers.clangsa.analyzer import ClangSA
from .analyzers.clangsa.statistics_collector import \
SpecialReturnValueCollector
from .analyzers.clangsa.statistics_collector import ReturnValueCollector

from .makefile import MakeFileCreator

LOG = get_logger('analyzer')

Expand Down Expand Up @@ -110,16 +113,20 @@ def __mgr_init():
signal.signal(signal.SIGINT, signal.SIG_IGN)


def __get_statistics_data(args, manager):
def __get_statistics_data(args):
""" Get statistics data. """
statistics_data = None

if 'stats_enabled' in args and args.stats_enabled:
statistics_data = manager.dict({
'stats_out_dir': os.path.join(args.output_path, "stats")})
statistics_data = {
'stats_out_dir': os.path.join(args.output_path, "stats")}

if 'stats_output' in args and args.stats_output:
statistics_data = manager.dict({'stats_out_dir':
args.stats_output})
statistics_data = {'stats_out_dir': args.stats_output}

if statistics_data:
statistics_data['stat_tmp_dir'] = \
os.path.join(statistics_data.get('stats_out_dir'), 'tmp')

if 'stats_min_sample_count' in args and statistics_data:
if args.stats_min_sample_count > 1:
Expand All @@ -142,6 +149,15 @@ def __get_statistics_data(args, manager):
return statistics_data


def __get_ctu_data(config_map, ctu_dir):
""" Get CTU data. """
ctu_capability = config_map[ClangSA.ANALYZER_NAME].ctu_capability
return {'ctu_dir': ctu_dir,
'ctu_func_map_cmd': ctu_capability.mapping_tool_path,
'ctu_func_map_file': ctu_capability.mapping_file_name,
'ctu_temp_fnmap_folder': 'tmpExternalFnMaps'}


def perform_analysis(args, skip_handler, context, actions, metadata_tool,
compile_cmd_count):
"""
Expand Down Expand Up @@ -237,6 +253,20 @@ def perform_analysis(args, skip_handler, context, actions, metadata_tool,

metadata_tool['analyzers'][analyzer] = metadata_info

if 'makefile' in args and args.makefile:
statistics_data = __get_statistics_data(args)

ctu_data = None
if ctu_collect or statistics_data:
ctu_data = __get_ctu_data(config_map, ctu_dir)

makefile_creator = MakeFileCreator(analyzers, args.output_path,
config_map, context, skip_handler,
ctu_collect, statistics_data,
ctu_data)
makefile_creator.create(actions)
return

if ctu_collect:
shutil.rmtree(ctu_dir, ignore_errors=True)
elif ctu_analyze and not os.path.exists(ctu_dir):
Expand All @@ -253,19 +283,14 @@ def perform_analysis(args, skip_handler, context, actions, metadata_tool,
actions_map = create_actions_map(actions, manager)

# Setting to not None value will enable statistical analysis features.
statistics_data = __get_statistics_data(args, manager)
statistics_data = __get_statistics_data(args)
if statistics_data:
statistics_data = manager.dict(statistics_data)

if ctu_collect or statistics_data:
ctu_data = None
if ctu_collect or ctu_analyze:
ctu_capability = config_map[ClangSA.ANALYZER_NAME].ctu_capability
ctu_data = manager.dict({'ctu_dir': ctu_dir,
'ctu_func_map_cmd':
ctu_capability.mapping_tool_path,
'ctu_func_map_file':
ctu_capability.mapping_file_name,
'ctu_temp_fnmap_folder':
'tmpExternalFnMaps'})
ctu_data = manager.dict(__get_ctu_data(config_map, ctu_dir))

pre_analyze = [a for a in actions
if a.analyzer_type == ClangSA.ANALYZER_NAME]
Expand Down
Loading

0 comments on commit d589767

Please sign in to comment.