From e05c8851d4562acf6b181282bdc3637a5e332dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Csord=C3=A1s?= Date: Fri, 13 Sep 2019 12:58:28 +0200 Subject: [PATCH] [tools] Warning to plist converter Create a new tool which can be used to convert `clang-tidy` and `clang sanitizer` output messages to `.plist` files. --- Makefile | 5 +- tools/warning-to-plist/.gitignore | 2 + tools/warning-to-plist/.noserc | 13 ++ tools/warning-to-plist/.pypirc | 10 ++ tools/warning-to-plist/LICENSE.txt | 39 +++++ tools/warning-to-plist/Makefile | 61 +++++++ tools/warning-to-plist/README.md | 42 +++++ .../requirements_py/dev/requirements.txt | 3 + tools/warning-to-plist/setup.py | 37 ++++ .../warning_to_plist/WarningToPlist.py | 123 ++++++++++++++ .../warning_to_plist/__init__.py | 5 + .../warning_to_plist/converter/__init__.py | 5 + .../converter/clang_tidy/__init__.py | 5 + .../converter/clang_tidy/output_parser.py | 126 ++++++++++++++ .../converter/clang_tidy/plist_converter.py | 33 ++++ .../converter/output_parser.py | 96 +++++++++++ .../converter/plist_converter.py | 160 ++++++++++++++++++ .../warning_to_plist/converter/report.py | 134 +++++++++++++++ 18 files changed, 898 insertions(+), 1 deletion(-) create mode 100644 tools/warning-to-plist/.gitignore create mode 100644 tools/warning-to-plist/.noserc create mode 100644 tools/warning-to-plist/.pypirc create mode 100644 tools/warning-to-plist/LICENSE.txt create mode 100644 tools/warning-to-plist/Makefile create mode 100644 tools/warning-to-plist/README.md create mode 100644 tools/warning-to-plist/requirements_py/dev/requirements.txt create mode 100644 tools/warning-to-plist/setup.py create mode 100755 tools/warning-to-plist/warning_to_plist/WarningToPlist.py create mode 100644 tools/warning-to-plist/warning_to_plist/__init__.py create mode 100644 tools/warning-to-plist/warning_to_plist/converter/__init__.py create mode 100644 tools/warning-to-plist/warning_to_plist/converter/clang_tidy/__init__.py create mode 100644 tools/warning-to-plist/warning_to_plist/converter/clang_tidy/output_parser.py create mode 100644 tools/warning-to-plist/warning_to_plist/converter/clang_tidy/plist_converter.py create mode 100644 tools/warning-to-plist/warning_to_plist/converter/output_parser.py create mode 100644 tools/warning-to-plist/warning_to_plist/converter/plist_converter.py create mode 100644 tools/warning-to-plist/warning_to_plist/converter/report.py diff --git a/Makefile b/Makefile index ff71672369..e8f7ad0d8b 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ clean_venv_dev: clean: clean_package clean_vendor -clean_package: clean_plist_to_html clean_tu_collector +clean_package: clean_plist_to_html clean_tu_collector clean_warning_to_plist rm -rf $(BUILD_DIR) find . -name "*.pyc" -delete @@ -150,6 +150,9 @@ clean_plist_to_html: clean_tu_collector: $(MAKE) -C $(CC_TOOLS)/tu_collector clean +clean_warning_to_plist: + $(MAKE) -C $(CC_TOOLS)/warning-to-plist clean + clean_travis: # Clean CodeChecker config files stored in the users home directory. rm -rf ~/.codechecker* diff --git a/tools/warning-to-plist/.gitignore b/tools/warning-to-plist/.gitignore new file mode 100644 index 0000000000..ae3b1573c7 --- /dev/null +++ b/tools/warning-to-plist/.gitignore @@ -0,0 +1,2 @@ +build/ +warning_to_plist.egg-info diff --git a/tools/warning-to-plist/.noserc b/tools/warning-to-plist/.noserc new file mode 100644 index 0000000000..512f4e1a08 --- /dev/null +++ b/tools/warning-to-plist/.noserc @@ -0,0 +1,13 @@ +[nosetests] + +# increase verbosity level +verbosity=3 + +# more detailed error messages on failed asserts +detailed-errors=1 + +# stop running tests on first error +stop=1 + +# do not capture stdout +#nocapture=1 diff --git a/tools/warning-to-plist/.pypirc b/tools/warning-to-plist/.pypirc new file mode 100644 index 0000000000..52d57ec25f --- /dev/null +++ b/tools/warning-to-plist/.pypirc @@ -0,0 +1,10 @@ +[distutils] +index-servers = + pypi + testpypi + +[pypi] +repository: https://upload.pypi.org/legacy/ + +[testpypi] +repository: https://test.pypi.org/legacy/ diff --git a/tools/warning-to-plist/LICENSE.txt b/tools/warning-to-plist/LICENSE.txt new file mode 100644 index 0000000000..d667a31750 --- /dev/null +++ b/tools/warning-to-plist/LICENSE.txt @@ -0,0 +1,39 @@ +University of Illinois/NCSA Open Source License + +Copyright (c) 2015 Ericsson. +All rights reserved. + +Developed by: + + CodeChecker Team + + Ericsson + + http://www.ericsson.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the CodeChecker Team, Ericsson, nor the names of + its contributors may be used to endorse or promote products derived + from this Software without specific prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + diff --git a/tools/warning-to-plist/Makefile b/tools/warning-to-plist/Makefile new file mode 100644 index 0000000000..d25c8708db --- /dev/null +++ b/tools/warning-to-plist/Makefile @@ -0,0 +1,61 @@ +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- +CURRENT_DIR = $(shell pwd) +ROOT = $(CURRENT_DIR) + +BUILD_DIR = $(CURRENT_DIR)/build +WARNING_TO_PLIST_DIR = $(BUILD_DIR)/warning_to_plist + +ACTIVATE_DEV_VENV ?= . venv_dev/bin/activate +ACTIVATE_RUNTIME_VENV ?= . venv/bin/activate + +VENV_DEV_REQ_FILE ?= requirements_py/dev/requirements.txt + +default: all + +all: package + +venv: + # Create a virtual environment which can be used to run the build package. + virtualenv -p python2 venv && $(ACTIVATE_RUNTIME_VENV) + +pip_dev_deps: + pip install -r $(VENV_DEV_REQ_FILE) + +venv_dev: + # Create a virtual environment for development. + virtualenv -p python2 venv_dev && \ + $(ACTIVATE_DEV_VENV) && pip install -r $(VENV_DEV_REQ_FILE) + +clean_venv_dev: + rm -rf venv_dev + +package: + # Install package in 'development mode'. + python setup.py develop + +build: + python setup.py build --build-purelib $(WARNING_TO_PLIST_DIR) + +dist: + # Create a source distribution. + python setup.py sdist + +upload_test: + # Upload package to the TestPyPI repository. + $(eval PKG_NAME := $(shell python setup.py --name)) + $(eval PKG_VERSION := $(shell python setup.py --version)) + twine upload -r testpypi dist/$(PKG_NAME)-$(PKG_VERSION).tar.gz + +upload: + # Upload package to the PyPI repository. + $(eval PKG_NAME := $(shell python setup.py --name)) + $(eval PKG_VERSION := $(shell python setup.py --version)) + twine upload -r pypi dist/$(PKG_NAME)-$(PKG_VERSION).tar.gz + +clean: + rm -rf $(BUILD_DIR) + rm -rf warning_to_plist.egg-info diff --git a/tools/warning-to-plist/README.md b/tools/warning-to-plist/README.md new file mode 100644 index 0000000000..1335c20e4a --- /dev/null +++ b/tools/warning-to-plist/README.md @@ -0,0 +1,42 @@ +# warning-to-plist +`warning-to-plist` is a Python tool which can be used to create a CodeChecker +report directory from the given code analyzer output which can be stored to +a CodeChecker server. + +## Install guide +```sh +# Create a Python virtualenv and set it as your environment. +make venv +source $PWD/venv/bin/activate + +# Build and install warning-to-plist package. +make package +``` + +## Usage +```sh +usage: warn-to-plist [-h] -o OUTPUT_DIR -t TYPE [-v] [file] + +Creates a CodeChecker report directory from the given code analyzer output +which can be stored to a CodeChecker web server. + +positional arguments: + file Code analyzer output result file which will be parsed + and used to generate a CodeChecker report directory. + If this parameter is not given the standard input will + be used. + +optional arguments: + -h, --help show this help message and exit + -o OUTPUT_DIR, --output OUTPUT_DIR + This directory will be used to generate CodeChecker + report directory files. + -t TYPE, --type TYPE Specify the format of the code analyzer output. + Currently supported output types are: clang-tidy. + -v, --verbose Set verbosity level. (default: False) +``` + +## License + +The project is licensed under University of Illinois/NCSA Open Source License. +See LICENSE.TXT for details. \ No newline at end of file diff --git a/tools/warning-to-plist/requirements_py/dev/requirements.txt b/tools/warning-to-plist/requirements_py/dev/requirements.txt new file mode 100644 index 0000000000..bf35683890 --- /dev/null +++ b/tools/warning-to-plist/requirements_py/dev/requirements.txt @@ -0,0 +1,3 @@ +nose==1.3.7 +pycodestyle==2.4.0 +pylint==1.9.4 diff --git a/tools/warning-to-plist/setup.py b/tools/warning-to-plist/setup.py new file mode 100644 index 0000000000..a13e2e0b96 --- /dev/null +++ b/tools/warning-to-plist/setup.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="warning-to-plist", + version="0.1.0", + author='CodeChecker Team (Ericsson)', + description="Parse and create HTML files from one or more '.plist' " + "result files.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/Ericsson/CodeChecker", + keywords=['warn-to-plist', 'plist-converter', 'plist'], + license='LICENSE.txt', + packages=setuptools.find_packages(), + include_package_data=True, + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: University of Illinois/NCSA Open Source License", + "Operating System :: MacOS", + "Operating System :: POSIX", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + ], + entry_points={ + 'console_scripts': [ + 'warning-to-plist = warning_to_plist.WarningToPlist:main' + ] + }, +) diff --git a/tools/warning-to-plist/warning_to_plist/WarningToPlist.py b/tools/warning-to-plist/warning_to_plist/WarningToPlist.py new file mode 100755 index 0000000000..328e8f79d4 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/WarningToPlist.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import argparse +import logging +import os +import sys + +from .converter.clang_tidy.plist_converter import ClangTidyPlistConverter + +LOG = logging.getLogger('WarningToPlist') + +msg_formatter = logging.Formatter('[%(levelname)s] - %(message)s') +log_handler = logging.StreamHandler() +log_handler.setFormatter(msg_formatter) +LOG.setLevel(logging.INFO) +LOG.addHandler(log_handler) + + +supported_converters = { + ClangTidyPlistConverter.TOOL_NAME: ClangTidyPlistConverter} + + +def output_to_plist(output, parser_type, output_dir): + """ Creates .plist files from the given output to the given output dir. """ + plist_converter = supported_converters[parser_type]() + messages = plist_converter.parse_messages(output) + + if not messages: + LOG.info("No '%s' results can be found in the given code analyzer " + "output.", parser_type) + sys.exit(0) + + converters = {} + for m in messages: + if m.path not in converters: + converters[m.path] = supported_converters[parser_type]() + converters[m.path].add_messages([m]) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + for c in converters: + file_name = os.path.basename(c) + out_file_name = '{0}_{1}.plist'.format(file_name, parser_type) + out_file = os.path.join(output_dir, out_file_name) + + LOG.info("Creating plist file: '%s'.", out_file) + LOG.debug(converters[c].plist) + + converters[c].write_to_file(out_file) + + +def __add_arguments_to_parser(parser): + """ Add arguments to the the given parser. """ + parser.add_argument('input', + type=str, + metavar='file', + nargs='?', + default=argparse.SUPPRESS, + help="Code analyzer output result file which will be " + "parsed and used to generate a CodeChecker " + "report directory. If this parameter is not " + "given the standard input will be used.") + + parser.add_argument('-o', '--output', + dest="output_dir", + required=True, + default=argparse.SUPPRESS, + help="This directory will be used to generate " + "CodeChecker report directory files.") + + parser.add_argument('-t', '--type', + dest='type', + metavar='TYPE', + required=True, + choices=supported_converters, + default=argparse.SUPPRESS, + help="Specify the format of the code analyzer output. " + "Currently supported output types are: " + + ', '.join(supported_converters) + ".") + + parser.add_argument('-v', '--verbose', + action='store_true', + dest='verbose', + help="Set verbosity level.") + + +def main(): + """ Warning to plist converter main command line. """ + parser = argparse.ArgumentParser( + prog="warn-to-plist", + description="Creates a CodeChecker report directory from the given " + "code analyzer output which can be stored to a " + "CodeChecker web server.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + __add_arguments_to_parser(parser) + + args = parser.parse_args() + + if 'verbose' in args and args.verbose: + LOG.setLevel(logging.DEBUG) + + if 'input' in args and args.input: + with open(args.input) as input_file: + output = input_file.readlines() + else: + output = sys.stdin.readlines() + + output_to_plist(output, args.type, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/tools/warning-to-plist/warning_to_plist/__init__.py b/tools/warning-to-plist/warning_to_plist/__init__.py new file mode 100644 index 0000000000..9f68a237a0 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/__init__.py @@ -0,0 +1,5 @@ +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- diff --git a/tools/warning-to-plist/warning_to_plist/converter/__init__.py b/tools/warning-to-plist/warning_to_plist/converter/__init__.py new file mode 100644 index 0000000000..9f68a237a0 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/converter/__init__.py @@ -0,0 +1,5 @@ +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- diff --git a/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/__init__.py b/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/__init__.py new file mode 100644 index 0000000000..9f68a237a0 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/__init__.py @@ -0,0 +1,5 @@ +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- diff --git a/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/output_parser.py b/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/output_parser.py new file mode 100644 index 0000000000..5d55a7b435 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/output_parser.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import logging +import os +import re + +from ..output_parser import Message, Event, OutputParser + +LOG = logging.getLogger('WarningToPlist') + + +class ClangTidyOutputParser(OutputParser): + """ Parser for clang-tidy console output. """ + + def __init__(self): + super(ClangTidyOutputParser, self).__init__() + + # Regex for parsing a clang-tidy message. + self.message_line_re = re.compile( + # File path followed by a ':'. + r'^(?P[\S ]+?):' + # Line number followed by a ':'. + r'(?P\d+?):' + # Column number followed by a ':' and a space. + r'(?P\d+?): ' + # Severity followed by a ':'. + r'(?P(error|warning)):' + # Checker message. + r'(?P[\S \t]+)\s*' + # Checker name. + r'\[(?P.*)\]') + + # Matches a note. + self.note_line_re = re.compile( + # File path followed by a ':'. + r'^(?P[\S ]+?):' + # Line number followed by a ':'. + r'(?P\d+?):' + # Column number followed by a ':' and a space. + r'(?P\d+?): ' + # Severity == note. + r'note:' + # Checker message. + r'(?P.*)') + + def parse_message(self, it, line): + """Parse the given line. + + Returns a (message, next_line) pair or throws a StopIteration. + The message could be None. + """ + match = self.message_line_re.match(line) + if match is None: + return None, next(it) + + message = Message( + os.path.abspath(match.group('path')), + int(match.group('line')), + int(match.group('column')), + match.group('message').strip(), + match.group('checker').strip()) + + try: + line = next(it) + line = self._parse_code(message, it, line) + line = self._parse_fixits(message, it, line) + line = self._parse_notes(message, it, line) + + return message, line + except StopIteration: + return message, '' + + def _parse_code(self, message, it, line): + # Eat code line. + if self.note_line_re.match(line) or self.message_line_re.match(line): + LOG.debug("Unexpected line: %s. Expected a code line!", line) + return line + + # Eat arrow line. + # FIXME: range support? + line = next(it) + if '^' not in line: + LOG.debug("Unexpected line: %s. Expected an arrow line!", line) + return line + + return next(it) + + def _parse_fixits(self, message, it, line): + """ Parses fixit messages. """ + + while self.message_line_re.match(line) is None and \ + self.note_line_re.match(line) is None: + message_text = line.strip() + + if message_text != '': + message.fixits.append(Event(message.path, message.line, + line.find(message_text) + 1, + message_text)) + line = next(it) + return line + + def _parse_notes(self, message, it, line): + """ Parses note messages. """ + + while self.message_line_re.match(line) is None: + match = self.note_line_re.match(line) + if match is None: + LOG.debug("Unexpected line: %s", line) + return next(it) + + message.events.append(Event(os.path.abspath(match.group('path')), + int(match.group('line')), + int(match.group('column')), + match.group('message').strip())) + line = next(it) + line = self._parse_code(message, it, line) + return line diff --git a/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/plist_converter.py b/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/plist_converter.py new file mode 100644 index 0000000000..74fa13e458 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/converter/clang_tidy/plist_converter.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +from ..plist_converter import PlistConverter +from .output_parser import ClangTidyOutputParser + + +class ClangTidyPlistConverter(PlistConverter): + """ Warning messages to plist converter. """ + + TOOL_NAME = 'clang-tidy' + + def parse_messages(self, output): + """ Parse the given output. """ + parser = ClangTidyOutputParser() + return parser.parse_messages(output) + + def _get_checker_category(self, checker): + """ Returns the check's category.""" + parts = checker.split('-') + return parts[0] if parts else 'unknown' + + def _get_analyzer_type(self): + """ Returns the analyzer type. """ + return self.TOOL_NAME diff --git a/tools/warning-to-plist/warning_to_plist/converter/output_parser.py b/tools/warning-to-plist/warning_to_plist/converter/output_parser.py new file mode 100644 index 0000000000..4ff84df674 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/converter/output_parser.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +from abc import ABCMeta, abstractmethod + + +def get_next(it): + """ Returns the next item from the iterator or return an empty string. """ + try: + return next(it) + except StopIteration: + return '' + + +class Event(object): + """ Represents an event message. """ + + def __init__(self, path, line, column, message): + self.path = path + self.line = line + self.column = column + self.message = message + + def __eq__(self, other): + return self.path == other.path and \ + self.line == other.line and \ + self.column == other.column and \ + self.message == other.message + + def __str__(self): + return 'path={0}, line={1}, column={2}, message={3}'.format( + self.path, self.line, self.column, self.message) + + +class Message(Event): + """ Represents a message with an optional event and fixit messages. """ + + def __init__(self, path, line, column, message, checker, events=None, + fixits=None): + super(Message, self).__init__(path, line, column, message) + self.checker = checker + self.events = events if events else [] + self.fixits = fixits if fixits else [] + + def __eq__(self, other): + return super(Message, self).__eq__(other) and \ + self.checker == other.checker and \ + self.events == other.events and \ + self.fixits == other.fixits + + def __str__(self): + return '%s, checker=%s, events=%s, fixits=%s' % \ + (super(Message, self).__str__(), self.checker, + [str(event) for event in self.events], + [str(fixit) for fixit in self.fixits]) + + +class OutputParser(object): + """ Warning message parser. """ + + __metaclass__ = ABCMeta + + def __init__(self): + self.messages = [] + + def parse_messages_from_file(self, path): + """ Parse output dump (redirected output). """ + with open(path, 'r') as file: + return self.parse_messages(file) + + def parse_messages(self, lines): + """ Parse the given output. """ + it = iter(lines) + try: + next_line = next(it) + while True: + message, next_line = self.parse_message(it, next_line) + if message: + self.messages.append(message) + except StopIteration: + pass + + return self.messages + + @abstractmethod + def parse_message(self, it, line): + """ Parse the given line. """ + raise NotImplementedError("Subclasses should implement this!") diff --git a/tools/warning-to-plist/warning_to_plist/converter/plist_converter.py b/tools/warning-to-plist/warning_to_plist/converter/plist_converter.py new file mode 100644 index 0000000000..d71d97bcc7 --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/converter/plist_converter.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +from abc import ABCMeta, abstractmethod +import copy +import json +import plistlib + +from .report import generate_report_hash + + +class PlistConverter(object): + """ Warning messages to plist converter. """ + + __metaclass__ = ABCMeta + + def __init__(self): + self.plist = { + 'files': [], + 'diagnostics': [] + } + + @abstractmethod + def parse_messages(self, output): + """ Parse the given output. """ + raise NotImplementedError("Subclasses should implement this!") + + def add_messages(self, messages): + """ Adds the given messages to the plist. """ + self._add_diagnostics(messages, self.plist['files']) + + def write_to_file(self, path): + """ Writes out the plist XML to the given path. """ + with open(path, 'wb') as file: + self.write(file) + + def write(self, file): + """ Writes out the plist XML using the given file object. """ + plistlib.writePlist(self.plist, file) + + def _create_location(self, msg, fmap): + """ Create a location section from the message. """ + return {'line': msg.line, + 'col': msg.column, + 'file': fmap[msg.path]} + + def _create_event(self, msg, fmap): + """ Create an event from the given message. """ + return {'kind': 'event', + 'location': self._create_location(msg, fmap), + 'depth': 0, + 'message': msg.message} + + def _create_edge(self, start_msg, end_msg, fmap): + """ Create an edge between the start and end messages. """ + start_loc = self._create_location(start_msg, fmap) + end_loc = self._create_location(end_msg, fmap) + return {'start': [start_loc, start_loc], + 'end': [end_loc, end_loc]} + + def _add_diagnostics(self, messages, files): + """ Adds the messages to the plist as diagnostics. """ + fmap = self._add_files_from_messages(messages) + for message in messages: + diag = self._create_diag(message, fmap, files) + self.plist['diagnostics'].append(diag) + + def _add_files_from_message(self, message, fmap): + """ Add new file from the given message. """ + try: + idx = self.plist['files'].index(message.path) + fmap[message.path] = idx + except ValueError: + fmap[message.path] = len(self.plist['files']) + self.plist['files'].append(message.path) + + def _add_files_from_messages(self, messages): + """ Add new file from the given messages. + + Adds the new files from the given message array to the plist's "files" + key, and returns a path to file index dictionary. + """ + fmap = {} + for message in messages: + self._add_files_from_message(message, fmap) + + # Collect file paths from the events. + for nt in message.events: + self._add_files_from_message(nt, fmap) + + return fmap + + @abstractmethod + def _get_checker_category(self, checker): + """ Returns the check's category.""" + raise NotImplementedError("Subclasses should implement this!") + + @abstractmethod + def _get_analyzer_type(self): + """ Returns the analyzer type. """ + raise NotImplementedError("Subclasses should implement this!") + + def _create_diag(self, message, fmap, files): + """ Creates a new plist diagnostic from the given message. """ + diag = {'location': self._create_location(message, fmap), + 'check_name': message.checker, + 'description': message.message, + 'category': self._get_checker_category(message.checker), + 'type': self._get_analyzer_type(), + 'path': []} + + self.__add_fixits(diag, message, fmap) + self.__add_events(diag, message, fmap) + + # The original message should be the last part of the path. This is + # displayed by quick check, and this is the main event displayed by + # the web interface. FIXME: notes and fixits should not be events. + diag['path'].append(self._create_event(message, fmap)) + + diag['issue_hash_content_of_line_in_context'] \ + = generate_report_hash(diag, + files[diag['location']['file']]) + + return diag + + def __add_fixits(self, diag, message, fmap): + """ Adds fixits as events to the diagnostics. """ + for fixit in message.fixits: + mf = copy.deepcopy(fixit) + mf.message = '%s (fixit)' % fixit.message + diag['path'].append(self._create_event(mf, fmap)) + + def __add_events(self, diag, message, fmap): + """ Adds events to the diagnostics. + + It also creates edges between the events. + """ + edges = [] + last = None + for event in message.events: + if last is not None: + edges.append(self._create_edge(last, event, fmap)) + + diag['path'].append(self._create_event(event, fmap)) + last = event + + # Add control items only if there is any. + if edges: + diag['path'].append({'kind': 'control', 'edges': edges}) + + def __str__(self): + return str(json.dumps(self.plist, indent=4, separators=(',', ': '))) diff --git a/tools/warning-to-plist/warning_to_plist/converter/report.py b/tools/warning-to-plist/warning_to_plist/converter/report.py new file mode 100644 index 0000000000..1a91cec3db --- /dev/null +++ b/tools/warning-to-plist/warning_to_plist/converter/report.py @@ -0,0 +1,134 @@ +# ------------------------------------------------------------------------- +# The CodeChecker Infrastructure +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# ------------------------------------------------------------------------- +""" +Parsers for the analyzer output formats (plist ...) should create this +Report which will be stored. + +Multiple bug identification hash-es can be generated. +All hash generation algorithms should be documented and implemented here. +""" + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import + +import hashlib +import io +import logging +import os + +LOG = logging.getLogger('WarningToPlist') + + +def get_line(file_name, line_no, errors='ignore'): + """ Return the given line from the file. + If line_no is larger than the number of lines in the file then empty + string returns. If the file can't be opened for read, the function also + returns empty string. + + Try to encode every file as utf-8 to read the line content do not depend + on the platform settings. By default locale.getpreferredencoding() is used + which depends on the platform. + + Changing the encoding error handling can influence the hash content! + """ + try: + with io.open(file_name, mode='r', + encoding='utf-8', + errors=errors) as source_file: + for line in source_file: + line_no -= 1 + if line_no == 0: + return line + return u'' + except IOError: + LOG.error("Failed to open file %s", file_name) + return u'' + + +def remove_whitespace(line_content, old_col): + """ Removes white spaces from the given line content. + + This function removes white spaces from the line content parameter and + calculates the new line location. + Returns the line content without white spaces and the new column number. + E.g.: + line_content = " int foo = 17; sizeof(43); " + ^ + |- bug_col = 18 + content_begin = " int foo = 17; " + content_begin_strip = "intfoo=17;" + line_strip_len = 18 - 10 => 8 + ''.join(line_content.split()) => "intfoo=17;sizeof(43);" + ^ + |- until_col - line_strip_len + 18 - 8 + = 10 + """ + content_begin = line_content[:old_col] + content_begin_strip = u''.join(content_begin.split()) + line_strip_len = len(content_begin) - len(content_begin_strip) + + return ''.join(line_content.split()), \ + old_col - line_strip_len + + +def generate_report_hash(main_section, source_file): + """ Generate unique hashes for reports. + + Hash generation algoritm for older plist versions where no issue hash was + generated or for the plists generated where the issue hash generation + feature is still missing. + + High level overview of the hash content: + * file_name from the main diag section + * checker message + * line content from the source file if can be read up + * column numbers from the main diag sections location + * all the whitespaces from the source content are removed + + """ + + try: + m_loc = main_section.get('location') + source_line = m_loc.get('line') + + from_col = m_loc.get('col') + until_col = m_loc.get('col') + + # WARNING!!! Changing the error handling type for encoding errors + # can influence the hash content! + line_content = get_line(source_file, source_line, errors='ignore') + + # Remove whitespaces so the hash will be independet of the + # source code indentation. + line_content, new_col = \ + remove_whitespace(line_content, from_col) + # Update the column number in sync with the + # removed whitespaces. + until_col = until_col - (from_col-new_col) + from_col = new_col + + if line_content == '' and not os.path.isfile(source_file): + LOG.error("Failed to include soruce line in the report hash.") + LOG.error('%s does not exists!', source_file) + + file_name = os.path.basename(source_file) + msg = main_section.get('description') + + hash_content = [file_name, + msg, + line_content, + str(from_col), + str(until_col)] + + string_to_hash = '|||'.join(hash_content) + return hashlib.md5(string_to_hash.encode()).hexdigest() + + except Exception as ex: + LOG.error("Hash generation failed") + LOG.error(ex) + return ''