port CodeChecker to Python3

run 2to3 and autopep8
2to3 -w -n analyzer bin codechecker_common scripts tools web

format almost everything:
autopep8 --select E501,E502,E303,E127,E122,E128,E131,E125 --max-line-length 80 -i -aaaa -r analyzer bin codechecker_common scripts tools web --exclude "*codechecker_server/database*,*codechecker_server/migrations*"

Pycodestyle and Pylint were updated and the errors were cleaned up for python3.

Add encoding to:
- subprocess calls
- named temporary file creation

Fix deprecated functions and methods from plistlib and imp.

Some analysis output files needed to be split up
because the order of the executed analyzers is not
deterministic (pool.map).

travis-ci update:
- linux to python 3.6
- osx to python 3.7 (the default python3 version)

Use pip3 instead of pip and use the python3 interpreter.
Machines where the default python or pip is python2 using
that can cause problems.

Remove all the unnecessary python interpreter from the
the modules where no main function is available.

Set minimal python3 version for the tools in the setup.py files.
Remove unnecessary module import checks for py2 py3 compatibility.
Update dependecies to use the stubs generated with thrift 0.11.0.

.travis.yml

@@ -10,25 +10,25 @@ matrix:
       name: "Analyzer, web (sqlite, psycopg2, pg8000) and tools test cases."
       sudo: required
       dist: bionic
-      python: "2.7"
+      python: "3.6"
       env: CC_MODULE=analyzer
     - os: linux
       name: "Server/client test cases (sqlite)"
       sudo: required
       dist: bionic
-      python: "2.7"
+      python: "3.6"
       env: DATABASE=sqlite
     - os: linux
       name: "Server/client test cases (psycopg2)"
       sudo: required
       dist: bionic
-      python: "2.7"
+      python: "3.6"
       env: DATABASE=psql_psycopg2
     - os: linux
       name: "Server/client test cases (pg8000)"
       sudo: required
       dist: bionic
-      python: "2.7"
+      python: "3.6"
       env: DATABASE=psql_pg8000
     - os: osx
       osx_image: xcode10
@@ -82,9 +82,9 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then while ! pg_isready; do sleep 1; done; createuser -s postgres; fi
 
 install:
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install --ignore-installed nose --user; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=/Users/travis/Library/Python/2.7/bin:$PATH; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install virtualenv; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip3 install --ignore-installed nose --user; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=/Users/travis/Library/Python/3.7/bin:$PATH; fi
+    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip3 install virtualenv; fi
 
     # Remove clang symlinks from the travis base install.
     # The update-alternatives changes will not change the used clang version without this.

Makefile

@@ -7,7 +7,7 @@ VENDOR_DIR = $(CURRENT_DIR)/vendor
 CC_BUILD_DIR = $(BUILD_DIR)/CodeChecker
 CC_BUILD_BIN_DIR = $(CC_BUILD_DIR)/bin
 CC_BUILD_WEB_DIR = $(CC_BUILD_DIR)/www
-CC_BUILD_LIB_DIR = $(CC_BUILD_DIR)/lib/python2.7
+CC_BUILD_LIB_DIR = $(CC_BUILD_DIR)/lib/python3
 
 CC_WEB = $(CURRENT_DIR)/web
 CC_SERVER = $(CC_WEB)/server/
@@ -51,7 +51,7 @@ package_tu_collector: build_tu_collector package_dir_structure
 	cp -rp $(CC_TOOLS)/tu_collector/build/tu_collector/tu_collector $(CC_BUILD_LIB_DIR) && \
 	chmod u+x $(CC_BUILD_LIB_DIR)/tu_collector/tu_collector.py && \
 	cd $(CC_BUILD_DIR) && \
-	ln -sf ../lib/python2.7/tu_collector/tu_collector.py bin/tu_collector
+	ln -sf ../lib/python3/tu_collector/tu_collector.py bin/tu_collector
 
 build_report_converter:
 	$(MAKE) -C $(ROOT)/tools/report-converter build
@@ -61,7 +61,7 @@ package_report_converter: build_report_converter package_dir_structure
 	cp -rp $(CC_TOOLS)/report-converter/build/report_converter/codechecker_report_converter $(CC_BUILD_LIB_DIR) && \
 	chmod u+x $(CC_BUILD_LIB_DIR)/codechecker_report_converter/cli.py && \
 	cd $(CC_BUILD_DIR) && \
-	ln -sf ../lib/python2.7/codechecker_report_converter/cli.py bin/report-converter
+	ln -sf ../lib/python3/codechecker_report_converter/cli.py bin/report-converter
 
 package: package_dir_structure set_git_commit_template package_plist_to_html package_tu_collector package_report_converter
 	BUILD_DIR=$(BUILD_DIR) BUILD_LOGGER_64_BIT_ONLY=$(BUILD_LOGGER_64_BIT_ONLY) $(MAKE) -C $(CC_ANALYZER) package_analyzer
@@ -111,17 +111,17 @@ standalone_package: venv package
 
 venv:
 	# Create a virtual environment which can be used to run the build package.
-	virtualenv -p python2 venv && \
+	virtualenv -p python3 venv && \
 		$(ACTIVATE_RUNTIME_VENV) && \
-		pip install -r $(CC_ANALYZER)/requirements.txt && \
-		pip install -r $(CC_WEB)/requirements.txt
+		pip3 install -r $(CC_ANALYZER)/requirements.txt && \
+		pip3 install -r $(CC_WEB)/requirements.txt
 
 venv_osx:
 	# Create a virtual environment which can be used to run the build package.
-	virtualenv -p python2.7 venv && \
+	virtualenv -p python3 venv && \
 		$(ACTIVATE_RUNTIME_VENV) && \
-		pip install -r $(CC_ANALYZER)/requirements_py/osx/requirements.txt && \
-		pip install -r $(CC_WEB)/requirements_py/osx/requirements.txt
+		pip3 install -r $(CC_ANALYZER)/requirements_py/osx/requirements.txt && \
+		pip3 install -r $(CC_WEB)/requirements_py/osx/requirements.txt
 
 clean_venv:
 	rm -rf venv
@@ -136,7 +136,7 @@ pip_dev_deps:
 
 venv_dev:
 	# Create a virtual environment for development.
-	virtualenv -p python2 venv_dev && \
+	virtualenv -p python3 venv_dev && \
 		$(ACTIVATE_DEV_VENV) && $(PIP_DEV_DEPS_CMD)
 
 clean_venv_dev:

analyzer/Makefile

@@ -4,7 +4,7 @@ CURRENT_DIR = $(shell pwd)
 BUILD_DIR ?= $(CURRENT_DIR)/build
 
 CC_BUILD_DIR = $(BUILD_DIR)/CodeChecker
-CC_BUILD_LIB_DIR = $(CC_BUILD_DIR)/lib/python2.7
+CC_BUILD_LIB_DIR = $(CC_BUILD_DIR)/lib/python3
 
 # Root of the repository.
 ROOT = $(CURRENT_DIR)/..
@@ -22,12 +22,12 @@ BUILD_LOGGER_64_BIT_ONLY ?= NO
 include tests/Makefile
 
 pip_dev_deps:
-	pip install -r $(VENV_DEV_REQ_FILE)
+	pip3 install -r $(VENV_DEV_REQ_FILE)
 
 venv_dev:
 	# Create a virtual environment for development.
-	virtualenv -p python2 venv_dev && \
-		$(ACTIVATE_DEV_VENV) && pip install -r $(VENV_DEV_REQ_FILE)
+	virtualenv -p python3 venv_dev && \
+		$(ACTIVATE_DEV_VENV) && pip3 install -r $(VENV_DEV_REQ_FILE)
 
 clean_venv_dev:
 	rm -rf venv_dev
@@ -52,7 +52,7 @@ package_tu_collector: build_tu_collector package_dir_structure
 	cp -rp $(CC_TOOLS)/tu_collector/build/tu_collector/tu_collector $(CC_BUILD_LIB_DIR) && \
 	chmod u+x $(CC_BUILD_LIB_DIR)/tu_collector/tu_collector.py && \
 	cd $(CC_BUILD_DIR) && \
-	ln -sf ../lib/python2.7/tu_collector/tu_collector.py bin/tu_collector
+	ln -sf ../lib/python3/tu_collector/tu_collector.py bin/tu_collector
 
 # This target should be used from the top level Makefile to build the package
 # together with the web part. This way we will not build plist-to-html

analyzer/bin/codechecker-analyze

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -------------------------------------------------------------------------
 #                     The CodeChecker Infrastructure
 #   This file is distributed under the University of Illinois Open Source
@@ -8,14 +8,14 @@
 Entry point for the analysis execution command.
 """
 
-import imp
+from importlib.util import spec_from_file_location
 import os
 
 THIS_PATH = os.path.dirname(os.path.abspath(__file__))
 CC = os.path.join(THIS_PATH, "CodeChecker")
 
 # Load CodeChecker from the current folder (the wrapper script (without .py))
-CodeChecker = imp.load_source('CodeChecker', CC)
+CodeChecker = spec_from_file_location('CodeChecker', CC).loader.load_module()
 
 # Execute CC's main script with the current subcommand.
 CodeChecker.main("analyze")

analyzer/bin/codechecker-analyzer-version

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -------------------------------------------------------------------------
 #                     The CodeChecker Infrastructure
 #   This file is distributed under the University of Illinois Open Source
@@ -8,14 +8,14 @@
 Entry point for the analysis execution command.
 """
 
-import imp
+from importlib.util import spec_from_file_location
 import os
 
 THIS_PATH = os.path.dirname(os.path.abspath(__file__))
 CC = os.path.join(THIS_PATH, "CodeChecker")
 
 # Load CodeChecker from the current folder (the wrapper script (without .py))
-CodeChecker = imp.load_source('CodeChecker', CC)
+CodeChecker = spec_from_file_location('CodeChecker', CC).loader.load_module()
 
 # Execute CC's main script with the current subcommand.
 CodeChecker.main("analyzer-version")

analyzer/bin/codechecker-analyzers

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -------------------------------------------------------------------------
 #                     The CodeChecker Infrastructure
 #   This file is distributed under the University of Illinois Open Source
@@ -8,14 +8,14 @@
 Entry point for the analyzers command.
 """
 
-import imp
+from importlib.util import spec_from_file_location
 import os
 
 THIS_PATH = os.path.dirname(os.path.abspath(__file__))
 CC = os.path.join(THIS_PATH, "CodeChecker")
 
 # Load CodeChecker from the current folder (the wrapper script (without .py))
-CodeChecker = imp.load_source('CodeChecker', CC)
+CodeChecker = spec_from_file_location('CodeChecker', CC).loader.load_module()
 
 # Execute CC's main script with the current subcommand.
 CodeChecker.main("analyzers")

analyzer/bin/codechecker-checkers

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -------------------------------------------------------------------------
 #                     The CodeChecker Infrastructure
 #   This file is distributed under the University of Illinois Open Source
@@ -8,14 +8,14 @@
 Entry point for the available checker listing command.
 """
 
-import imp
+from importlib.util import spec_from_file_location
 import os
 
 THIS_PATH = os.path.dirname(os.path.abspath(__file__))
 CC = os.path.join(THIS_PATH, "CodeChecker")
 
 # Load CodeChecker from the current folder (the wrapper script (without .py))
-CodeChecker = imp.load_source('CodeChecker', CC)
+CodeChecker = spec_from_file_location('CodeChecker', CC).loader.load_module()
 
 # Execute CC's main script with the current subcommand.
 CodeChecker.main("checkers")

analyzer/bin/codechecker-log

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -------------------------------------------------------------------------
 #                     The CodeChecker Infrastructure
 #   This file is distributed under the University of Illinois Open Source
@@ -8,14 +8,14 @@
 Entry point for the build logger command.
 """
 
-import imp
+from importlib.util import spec_from_file_location
 import os
 
 THIS_PATH = os.path.dirname(os.path.abspath(__file__))
 CC = os.path.join(THIS_PATH, "CodeChecker")
 
 # Load CodeChecker from the current folder (the wrapper script (without .py))
-CodeChecker = imp.load_source('CodeChecker', CC)
+CodeChecker = spec_from_file_location('CodeChecker', CC).loader.load_module()
 
 # Execute CC's main script with the current subcommand.
 CodeChecker.main("log")

analyzer/bin/codechecker-parse

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -------------------------------------------------------------------------
 #                     The CodeChecker Infrastructure
 #   This file is distributed under the University of Illinois Open Source
@@ -8,14 +8,14 @@
 Entry point for the parse (analysis results to stdout) command.
 """
 
-import imp
+from importlib.util import spec_from_file_location
 import os
 
 THIS_PATH = os.path.dirname(os.path.abspath(__file__))
 CC = os.path.join(THIS_PATH, "CodeChecker")
 
 # Load CodeChecker from the current folder (the wrapper script (without .py))
-CodeChecker = imp.load_source('CodeChecker', CC)
+CodeChecker = spec_from_file_location('CodeChecker', CC).loader.load_module()
 
 # Execute CC's main script with the current subcommand.
 CodeChecker.main("parse")

analyzer/codechecker_analyzer/analysis_manager.py

@@ -5,9 +5,7 @@
 # -------------------------------------------------------------------------
 """
 """
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
+
 
 import glob
 import multiprocessing
@@ -115,7 +113,7 @@ def worker_result_handler(results, metadata, output_path, analyzer_binaries):
     # threads would be more error prone.
     source_map = {}
     for f in glob.glob(os.path.join(output_path, "*.source")):
-        with open(f, 'r') as sfile:
+        with open(f, 'r', encoding="utf-8", errors="ignore") as sfile:
             source_map[f[:-7]] = sfile.read().strip()
         os.remove(f)
 
@@ -142,11 +140,13 @@ def init_worker(checked_num, action_num):
 def save_output(base_file_name, out, err):
     try:
         if out:
-            with open(base_file_name + ".stdout.txt", 'w') as outf:
+            with open(base_file_name + ".stdout.txt", 'w',
+                      encoding="utf-8", errors="ignore") as outf:
                 outf.write(out)
 
         if err:
-            with open(base_file_name + ".stderr.txt", 'w') as outf:
+            with open(base_file_name + ".stderr.txt", 'w',
+                      encoding="utf-8", errors="ignore") as outf:
                 outf.write(err)
     except IOError as ioerr:
         LOG.debug("Failed to save analyzer output")
@@ -158,7 +158,8 @@ def save_metadata(result_file, analyzer_result_file, analyzed_source_file):
     Save some extra information next to the plist, .source
     acting as an extra metadata file.
     """
-    with open(result_file + ".source", 'w') as orig:
+    with open(result_file + ".source", 'w',
+              encoding="utf-8", errors="ignore") as orig:
         orig.write(analyzed_source_file.replace(r'\ ', ' ') + "\n")
 
     if os.path.exists(analyzer_result_file) and \
@@ -313,7 +314,7 @@ def handle_failure(source_analyzer, rh, zip_file, result_base, actions_map):
     # TODO: What about the dependencies of the other_files?
     tu_collector.add_sources_to_zip(
         zip_file,
-        map(lambda path: os.path.join(action.directory, path), other_files))
+        [os.path.join(action.directory, path) for path in other_files])
 
     with zipfile.ZipFile(zip_file, 'a') as archive:
         LOG.debug("[ZIP] Writing analyzer STDOUT to /stdout")

analyzer/codechecker_analyzer/analyzer.py

@@ -6,9 +6,7 @@
 """
 Prepare and start different analysis types
 """
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
+
 
 from multiprocessing.managers import SyncManager
 import os
@@ -77,13 +75,22 @@ def __get_analyzer_version(context, analyzer_config_map):
     versions = {}
     for _, analyzer_cfg in analyzer_config_map.items():
         analyzer_bin = analyzer_cfg.analyzer_binary
-        version = [analyzer_bin, u' --version']
+        version = [analyzer_bin, ' --version']
         try:
-            output = subprocess.check_output(shlex.split(' '.join(version)),
-                                             env=check_env,
-                                             universal_newlines=True)
+            output = subprocess.check_output(
+                shlex.split(
+                    ' '.join(version)),
+                env=check_env,
+                universal_newlines=True,
+                encoding="utf-8",
+                errors="ignore")
             versions[analyzer_bin] = output
-        except (subprocess.CalledProcessError, OSError) as oerr:
+        except subprocess.CalledProcessError as oerr:
+            LOG.warning("Failed to get analyzer version: %s",
+                        ' '.join(version))
+            LOG.warning(oerr.output)
+            LOG.warning(oerr.stderr)
+        except OSError as oerr:
             LOG.warning("Failed to get analyzer version: %s",
                         ' '.join(version))
             LOG.warning(oerr.strerror)

analyzer/codechecker_analyzer/analyzer_context.py

@@ -7,11 +7,8 @@
 Context to store package related information.
 """
 
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
 
-from collections import Mapping
+from collections.abc import Mapping
 # pylint: disable=no-name-in-module
 from distutils.spawn import find_executable
 
@@ -203,7 +200,7 @@ def logger_lib_name(self):
 
     @property
     def path_plist_to_html_dist(self):
-        return os.path.join(self.package_root, 'lib', 'python2.7',
+        return os.path.join(self.package_root, 'lib', 'python3',
                             'plist_to_html', 'static')
 
     @property