Merge pull request #121 from melexis/code-quality

Generate Code Quality report for GitLab CI

README.rst

@@ -289,6 +289,7 @@ input file. When this setting is missing, the default value ``true`` is used.
 .. _`Robot Framework`: https://robotframework.org/
 .. _`--xunit report.xml`: https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#xunit-compatible-result-file
 
+.. _configuration_file:
 
 ----------------------------------
 Configuration File to Pass Options
@@ -302,11 +303,13 @@ Configuration file is in JSON or YAML_ format with a simple structure.
     {
         "sphinx": {
             "enabled": false,
+            "cq_default_path": "doc/source/conf.py",
             "min": 0,
             "max": 0
         },
         "doxygen": {
             "enabled": false,
+            "cq_default_path": "doc/doxygen/Doxyfile",
             "min": 0,
             "max": 0
         },
@@ -414,6 +417,16 @@ Example entries:
     JUnit/RobotFramework:
       test_warn_plugin_double_fail.myfirstfai1ure: Is our warnings plugin able to trace this random failure msg?
 
+Code Quality Report
+-------------------
+
+Use `-C, --code-quality` to let the plugin generate `a Code Quality report`_ for GitLab CI. All counted
+Sphinx, Doxygen and XMLRunner will be included. Other checker types are not supported by this feature. The report is
+a JSON file that implements `a subset of the Code Climate spec`_. Declare this file `as an artifact`_ of the
+`code_quality` CI job.
+If a warning doesn't contain a path, `"cq_default_path"` from the configuration_file_ will be used.
+If not configured, `.gitlab-ci.yml` will be used as a fallback path.
+
 =======================
 Issues and New Features
 =======================
@@ -431,3 +444,6 @@ There is a Contribution guide available if you would like to get involved in
 development of the plugin. We encourage anyone to contribute to our repository.
 
 .. _YAML: https://yaml.org/spec/1.2.2/
+.. _a Code Quality report: https://docs.gitlab.com/ee/ci/testing/code_quality.html
+.. _a subset of the Code Climate spec: https://docs.gitlab.com/ee/ci/testing/code_quality.html#implement-a-custom-tool
+.. _as an artifact: https://docs.gitlab.com/ee/ci/testing/code_quality.html#download-output-in-json-format

src/mlx/regex_checker.py

@@ -1,14 +1,16 @@
+import hashlib
 import re
+from pathlib import Path
 
 from mlx.warnings_checker import WarningsChecker
 
-DOXYGEN_WARNING_REGEX = r"(?:((?:[/.]|[A-Za-z]).+?):(-?\d+):\s*([Ww]arning|[Ee]rror)|<.+>:-?\d+(?::\s*([Ww]arning|[Ee]rror))?): (.+(?:(?!\s*(?:[Nn]otice|[Ww]arning|[Ee]rror): )[^/<\n][^:\n][^/\n].+)*)|\s*\b([Nn]otice|[Ww]arning|[Ee]rror): (?!notes)(.+)\n?"
+DOXYGEN_WARNING_REGEX = r"(?:(?P<path1>(?:[/.]|[A-Za-z]).+?):(?P<line1>-?\d+):\s*(?P<severity1>[Ww]arning|[Ee]rror)|<.+>:(?P<line2>-?\d+)(?::\s*(?P<severity2>[Ww]arning|[Ee]rror))?): (?P<description1>.+(?:(?!\s*([Nn]otice|[Ww]arning|[Ee]rror): )[^/<\n][^:\n][^/\n].+)*)|\s*\b(?P<severity3>[Nn]otice|[Ww]arning|[Ee]rror): (?!notes)(?P<description2>.+)\n?"
 doxy_pattern = re.compile(DOXYGEN_WARNING_REGEX)
 
-SPHINX_WARNING_REGEX = r"(?m)^(?:(.+?:(?:\d+|None)?):?\s*)?(DEBUG|INFO|WARNING|ERROR|SEVERE|CRITICAL):\s*(.+)$"
+SPHINX_WARNING_REGEX = r"(?m)^(?:((?P<path1>.+?):(?P<line1>\d+|None)?):?\s*)?(?P<severity1>DEBUG|INFO|WARNING|ERROR|SEVERE|CRITICAL):\s*(?P<description1>.+)$"
 sphinx_pattern = re.compile(SPHINX_WARNING_REGEX)
 
-PYTHON_XMLRUNNER_REGEX = r"(\s*(ERROR|FAILED) (\[\d+\.\d{3}s\]: \s*(.+)))\n?"
+PYTHON_XMLRUNNER_REGEX = r"(\s*(?P<severity1>ERROR|FAILED) (\[\d+\.\d{3}s\]: \s*(?P<description1>.+)))\n?"
 xmlrunner_pattern = re.compile(PYTHON_XMLRUNNER_REGEX)
 
 COVERITY_WARNING_REGEX = r"(?:((?:[/.]|[A-Za-z]).+?):(-?\d+):) (CID) \d+ \(#(?P<curr>\d+) of (?P<max>\d+)\): (?P<checker>.+\)): (?P<classification>\w+), *(.+)\n?"
@@ -18,6 +20,16 @@
 class RegexChecker(WarningsChecker):
     name = 'regex'
     pattern = None
+    SEVERITY_MAP = {
+        'debug': 'info',
+        'info': 'info',
+        'notice': 'info',
+        'warning': 'major',
+        'error': 'critical',
+        'severe': 'critical',
+        'critical': 'critical',
+        'failed': 'critical',
+    }
 
     def check(self, content):
         ''' Function for counting the number of warnings in a specific text
@@ -33,6 +45,47 @@ def check(self, content):
             self.count += 1
             self.counted_warnings.append(match_string)
             self.print_when_verbose(match_string)
+            if self.cq_enabled:
+                self.add_code_quality_finding(match)
+
+    def add_code_quality_finding(self, match):
+        finding = {
+            "severity": "major",
+            "location": {
+                "path": self.cq_default_path,
+                "lines": {
+                    "begin": 1,
+                }
+            }
+        }
+        groups = {name: result for name, result in match.groupdict().items() if result}
+        for name, result in groups.items():
+            if name.startswith("description"):
+                finding["description"] = result
+                break
+        else:
+            return  # no description was found, which is the bare minimum
+        for name, result in groups.items():
+            if name.startswith("severity"):
+                finding["severity"] = self.SEVERITY_MAP[result.lower()]
+                break
+        for name, result in groups.items():
+            if name.startswith("path"):
+                path = Path(result)
+                if path.is_absolute():
+                    path = path.relative_to(Path.cwd())
+                finding["location"]["path"] = str(path)
+                break
+        for name, result in groups.items():
+            if name.startswith("line"):
+                try:
+                    lineno = int(result, 0)
+                except (TypeError, ValueError):
+                    lineno = 1
+                finding["location"]["lines"]["begin"] = lineno
+                break
+        finding["fingerprint"] = hashlib.md5(str(finding).encode('utf8')).hexdigest()
+        self.cq_findings.append(finding)
 
 
 class CoverityChecker(RegexChecker):

src/mlx/warnings.py

@@ -21,16 +21,18 @@
 
 class WarningsPlugin:
 
-    def __init__(self, verbose=False, config_file=None):
+    def __init__(self, verbose=False, config_file=None, cq_enabled=False):
         '''
         Function for initializing the parsers
 
         Args:
             verbose (bool): optional - enable verbose logging
             config_file (Path): optional - configuration file with setup
+            cq_enabled (bool): optional - enable generation of Code Quality report
         '''
         self.activated_checkers = {}
         self.verbose = verbose
+        self.cq_enabled = cq_enabled
         self.public_checkers = [SphinxChecker(self.verbose), DoxyChecker(self.verbose), JUnitChecker(self.verbose),
                                 XMLRunnerChecker(self.verbose), CoverityChecker(self.verbose),
                                 RobotChecker(self.verbose)]
@@ -55,7 +57,7 @@ def activate_checker(self, checker):
         Args:
             checker (WarningsChecker): checker object
         '''
-        checker.reset()
+        checker.cq_enabled = self.cq_enabled and checker.name in ('doxygen', 'sphinx', 'xmlrunner')
         self.activated_checkers[checker.name] = checker
 
     def activate_checker_name(self, name):
@@ -201,6 +203,19 @@ def write_counted_warnings(self, out_file):
             for checker in self.activated_checkers.values():
                 open_file.write("\n".join(checker.counted_warnings) + "\n")
 
+    def write_code_quality_report(self, out_file):
+        ''' Generates the Code Quality report artifact as a JSON file that implements a subset of the Code Climate spec
+
+        Args:
+            out_file (str): Location for the output file
+        '''
+        results = []
+        for checker in self.activated_checkers.values():
+            results.extend(checker.cq_findings)
+        content = json.dumps(results, indent=4, sort_keys=False)
+        with open(out_file, 'w', encoding='utf-8', newline='\n') as open_file:
+            open_file.write(f"{content}\n")
+
 
 def warnings_wrapper(args):
     parser = argparse.ArgumentParser(prog='mlx-warnings')
@@ -226,6 +241,8 @@ def warnings_wrapper(args):
                         help="Sphinx checker will include warnings matching (RemovedInSphinx\\d+Warning) regex")
     parser.add_argument('-o', '--output',
                         help='Output file that contains all counted warnings')
+    parser.add_argument('-C', '--code-quality',
+                        help='Output Code Quality report artifact for GitLab CI')
     parser.add_argument('-v', '--verbose', dest='verbose', action='store_true')
     parser.add_argument('--command', dest='command', action='store_true',
                         help='Treat program arguments as command to execute to obtain data')
@@ -237,6 +254,7 @@ def warnings_wrapper(args):
                         help='Possible not-used flags from above are considered as command flags')
 
     args = parser.parse_args(args)
+    code_quality_enabled = bool(args.code_quality)
 
     # Read config file
     if args.configfile is not None:
@@ -245,9 +263,9 @@ def warnings_wrapper(args):
         if checker_flags or warning_args:
             print("Configfile cannot be provided with other arguments")
             sys.exit(2)
-        warnings = WarningsPlugin(verbose=args.verbose, config_file=args.configfile)
+        warnings = WarningsPlugin(verbose=args.verbose, config_file=args.configfile, cq_enabled=code_quality_enabled)
     else:
-        warnings = WarningsPlugin(verbose=args.verbose)
+        warnings = WarningsPlugin(verbose=args.verbose, cq_enabled=code_quality_enabled)
         if args.sphinx:
             warnings.activate_checker_name('sphinx')
         if args.doxygen:
@@ -294,6 +312,8 @@ def warnings_wrapper(args):
     warnings.return_count()
     if args.output:
         warnings.write_counted_warnings(args.output)
+    if args.code_quality:
+        warnings.write_code_quality_report(args.code_quality)
     return warnings.return_check_limits()
 
 

src/mlx/warnings_checker.py

@@ -13,16 +13,15 @@ def __init__(self, verbose=False):
             verbose (bool): Enable/disable verbose logging
         '''
         self.verbose = verbose
-        self.reset()
-        self.exclude_patterns = []
-        self.include_patterns = []
-
-    def reset(self):
-        ''' Reset function (resets min, max and counter values) '''
         self.count = 0
         self.warn_min = 0
         self.warn_max = 0
         self._counted_warnings = []
+        self.cq_findings = []
+        self.cq_enabled = False
+        self.cq_default_path = '.gitlab-ci.yml'
+        self.exclude_patterns = []
+        self.include_patterns = []
 
     @property
     def counted_warnings(self):
@@ -153,6 +152,8 @@ def parse_config(self, config):
         self.set_maximum(int(config['max']))
         self.set_minimum(int(config['min']))
         self.add_patterns(config.get("exclude"), self.exclude_patterns)
+        if 'cq_default_path' in config:
+            self.cq_default_path = config['cq_default_path']
 
     def _is_excluded(self, content):
         ''' Checks if the specific text must be excluded based on the configured regexes for exclusion and inclusion.

tests/test_config.py

@@ -16,15 +16,15 @@ class TestConfig(TestCase):
     def test_configfile_parsing(self):
         warnings = WarningsPlugin(config_file=(TEST_IN_DIR / "config_example.json"))
         warnings.check('testfile.c:6: warning: group test: ignoring title "Some test functions" that does not match old title "Some freaky test functions"')
-        self.assertEqual(warnings.return_count(), 0)
+        self.assertEqual(warnings.return_count(), 1)
         warnings.check('<testcase classname="dummy_class" name="dummy_name"><failure message="some random message from test case" /></testcase>')
-        self.assertEqual(warnings.return_count(), 0)
-        warnings.check("/home/bljah/test/index.rst:5: WARNING: toctree contains reference to nonexisting document u'installation'")
         self.assertEqual(warnings.return_count(), 1)
+        warnings.check("/home/bljah/test/index.rst:5: WARNING: toctree contains reference to nonexisting document u'installation'")
+        self.assertEqual(warnings.return_count(), 2)
         warnings.check('This should not be treated as warning2')
-        self.assertEqual(warnings.return_count(), 1)
+        self.assertEqual(warnings.return_count(), 2)
         warnings.check('ERROR [0.000s]: test_some_error_test (something.anything.somewhere)')
-        self.assertEqual(warnings.return_count(), 1)
+        self.assertEqual(warnings.return_count(), 3)
 
     def _helper_exclude(self, warnings):
         with patch('sys.stdout', new=StringIO()) as verbose_output:

tests/test_in/code_quality.json

@@ -0,0 +1,79 @@
+[
+    {
+        "severity": "major",
+        "location": {
+            "path": "git/test/index.rst",
+            "lines": {
+                "begin": 1
+            }
+        },
+        "description": "toctree contains reference to nonexisting document u'installation'",
+        "fingerprint": "4112c9fb48c96c3d2d0f957215e2dd75"
+    },
+    {
+        "severity": "major",
+        "location": {
+            "path": "doc/source/conf.py",
+            "lines": {
+                "begin": 1
+            }
+        },
+        "description": "List item 'CL-UNDEFINED_CL_ITEM' in merge/pull request 138 is not defined as a checklist-item.",
+        "fingerprint": "b4a2d90bef6fb0a6cad87463d5830080"
+    },
+    {
+        "severity": "info",
+        "location": {
+            "path": "doc/doxygen/Doxyfile",
+            "lines": {
+                "begin": 1
+            }
+        },
+        "description": "Output directory `doc/doxygen/framework' does not exist. I have created it for you.",
+        "fingerprint": "256cb9d4b9b3c05ba755e5da30b2afb2"
+    },
+    {
+        "severity": "critical",
+        "location": {
+            "path": "helper/SimpleTimer.h",
+            "lines": {
+                "begin": 19
+            }
+        },
+        "description": "Unexpected character `\"'",
+        "fingerprint": "d71d9f49bd308ace916bce3c54f430c6"
+    },
+    {
+        "severity": "major",
+        "location": {
+            "path": "doc/doxygen/Doxyfile",
+            "lines": {
+                "begin": 1
+            }
+        },
+        "description": "The following parameters of sofa::component::odesolver::EulerKaapiSolver::v_peq(VecId v, VecId a, double f) are not documented:",
+        "fingerprint": "9c91370059e40eb07289790f2b9d6a75"
+    },
+    {
+        "severity": "critical",
+        "location": {
+            "path": "doc/doxygen/Doxyfile",
+            "lines": {
+                "begin": 1
+            }
+        },
+        "description": "Could not read image `/home/user/myproject/html/struct_foo_graph.png' generated by dot!",
+        "fingerprint": "bdd0d0b7cc25b1f4414f630c2351bf3c"
+    },
+    {
+        "severity": "critical",
+        "location": {
+            "path": ".gitlab-ci.yml",
+            "lines": {
+                "begin": 1
+            }
+        },
+        "description": "test_some_error_test (something.anything.somewhere)'",
+        "fingerprint": "cd09ae46ee5361570fd59de78b454e11"
+    }
+]

tests/test_in/config_example.json

@@ -1,11 +1,13 @@
 {
     "sphinx": {
         "enabled": true,
+        "cq_default_path": "doc/source/conf.py",
         "min": 0,
         "max": 0
     },
     "doxygen": {
-        "enabled": false,
+        "enabled": true,
+        "cq_default_path": "doc/doxygen/Doxyfile",
         "min": 0,
         "max": 0
     },
@@ -15,12 +17,12 @@
         "max": 0
     },
     "xmlrunner": {
-        "enabled": false,
+        "enabled": true,
         "min": 0,
         "max": 0
     },
     "coverity": {
-        "enabled": false,
+        "enabled": true,
         "min": 0,
         "max": 0
     },

tests/test_in/mixed_warnings.txt

@@ -0,0 +1,19 @@
+# Doxygen
+Notice: Output directory `doc/doxygen/framework' does not exist. I have created it for you.
+/home/user/myproject/helper/SimpleTimer.h:19: Error: Unexpected character `"'
+<v_peq>:1: Warning: The following parameters of sofa::component::odesolver::EulerKaapiSolver::v_peq(VecId v, VecId a, double f) are not documented:
+  parameter 'v'
+  parameter 'a'
+error: Could not read image `/home/user/myproject/html/struct_foo_graph.png' generated by dot!
+
+# Sphinx
+/usr/local/lib/python3.7/dist-packages/sphinx_rtd_theme/search.html:20: RemovedInSphinx30Warning: To modify script_files in the theme is deprecated. Please insert a <script> tag directly in your theme instead.
+git/test/index.rst:None: WARNING: toctree contains reference to nonexisting document u'installation'
+WARNING: List item 'CL-UNDEFINED_CL_ITEM' in merge/pull request 138 is not defined as a checklist-item.
+
+# XMLRunner
+'ERROR [0.000s]: test_some_error_test (something.anything.somewhere)'
+
+# Coverity
+/src/somefile.c:82: CID 113396 (#2 of 2): Coding standard violation (MISRA C-2012 Rule 10.1): Unclassified, Unspecified, Undecided, owner is nobody, first detected on 2017-07-27.
+