Add Python3 pre-commit config checks with pylint and mypy

Signed-off-by: Bernhard Kaindl <[email protected]>

.github/workflows/build.yml

@@ -3,6 +3,41 @@ name: build
 on: [push, pull_request]
 
 jobs:
+  python-tests:
+    name: "Python Tests"
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: pre-commit checks - setup cache
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pre-commit
+        key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+
+    - name: pre-commit checks - run checks
+      uses: pre-commit/[email protected]
+      env:
+        # For merging PRs to master, skip the no-commit-to-branch check:
+        SKIP: no-commit-to-branch
+
+    - name: Install Python2 dependencies
+      run: |
+        #: Install Python 2.7 from Ubuntu 20.04 using apt-get install
+        sudo apt-get update && sudo apt-get install -y python2
+        curl -sSL https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py
+        python2 get-pip.py
+        if [ -f requirements.txt ]; then pip2 install -r requirements.txt; fi
+        if [ -f requirements-dev.txt ]; then pip2 install -r requirements-dev.txt; fi
+        pip2 install pytest pylint==1.9.4
+
+    - name: Run pylint-1.9.4
+      run: python2 -m pylint xtf-runner build/*.py
+
+    - name: Run python2 -m pytest to execute all unit and integration tests
+      run: python2 -m pytest -v -rA
+
   build:
 
     strategy:
@@ -46,6 +81,20 @@ jobs:
 
     - uses: actions/checkout@v3
 
+    - name: pre-commit checks - setup cache
+      uses: actions/cache@v3
+      if: ${{ matrix.compiler == 'gcc-13' }}
+      with:
+        path: ~/.cache/pre-commit
+        key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+
+    - name: pre-commit checks - run checks
+      uses: pre-commit/[email protected]
+      if: ${{ matrix.compiler == 'gcc-13' }}
+      env:
+        # For merging PRs to master, skip the no-commit-to-branch check:
+        SKIP: no-commit-to-branch
+
     - name: Build
       run: |
         # Select appropriate LLVM= or CC=

.pre-commit-config.yaml

@@ -0,0 +1,105 @@
+#
+# This is the configuration file of the pre-commit framework for this repository:
+# https://pypi.org/project/pre-commit
+#
+# pre-commit runs in the GitHub Workflow of this project on each push and PR.
+# Additionally, you can run it locally for faster fixing of issues using:
+# $ pip3 install pre-commit
+#
+# On the initial run, pre-commit downloads its checkers, subsequent runs are fast:
+#
+# $ pre-commit run    # automatically checks which checks are needed.
+# $ pre-commit run -a # runs all fixes and checks, even when not needed
+#
+# When this works, you can enable it as the pre-commit hook for this git clone:
+# $ pre-commit install
+#
+# Then, the fixups and checks defined below run by default when you commit:
+#
+# To commit with all pre-commit hooks skipped (for exceptional cases):
+# $ git commit --no-verify
+#
+# To commit with some checks skipped:
+# $ SKIP=mypy,pylint git commit -m "unchecked emergency commit"
+#
+# For more customisation, see https://pre-commit.com/#temporarily-disabling-hooks
+# and https://pre-commit.com/#confining-hooks-to-run-at-certain-stages (e.g push)
+# All hooks: https://pre-commit.com/hooks.html
+#
+fail_fast: false
+default_stages: [commit, push]
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    # https://github.com/pre-commit/pre-commit-hooks/blob/main/README.md:
+    hooks:
+    -   id: no-commit-to-branch
+        name: "ensure that you don't commit to the local master branch"
+        args: [--branch, master]
+        always_run: true
+    -   id: trailing-whitespace
+        name: 'check and fix files to have no trailing whitespace'
+        exclude: install-sh
+    -   id: end-of-file-fixer
+        name: 'check and fix that files have a trailing newline'
+        exclude: |
+            (?x)^(
+                arch/x86/include/arch/msr-index.h
+            )
+    -   id: mixed-line-ending
+        args: ['--fix=lf']
+        name: 'check and fix that line endings are line feeds'
+    -   id: check-added-large-files
+        args: ['--maxkb=12']
+        name: 'check that no large files are added'
+    -   id: check-executables-have-shebangs
+    -   id: debug-statements
+        name: 'check for debugger imports and breakpoint calls'
+    -   id: check-shebang-scripts-are-executable
+        exclude: XSConsoleImporter.py
+    -   id: check-merge-conflict
+    -   id: check-yaml
+        name: 'check the syntax of yaml files'
+-   repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
+    hooks:
+    -   id: autoflake
+        name: remove unused variables and imports
+        args: [
+            "--in-place",
+            "--remove-unused-variables",
+            "--remove-all-unused-imports"
+        ]
+        language: python
+        files: \.py$
+-   repo: https://github.com/akaihola/darker
+    rev: 1.7.2
+    hooks:
+    -   id: darker
+        name: format staged changes like black and isort would format them
+        args: [--skip-string-normalization, --isort, -tpy36]
+        additional_dependencies:
+          - isort
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.7.0
+    hooks:
+    -   id: mypy
+        name: run mypy to check e.g. that all expected arguments are passed to functions etc
+        additional_dependencies: [types-simplejson]
+-   repo: https://github.com/pycqa/pylint
+    rev: v3.0.2
+    hooks:
+    -   id: pylint
+        name: run pylint
+        args: [
+            --jobs=2,
+        ]
+        log_file: ".git/pre-commit-pylint.log"
+-   repo: local
+    hooks:
+    -   id: git-diff # For reference: https://github.com/pre-commit/pre-commit/issues/1712
+        name: When pre-commit fixers made changes, run 'git diff' to show the changes
+        entry: git diff --exit-code  # When changes were made, the exit code is needed to show them
+        language: system
+        pass_filenames: false
+        always_run: true

.pylintrc

@@ -48,7 +48,21 @@ extension-pkg-whitelist=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=bad-whitespace, bad-continuation, global-statement, star-args
+disable=bad-whitespace, bad-continuation, global-statement, star-args,
+    # Not real problems, returns on the same indentation level can be easier:
+    consider-using-with,
+    no-else-raise,
+    no-else-return,
+    multiple-imports,
+    len-as-condition,
+    # For Python3-only projects:
+    consider-using-f-string,
+    deprecated-module,
+    unrecognized-option,
+    unspecified-encoding,
+    use-implicit-booleaness-not-len,
+    useless-object-inheritance,
+    useless-option-value,
 
 
 [REPORTS]
@@ -337,4 +351,4 @@ max-public-methods=20
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

build/mkcfg.py

@@ -9,7 +9,7 @@
 import sys, os
 
 # Usage: mkcfg.py $OUT $DEFAULT-CFG $EXTRA-CFG $VARY-CFG
-_, out, defcfg, vcpus, extracfg, varycfg = sys.argv
+_, out, defcfg, vcpus, extracfg, varycfg = sys.argv  # pylint: disable=unbalanced-tuple-unpacking
 
 # Evaluate environment and name from $OUT
 _, env, name = out.split('.')[0].split('-', 2)

build/mkinfo.py

@@ -1,10 +1,43 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+"""
+mkcfg.py - Generate a configuration JSON file based on provided parameters.
+
+Usage:
+    python mkcfg.py $OUT $NAME $CATEGORY $ENVS $VARIATIONS
+
+Arguments:
+
+    $OUT:        Path to the output file where the generated JSON configuration
+                 will be saved.
+    $NAME:       Name to be assigned in the configuration.
+    $CATEGORY:   Category designation in the configuration.
+    $ENVS:       Optional space-separated list of environments (can be empty).
+    $VARIATIONS: Optional space-separated list of variations (can be empty).
+
+Description:
+
+    This script generates a JSON configuration file using provided parameters
+    and saves it to the specified output file. The generated JSON structure
+    includes fields for 'name', 'category', 'environments', and 'variations'.
+    The 'environments' and 'variations' fields can be populated with
+    space-separated lists if corresponding arguments ($ENVS and $VARIATIONS)
+    are provided.
+
+Example:
+
+    python mkcfg.py config.json ExampleConfig Utilities prod dev test
+
+    This example will create a configuration file named 'config.json' with
+       'name' as 'ExampleConfig',
+       'category' as 'Utilities', and
+       'environments' as ['prod', 'dev', 'test'].
+"""
 
 import sys, json
 
 # Usage: mkcfg.py $OUT $NAME $CATEGORY $ENVS $VARIATIONS
-_, out, name, cat, envs, variations = sys.argv
+_, out, name, cat, envs, variations = sys.argv  # pylint: disable=unbalanced-tuple-unpacking
 
 template = {
     "name": name,

xtf-runner

@@ -13,15 +13,16 @@ import sys, os, os.path as path
 
 from optparse import OptionParser
 from subprocess import Popen, PIPE, call as subproc_call
+from typing import Dict  # pylint: disable=unused-import
 
 try:
     import json
 except ImportError:
-    import simplejson as json
+    import simplejson as json  # type:ignore[no-redef]
 
 # Python 2/3 compatibility
 if sys.version_info >= (3, ):
-    basestring = str
+    basestring = str  # pylint: disable=invalid-name,redefined-builtin
 
 # All results of a test, keep in sync with C code report.h.
 # Notes:
@@ -80,8 +81,7 @@ class TestInstance(object):
     def __repr__(self):
         if not self.variation:
             return "test-{0}-{1}".format(self.env, self.name)
-        else:
-            return "test-{0}-{1}~{2}".format(self.env, self.name, self.variation)
+        return "test-{0}-{1}~{2}".format(self.env, self.name, self.variation)
 
     def __hash__(self):
         return hash(repr(self))
@@ -92,8 +92,8 @@ class TestInstance(object):
     def __ne__(self, other):
         return repr(self) != repr(other)
 
-    def __cmp__(self, other):
-        return cmp(repr(self), repr(other))
+    def __cmp__(self, other):  # Not used by Python3
+        return cmp(repr(self), repr(other))  # pylint: disable=undefined-variable
 
 
 class TestInfo(object):
@@ -235,7 +235,7 @@ def parse_test_instance_string(arg):
 
 
 # Cached test json from disk
-_all_test_info = {}
+_all_test_info = {}  # type: Dict[str, str]
 
 def get_all_test_info():
     """ Open and collate each info.json """