From b6bb14d70c14d985d28645360d8c44fef2b206eb Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 5 Jan 2024 12:00:00 +0100 Subject: [PATCH] Add Python3 pre-commit config checks with pylint and mypy Signed-off-by: Bernhard Kaindl --- .github/workflows/build.yml | 49 +++++++++++++++++ .pre-commit-config.yaml | 105 ++++++++++++++++++++++++++++++++++++ .pylintrc | 18 ++++++- build/mkcfg.py | 2 +- build/mkinfo.py | 35 +++++++++++- include/xen/sysctl.h | 0 xtf-runner | 14 ++--- 7 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 .pre-commit-config.yaml mode change 100644 => 100755 build/mkcfg.py mode change 100644 => 100755 build/mkinfo.py mode change 100755 => 100644 include/xen/sysctl.h diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fac842d5..d2fac6ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,41 @@ name: build on: [push, pull_request] jobs: + python-tests: + name: "Python Tests" + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: pre-commit checks - setup cache + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: pre-commit checks - run checks + uses: pre-commit/action@v3.0.0 + env: + # For merging PRs to master, skip the no-commit-to-branch check: + SKIP: no-commit-to-branch + + - name: Install Python2 dependencies + run: | + #: Install Python 2.7 from Ubuntu 20.04 using apt-get install + sudo apt-get update && sudo apt-get install -y python2 + curl -sSL https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py + python2 get-pip.py + if [ -f requirements.txt ]; then pip2 install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip2 install -r requirements-dev.txt; fi + pip2 install pytest pylint==1.9.4 + + - name: Run pylint-1.9.4 + run: python2 -m pylint xtf-runner build/*.py + + - name: Run python2 -m pytest to execute all unit and integration tests + run: python2 -m pytest -v -rA + build: strategy: @@ -46,6 +81,20 @@ jobs: - uses: actions/checkout@v3 + - name: pre-commit checks - setup cache + uses: actions/cache@v3 + if: ${{ matrix.compiler == 'gcc-13' }} + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: pre-commit checks - run checks + uses: pre-commit/action@v3.0.0 + if: ${{ matrix.compiler == 'gcc-13' }} + env: + # For merging PRs to master, skip the no-commit-to-branch check: + SKIP: no-commit-to-branch + - name: Build run: | # Select appropriate LLVM= or CC= diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c44e15aa --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,105 @@ +# +# This is the configuration file of the pre-commit framework for this repository: +# https://pypi.org/project/pre-commit +# +# pre-commit runs in the GitHub Workflow of this project on each push and PR. +# Additionally, you can run it locally for faster fixing of issues using: +# $ pip3 install pre-commit +# +# On the initial run, pre-commit downloads its checkers, subsequent runs are fast: +# +# $ pre-commit run # automatically checks which checks are needed. +# $ pre-commit run -a # runs all fixes and checks, even when not needed +# +# When this works, you can enable it as the pre-commit hook for this git clone: +# $ pre-commit install +# +# Then, the fixups and checks defined below run by default when you commit: +# +# To commit with all pre-commit hooks skipped (for exceptional cases): +# $ git commit --no-verify +# +# To commit with some checks skipped: +# $ SKIP=mypy,pylint git commit -m "unchecked emergency commit" +# +# For more customisation, see https://pre-commit.com/#temporarily-disabling-hooks +# and https://pre-commit.com/#confining-hooks-to-run-at-certain-stages (e.g push) +# All hooks: https://pre-commit.com/hooks.html +# +fail_fast: false +default_stages: [commit, push] +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + # https://github.com/pre-commit/pre-commit-hooks/blob/main/README.md: + hooks: + - id: no-commit-to-branch + name: "ensure that you don't commit to the local master branch" + args: [--branch, master] + always_run: true + - id: trailing-whitespace + name: 'check and fix files to have no trailing whitespace' + exclude: install-sh + - id: end-of-file-fixer + name: 'check and fix that files have a trailing newline' + exclude: | + (?x)^( + arch/x86/include/arch/msr-index.h + ) + - id: mixed-line-ending + args: ['--fix=lf'] + name: 'check and fix that line endings are line feeds' + - id: check-added-large-files + args: ['--maxkb=12'] + name: 'check that no large files are added' + - id: check-executables-have-shebangs + - id: debug-statements + name: 'check for debugger imports and breakpoint calls' + - id: check-shebang-scripts-are-executable + exclude: XSConsoleImporter.py + - id: check-merge-conflict + - id: check-yaml + name: 'check the syntax of yaml files' +- repo: https://github.com/PyCQA/autoflake + rev: v2.2.1 + hooks: + - id: autoflake + name: remove unused variables and imports + args: [ + "--in-place", + "--remove-unused-variables", + "--remove-all-unused-imports" + ] + language: python + files: \.py$ +- repo: https://github.com/akaihola/darker + rev: 1.7.2 + hooks: + - id: darker + name: format staged changes like black and isort would format them + args: [--skip-string-normalization, --isort, -tpy36] + additional_dependencies: + - isort +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.7.0 + hooks: + - id: mypy + name: run mypy to check e.g. that all expected arguments are passed to functions etc + additional_dependencies: [types-simplejson] +- repo: https://github.com/pycqa/pylint + rev: v3.0.2 + hooks: + - id: pylint + name: run pylint + args: [ + --jobs=2, + ] + log_file: ".git/pre-commit-pylint.log" +- repo: local + hooks: + - id: git-diff # For reference: https://github.com/pre-commit/pre-commit/issues/1712 + name: When pre-commit fixers made changes, run 'git diff' to show the changes + entry: git diff --exit-code # When changes were made, the exit code is needed to show them + language: system + pass_filenames: false + always_run: true diff --git a/.pylintrc b/.pylintrc index 7c7e50c4..47a0d8ff 100644 --- a/.pylintrc +++ b/.pylintrc @@ -48,7 +48,21 @@ extension-pkg-whitelist= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=bad-whitespace, bad-continuation, global-statement, star-args +disable=bad-whitespace, bad-continuation, global-statement, star-args, + # Not real problems, returns on the same indentation level can be easier: + consider-using-with, + no-else-raise, + no-else-return, + multiple-imports, + len-as-condition, + # For Python3-only projects: + consider-using-f-string, + deprecated-module, + unrecognized-option, + unspecified-encoding, + use-implicit-booleaness-not-len, + useless-object-inheritance, + useless-option-value, [REPORTS] @@ -337,4 +351,4 @@ max-public-methods=20 # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/build/mkcfg.py b/build/mkcfg.py old mode 100644 new mode 100755 index 7f8e3d52..631992be --- a/build/mkcfg.py +++ b/build/mkcfg.py @@ -9,7 +9,7 @@ import sys, os # Usage: mkcfg.py $OUT $DEFAULT-CFG $EXTRA-CFG $VARY-CFG -_, out, defcfg, vcpus, extracfg, varycfg = sys.argv +_, out, defcfg, vcpus, extracfg, varycfg = sys.argv # pylint: disable=unbalanced-tuple-unpacking # Evaluate environment and name from $OUT _, env, name = out.split('.')[0].split('-', 2) diff --git a/build/mkinfo.py b/build/mkinfo.py old mode 100644 new mode 100755 index 50819e2c..8df3970d --- a/build/mkinfo.py +++ b/build/mkinfo.py @@ -1,10 +1,43 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +mkcfg.py - Generate a configuration JSON file based on provided parameters. + +Usage: + python mkcfg.py $OUT $NAME $CATEGORY $ENVS $VARIATIONS + +Arguments: + + $OUT: Path to the output file where the generated JSON configuration + will be saved. + $NAME: Name to be assigned in the configuration. + $CATEGORY: Category designation in the configuration. + $ENVS: Optional space-separated list of environments (can be empty). + $VARIATIONS: Optional space-separated list of variations (can be empty). + +Description: + + This script generates a JSON configuration file using provided parameters + and saves it to the specified output file. The generated JSON structure + includes fields for 'name', 'category', 'environments', and 'variations'. + The 'environments' and 'variations' fields can be populated with + space-separated lists if corresponding arguments ($ENVS and $VARIATIONS) + are provided. + +Example: + + python mkcfg.py config.json ExampleConfig Utilities prod dev test + + This example will create a configuration file named 'config.json' with + 'name' as 'ExampleConfig', + 'category' as 'Utilities', and + 'environments' as ['prod', 'dev', 'test']. +""" import sys, json # Usage: mkcfg.py $OUT $NAME $CATEGORY $ENVS $VARIATIONS -_, out, name, cat, envs, variations = sys.argv +_, out, name, cat, envs, variations = sys.argv # pylint: disable=unbalanced-tuple-unpacking template = { "name": name, diff --git a/include/xen/sysctl.h b/include/xen/sysctl.h old mode 100755 new mode 100644 diff --git a/xtf-runner b/xtf-runner index 94ed1764..1a9d5171 100755 --- a/xtf-runner +++ b/xtf-runner @@ -13,15 +13,16 @@ import sys, os, os.path as path from optparse import OptionParser from subprocess import Popen, PIPE, call as subproc_call +from typing import Dict # pylint: disable=unused-import try: import json except ImportError: - import simplejson as json + import simplejson as json # type:ignore[no-redef] # Python 2/3 compatibility if sys.version_info >= (3, ): - basestring = str + basestring = str # pylint: disable=invalid-name,redefined-builtin # All results of a test, keep in sync with C code report.h. # Notes: @@ -80,8 +81,7 @@ class TestInstance(object): def __repr__(self): if not self.variation: return "test-{0}-{1}".format(self.env, self.name) - else: - return "test-{0}-{1}~{2}".format(self.env, self.name, self.variation) + return "test-{0}-{1}~{2}".format(self.env, self.name, self.variation) def __hash__(self): return hash(repr(self)) @@ -92,8 +92,8 @@ class TestInstance(object): def __ne__(self, other): return repr(self) != repr(other) - def __cmp__(self, other): - return cmp(repr(self), repr(other)) + def __cmp__(self, other): # Not used by Python3 + return cmp(repr(self), repr(other)) # pylint: disable=undefined-variable class TestInfo(object): @@ -235,7 +235,7 @@ def parse_test_instance_string(arg): # Cached test json from disk -_all_test_info = {} +_all_test_info = {} # type: Dict[str, str] def get_all_test_info(): """ Open and collate each info.json """