Update submodules & prepare for pypi distribution (#11)

- update submodules
- update docs
- use cibuildwheels to only build aarch64 wheels
- check type hinting with mypy tool
- update stubs
- update CI workflows
- allow for manually triggering the workflows.
- match pylint problems to diff using github annotations.
- trim sdist by manually specifying the MANIFEST
- update scanner.py
- change the version str for non-tagged commits
- build CI upload sdist to testPyPi (only from main)
- remove ninja from build reqs in toml

.github/workflows/build.yml

@@ -1,13 +1,13 @@
 name: Build CI
 
-on: [pull_request, push]
+on: [pull_request, push, workflow_dispatch]
 
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: 3.9
 
@@ -18,40 +18,67 @@ jobs:
           fetch-depth: 0
 
       - name: Install dependencies
+        # cmake ships with the ubuntu-latest runner
         run: |
           sudo apt-get install python3-dev graphviz
-          python3 -m pip install --upgrade pip pylint
+          python3 -m pip install --upgrade pip pylint mypy
           python3 -m pip install -r docs/requirements.txt -r requirements.txt
 
       - name: Build package for docs extraction and linting examples
-        id: wheel
         run: |
-          python3 setup.py bdist_wheel
-          python3 -m pip install $(ls dist/pyrf24-*.whl)
+          python3 setup.py sdist bdist_wheel
+          python3 -m pip install dist/pyrf24-*.whl
+
+      - name: check python typing
+        run: mypy src
 
       - name: check python examples PEP8 compliance
-        run: |
-          pylint examples/*.py --disable=duplicate-code
         # duplicate-code error gets flagged because some examples use similar code snippets
+        run: pylint examples/*.py src/pyrf24/*.py --disable=duplicate-code --output-format=json:pylint.json
+
+      - name: Match pylint problems to diff via github annotations
+        run: python ./.github/workflows/pylint_matcher.py pylint.json
 
       - name: Build docs
         working-directory: docs
         run: sphinx-build -E -W -b html . _build/html
 
       - name: Save built docs as artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "pyRF24_docs"
           path: ${{ github.workspace }}/docs/_build/html
 
       - name: upload to github pages
+        # only publish doc changes from main branch
         if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs/_build/html
 
+      - name: Save distributable wheels as artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: "pyRF24_pkg_dist"
+          path: ${{ github.workspace }}/dist
+
+      - name: Create sdist for testPyPi
+        # Doing this again to ensure a fresh build env.
+        # We don't need to upload a bdist as we can check that locally, but
+        # sdist is used by the piwheels project to create a bdist for RPi OS
+        run: |
+          rm -r dist/ _skbuild
+          python3 setup.py sdist
+
       - name: Validate distribution
         run: |
           python3 -m pip install twine
-          twine check dist/*
+          python3 -m twine check dist/*
+
+      - name: upload to test PyPi
+        if: github.event_name == 'push' && github.repository == 'nRF24/pyRF24' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.test_pypi_token }}
+        run: python3 -m twine upload -r testpypi dist/*

.github/workflows/pylint_matcher.py

@@ -0,0 +1,69 @@
+"""Parse output from clang-tidy's stdout"""
+import argparse
+import pathlib
+import json
+from typing import Dict, Union, List
+import logging
+
+log_commander = logging.getLogger("PylintMatcher")
+log_commander.setLevel(logging.DEBUG)  # be sure that log commands are output
+console_handler = logging.StreamHandler()  # Create special stdout stream handler
+console_handler.setFormatter(logging.Formatter("%(message)s"))  # no formatted log cmds
+log_commander.addHandler(console_handler)  # Use special handler for log_commander
+log_commander.propagate = False  # prevent duplicate messages in the parent logger obj
+
+
+def annotate_pylint_note(obj: Dict[str, Union[int, str]]) -> str:
+    """Translate a 1 notification from pylint to github's checks API.
+
+    :param dict obj: The JSON object output by pylint (for 1 notification).
+        A typical JSON object output from pylint looks like:
+
+        .. code-block:: json
+
+            {
+                "type": "error",
+                "module": "basic_test",
+                "obj": "",
+                "line": 3,
+                "column": 19,
+                "path": "tests/basic_test.py",
+                "symbol": "syntax-error",
+                "message": "invalid syntax (<unknown>, line 3)",
+                "message-id": "E0001"
+            }
+
+    :Returns:
+        A `str` that can be used by github's workflow log commands.
+    """
+    priority = {
+        "convention": "notice",
+        "refactor": "notice",
+        "warning": "warning",
+        "error": "error",
+        "fatal": "error",
+    }
+    return (
+        "::{level} file={path},line={line},title={path}:{line}:{col} {symbol} [{code}]"
+        "::{msg}".format(
+            level=priority[obj["type"]],
+            path=obj["path"],
+            line=obj["line"],
+            col=obj["column"],
+            symbol=obj["symbol"],
+            code=obj["message-id"],
+            msg=obj["message"],
+        )
+    )
+
+
+if __name__ == "__main__":
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument("json_output", type=pathlib.Path)
+    args = arg_parser.parse_args()
+
+    pylint_result: List[Dict[str, Union[int, str]]] = json.loads(
+        pathlib.Path(args.json_output).read_text(encoding="utf-8")
+    )
+    for note in pylint_result:
+        log_commander.info(annotate_pylint_note(note))

.github/workflows/release.yml

@@ -1,20 +1,21 @@
 name: Release Actions
 
 on:
+  workflow_dispatch:
   release:
     types: [published]
 
 jobs:
   upload-pypi:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - name: Set up Python
+        uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Checkout Current Repo
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 0
@@ -24,20 +25,38 @@ jobs:
           sudo apt install python3-dev
           python3 -m pip install --upgrade pip
           python3 -m pip install -r requirements.txt
+          python3 -m pip install twine
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: aarch64
+
+      - name: Build wheels with cibuildwheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_ARCHS_LINUX: aarch64
+          CIBW_SKIP: cp36* pp* cp311*
+
+      - name: Move cross-compiled wheels to dist folder
+        run: |
+          mkdir -p dist
+          mv ./wheelhouse/*.whl ${{ github.workspace }}/dist/
 
       - name: build binary distributable wheel
-        run: python setup.py bdist_wheel
+        # sdist for non-supprted platforms will serve as a stub lib install
+        run: python setup.py sdist
 
       - name: Save distributable wheels as artifacts
         uses: actions/upload-artifact@v2
         with:
           name: "pyRF24_pkg_dist"
           path: ${{ github.workspace }}/dist
 
-      ###### We need a self-hosted runner to build distributable wheels for armv7l
-      # - name: Publish to PyPi
-      #   env:
-      #     TWINE_USERNAME: __token__
-      #     TWINE_PASSWORD: ${{ secrets.pypi_token }}
-      #   run: |
-      #     twine upload dist/*
+      - name: Publish to PyPi
+        # only upload distributions to PyPi when triggered by a published release
+        if: github.event_name == 'release'
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.pypi_token }}
+        run: twine upload dist/*

CMakeLists.txt

@@ -7,9 +7,10 @@ if(SKBUILD)
   message(STATUS "This project is being built using scikit-build & pybind11")
 endif()
 
+include(cmake/using_flags.cmake)
+
 add_subdirectory(pybind11)
 add_subdirectory(RF24/utility) # configure the RF24_DRIVER
-include(cmake/using_flags.cmake)
 
 ################################# RF24 #############################
 

MANIFEST.in

@@ -0,0 +1,20 @@
+recursive-include RF24* *
+recursive-include cmake *
+recursive-include src/pyrf24 *.cpp *.typed
+recursive-include pybind11 *
+recursive-exclude */.github *
+recursive-exclude */docs *
+recursive-exclude */examples* *
+recursive-exclude */images *
+recursive-exclude */tests *
+recursive-exclude */build *
+recursive-exclude */cmake/toolchains *.cmake
+recursive-exclude RF24/utility/AT* *.h *.cpp *.c
+recursive-exclude RF24/utility/Teensy *.h
+recursive-exclude RF24/utility/Template* *.h
+recursive-exclude RF24/utility/rp2 *.h *.cpp *.txt
+recursive-exclude RF24*/pyRF24* *
+recursive-exclude RF24Network/RPi/pyRF24Network *
+global-exclude .clang-format .clang-tidy .git* library.json library.properties Doxyfile keywords.txt Makefile* doxygen-custom.css **.yml **.yaml *.pdf .pylintrc
+exclude */*.md */README* RF24/utility/includes.h RF24/configure
+include pyproject.toml setup.py CMakeLists.txt

README.rst

@@ -63,9 +63,20 @@ To build this python package locally, you need to have cloned this library's rep
 
     git clone --recurse-submodules https://github.com/nRF24/pyRF24.git
     cd pyRF24
-    python -m pip install .
+    python -m pip install . -v
+
+.. hint::
+    For consecutive build attempts, it is recommended to delete any previous build artifacts
+    before attempting to build again.
+
+    .. code-block:: bash
+
+        rm -r _skbuild/ dist/
 
 .. note::
+    The ``-v`` is optional. Here, we use it to show that pip isn't frozen during the
+    build process.
+
     Installing the package can take a long time, and you might think that pip is frozen
     on the step labeled "Building wheel for pyrf24 (pyproject.toml)". Just wait for about
     5 minutes (maybe longer on older/slower variants of Raspberry Pi).
@@ -81,7 +92,7 @@ it is appropriate to pass an additional argument to the install command:
 
 .. code-block:: bash
 
-    python setup.py install -DRF24_DRIVER=RPi
+    python setup.py bdist_wheel -DRF24_DRIVER=RPi
 
 Building a wheel
 -----------------
@@ -110,7 +121,7 @@ same version of CPython, CPU architecture, and C standard lib.
 
        .. code-block:: bash
 
-           rm -r _skbuild/
+           rm -r _skbuild/ dist/
 
 3. To install a built wheel, simply pass the wheel's path and file name to ``pip install``:
 
@@ -120,7 +131,12 @@ same version of CPython, CPU architecture, and C standard lib.
 
    Where the following would be replaced accordingly:
 
-   - ``MAJOR.MINOR.PATCH`` is the current version of the pyrf24 package
+   - ``MAJOR.MINOR.PATCH`` is the current version of the pyrf24 package.
+
+     - If not building a tagged commit, then the version will describe the commit relative to
+       the number of commits since the latest tag. For example, ``0.1.1.post1.dev3`` is
+       the third commit (``dev3``) since the first "post release" (``post1``) after the
+       tagged version ``0.1.1``. This adhere's to `PEP440 <https://peps.python.org/pep-0440>`_.
    - ``cp3X`` is the version of python used to build the wheel (ie ``cp39`` for CPython 3.9)
      The second occurrence of ``cp3X`` describes the CPython ABI compatibility.
    - ``ARCH`` is the architecture type of the CPU. This corresponds to the compiler used.

cmake/using_flags.cmake

@@ -22,11 +22,18 @@ option(MESH_NOMASTER "exclude compiling code that is strictly for master nodes f
 option(MESH_DEBUG "enable/disable debugging output for RF24Mesh lib" OFF)
 option(MESH_DEBUG_MINIMAL "enable/disable minimal debugging output for RF24Mesh lib" OFF)
 
+# Disabling IRQ support should be always done because
+# IRQ support can be handled in python with different libs.
+option(RF24_NO_INTERRUPT "disable IRQ support (dependent on pigpio)" ON)
+# does not affect pigpio driver though
 
 ###############################################
 # function to apply flags to applicable targets
 function(apply_flags target)
     # apply RF24 flags to all targets
+    if(RF24_NO_INTERRUPT)
+        target_compile_definitions(${target} PUBLIC RF24_NO_INTERRUPT)
+    endif()
     if(RF24_DEBUG)
         message(STATUS "RF24_DEBUG asserted for ${target}")
         target_compile_definitions(${target} PUBLIC SERIAL_DEBUG)

docs/index.rst

@@ -29,4 +29,3 @@ Indices and tables
 
 * :ref:`genindex`
 * :ref:`modindex`
-* :ref:`search`