Maintenance: Improve demo documentation and fix minor linting issues (#…

…319)

Also fix #313 

Co-authored-by: Pavel Kirienko <[email protected]>

README.md

@@ -1,7 +1,7 @@
 Full-featured Cyphal stack in Python
 ====================================
 
-[![Build status](https://ci.appveyor.com/api/projects/status/2vv83afj3dxqibi5/branch/master?svg=true)](https://ci.appveyor.com/project/Zubax/pycyphal/branch/master) [![RTFD](https://readthedocs.org/projects/pycyphal/badge/)](https://pycyphal.readthedocs.io/) [![Coverage Status](https://coveralls.io/repos/github/OpenCyphal/pycyphal/badge.svg)](https://coveralls.io/github/OpenCyphal/pycyphal) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=PyCyphal&metric=alert_status)](https://sonarcloud.io/dashboard?id=PyCyphal) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=PyCyphal&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=PyCyphal) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=PyCyphal&metric=ncloc)](https://sonarcloud.io/dashboard?id=PyCyphal) [![PyPI - Version](https://img.shields.io/pypi/v/pycyphal.svg)](https://pypi.org/project/pycyphal/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Forum](https://img.shields.io/discourse/https/forum.opencyphal.org/users.svg)](https://forum.opencyphal.org)
+[![Test and Release PyCyphal](https://github.com/OpenCyphal/pycyphal/actions/workflows/test-and-release.yml/badge.svg)](https://github.com/OpenCyphal/pycyphal/actions/workflows/test-and-release.yml) [![RTFD](https://readthedocs.org/projects/pycyphal/badge/)](https://pycyphal.readthedocs.io/) [![Coverage Status](https://coveralls.io/repos/github/OpenCyphal/pycyphal/badge.svg)](https://coveralls.io/github/OpenCyphal/pycyphal) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=PyCyphal&metric=alert_status)](https://sonarcloud.io/dashboard?id=PyCyphal) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=PyCyphal&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=PyCyphal) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=PyCyphal&metric=ncloc)](https://sonarcloud.io/dashboard?id=PyCyphal) [![PyPI - Version](https://img.shields.io/pypi/v/pycyphal.svg)](https://pypi.org/project/pycyphal/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Forum](https://img.shields.io/discourse/https/forum.opencyphal.org/users.svg)](https://forum.opencyphal.org)
 
 PyCyphal is a full-featured implementation of the Cyphal protocol stack intended for non-embedded, user-facing applications such as GUI software, diagnostic tools, automation scripts, prototypes, and various R&D cases.
 

demo/demo_app.py

@@ -4,10 +4,9 @@
 
 import os
 import sys
-import pathlib
 import asyncio
 import logging
-import pycyphal
+import pycyphal  # Importing PyCyphal will automatically install the import hook for DSDL compilation.
 
 # DSDL files are automatically compiled by pycyphal import hook from sources pointed by CYPHAL_PATH env variable.
 import sirius_cyber_corp  # This is our vendor-specific root namespace. Custom data types.

demo/plant.py

@@ -7,9 +7,9 @@
 
 import time
 import asyncio
-import pycyphal
+import pycyphal  # Importing PyCyphal will automatically install the import hook for DSDL compilation.
 
-# Import DSDL's after pycyphal import hook is installed
+# Import DSDLs after pycyphal import hook is installed.
 import uavcan.si.unit.voltage
 import uavcan.si.sample.temperature
 import uavcan.time

docs/conf.py

@@ -15,7 +15,6 @@
 import pathlib
 import inspect
 import datetime
-import importlib
 import subprocess
 
 
@@ -28,24 +27,13 @@
 APIDOC_GENERATED_ROOT = pathlib.Path("api")
 DOC_ROOT = pathlib.Path(__file__).absolute().parent
 REPOSITORY_ROOT = DOC_ROOT.parent
-
-# The generated files are not documented, but they must be importable to import the target package.
 DSDL_GENERATED_ROOT = REPOSITORY_ROOT / ".compiled"
-PUBLIC_REGULATED_DATA_TYPES_ROOT = REPOSITORY_ROOT / "demo" / "public_regulated_data_types"
-
 sys.path.insert(0, str(REPOSITORY_ROOT))
-sys.path.insert(0, str(DSDL_GENERATED_ROOT))
 
-import pycyphal  # pylint: disable=wrong-import-position
+import pycyphal
 
-try:
-    import pycyphal.application
-except (ImportError, AttributeError) as ex:
-    print("Generating DSDL packages because:", ex)
-    DSDL_GENERATED_ROOT.mkdir(parents=True, exist_ok=True)
-    pycyphal.dsdl.compile(PUBLIC_REGULATED_DATA_TYPES_ROOT / "uavcan", [], DSDL_GENERATED_ROOT)
-    importlib.invalidate_caches()
-    import pycyphal.application
+pycyphal.dsdl.install_import_hook([REPOSITORY_ROOT / "demo" / "public_regulated_data_types"], DSDL_GENERATED_ROOT)
+import pycyphal.application  # This may trigger DSDL compilation.
 
 assert "/site-packages/" not in pycyphal.__file__, "Wrong import source"
 
@@ -212,7 +200,7 @@ def report_exception(exc: Exception) -> None:
     return f"https://github.com/{GITHUB_USER_REPO[0]}/{GITHUB_USER_REPO[1]}/blob/{GIT_HASH}/{path}"
 
 
-for p in map(str, [DSDL_GENERATED_ROOT, REPOSITORY_ROOT]):
+for p in map(str, [REPOSITORY_ROOT]):
     if os.environ.get("PYTHONPATH"):
         os.environ["PYTHONPATH"] += os.path.pathsep + p
     else:

docs/pages/architecture.rst

@@ -239,6 +239,7 @@ The default import hook can be disabled by setting the ``PYCYPHAL_NO_IMPORT_HOOK
 The main API entries are:
 
 - :func:`pycyphal.dsdl.compile` --- transcompiles a DSDL namespace into a Python package.
+  Normally, one should rely on the import hook instead of invoking this directly.
 
 - :func:`pycyphal.dsdl.serialize` and :func:`pycyphal.dsdl.deserialize` --- serialize and deserialize
   an instance of an autogenerated class.
@@ -269,10 +270,9 @@ Submodule :mod:`pycyphal.application` provides the top-level API for the applica
 standard application-layer functions defined by the Cyphal Specification (chapter 5 *Application layer*).
 The **main entry point of the library** is :func:`pycyphal.application.make_node`.
 
-This submodule requires the standard DSDL namespace ``uavcan`` to be compiled first (see :func:`pycyphal.dsdl.compile`),
-so it is not auto-imported.
+This submodule requires the standard DSDL namespace ``uavcan`` to be compiled, so it is not auto-imported.
 A typical usage scenario is to either distribute compiled DSDL namespaces together with the application,
-or to generate them lazily before importing this submodule.
+or to generate them lazily relying on the import hook.
 
 Chapter :ref:`demo` contains a complete usage example.
 

docs/pages/demo.rst

@@ -22,7 +22,8 @@ The document is arranged as follows:
 You are expected to be familiar with terms like *Cyphal node*, *DSDL*, *subject-ID*, *RPC-service*.
 If not, skim through the `Cyphal Guide <https://opencyphal.org/guide>`_ first.
 
-If you want to follow along, :ref:`install PyCyphal <installation>` and switch to a new directory before continuing.
+If you want to follow along, :ref:`install PyCyphal <installation>` and
+switch to a new directory (``~/pycyphal-demo``) before continuing.
 
 
 DSDL definitions
@@ -62,17 +63,19 @@ For the sake of clarity, move the custom DSDL root namespace directory ``sirius_
 that we created above into ``custom_data_types/``.
 You should end up with the following directory structure::
 
-    custom_data_types/
-        sirius_cyber_corp/                          # Created in the previous section
-            PerformLinearLeastSquaresFit.1.0.dsdl
-            PointXY.1.0.dsdl
-    public_regulated_data_types/                    # Clone from git
-        uavcan/                                     # The standard DSDL namespace
+    pycyphal-demo/
+        custom_data_types/
+            sirius_cyber_corp/                          # Created in the previous section
+                PerformLinearLeastSquaresFit.1.0.dsdl
+                PointXY.1.0.dsdl
+        public_regulated_data_types/                    # Clone from git
+            uavcan/                                     # The standard DSDL namespace
+                ...
             ...
-        ...
-    demo_app.py                                     # The thermostat node script
+        demo_app.py                                     # The thermostat node script
 
-``CYPHAL_PATH`` should contain a list to all the paths where the DSDL root namespace directories are to be found
+The ``CYPHAL_PATH`` environment variable should contain the list of paths where the
+DSDL root namespace directories are to be found
 (be sure to modify the values to match your environment):
 
 ..  code-block:: sh
@@ -225,15 +228,12 @@ You will need to open a couple of new terminal sessions now.
 
 If you don't have Yakut installed on your system yet, install it now by following its documentation.
 
-Yakut requires us to compile our DSDL namespaces beforehand using ``yakut compile``:
+Yakut also needs to know where the DSDL files are located, this is specified via the same ``CYPHAL_PATH``
+environment variable (this is a standard variable that many Cyphal tools rely on):
 
 .. code-block:: sh
 
-    yakut compile  custom_data_types/sirius_cyber_corp  public_regulated_data_types/uavcan
-
-The outputs will be stored in the current working directory.
-If you decided to change the working directory or move the compilation outputs,
-make sure to export the ``YAKUT_PATH`` environment variable pointing to the correct location.
+    export CYPHAL_PATH="$HOME/pycyphal-demo/custom_data_types:$HOME/pycyphal-demo/public_regulated_data_types"
 
 The commands shown later need to operate on the same network as the demo.
 Earlier we configured the demo to use Cyphal/UDP via the localhost interface.
@@ -248,6 +248,7 @@ launch the following in a new terminal and leave it running (``y`` is a convenie
 
 ..  code-block:: sh
 
+    export CYPHAL_PATH="$HOME/pycyphal-demo/custom_data_types:$HOME/pycyphal-demo/public_regulated_data_types"
     export UAVCAN__UDP__IFACE=127.0.0.1
     y sub --with-metadata uavcan.node.heartbeat uavcan.diagnostic.record    # You should see heartbeats
 
@@ -256,13 +257,15 @@ Launch another subscriber to see the published voltage command (it is not going
 
 ..  code-block:: sh
 
+    export CYPHAL_PATH="$HOME/pycyphal-demo/custom_data_types:$HOME/pycyphal-demo/public_regulated_data_types"
     export UAVCAN__UDP__IFACE=127.0.0.1
     y sub 2347:uavcan.si.unit.voltage.scalar --redraw       # Prints nothing.
 
 And publish the setpoint along with the measurement (process variable):
 
 ..  code-block:: sh
 
+    export CYPHAL_PATH="$HOME/pycyphal-demo/custom_data_types:$HOME/pycyphal-demo/public_regulated_data_types"
     export UAVCAN__UDP__IFACE=127.0.0.1
     export UAVCAN__NODE__ID=111         # We need a node-ID to publish messages properly
     y pub --count=10 2345:uavcan.si.unit.temperature.scalar   250 \
@@ -405,16 +408,10 @@ that allows one to define process groups and conveniently manage them as a singl
 The language comes with a user-friendly syntax for managing Cyphal registers.
 Those familiar with ROS may find it somewhat similar to *roslaunch*.
 
-The following orchestration file (orc-file) ``launch.orc.yaml`` does this:
-
-- Compiles two DSDL namespaces: the standard ``uavcan`` and the custom ``sirius_cyber_corp``.
-  If they are already compiled, this step is skipped.
-
-- When compilation is done, the two applications are launched.
-  Be sure to stop the first script if it is still running!
-
-- Aside from the applications, a couple of diagnostic processes are started as well.
-  A setpoint publisher will command the thermostat to drive the plant to the specified temperature.
+The following orchestration file (orc-file) ``launch.orc.yaml`` launches the two applications
+(be sure to stop the first script if it is still running!)
+along with a couple of diagnostic processes that monitor the network.
+A setpoint publisher that will command the thermostat to drive the plant to the specified temperature is also started.
 
 The orchestrator runs everything concurrently, but *join statements* are used to enforce sequential execution as needed.
 The first process to fail (that is, exit with a non-zero code) will bring down the entire *composition*.

pycyphal/_version.py

@@ -1 +1 @@
-__version__ = "1.17.1"
+__version__ = "1.17.2"

pycyphal/dsdl/_import_hook.py

@@ -5,7 +5,7 @@
 import sys
 import os
 from types import ModuleType
-from typing import Iterable, Optional, Sequence, Union
+from typing import Iterable, Optional, Sequence, Union, List
 import pathlib
 import keyword
 import re
@@ -25,7 +25,7 @@
 
 def root_namespace_from_module_name(module_name: str) -> str:
     """
-    Tranlates python module name to DSDL root namespace.
+    Translates python module name to DSDL root namespace.
     This handles special case where root namespace is a python keyword by removing trailing underscore.
     """
     if module_name.endswith("_") and keyword.iskeyword(module_name[-1]):
@@ -48,7 +48,7 @@ def __init__(
         self.lookup_directories = list(map(str, lookup_directories))
         self.output_directory = output_directory
         self.allow_unregulated_fixed_port_id = allow_unregulated_fixed_port_id
-        self.root_namespace_directories: Sequence[pathlib.Path] = []
+        self.root_namespace_directories: List[pathlib.Path] = []
 
         # Build a list of root namespace directories from lookup directories.
         # Any dir inside any of the lookup directories is considered a root namespace if it matches regex

pycyphal/dsdl/_support_wrappers.py

@@ -6,7 +6,7 @@
 autogenerated code unless explicitly requested by the application.
 """
 
-from typing import TypeVar, Type, Sequence, cast, Any, Iterable, Optional, Dict
+from typing import TypeVar, Type, Sequence, Any, Iterable, Optional, Dict
 import pydsdl
 
 

pycyphal/transport/can/media/socketcan/_socketcan.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2019 OpenCyphal
 # This software is distributed under the terms of the MIT License.
 # Author: Pavel Kirienko <[email protected]>
+# pylint: disable=duplicate-code
 
 import enum
 import time

pycyphal/transport/can/media/socketcand/_socketcand.py

@@ -157,7 +157,8 @@ def _transmit_thread_worker(self) -> None:
                     tx.loop.call_soon_threadsafe(partial(tx.future.set_exception, ex))
         except Exception as ex:
             _logger.critical(
-                "Unhandled exception in transmit thread, transmission thread stopped and transmission is no longer possible: %s",
+                "Unhandled exception in transmit thread, "
+                "transmission thread stopped and transmission is no longer possible: %s",
                 ex,
                 exc_info=True,
             )

tests/transport/can/media/_socketcand.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2023 OpenCyphal
 # This software is distributed under the terms of the MIT License.
+# pylint: disable=protected-access,duplicate-code
 
 import sys
 import typing
@@ -9,7 +10,7 @@
 import pytest
 
 from pycyphal.transport import Timestamp
-from pycyphal.transport.can.media import Envelope, DataFrame, FrameFormat, FilterConfiguration
+from pycyphal.transport.can.media import Envelope, DataFrame, FrameFormat
 from pycyphal.transport.can.media.socketcand import SocketcandMedia
 
 if sys.platform != "linux":  # pragma: no cover
@@ -23,7 +24,7 @@ def _start_socketcand() -> typing.Generator[None, None, None]:
     # starting a socketcand daemon in background
     cmd = ["socketcand", "-i", "vcan0", "-l", "lo", "-p", "29536"]
 
-    socketcand = subprocess.Popen(
+    socketcand = subprocess.Popen(  # pylint: disable=consider-using-with
         cmd,
         encoding="utf8",
         stdout=subprocess.PIPE,

tests/transport/redundant/_session_input.py

@@ -2,7 +2,6 @@
 # This software is distributed under the terms of the MIT License.
 # Author: Pavel Kirienko <[email protected]>
 
-import time
 import asyncio
 import pytest
 import pycyphal
@@ -96,7 +95,7 @@ async def add_inferior(inferior: pycyphal.transport.InputSession) -> None:
     assert inf_b.transfer_id_timeout == pytest.approx(1.1)
 
     # Redundant reception - new transfers accepted because the iface switch timeout is exceeded.
-    time.sleep(ses.transfer_id_timeout)  # Just to make sure that it is REALLY exceeded.
+    await asyncio.sleep(ses.transfer_id_timeout)  # Just to make sure that it is REALLY exceeded.
     assert await tx_b.send(
         Transfer(
             timestamp=Timestamp.now(),
@@ -132,7 +131,7 @@ async def add_inferior(inferior: pycyphal.transport.InputSession) -> None:
     assert tr.fragmented_payload == [memoryview(b"ghi")]
     assert tr.inferior_session == inf_b
 
-    assert None is await ses.receive(asyncio.get_running_loop().time() + 1.0)  # Nothing left to read now.
+    assert None is await ses.receive(asyncio.get_running_loop().time() + 0.1)  # Nothing left to read now.
 
     # This one will be rejected because wrong iface and the switch timeout is not yet exceeded.
     assert await tx_a.send(
@@ -144,7 +143,7 @@ async def add_inferior(inferior: pycyphal.transport.InputSession) -> None:
         ),
         asyncio.get_running_loop().time() + 1.0,
     )
-    assert None is await ses.receive(asyncio.get_running_loop().time() + 1.0)
+    assert None is await ses.receive(asyncio.get_running_loop().time() + 0.1)
 
     # Transfer-ID timeout reconfiguration.
     ses.transfer_id_timeout = 3.0