Merge pull request #229 from ska-sa/pre-commit

Apply black, isort, pyupgrade, pre-commit

.github/workflows/test.yml

@@ -111,7 +111,8 @@ jobs:
       - run: pip install -v .
       - name: Run tests
         run: ./.ci/py-tests.sh
-      - run: flake8
+      # Skip pip-compile because not all configurations have a python3.8 binary
+      - run: SKIP=pip-compile pre-commit run --all-files
       - name: Show ccache stats
         run: ccache -s
 

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort
+  - repo: https://github.com/psf/black
+    rev: 23.7.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: check-executables-have-shebangs
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: trailing-whitespace
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.4.1
+    hooks:
+      - id: mypy
+        # Passing filenames to mypy can do odd things. See
+        # https://github.com/pre-commit/mirrors-mypy/issues/33.
+        # mypy.ini determines the set of files that will actually be checked.
+        pass_filenames: false
+        # The pre-commit hook passes some options, but we set options in mypy.ini.
+        args: []
+        # The pre-commit hook only has python, not pyi.
+        types: []
+        types_or: [python, pyi]
+        additional_dependencies: [
+          'numpy==1.24.4'
+        ]
+  - repo: https://github.com/jazzband/pip-tools
+    rev: 7.1.0
+    hooks:
+      - id: pip-compile
+        args: [requirements.in]
+        language_version: python3.8
+        files: '^requirements\.(?:in|txt)$'

doc/changelog.rst

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+.. rubric:: Development version
+
+- Reformat the Python codebase using black_ and isort_.
+- Add `pre-commit`_ configuration.
+
+.. _black: https://black.readthedocs.io/en/stable/
+.. _isort: https://pycqa.github.io/isort/
+.. _pre-commit: https://pre-commit.com/
+
 .. rubric:: 3.12.0
 
 - Add support for :doc:`recv-chunk-group` to assemble chunks in parallel.

doc/conf.py

@@ -19,32 +19,34 @@
 #
 # import os
 # import sys
-sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath(".."))
 
 # Build doxygen first on readthedocs
-rtd = os.environ.get('READTHEDOCS') == 'True'
+rtd = os.environ.get("READTHEDOCS") == "True"
 if rtd:
-    subprocess.check_call(['doxygen'])
+    subprocess.check_call(["doxygen"])
     # ReadTheDocs can't build the extension (no Boost), so we have to mock it
     # See http://docs.readthedocs.io/en/latest/faq.html#i-get-import-errors-on-libraries-that-depend-on-c-modules
-    autodoc_mock_imports = ['spead2._spead2']
+    autodoc_mock_imports = ["spead2._spead2"]
 
 # -- Project information -----------------------------------------------------
 
-project = 'spead2'
-copyright = '2015–2021, National Research Foundation (SARAO)'
-author = 'National Research Foundation (SARAO)'
+project = "spead2"
+copyright = "2015–2021, National Research Foundation (SARAO)"
+author = "National Research Foundation (SARAO)"
+
 
 def get_version():
     globals_ = {}
     root = os.path.dirname(os.path.dirname(__file__))
-    with open(os.path.join(root, 'src', 'spead2', '_version.py')) as f:
+    with open(os.path.join(root, "src", "spead2", "_version.py")) as f:
         code = f.read()
     exec(code, globals_)
-    release = globals_['__version__']
-    match = re.match('^(\d+)\.(\d+)', release)
+    release = globals_["__version__"]
+    match = re.match(r"^(\d+)\.(\d+)", release)
     return match.group(0), release
 
+
 version, release = get_version()
 
 # -- General configuration ---------------------------------------------------
@@ -53,40 +55,40 @@ def get_version():
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.napoleon',
-    'sphinx_rtd_theme',
-    'breathe'
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.napoleon",
+    "sphinx_rtd_theme",
+    "breathe",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-breathe_projects = {'spead2': './doxygen/xml'}
-breathe_default_project = 'spead2'
+breathe_projects = {"spead2": "./doxygen/xml"}
+breathe_default_project = "spead2"
 
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/3/', None),
-    'numpy': ('https://numpy.org/doc/stable/', None),
-    'scipy': ('https://docs.scipy.org/doc/scipy/', None),
-    'numba': ('https://numba.readthedocs.io/en/latest/', None)
+    "python": ("https://docs.python.org/3/", None),
+    "numpy": ("https://numpy.org/doc/stable/", None),
+    "scipy": ("https://docs.scipy.org/doc/scipy/", None),
+    "numba": ("https://numba.readthedocs.io/en/latest/", None),
 }
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]

doc/license.rst

@@ -13,4 +13,3 @@ details.
 
 You should have received a copy of the GNU Lesser General Public License
 along with this program.  If not, see http://www.gnu.org/licenses/.
-

examples/recv_chunk_group_example.py

@@ -19,14 +19,14 @@
 # ringbuffers. To test it, run
 # spead2_send localhost:8888 localhost:8889 --heaps 1000 --heap-size 65536 --rate 10
 
-from spead2.numba import intp_to_voidptr
-import spead2.recv
-from spead2.recv.numba import chunk_place_data
-
 import numba
-from numba import types
 import numpy as np
 import scipy
+from numba import types
+
+import spead2.recv
+from spead2.numba import intp_to_voidptr
+from spead2.recv.numba import chunk_place_data
 
 HEAP_PAYLOAD_SIZE = 65536
 HEAPS_PER_CHUNK = 64
@@ -49,39 +49,30 @@ def chunk_place(data_ptr, data_size):
 def main():
     NUM_STREAMS = 2
     MAX_CHUNKS = 4
-    place_callback = scipy.LowLevelCallable(
-        chunk_place.ctypes,
-        signature='void (void *, size_t)'
-    )
+    place_callback = scipy.LowLevelCallable(chunk_place.ctypes, signature="void (void *, size_t)")
     chunk_config = spead2.recv.ChunkStreamConfig(
         items=[spead2.HEAP_CNT_ID, spead2.HEAP_LENGTH_ID],
         max_chunks=MAX_CHUNKS,
-        place=place_callback)
+        place=place_callback,
+    )
     group_config = spead2.recv.ChunkStreamGroupConfig(max_chunks=MAX_CHUNKS)
     data_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS)
     free_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS)
     group = spead2.recv.ChunkStreamRingGroup(group_config, data_ring, free_ring)
     for _ in range(NUM_STREAMS):
-        group.emplace_back(
-            spead2.ThreadPool(),
-            spead2.recv.StreamConfig(),
-            chunk_config
-        )
+        group.emplace_back(spead2.ThreadPool(), spead2.recv.StreamConfig(), chunk_config)
     for _ in range(MAX_CHUNKS):
         chunk = spead2.recv.Chunk(
-            present=np.empty(HEAPS_PER_CHUNK, np.uint8),
-            data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8)
+            present=np.empty(HEAPS_PER_CHUNK, np.uint8), data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8)
         )
         group.add_free_chunk(chunk)
     for i in range(NUM_STREAMS):
-        group[i].add_udp_reader(8888 + i, buffer_size=1024 * 1024, bind_hostname='127.0.0.1')
+        group[i].add_udp_reader(8888 + i, buffer_size=1024 * 1024, bind_hostname="127.0.0.1")
     for chunk in data_ring:
         n_present = np.sum(chunk.present)
-        print(
-            f"Received chunk {chunk.chunk_id} with "
-            f"{n_present} / {HEAPS_PER_CHUNK} heaps")
+        print(f"Received chunk {chunk.chunk_id} with {n_present} / {HEAPS_PER_CHUNK} heaps")
         group.add_free_chunk(chunk)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

examples/recv_chunk_ring_example.py

@@ -19,14 +19,14 @@
 # test it, run
 # spead2_send localhost:8888 --heaps 1000 --heap-size 65536 --rate 10
 
-from spead2.numba import intp_to_voidptr
-import spead2.recv
-from spead2.recv.numba import chunk_place_data
-
 import numba
-from numba import types
 import numpy as np
 import scipy
+from numba import types
+
+import spead2.recv
+from spead2.numba import intp_to_voidptr
+from spead2.recv.numba import chunk_place_data
 
 HEAP_PAYLOAD_SIZE = 65536
 HEAPS_PER_CHUNK = 64
@@ -48,36 +48,28 @@ def chunk_place(data_ptr, data_size):
 
 def main():
     MAX_CHUNKS = 4
-    place_callback = scipy.LowLevelCallable(
-        chunk_place.ctypes,
-        signature='void (void *, size_t)'
-    )
+    place_callback = scipy.LowLevelCallable(chunk_place.ctypes, signature="void (void *, size_t)")
     chunk_config = spead2.recv.ChunkStreamConfig(
         items=[spead2.HEAP_CNT_ID, spead2.HEAP_LENGTH_ID],
         max_chunks=MAX_CHUNKS,
-        place=place_callback)
+        place=place_callback,
+    )
     data_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS)
     free_ring = spead2.recv.ChunkRingbuffer(MAX_CHUNKS)
     stream = spead2.recv.ChunkRingStream(
-        spead2.ThreadPool(),
-        spead2.recv.StreamConfig(),
-        chunk_config,
-        data_ring,
-        free_ring)
+        spead2.ThreadPool(), spead2.recv.StreamConfig(), chunk_config, data_ring, free_ring
+    )
     for i in range(MAX_CHUNKS):
         chunk = spead2.recv.Chunk(
-            present=np.empty(HEAPS_PER_CHUNK, np.uint8),
-            data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8)
+            present=np.empty(HEAPS_PER_CHUNK, np.uint8), data=np.empty(CHUNK_PAYLOAD_SIZE, np.uint8)
         )
         stream.add_free_chunk(chunk)
-    stream.add_udp_reader(8888, buffer_size=1024 * 1024, bind_hostname='127.0.0.1')
+    stream.add_udp_reader(8888, buffer_size=1024 * 1024, bind_hostname="127.0.0.1")
     for chunk in data_ring:
         n_present = np.sum(chunk.present)
-        print(
-            f"Received chunk {chunk.chunk_id} with "
-            f"{n_present} / {HEAPS_PER_CHUNK} heaps")
+        print(f"Received chunk {chunk.chunk_id} with {n_present} / {HEAPS_PER_CHUNK} heaps")
         stream.add_free_chunk(chunk)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

examples/test_recv.py

@@ -15,20 +15,21 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+
 import spead2
 import spead2.recv
-import logging
 
 logging.basicConfig(level=logging.INFO)
 
 thread_pool = spead2.ThreadPool()
 stream = spead2.recv.Stream(
     thread_pool,
-    spead2.recv.StreamConfig(memory_allocator=spead2.MemoryPool(16384, 26214400, 12, 8))
+    spead2.recv.StreamConfig(memory_allocator=spead2.MemoryPool(16384, 26214400, 12, 8)),
 )
 del thread_pool
 if 0:
-    with open('junkspeadfile', 'rb') as f:
+    with open("junkspeadfile", "rb") as f:
         text = f.read()
     stream.add_buffer_reader(text)
 else:

examples/test_send.py

@@ -15,21 +15,24 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import spead2
-import spead2.send
 import logging
+
 import numpy as np
 
+import spead2
+import spead2.send
+
 logging.basicConfig(level=logging.INFO)
 
 thread_pool = spead2.ThreadPool()
 stream = spead2.send.UdpStream(
-    thread_pool, [("127.0.0.1", 8888)], spead2.send.StreamConfig(rate=1e7))
+    thread_pool, [("127.0.0.1", 8888)], spead2.send.StreamConfig(rate=1e7)
+)
 del thread_pool
 
 shape = (40, 50)
 ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48, 0))
-item = ig.add_item(0x1234, 'foo', 'a foo item', shape=shape, dtype=np.int32)
+item = ig.add_item(0x1234, "foo", "a foo item", shape=shape, dtype=np.int32)
 item.value = np.zeros(shape, np.int32)
 stream.send_heap(ig.get_heap())
 stream.send_heap(ig.get_end())