2024 maintenance (#814)

* Fix issue #504, #795, #803.

* Support python 3.13 and adopt python 3.12 everywhere in the CI except for unit test which are ran on all supported python versions and platforms.

.github/workflows/pydoctor_primer.yaml

@@ -28,7 +28,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install pydoctor_primer
         run: |
           python -m pip install -U pip

.github/workflows/system.yaml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up CPython
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
 
     - name: Install tox
       run: |

.github/workflows/unit.yaml

@@ -19,14 +19,9 @@ jobs:
 
     strategy:
       matrix:
-        # Re-enable 3.13-dev when https://github.com/zopefoundation/zope.interface/issues/292 is fixed
-        python-version: [pypy-3.8, 3.8, 3.9, '3.10', 3.11, '3.12']
-        os: [ubuntu-22.04]
-        include:
-          - os: windows-latest
-            python-version: 3.11
-          - os: macos-latest
-            python-version: 3.11
+        python-version: ['pypy-3.8', 'pypy-3.9', 'pypy-3.10', 
+                         '3.8', '3.9', '3.10', '3.11', '3.12', '3.13.0-rc.2']
+        os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:
     - uses: actions/checkout@v4
@@ -51,8 +46,7 @@ jobs:
       run: |
         tox -e test
 
-    - name: Run unit tests with latest Twisted version (only for python 3.8 and later)
-      if: matrix.python-version != '3.7' && matrix.python-version != 'pypy-3.7'
+    - name: Run unit tests with latest Twisted version
       run: |
         tox -e test-latest-twisted
 

README.rst

@@ -76,7 +76,11 @@ What's New?
 in development
 ^^^^^^^^^^^^^^
 
+* Drop Python 3.7 and support Python 3.13.
 * Trigger a warning when several docstrings are detected for the same object.
+* Improve typing of docutils related code.
+* Run unit tests on all supported combinations of Python versions and platforms, including PyPy for Windows. Previously, tests where ran on all supported Python version for Linux, but not for MacOS and Windows.
+* Replace the deprecated dependency appdirs with platformdirs.
 * Fix WinError caused by the failure of the symlink creation process.
   Pydoctor should now run on windows without the need to be administrator.
 

pydoctor/epydoc/docutils.py

@@ -3,7 +3,7 @@
 """
 from __future__ import annotations
 
-from typing import Iterable, Iterator, Optional
+from typing import Iterable, Iterator, Optional, TypeVar, cast
 
 import optparse
 
@@ -25,7 +25,7 @@ def new_document(source_path: str, settings: Optional[optparse.Values] = None) -
     # the default settings. Otherwise we let new_document figure it out.
     if settings is None and docutils_version_info >= (0,19):
         if _DEFAULT_DOCUTILS_SETTINGS is None:
-            _DEFAULT_DOCUTILS_SETTINGS = frontend.get_default_settings() # type:ignore[attr-defined]
+            _DEFAULT_DOCUTILS_SETTINGS = frontend.get_default_settings()
 
         settings = _DEFAULT_DOCUTILS_SETTINGS
 
@@ -41,10 +41,11 @@ def _set_nodes_parent(nodes: Iterable[nodes.Node], parent: nodes.Element) -> Ite
         node.parent = parent
         yield node
 
-def set_node_attributes(node: nodes.Node, 
+TNode = TypeVar('TNode', bound=nodes.Node)
+def set_node_attributes(node: TNode, 
                         document: Optional[nodes.document] = None, 
                         lineno: Optional[int] = None, 
-                        children: Optional[Iterable[nodes.Node]] = None) -> nodes.Node:
+                        children: Optional[Iterable[nodes.Node]] = None) -> TNode:
     """
     Set the attributes of a Node and return the modified node.
     This is required to manually construct a docutils document that is consistent.
@@ -68,29 +69,34 @@ def set_node_attributes(node: nodes.Node,
 
     return node
 
-def build_table_of_content(node: nodes.Node, depth: int, level: int = 0) -> Optional[nodes.Node]:
+def build_table_of_content(node: nodes.Element, depth: int, level: int = 0) -> nodes.Element | None:
     """
     Simplified from docutils Contents transform. 
 
     All section nodes MUST have set attribute 'ids' to a list of strings.
     """
 
-    def _copy_and_filter(node: nodes.Node) -> nodes.Node:
+    def _copy_and_filter(node: nodes.Element) -> nodes.Element:
         """Return a copy of a title, with references, images, etc. removed."""
-        visitor = parts.ContentsFilter(node.document)
+        if (doc:=node.document) is None:
+            raise AssertionError(f'missing document attribute on {node}')
+        visitor = parts.ContentsFilter(doc)
         node.walkabout(visitor)
-        return visitor.get_entry_text()
+        #                                 the stubs are currently imcomplete, 2024.
+        return visitor.get_entry_text() # type:ignore
 
     level += 1
     sections = [sect for sect in node if isinstance(sect, nodes.section)]
     entries = []
+    if (doc:=node.document) is None:
+        raise AssertionError(f'missing document attribute on {node}')
+
     for section in sections:
-        title = section[0]
+        title = cast(nodes.Element, section[0]) # the first element of a section is the header.
         entrytext = _copy_and_filter(title)
         reference = nodes.reference('', '', refid=section['ids'][0],
                                     *entrytext)
-        ref_id = node.document.set_id(reference,
-                                    suggested_prefix='toc-entry')
+        ref_id = doc.set_id(reference, suggested_prefix='toc-entry')
         entry = nodes.paragraph('', '', reference)
         item = nodes.list_item('', entry)
         if title.next_node(nodes.reference) is None:
@@ -105,7 +111,7 @@ def _copy_and_filter(node: nodes.Node) -> nodes.Node:
     else:
         return None
 
-def get_lineno(node: nodes.Node) -> int:
+def get_lineno(node: nodes.Element) -> int:
     """
     Get the 0-based line number for a docutils `nodes.title_reference`.
 
@@ -114,7 +120,7 @@ def get_lineno(node: nodes.Node) -> int:
     """
     # Fixes https://github.com/twisted/pydoctor/issues/237
 
-    def get_first_parent_lineno(_node: Optional[nodes.Node]) -> int:
+    def get_first_parent_lineno(_node: nodes.Element | None) -> int:
         if _node is None:
             return 0
 
@@ -143,7 +149,7 @@ def get_first_parent_lineno(_node: Optional[nodes.Node]) -> int:
     else:
         line = get_first_parent_lineno(node.parent)
 
-    return line # type:ignore[no-any-return]
+    return line
 
 class wbr(nodes.inline):
     """

pydoctor/epydoc/markup/__init__.py

@@ -430,14 +430,14 @@ def visit_document(self, node: nodes.Node) -> None:
 
     _SENTENCE_RE_SPLIT = re.compile(r'( *[\.\?!][\'"\)\]]* *)')
 
-    def visit_paragraph(self, node: nodes.Node) -> None:
+    def visit_paragraph(self, node: nodes.paragraph) -> None:
         if self.summary is not None:
             # found a paragraph after the first one
             self.other_docs = True
             raise nodes.StopTraversal()
 
         summary_doc = new_document('summary')
-        summary_pieces = []
+        summary_pieces: list[nodes.Node] = []
 
         # Extract the first sentences from the first paragraph until maximum number 
         # of characters is reach or until the end of the paragraph.

pydoctor/epydoc/markup/_pyval_repr.py

@@ -390,24 +390,25 @@ def _trim_result(self, result: List[nodes.Node], num_chars: int) -> None:
         while num_chars > 0:
             if not result: 
                 return
-            if isinstance(result[-1], nodes.Element):
-                if len(result[-1].children) >= 1:
-                    data = result[-1][-1].astext()
+            if isinstance(r1:=result[-1], nodes.Element):
+                if len(r1.children) >= 1:
+                    data = r1[-1].astext()
                     trim = min(num_chars, len(data))
-                    result[-1][-1] = nodes.Text(data[:-trim])
-                    if not result[-1][-1].astext(): 
-                        if len(result[-1].children) == 1:
+                    r1[-1] = nodes.Text(data[:-trim])
+                    if not r1[-1].astext(): 
+                        if len(r1.children) == 1:
                             result.pop()
                         else:
-                            result[-1].pop()
+                            r1.pop()
                 else:
                     trim = 0
                     result.pop()
                 num_chars -= trim
             else:
                 # Must be Text if it's not an Element
-                trim = min(num_chars, len(result[-1]))
-                result[-1] = nodes.Text(result[-1].astext()[:-trim])
+                assert isinstance(r1, nodes.Text)
+                trim = min(num_chars, len(r1))
+                result[-1] = nodes.Text(r1.astext()[:-trim])
                 if not result[-1].astext(): 
                     result.pop()
                 num_chars -= trim
@@ -1012,6 +1013,7 @@ def _output(self, s: AnyStr, css_class: Optional[str],
             # If the segment fits on the current line, then just call
             # markup to tag it, and store the result.
             # Don't break links into separate segments, neither quotes.
+            element: nodes.Node
             if (self.linelen is None or 
                 state.charpos + segment_len <= self.linelen 
                 or link is True 

pydoctor/epydoc/markup/_types.py

@@ -5,7 +5,7 @@
 """
 from __future__ import annotations
 
-from typing import Callable, Dict, List, Tuple, Union
+from typing import Any, Callable, Dict, List, Tuple, Union, cast
 
 from pydoctor.epydoc.markup import DocstringLinker, ParseError, ParsedDocstring, get_parser_by_name
 from pydoctor.node2stan import node2stan
@@ -21,8 +21,9 @@ class ParsedTypeDocstring(TypeDocstring, ParsedDocstring):
     """
 
     FIELDS = ('type', 'rtype', 'ytype', 'returntype', 'yieldtype')
-
-    _tokens: List[Tuple[Union[str, nodes.Node], TokenType]]
+
+    #                                                   yes this overrides the superclass type!
+    _tokens: list[tuple[str | nodes.Node, TokenType]] # type: ignore
 
     def __init__(self, annotation: Union[nodes.document, str],
                  warns_on_unknown_tokens: bool = False, lineno: int = 0) -> None:
@@ -31,7 +32,8 @@ def __init__(self, annotation: Union[nodes.document, str],
             TypeDocstring.__init__(self, '', warns_on_unknown_tokens)
 
             _tokens = self._tokenize_node_type_spec(annotation)
-            self._tokens = self._build_tokens(_tokens)
+            self._tokens = cast('list[tuple[str | nodes.Node, TokenType]]', 
+                                self._build_tokens(_tokens))
             self._trigger_warnings()
         else:
             TypeDocstring.__init__(self, annotation, warns_on_unknown_tokens)
@@ -82,8 +84,8 @@ def _warn_not_supported(n:nodes.Node) -> None:
 
         return tokens
 
-    def _convert_obj_tokens_to_stan(self, tokens: List[Tuple[Union[str, nodes.Node], TokenType]], 
-                                    docstring_linker: DocstringLinker) -> List[Tuple[Union[str, Tag, nodes.Node], TokenType]]:
+    def _convert_obj_tokens_to_stan(self, tokens: List[Tuple[Any, TokenType]], 
+                                    docstring_linker: DocstringLinker) -> list[tuple[Any, TokenType]]:
         """
         Convert L{TokenType.OBJ} and PEP 484 like L{TokenType.DELIMITER} type to stan, merge them together. Leave the rest untouched. 
 
@@ -96,12 +98,13 @@ def _convert_obj_tokens_to_stan(self, tokens: List[Tuple[Union[str, nodes.Node],
         @param tokens: List of tuples: C{(token, type)}
         """
 
-        combined_tokens: List[Tuple[Union[str, Tag], TokenType]] = []
+        combined_tokens: list[tuple[Any, TokenType]] = []
 
         open_parenthesis = 0
         open_square_braces = 0
 
         for _token, _type in tokens:
+            # The actual type of_token is str | Tag | Node. 
 
             if (_type is TokenType.DELIMITER and _token in ('[', '(', ')', ']')) \
                or _type is TokenType.OBJ: 

pydoctor/epydoc/markup/epytext.py

@@ -1425,7 +1425,9 @@ def _to_node(self, tree: Element) -> Iterable[nodes.Node]:
             yield set_node_attributes(nodes.inline('', ''), document=self._document, children=variables)
         elif tree.tag == 'target':
             value, = variables
-            yield set_node_attributes(nodes.Text(value), document=self._document)
+            if not isinstance(value, nodes.Text):
+                raise AssertionError("target contents must be a simple text.")
+            yield set_node_attributes(value, document=self._document)
         elif tree.tag == 'italic':
             yield set_node_attributes(nodes.emphasis('', ''), document=self._document, children=variables)
         elif tree.tag == 'math':
@@ -1445,7 +1447,9 @@ def _to_node(self, tree: Element) -> Iterable[nodes.Node]:
         elif tree.tag == 'literalblock':
             yield set_node_attributes(nodes.literal_block('', ''), document=self._document, children=variables)
         elif tree.tag == 'doctestblock':
-            yield set_node_attributes(nodes.doctest_block(tree.children[0], tree.children[0]), document=self._document)
+            if not isinstance(contents:=tree.children[0], str):
+                raise AssertionError("doctest block contents is not a string")
+            yield set_node_attributes(nodes.doctest_block(contents, contents), document=self._document)
         elif tree.tag in ('fieldlist', 'tag', 'arg'):
             raise AssertionError("There should not be any field lists left")
         elif tree.tag == 'section':