Add sphinx.ext.apidoc extension

doc/usage/extensions/apidoc.rst

@@ -0,0 +1,126 @@
+.. _ext-apidoc:
+
+:mod:`sphinx.ext.apidoc` -- Generate API documentation from Python packages
+===========================================================================
+
+.. py:module:: sphinx.ext.apidoc
+   :synopsis: Generate API documentation from Python modules
+
+.. index:: pair: automatic; documentation
+.. index:: pair: generation; documentation
+.. index:: pair: generate; documentation
+
+.. versionadded:: 8.2
+
+.. role:: code-py(code)
+   :language: Python
+
+:mod:`sphinx.ext.apidoc` is a tool for automatic generation
+of Sphinx sources from Python packages.
+It provides the :program:`sphinx-apidoc` command-line tool as an extension,
+allowing it to be run during the Sphinx build process.
+
+The extension writes generated source files to a provided directory,
+which are then read by Sphinx using the :mod:`sphinx.ext.autodoc` extension.
+
+.. warning::
+
+   :mod:`sphinx.ext.apidoc` generates source files that
+   use :mod:`sphinx.ext.autodoc` to document all found modules.
+   If any modules have side effects on import,
+   these will be executed by ``autodoc`` when :program:`sphinx-build` is run.
+
+   If you document scripts (as opposed to library modules),
+   make sure their main routine is protected by
+   an ``if __name__ == '__main__'`` condition.
+
+
+Configuration
+-------------
+
+The apidoc extension uses the following configuration values:
+
+.. confval:: apidoc_modules
+   :no-index:
+   :type: :code-py:`Sequence[dict[str, Any]]`
+   :default: :code-py:`()`
+
+   A list or sequence of dictionaries describing modules to document.
+
+   For example:
+
+   .. code-block:: python
+
+      apidoc_modules = [
+          {'destination': 'source/', 'path': 'path/to/module'},
+          {
+              'destination': 'source/',
+              'path': 'path/to/another_module',
+              'exclude_patterns': ['**/test*'],
+              'maxdepth': 4,
+              'followlinks': False,
+              'separatemodules': False,
+              'includeprivate': False,
+              'noheadings': False,
+              'modulefirst': False,
+              'implicit_namespaces': False,
+              'automodule_options': {
+                  'members', 'show-inheritance', 'undoc-members'
+              },
+          },
+      ]
+
+
+   Valid keys are:
+
+   :code-py:`'destination'`
+     The output directory for generated files (**required**).
+     This must be relative to the source directory,
+     and will be created if it does not exist.
+
+   :code-py:`'path'`
+     The path to the module to document (**required**).
+     This must be absolute or relative to the configuration directory.
+
+   :code-py:`'exclude_patterns'`
+     A sequence of patterns to exclude from generation.
+     These may be literal paths or :py:mod:`fnmatch`-style patterns.
+     Defaults to :code-py:`()`.
+
+   :code-py:`'maxdepth'`
+     The maximum depth of submodules to show in the generated table of contents.
+     Defaults to :code-py:`4`.
+
+   :code-py:`'followlinks'`
+     Follow symbolic links.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'separatemodules'`
+     Put documentation for each module on an individual page.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'includeprivate'`
+     Generate documentation for '_private' modules with leading underscores.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'noheadings'`
+     Do not create headings for the modules/packages.
+     Useful when source docstrings already contain headings.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'modulefirst'`
+     Place module documentation before submodule documentation.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'implicit_namespaces'`
+     By default sphinx-apidoc processes sys.path searching for modules only.
+     Python 3.3 introduced :pep:`420` implicit namespaces that allow module path
+     structures such as ``foo/bar/module.py`` or ``foo/bar/baz/__init__.py``
+     (notice that ``bar`` and ``foo`` are namespaces, not modules).
+
+     Interpret module paths using :pep:`420` implicit namespaces.
+     Defaults to :code-py:`False`.
+
+   :code-py:`'automodule_options'`
+     Options to pass to generated :rst:dir:`automodule` directives.
+     Defaults to :code-py:`{'members', 'show-inheritance', 'undoc-members'}`.

doc/usage/extensions/index.rst

@@ -21,6 +21,7 @@ These extensions are built in and can be activated by respective entries in the
 .. toctree::
    :maxdepth: 1
 
+   apidoc
    autodoc
    autosectionlabel
    autosummary

sphinx/ext/apidoc/__init__.py

@@ -13,9 +13,26 @@
 
 from typing import TYPE_CHECKING
 
+import sphinx
 from sphinx.ext.apidoc._cli import main
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
-__all__: Sequence[str] = ('main',)
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+__all__: Sequence[str] = 'main', 'setup'
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    from sphinx.ext.apidoc._extension import run_apidoc
+
+    # Require autodoc
+    app.setup_extension('sphinx.ext.autodoc')
+    app.add_config_value('apidoc_modules', (), 'env', types=frozenset((list, tuple)))
+    app.connect('builder-inited', run_apidoc)
+    return {
+        'version': sphinx.__display_version__,
+        'parallel_read_safe': True,
+    }

sphinx/ext/apidoc/_extension.py

@@ -0,0 +1,224 @@
+"""Sphinx extension for auto-generating API documentation."""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from sphinx.ext.apidoc._generate import create_modules_toc_file, recurse_tree
+from sphinx.ext.apidoc._shared import LOGGER, ApidocOptions, _remove_old_files
+from sphinx.locale import __
+from sphinx.util.console import bold
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from sphinx.application import Sphinx
+
+_BOOL_KEYS = frozenset({
+    'followlinks',
+    'separatemodules',
+    'includeprivate',
+    'noheadings',
+    'modulefirst',
+    'implicit_namespaces',
+})
+_ALLOWED_KEYS = _BOOL_KEYS | frozenset({
+    'path',
+    'destination',
+    'exclude_patterns',
+    'automodule_options',
+    'maxdepth',
+})
+
+
+def run_apidoc(app: Sphinx) -> None:
+    """Run the apidoc extension."""
+    apidoc_modules: Sequence[dict[str, Any]] = app.config.apidoc_modules
+    srcdir: Path = app.srcdir
+    confdir: Path = app.confdir
+
+    LOGGER.info(bold(__('Running apidoc')))
+
+    module_options: dict[str, Any]
+    for i, module_options in enumerate(apidoc_modules):
+        _run_apidoc_module(i, options=module_options, srcdir=srcdir, confdir=confdir)
+
+
+def _run_apidoc_module(
+    i: int, *, options: dict[str, Any], srcdir: Path, confdir: Path
+) -> None:
+    args = _parse_module_options(i, options=options, srcdir=srcdir, confdir=confdir)
+    if args is None:
+        return
+
+    exclude_patterns_compiled: list[re.Pattern[str]] = [
+        re.compile(fnmatch.translate(exclude)) for exclude in args.exclude_pattern
+    ]
+
+    written_files, modules = recurse_tree(
+        args.module_path, exclude_patterns_compiled, args, args.templatedir
+    )
+    if args.tocfile:
+        written_files.append(
+            create_modules_toc_file(modules, args, args.tocfile, args.templatedir)
+        )
+    if args.remove_old:
+        _remove_old_files(written_files, args.destdir, args.suffix)
+
+
+def _parse_module_options(
+    i: int, *, options: dict[str, Any], srcdir: Path, confdir: Path
+) -> ApidocOptions | None:
+    if not isinstance(options, dict):
+        LOGGER.warning(__('apidoc_modules item %i must be a dict'), i, type='apidoc')
+        return None
+
+    # module path should be absolute or relative to the conf directory
+    try:
+        path = Path(os.fspath(options['path']))
+    except KeyError:
+        LOGGER.warning(
+            __("apidoc_modules item %i must have a 'path' key"), i, type='apidoc'
+        )
+        return None
+    except TypeError:
+        LOGGER.warning(
+            __("apidoc_modules item %i 'path' must be a string"), i, type='apidoc'
+        )
+        return None
+    module_path = confdir / path
+    if not module_path.is_dir():
+        LOGGER.warning(
+            __("apidoc_modules item %i 'path' is not an existing folder: %s"),
+            i,
+            module_path,
+            type='apidoc',
+        )
+        return None
+
+    # destination path should be relative to the source directory
+    try:
+        destination = Path(os.fspath(options['destination']))
+    except KeyError:
+        LOGGER.warning(
+            __("apidoc_modules item %i must have a 'destination' key"),
+            i,
+            type='apidoc',
+        )
+        return None
+    except TypeError:
+        LOGGER.warning(
+            __("apidoc_modules item %i 'destination' must be a string"),
+            i,
+            type='apidoc',
+        )
+        return None
+    if destination.is_absolute():
+        LOGGER.warning(
+            __("apidoc_modules item %i 'destination' should be a relative path"),
+            i,
+            type='apidoc',
+        )
+        return None
+    dest_path = srcdir / destination
+    try:
+        dest_path.mkdir(parents=True, exist_ok=True)
+    except OSError as exc:
+        LOGGER.warning(
+            __('apidoc_modules item %i cannot create destination directory: %s'),
+            i,
+            exc.strerror,
+            type='apidoc',
+        )
+        return None
+
+    # exclude patterns should be absolute or relative to the conf directory
+    exclude_patterns: list[str] = [
+        str(confdir / pattern)
+        for pattern in _check_list_of_strings(i, options, key='exclude_patterns')
+    ]
+
+    # TODO template_dir
+
+    maxdepth = 4
+    if 'maxdepth' in options:
+        if not isinstance(options['maxdepth'], int):
+            LOGGER.warning(
+                __("apidoc_modules item %i '%s' must be an int"),
+                i,
+                'maxdepth',
+                type='apidoc',
+            )
+        else:
+            maxdepth = options['maxdepth']
+
+    extra_options = {}
+    for key in sorted(_BOOL_KEYS):
+        if key not in options:
+            continue
+        if not isinstance(options[key], bool):
+            LOGGER.warning(
+                __("apidoc_modules item %i '%s' must be a boolean"),
+                i,
+                key,
+                type='apidoc',
+            )
+            continue
+        extra_options[key] = options[key]
+
+    if _options := _check_list_of_strings(i, options, key='automodule_options'):
+        automodule_options = set(_options)
+    else:
+        # TODO per-module automodule_options
+        automodule_options = {'members', 'undoc-members', 'show-inheritance'}
+
+    if diff := set(options) - _ALLOWED_KEYS:
+        LOGGER.warning(
+            __('apidoc_modules item %i has unexpected keys: %s'),
+            i,
+            ', '.join(sorted(diff)),
+            type='apidoc',
+        )
+
+    return ApidocOptions(
+        destdir=dest_path,
+        module_path=module_path,
+        exclude_pattern=exclude_patterns,
+        automodule_options=automodule_options,
+        maxdepth=maxdepth,
+        quiet=True,
+        **extra_options,
+    )
+
+
+def _check_list_of_strings(
+    index: int, options: dict[str, Any], *, key: str
+) -> list[str]:
+    """Check that a key's value is a list of strings in the options.
+
+    :returns: the value of the key, or the empty list if invalid.
+    """
+    if key not in options:
+        return []
+    if not isinstance(options[key], list | tuple | set | frozenset):
+        LOGGER.warning(
+            __("apidoc_modules item %i '%s' must be a sequence"),
+            index,
+            key,
+            type='apidoc',
+        )
+        return []
+    for item in options[key]:
+        if not isinstance(item, str):
+            LOGGER.warning(
+                __("apidoc_modules item %i '%s' must contain strings"),
+                index,
+                key,
+                type='apidoc',
+            )
+            return []
+    return options[key]