From 73f7af2f86b8c5b7c3c3de40db21feea4be26f18 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 15 Jun 2023 00:26:41 +0200 Subject: [PATCH] MNT: update vendored docs files Copied from numpydoc 1.5.0. --- docs/sphinxext/docscrape.py | 631 +++++++++++++++++++---------- docs/sphinxext/docscrape_sphinx.py | 455 +++++++++++++++------ docs/sphinxext/numpydoc.py | 463 ++++++++++++++++----- 3 files changed, 1102 insertions(+), 447 deletions(-) diff --git a/docs/sphinxext/docscrape.py b/docs/sphinxext/docscrape.py index 470badd3352..e5c07f59ded 100644 --- a/docs/sphinxext/docscrape.py +++ b/docs/sphinxext/docscrape.py @@ -1,33 +1,48 @@ """Extract reference documentation from the NumPy source tree. """ -from __future__ import division, absolute_import, print_function - import inspect import textwrap import re import pydoc from warnings import warn -import collections +from collections import namedtuple +from collections.abc import Callable, Mapping +import copy import sys -class Reader(object): - """A line-based string reader. +# TODO: Remove try-except when support for Python 3.7 is dropped +try: + from functools import cached_property +except ImportError: # cached_property added in Python 3.8 + cached_property = property + + +def strip_blank_lines(l): + "Remove leading and trailing blank lines from a list of lines" + while l and not l[0].strip(): + del l[0] + while l and not l[-1].strip(): + del l[-1] + return l + + +class Reader: + """A line-based string reader.""" - """ def __init__(self, data): """ Parameters ---------- data : str - String with lines separated by '\n'. + String with lines separated by '\\n'. """ if isinstance(data, list): self._str = data else: - self._str = data.split('\n') # store string as list of lines + self._str = data.split("\n") # store string as list of lines self.reset() @@ -43,10 +58,10 @@ def read(self): self._l += 1 return out else: - return '' + return "" def seek_next_non_empty_line(self): - for l in self[self._l:]: + for l in self[self._l :]: if l.strip(): break else: @@ -59,10 +74,10 @@ def read_to_condition(self, condition_func): start = self._l for line in self[start:]: if condition_func(line): - return self[start:self._l] + return self[start : self._l] self._l += 1 if self.eof(): - return self[start:self._l+1] + return self[start : self._l + 1] return [] def read_to_next_empty_line(self): @@ -75,52 +90,78 @@ def is_empty(line): def read_to_next_unindented_line(self): def is_unindented(line): - return (line.strip() and (len(line.lstrip()) == len(line))) + return line.strip() and (len(line.lstrip()) == len(line)) + return self.read_to_condition(is_unindented) def peek(self, n=0): if self._l + n < len(self._str): return self[self._l + n] else: - return '' + return "" def is_empty(self): - return not ''.join(self._str).strip() + return not "".join(self._str).strip() -class NumpyDocString(collections.Mapping): - def __init__(self, docstring, config={}): - docstring = textwrap.dedent(docstring).split('\n') +class ParseError(Exception): + def __str__(self): + message = self.args[0] + if hasattr(self, "docstring"): + message = f"{message} in {self.docstring!r}" + return message + + +Parameter = namedtuple("Parameter", ["name", "type", "desc"]) + + +class NumpyDocString(Mapping): + """Parses a numpydoc string to an abstract representation + + Instances define a mapping from section title to structured data. + + """ + + sections = { + "Signature": "", + "Summary": [""], + "Extended Summary": [], + "Parameters": [], + "Returns": [], + "Yields": [], + "Receives": [], + "Raises": [], + "Warns": [], + "Other Parameters": [], + "Attributes": [], + "Methods": [], + "See Also": [], + "Notes": [], + "Warnings": [], + "References": "", + "Examples": "", + "index": {}, + } + + def __init__(self, docstring, config=None): + orig_docstring = docstring + docstring = textwrap.dedent(docstring).split("\n") self._doc = Reader(docstring) - self._parsed_data = { - 'Signature': '', - 'Summary': [''], - 'Extended Summary': [], - 'Parameters': [], - 'Returns': [], - 'Yields': [], - 'Raises': [], - 'Warns': [], - 'Other Parameters': [], - 'Attributes': [], - 'Methods': [], - 'See Also': [], - 'Notes': [], - 'Warnings': [], - 'References': '', - 'Examples': '', - 'index': {} - } - - self._parse() + self._parsed_data = copy.deepcopy(self.sections) + + try: + self._parse() + except ParseError as e: + e.docstring = orig_docstring + raise def __getitem__(self, key): return self._parsed_data[key] def __setitem__(self, key, val): if key not in self._parsed_data: - warn("Unknown section %s" % key) + self._error_location(f"Unknown section {key}", error=False) else: self._parsed_data[key] = val @@ -138,11 +179,17 @@ def _is_at_section(self): l1 = self._doc.peek().strip() # e.g. Parameters - if l1.startswith('.. index::'): + if l1.startswith(".. index::"): return True l2 = self._doc.peek(1).strip() # ---------- or ========== - return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1)) + if len(l2) >= 3 and (set(l2) in ({"-"}, {"="})) and len(l2) != len(l1): + snip = "\n".join(self._doc._str[:2]) + "..." + self._error_location( + f"potentially wrong underline length... \n{l1} \n{l2} in \n{snip}", + error=False, + ) + return l2.startswith("-" * len(l1)) or l2.startswith("=" * len(l1)) def _strip(self, doc): i = 0 @@ -155,14 +202,14 @@ def _strip(self, doc): if line.strip(): break - return doc[i:len(doc)-j] + return doc[i : len(doc) - j] def _read_to_next_section(self): section = self._doc.read_to_next_empty_line() while not self._is_at_section() and not self._doc.eof(): if not self._doc.peek(-1).strip(): # previous line was empty - section += [''] + section += [""] section += self._doc.read_to_next_empty_line() @@ -173,32 +220,78 @@ def _read_sections(self): data = self._read_to_next_section() name = data[0].strip() - if name.startswith('..'): # index section + if name.startswith(".."): # index section yield name, data[1:] elif len(data) < 2: yield StopIteration else: yield name, self._strip(data[2:]) - def _parse_param_list(self, content): + def _parse_param_list(self, content, single_element_is_type=False): + content = dedent_lines(content) r = Reader(content) params = [] while not r.eof(): header = r.read().strip() - if ' : ' in header: - arg_name, arg_type = header.split(' : ')[:2] + if " : " in header: + arg_name, arg_type = header.split(" : ", maxsplit=1) else: - arg_name, arg_type = header, '' + # NOTE: param line with single element should never have a + # a " :" before the description line, so this should probably + # warn. + if header.endswith(" :"): + header = header[:-2] + if single_element_is_type: + arg_name, arg_type = "", header + else: + arg_name, arg_type = header, "" desc = r.read_to_next_unindented_line() desc = dedent_lines(desc) + desc = strip_blank_lines(desc) - params.append((arg_name, arg_type, desc)) + params.append(Parameter(arg_name, arg_type, desc)) return params - _name_rgx = re.compile(r"^\s*(:(?P\w+):`(?P[a-zA-Z0-9_.-]+)`|" - r" (?P[a-zA-Z0-9_.-]+))\s*", re.X) + # See also supports the following formats. + # + # + # SPACE* COLON SPACE+ SPACE* + # ( COMMA SPACE+ )+ (COMMA | PERIOD)? SPACE* + # ( COMMA SPACE+ )* SPACE* COLON SPACE+ SPACE* + + # is one of + # + # COLON COLON BACKTICK BACKTICK + # where + # is a legal function name, and + # is any nonempty sequence of word characters. + # Examples: func_f1 :meth:`func_h1` :obj:`~baz.obj_r` :class:`class_j` + # is a string describing the function. + + _role = r":(?P(py:)?\w+):" + _funcbacktick = r"`(?P(?:~\w+\.)?[a-zA-Z0-9_\.-]+)`" + _funcplain = r"(?P[a-zA-Z0-9_\.-]+)" + _funcname = r"(" + _role + _funcbacktick + r"|" + _funcplain + r")" + _funcnamenext = _funcname.replace("role", "rolenext") + _funcnamenext = _funcnamenext.replace("name", "namenext") + _description = r"(?P\s*:(\s+(?P\S+.*))?)?\s*$" + _func_rgx = re.compile(r"^\s*" + _funcname + r"\s*") + _line_rgx = re.compile( + r"^\s*" + + r"(?P" + + _funcname # group for all function names + + r"(?P([,]\s+" + + _funcnamenext + + r")*)" + + r")" + + r"(?P[,\.])?" # end of "allfuncs" + + _description # Some function lists have a trailing comma (or period) '\s*' + ) + + # Empty elements are replaced with '..' + empty_description = ".." def _parse_see_also(self, content): """ @@ -208,52 +301,52 @@ def _parse_see_also(self, content): func_name1, func_name2, :meth:`func_name`, func_name3 """ + + content = dedent_lines(content) + items = [] def parse_item_name(text): - """Match ':role:`name`' or 'name'""" - m = self._name_rgx.match(text) - if m: - g = m.groups() - if g[1] is None: - return g[3], None - else: - return g[2], g[1] - raise ValueError("%s is not a item name" % text) + """Match ':role:`name`' or 'name'.""" + m = self._func_rgx.match(text) + if not m: + self._error_location(f"Error parsing See Also entry {line!r}") + role = m.group("role") + name = m.group("name") if role else m.group("name2") + return name, role, m.end() - def push_item(name, rest): - if not name: - return - name, role = parse_item_name(name) - items.append((name, list(rest), role)) - del rest[:] - - current_func = None rest = [] - for line in content: if not line.strip(): continue - m = self._name_rgx.match(line) - if m and line[m.end():].strip().startswith(':'): - push_item(current_func, rest) - current_func, line = line[:m.end()], line[m.end():] - rest = [line.split(':', 1)[1].strip()] - if not rest[0]: - rest = [] - elif not line.startswith(' '): - push_item(current_func, rest) - current_func = None - if ',' in line: - for func in line.split(','): - if func.strip(): - push_item(func, []) - elif line.strip(): - current_func = line - elif current_func is not None: + line_match = self._line_rgx.match(line) + description = None + if line_match: + description = line_match.group("desc") + if line_match.group("trailing") and description: + self._error_location( + "Unexpected comma or period after function list at index %d of " + 'line "%s"' % (line_match.end("trailing"), line), + error=False, + ) + if not description and line.startswith(" "): rest.append(line.strip()) - push_item(current_func, rest) + elif line_match: + funcs = [] + text = line_match.group("allfuncs") + while True: + if not text.strip(): + break + name, role, match_end = parse_item_name(text) + funcs.append((name, role)) + text = text[match_end:].strip() + if text and text[0] == ",": + text = text[1:].strip() + rest = list(filter(None, [description])) + items.append((funcs, rest)) + else: + self._error_location(f"Error parsing See Also entry {line!r}") return items def _parse_index(self, section, content): @@ -262,17 +355,18 @@ def _parse_index(self, section, content): :refguide: something, else, and more """ + def strip_each_in(lst): return [s.strip() for s in lst] out = {} - section = section.split('::') + section = section.split("::") if len(section) > 1: - out['default'] = strip_each_in(section[1].split(','))[0] + out["default"] = strip_each_in(section[1].split(","))[0] for line in content: - line = line.split(':') + line = line.split(":") if len(line) > 2: - out[line[1]] = strip_each_in(line[2].split(',')) + out[line[1]] = strip_each_in(line[2].split(",")) return out def _parse_summary(self): @@ -284,87 +378,124 @@ def _parse_summary(self): while True: summary = self._doc.read_to_next_empty_line() summary_str = " ".join([s.strip() for s in summary]).strip() - if re.compile('^([\w., ]+=)?\s*[\w\.]+\(.*\)$').match(summary_str): - self['Signature'] = summary_str + compiled = re.compile(r"^([\w., ]+=)?\s*[\w\.]+\(.*\)$") + if compiled.match(summary_str): + self["Signature"] = summary_str if not self._is_at_section(): continue break if summary is not None: - self['Summary'] = summary + self["Summary"] = summary if not self._is_at_section(): - self['Extended Summary'] = self._read_to_next_section() + self["Extended Summary"] = self._read_to_next_section() def _parse(self): self._doc.reset() self._parse_summary() sections = list(self._read_sections()) - section_names = set([section for section, content in sections]) + section_names = {section for section, content in sections} - has_returns = 'Returns' in section_names - has_yields = 'Yields' in section_names + has_returns = "Returns" in section_names + has_yields = "Yields" in section_names # We could do more tests, but we are not. Arbitrarily. if has_returns and has_yields: - msg = 'Docstring contains both a Returns and Yields section.' + msg = "Docstring contains both a Returns and Yields section." + raise ValueError(msg) + if not has_yields and "Receives" in section_names: + msg = "Docstring contains a Receives section but not Yields." raise ValueError(msg) for (section, content) in sections: - if not section.startswith('..'): - section = (s.capitalize() for s in section.split(' ')) - section = ' '.join(section) - if section in ('Parameters', 'Returns', 'Yields', 'Raises', - 'Warns', 'Other Parameters', 'Attributes', - 'Methods'): + if not section.startswith(".."): + section = (s.capitalize() for s in section.split(" ")) + section = " ".join(section) + if self.get(section): + self._error_location( + "The section %s appears twice in %s" + % (section, "\n".join(self._doc._str)) + ) + + if section in ("Parameters", "Other Parameters", "Attributes", "Methods"): self[section] = self._parse_param_list(content) - elif section.startswith('.. index::'): - self['index'] = self._parse_index(section, content) - elif section == 'See Also': - self['See Also'] = self._parse_see_also(content) + elif section in ("Returns", "Yields", "Raises", "Warns", "Receives"): + self[section] = self._parse_param_list( + content, single_element_is_type=True + ) + elif section.startswith(".. index::"): + self["index"] = self._parse_index(section, content) + elif section == "See Also": + self["See Also"] = self._parse_see_also(content) else: self[section] = content + @property + def _obj(self): + if hasattr(self, "_cls"): + return self._cls + elif hasattr(self, "_f"): + return self._f + return None + + def _error_location(self, msg, error=True): + if self._obj is not None: + # we know where the docs came from: + try: + filename = inspect.getsourcefile(self._obj) + except TypeError: + filename = None + # Make UserWarning more descriptive via object introspection. + # Skip if introspection fails + name = getattr(self._obj, "__name__", None) + if name is None: + name = getattr(getattr(self._obj, "__class__", None), "__name__", None) + if name is not None: + msg += f" in the docstring of {name}" + msg += f" in {filename}." if filename else "" + if error: + raise ValueError(msg) + else: + warn(msg) + # string conversion routines - def _str_header(self, name, symbol='-'): - return [name, len(name)*symbol] + def _str_header(self, name, symbol="-"): + return [name, len(name) * symbol] def _str_indent(self, doc, indent=4): - out = [] - for line in doc: - out += [' '*indent + line] - return out + return [" " * indent + line for line in doc] def _str_signature(self): - if self['Signature']: - return [self['Signature'].replace('*', '\*')] + [''] - else: - return [''] + if self["Signature"]: + return [self["Signature"].replace("*", r"\*")] + [""] + return [""] def _str_summary(self): - if self['Summary']: - return self['Summary'] + [''] - else: - return [] + if self["Summary"]: + return self["Summary"] + [""] + return [] def _str_extended_summary(self): - if self['Extended Summary']: - return self['Extended Summary'] + [''] - else: - return [] + if self["Extended Summary"]: + return self["Extended Summary"] + [""] + return [] def _str_param_list(self, name): out = [] if self[name]: out += self._str_header(name) - for param, param_type, desc in self[name]: - if param_type: - out += ['%s : %s' % (param, param_type)] - else: - out += [param] - out += self._str_indent(desc) - out += [''] + for param in self[name]: + parts = [] + if param.name: + parts.append(param.name) + if param.type: + parts.append(param.type) + out += [" : ".join(parts)] + if param.desc and "".join(param.desc).strip(): + out += self._str_indent(param.desc) + out += [""] return out def _str_section(self, name): @@ -372,69 +503,81 @@ def _str_section(self, name): if self[name]: out += self._str_header(name) out += self[name] - out += [''] + out += [""] return out def _str_see_also(self, func_role): - if not self['See Also']: + if not self["See Also"]: return [] out = [] out += self._str_header("See Also") + out += [""] last_had_desc = True - for func, desc, role in self['See Also']: - if role: - link = ':%s:`%s`' % (role, func) - elif func_role: - link = ':%s:`%s`' % (func_role, func) - else: - link = "`%s`_" % func - if desc or last_had_desc: - out += [''] - out += [link] - else: - out[-1] += ", %s" % link + for funcs, desc in self["See Also"]: + assert isinstance(funcs, list) + links = [] + for func, role in funcs: + if role: + link = f":{role}:`{func}`" + elif func_role: + link = f":{func_role}:`{func}`" + else: + link = f"`{func}`_" + links.append(link) + link = ", ".join(links) + out += [link] if desc: - out += self._str_indent([' '.join(desc)]) + out += self._str_indent([" ".join(desc)]) last_had_desc = True else: last_had_desc = False - out += [''] + out += self._str_indent([self.empty_description]) + + if last_had_desc: + out += [""] + out += [""] return out def _str_index(self): - idx = self['index'] + idx = self["index"] out = [] - out += ['.. index:: %s' % idx.get('default', '')] + output_index = False + default_index = idx.get("default", "") + if default_index: + output_index = True + out += [f".. index:: {default_index}"] for section, references in idx.items(): - if section == 'default': + if section == "default": continue - out += [' :%s: %s' % (section, ', '.join(references))] - return out + output_index = True + out += [f" :{section}: {', '.join(references)}"] + if output_index: + return out + return "" - def __str__(self, func_role=''): + def __str__(self, func_role=""): out = [] out += self._str_signature() out += self._str_summary() out += self._str_extended_summary() - for param_list in ('Parameters', 'Returns', 'Yields', - 'Other Parameters', 'Raises', 'Warns'): + for param_list in ( + "Parameters", + "Returns", + "Yields", + "Receives", + "Other Parameters", + "Raises", + "Warns", + ): out += self._str_param_list(param_list) - out += self._str_section('Warnings') + out += self._str_section("Warnings") out += self._str_see_also(func_role) - for s in ('Notes', 'References', 'Examples'): + for s in ("Notes", "References", "Examples"): out += self._str_section(s) - for param_list in ('Attributes', 'Methods'): + for param_list in ("Attributes", "Methods"): out += self._str_param_list(param_list) out += self._str_index() - return '\n'.join(out) - - -def indent(str, indent=4): - indent_str = ' '*indent - if str is None: - return indent_str - lines = str.split('\n') - return '\n'.join(indent_str + l for l in lines) + return "\n".join(out) def dedent_lines(lines): @@ -442,78 +585,71 @@ def dedent_lines(lines): return textwrap.dedent("\n".join(lines)).split("\n") -def header(text, style='-'): - return text + '\n' + style*len(text) + '\n' - - class FunctionDoc(NumpyDocString): - def __init__(self, func, role='func', doc=None, config={}): + def __init__(self, func, role="func", doc=None, config=None): self._f = func self._role = role # e.g. "func" or "meth" if doc is None: if func is None: raise ValueError("No function or docstring given") - doc = inspect.getdoc(func) or '' - NumpyDocString.__init__(self, doc) - - if not self['Signature'] and func is not None: - func, func_name = self.get_func() - try: - # try to read signature - if sys.version_info[0] >= 3: - argspec = inspect.getfullargspec(func) - else: - argspec = inspect.getargspec(func) - argspec = inspect.formatargspec(*argspec) - argspec = argspec.replace('*', '\*') - signature = '%s%s' % (func_name, argspec) - except TypeError as e: - signature = '%s()' % func_name - self['Signature'] = signature + doc = inspect.getdoc(func) or "" + if config is None: + config = {} + NumpyDocString.__init__(self, doc, config) def get_func(self): - func_name = getattr(self._f, '__name__', self.__class__.__name__) + func_name = getattr(self._f, "__name__", self.__class__.__name__) if inspect.isclass(self._f): - func = getattr(self._f, '__call__', self._f.__init__) + func = getattr(self._f, "__call__", self._f.__init__) else: func = self._f return func, func_name def __str__(self): - out = '' + out = "" func, func_name = self.get_func() - signature = self['Signature'].replace('*', '\*') - roles = {'func': 'function', - 'meth': 'method'} + roles = {"func": "function", "meth": "method"} if self._role: if self._role not in roles: - print("Warning: invalid role %s" % self._role) - out += '.. %s:: %s\n \n\n' % (roles.get(self._role, ''), - func_name) + print(f"Warning: invalid role {self._role}") + out += f".. {roles.get(self._role, '')}:: {func_name}\n \n\n" - out += super(FunctionDoc, self).__str__(func_role=self._role) + out += super().__str__(func_role=self._role) return out +class ObjDoc(NumpyDocString): + def __init__(self, obj, doc=None, config=None): + self._f = obj + if config is None: + config = {} + NumpyDocString.__init__(self, doc, config=config) + + class ClassDoc(NumpyDocString): - extra_public_methods = ['__call__'] + extra_public_methods = ["__call__"] - def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc, - config={}): + def __init__(self, cls, doc=None, modulename="", func_doc=FunctionDoc, config=None): if not inspect.isclass(cls) and cls is not None: - raise ValueError("Expected a class or None, but got %r" % cls) + raise ValueError(f"Expected a class or None, but got {cls!r}") self._cls = cls - self.show_inherited_members = config.get( - 'show_inherited_class_members', True) + if "sphinx" in sys.modules: + from sphinx.ext.autodoc import ALL + else: + ALL = object() + + if config is None: + config = {} + self.show_inherited_members = config.get("show_inherited_class_members", True) - if modulename and not modulename.endswith('.'): - modulename += '.' + if modulename and not modulename.endswith("."): + modulename += "." self._mod = modulename if doc is None: @@ -523,21 +659,31 @@ def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc, NumpyDocString.__init__(self, doc) - if config.get('show_class_members', True): + _members = config.get("members", []) + if _members is ALL: + _members = None + _exclude = config.get("exclude-members", []) + + if config.get("show_class_members", True) and _exclude is not ALL: + def splitlines_x(s): if not s: return [] else: return s.splitlines() - for field, items in [('Methods', self.methods), - ('Attributes', self.properties)]: + for field, items in [ + ("Methods", self.methods), + ("Attributes", self.properties), + ]: if not self[field]: doc_list = [] for name in sorted(items): + if name in _exclude or (_members and name not in _members): + continue try: doc_item = pydoc.getdoc(getattr(self._cls, name)) - doc_list.append((name, '', splitlines_x(doc_item))) + doc_list.append(Parameter(name, "", splitlines_x(doc_item))) except AttributeError: pass # method doesn't exist self[field] = doc_list @@ -546,21 +692,33 @@ def splitlines_x(s): def methods(self): if self._cls is None: return [] - return [name for name, func in inspect.getmembers(self._cls) - if ((not name.startswith('_') - or name in self.extra_public_methods) - and isinstance(func, collections.Callable) - and self._is_show_member(name))] + return [ + name + for name, func in inspect.getmembers(self._cls) + if ( + (not name.startswith("_") or name in self.extra_public_methods) + and isinstance(func, Callable) + and self._is_show_member(name) + ) + ] @property def properties(self): if self._cls is None: return [] - return [name for name, func in inspect.getmembers(self._cls) - if (not name.startswith('_') and - (func is None or isinstance(func, property) or - inspect.isgetsetdescriptor(func)) - and self._is_show_member(name))] + return [ + name + for name, func in inspect.getmembers(self._cls) + if ( + not name.startswith("_") + and ( + func is None + or isinstance(func, (property, cached_property)) + or inspect.isdatadescriptor(func) + ) + and self._is_show_member(name) + ) + ] def _is_show_member(self, name): if self.show_inherited_members: @@ -568,3 +726,26 @@ def _is_show_member(self, name): if name not in self._cls.__dict__: return False # class member is inherited, we do not show it return True + + +def get_doc_object(obj, what=None, doc=None, config=None): + if what is None: + if inspect.isclass(obj): + what = "class" + elif inspect.ismodule(obj): + what = "module" + elif isinstance(obj, Callable): + what = "function" + else: + what = "object" + if config is None: + config = {} + + if what == "class": + return ClassDoc(obj, func_doc=FunctionDoc, doc=doc, config=config) + elif what in ("function", "method"): + return FunctionDoc(obj, doc=doc, config=config) + else: + if doc is None: + doc = pydoc.getdoc(obj) + return ObjDoc(obj, doc, config=config) diff --git a/docs/sphinxext/docscrape_sphinx.py b/docs/sphinxext/docscrape_sphinx.py index e44e770ef86..9a62cff9ce7 100644 --- a/docs/sphinxext/docscrape_sphinx.py +++ b/docs/sphinxext/docscrape_sphinx.py @@ -1,58 +1,225 @@ -import re, inspect, textwrap, pydoc +import re +import inspect +import textwrap +import pydoc +from collections.abc import Callable +import os + +from jinja2 import FileSystemLoader +from jinja2.sandbox import SandboxedEnvironment import sphinx -from docscrape import NumpyDocString, FunctionDoc, ClassDoc +from sphinx.jinja2glue import BuiltinTemplateLoader + +from .docscrape import NumpyDocString, FunctionDoc, ClassDoc, ObjDoc +from .xref import make_xref + + +IMPORT_MATPLOTLIB_RE = r"\b(import +matplotlib|from +matplotlib +import)\b" + class SphinxDocString(NumpyDocString): - def __init__(self, docstring, config={}): - self.use_plots = config.get('use_plots', False) + def __init__(self, docstring, config=None): + if config is None: + config = {} NumpyDocString.__init__(self, docstring, config=config) + self.load_config(config) + + def load_config(self, config): + self.use_plots = config.get("use_plots", False) + self.class_members_toctree = config.get("class_members_toctree", True) + self.attributes_as_param_list = config.get("attributes_as_param_list", True) + self.xref_param_type = config.get("xref_param_type", False) + self.xref_aliases = config.get("xref_aliases", dict()) + self.xref_ignore = config.get("xref_ignore", set()) + self.template = config.get("template", None) + if self.template is None: + template_dirs = [os.path.join(os.path.dirname(__file__), "templates")] + template_loader = FileSystemLoader(template_dirs) + template_env = SandboxedEnvironment(loader=template_loader) + self.template = template_env.get_template("numpydoc_docstring.rst") # string conversion routines - def _str_header(self, name, symbol='`'): - return ['.. rubric:: ' + name, ''] + def _str_header(self, name, symbol="`"): + return [".. rubric:: " + name, ""] def _str_field_list(self, name): - return [':' + name + ':'] + return [":" + name + ":"] def _str_indent(self, doc, indent=4): out = [] for line in doc: - out += [' '*indent + line] + out += [" " * indent + line] return out def _str_signature(self): - return [''] - if self['Signature']: - return ['``%s``' % self['Signature']] + [''] - else: - return [''] + return [""] def _str_summary(self): - return self['Summary'] + [''] + return self["Summary"] + [""] def _str_extended_summary(self): - return self['Extended Summary'] + [''] + return self["Extended Summary"] + [""] + + def _str_returns(self, name="Returns"): + named_fmt = "**%s** : %s" + unnamed_fmt = "%s" - def _str_param_list(self, name): out = [] if self[name]: out += self._str_field_list(name) - out += [''] - for param,param_type,desc in self[name]: - out += self._str_indent(['**%s** : %s' % (param.strip(), - param_type)]) - out += [''] - out += self._str_indent(desc,8) - out += [''] + out += [""] + for param in self[name]: + param_type = param.type + if param_type and self.xref_param_type: + param_type = make_xref( + param_type, self.xref_aliases, self.xref_ignore + ) + if param.name: + out += self._str_indent( + [named_fmt % (param.name.strip(), param_type)] + ) + else: + out += self._str_indent([unnamed_fmt % param_type.strip()]) + if not param.desc: + out += self._str_indent([".."], 8) + else: + out += self._str_indent(param.desc, 8) + out += [""] return out - @property - def _obj(self): - if hasattr(self, '_cls'): - return self._cls - elif hasattr(self, '_f'): - return self._f - return None + def _escape_args_and_kwargs(self, name): + if name[:2] == "**": + return r"\*\*" + name[2:] + elif name[:1] == "*": + return r"\*" + name[1:] + else: + return name + + def _process_param(self, param, desc, fake_autosummary): + """Determine how to display a parameter + + Emulates autosummary behavior if fake_autosummary + + Parameters + ---------- + param : str + The name of the parameter + desc : list of str + The parameter description as given in the docstring. This is + ignored when autosummary logic applies. + fake_autosummary : bool + If True, autosummary-style behaviour will apply for params + that are attributes of the class and have a docstring. + + Returns + ------- + display_param : str + The marked up parameter name for display. This may include a link + to the corresponding attribute's own documentation. + desc : list of str + A list of description lines. This may be identical to the input + ``desc``, if ``autosum is None`` or ``param`` is not a class + attribute, or it will be a summary of the class attribute's + docstring. + + Notes + ----- + This does not have the autosummary functionality to display a method's + signature, and hence is not used to format methods. It may be + complicated to incorporate autosummary's signature mangling, as it + relies on Sphinx's plugin mechanism. + """ + param = self._escape_args_and_kwargs(param.strip()) + # param = param.strip() + # XXX: If changing the following, please check the rendering when param + # ends with '_', e.g. 'word_' + # See https://github.com/numpy/numpydoc/pull/144 + display_param = f"**{param}**" + + if not fake_autosummary: + return display_param, desc + + param_obj = getattr(self._obj, param, None) + if not ( + callable(param_obj) + or isinstance(param_obj, property) + or inspect.isgetsetdescriptor(param_obj) + or inspect.ismemberdescriptor(param_obj) + ): + param_obj = None + obj_doc = pydoc.getdoc(param_obj) + + if not (param_obj and obj_doc): + return display_param, desc + + prefix = getattr(self, "_name", "") + if prefix: + link_prefix = f"{prefix}." + else: + link_prefix = "" + + # Referenced object has a docstring + display_param = f":obj:`{param} <{link_prefix}{param}>`" + if obj_doc: + # Overwrite desc. Take summary logic of autosummary + desc = re.split(r"\n\s*\n", obj_doc.strip(), 1)[0] + # XXX: Should this have DOTALL? + # It does not in autosummary + m = re.search(r"^([A-Z].*?\.)(?:\s|$)", " ".join(desc.split())) + if m: + desc = m.group(1).strip() + else: + desc = desc.partition("\n")[0] + desc = desc.split("\n") + return display_param, desc + + def _str_param_list(self, name, fake_autosummary=False): + """Generate RST for a listing of parameters or similar + + Parameter names are displayed as bold text, and descriptions + are in definition lists. + + Parameters + ---------- + name : str + Section name (e.g. Parameters) + fake_autosummary : bool + When True, the parameter names may correspond to attributes of the + object beign documented, usually ``property`` instances on a class. + In this case, names will be linked to fuller descriptions. + + Returns + ------- + rst : list of str + """ + out = [] + if self[name]: + out += self._str_field_list(name) + out += [""] + for param in self[name]: + display_param, desc = self._process_param( + param.name, param.desc, fake_autosummary + ) + parts = [] + if display_param: + parts.append(display_param) + param_type = param.type + if param_type: + param_type = param.type + if self.xref_param_type: + param_type = make_xref( + param_type, self.xref_aliases, self.xref_ignore + ) + parts.append(param_type) + out += self._str_indent([" : ".join(parts)]) + + if not desc: + # empty definition + desc = [".."] + out += self._str_indent(desc, 8) + out += [""] + + return out def _str_member_list(self, name): """ @@ -62,164 +229,210 @@ def _str_member_list(self, name): """ out = [] if self[name]: - out += ['.. rubric:: %s' % name, ''] - prefix = getattr(self, '_name', '') + out += [f".. rubric:: {name}", ""] + prefix = getattr(self, "_name", "") if prefix: - prefix = '~%s.' % prefix + prefix = f"~{prefix}." autosum = [] others = [] - for param, param_type, desc in self[name]: - param = param.strip() - if not self._obj or hasattr(self._obj, param): - autosum += [" %s%s" % (prefix, param)] + for param in self[name]: + param = param._replace(name=param.name.strip()) + + # Check if the referenced member can have a docstring or not + param_obj = getattr(self._obj, param.name, None) + if not ( + callable(param_obj) + or isinstance(param_obj, property) + or inspect.isdatadescriptor(param_obj) + ): + param_obj = None + + if param_obj and pydoc.getdoc(param_obj): + # Referenced object has a docstring + autosum += [f" {prefix}{param.name}"] else: - others.append((param, param_type, desc)) + others.append(param) if autosum: - out += ['.. autosummary::', ' :toctree:', ''] - out += autosum + out += [".. autosummary::"] + if self.class_members_toctree: + out += [" :toctree:"] + out += [""] + autosum if others: - maxlen_0 = max([len(x[0]) for x in others]) - maxlen_1 = max([len(x[1]) for x in others]) - hdr = "="*maxlen_0 + " " + "="*maxlen_1 + " " + "="*10 - fmt = '%%%ds %%%ds ' % (maxlen_0, maxlen_1) - n_indent = maxlen_0 + maxlen_1 + 4 - out += [hdr] - for param, param_type, desc in others: - out += [fmt % (param.strip(), param_type)] - out += self._str_indent(desc, n_indent) + maxlen_0 = max(3, max(len(p.name) + 4 for p in others)) + hdr = "=" * maxlen_0 + " " + "=" * 10 + fmt = "%%%ds %%s " % (maxlen_0,) + out += ["", "", hdr] + for param in others: + name = "**" + param.name.strip() + "**" + desc = " ".join(x.strip() for x in param.desc).strip() + if param.type: + desc = f"({param.type}) {desc}" + out += [fmt % (name, desc)] out += [hdr] - out += [''] + out += [""] return out def _str_section(self, name): out = [] if self[name]: out += self._str_header(name) - out += [''] content = textwrap.dedent("\n".join(self[name])).split("\n") out += content - out += [''] + out += [""] return out def _str_see_also(self, func_role): out = [] - if self['See Also']: - see_also = super(SphinxDocString, self)._str_see_also(func_role) - out = ['.. seealso::', ''] + if self["See Also"]: + see_also = super()._str_see_also(func_role) + out = [".. seealso::", ""] out += self._str_indent(see_also[2:]) return out def _str_warnings(self): out = [] - if self['Warnings']: - out = ['.. warning::', ''] - out += self._str_indent(self['Warnings']) + if self["Warnings"]: + out = [".. warning::", ""] + out += self._str_indent(self["Warnings"]) + out += [""] return out def _str_index(self): - idx = self['index'] + idx = self["index"] out = [] if len(idx) == 0: return out - out += ['.. index:: %s' % idx.get('default','')] - for section, references in idx.iteritems(): - if section == 'default': + out += [f".. index:: {idx.get('default', '')}"] + for section, references in idx.items(): + if section == "default": continue - elif section == 'refguide': - out += [' single: %s' % (', '.join(references))] + elif section == "refguide": + out += [f" single: {', '.join(references)}"] else: - out += [' %s: %s' % (section, ','.join(references))] + out += [f" {section}: {','.join(references)}"] + out += [""] return out def _str_references(self): out = [] - if self['References']: - out += self._str_header('References') - if isinstance(self['References'], str): - self['References'] = [self['References']] - out.extend(self['References']) - out += [''] + if self["References"]: + out += self._str_header("References") + if isinstance(self["References"], str): + self["References"] = [self["References"]] + out.extend(self["References"]) + out += [""] # Latex collects all references to a separate bibliography, # so we need to insert links to it - if sphinx.__version__ >= "0.6": - out += ['.. only:: latex',''] - else: - out += ['.. latexonly::',''] + out += [".. only:: latex", ""] items = [] - for line in self['References']: - m = re.match(r'.. \[([a-z0-9._-]+)\]', line, re.I) + for line in self["References"]: + m = re.match(r".. \[([a-z0-9._-]+)\]", line, re.I) if m: items.append(m.group(1)) - out += [' ' + ", ".join(["[%s]_" % item for item in items]), ''] + out += [" " + ", ".join([f"[{item}]_" for item in items]), ""] return out def _str_examples(self): - examples_str = "\n".join(self['Examples']) + examples_str = "\n".join(self["Examples"]) - if (self.use_plots and 'import matplotlib' in examples_str - and 'plot::' not in examples_str): + if ( + self.use_plots + and re.search(IMPORT_MATPLOTLIB_RE, examples_str) + and "plot::" not in examples_str + ): out = [] - out += self._str_header('Examples') - out += ['.. plot::', ''] - out += self._str_indent(self['Examples']) - out += [''] + out += self._str_header("Examples") + out += [".. plot::", ""] + out += self._str_indent(self["Examples"]) + out += [""] return out else: - return self._str_section('Examples') + return self._str_section("Examples") def __str__(self, indent=0, func_role="obj"): - out = [] - out += self._str_signature() - out += self._str_index() + [''] - out += self._str_summary() - out += self._str_extended_summary() - for param_list in ('Parameters', 'Returns', 'Other Parameters', - 'Raises', 'Warns'): - out += self._str_param_list(param_list) - out += self._str_warnings() - out += self._str_see_also(func_role) - out += self._str_section('Notes') - out += self._str_references() - out += self._str_examples() - for param_list in ('Attributes', 'Methods'): - out += self._str_member_list(param_list) - out = self._str_indent(out,indent) - return '\n'.join(out) + ns = { + "signature": self._str_signature(), + "index": self._str_index(), + "summary": self._str_summary(), + "extended_summary": self._str_extended_summary(), + "parameters": self._str_param_list("Parameters"), + "returns": self._str_returns("Returns"), + "yields": self._str_returns("Yields"), + "receives": self._str_returns("Receives"), + "other_parameters": self._str_param_list("Other Parameters"), + "raises": self._str_returns("Raises"), + "warns": self._str_returns("Warns"), + "warnings": self._str_warnings(), + "see_also": self._str_see_also(func_role), + "notes": self._str_section("Notes"), + "references": self._str_references(), + "examples": self._str_examples(), + "attributes": self._str_param_list("Attributes", fake_autosummary=True) + if self.attributes_as_param_list + else self._str_member_list("Attributes"), + "methods": self._str_member_list("Methods"), + } + ns = {k: "\n".join(v) for k, v in ns.items()} + + rendered = self.template.render(**ns) + return "\n".join(self._str_indent(rendered.split("\n"), indent)) + class SphinxFunctionDoc(SphinxDocString, FunctionDoc): - def __init__(self, obj, doc=None, config={}): - self.use_plots = config.get('use_plots', False) + def __init__(self, obj, doc=None, config=None): + if config is None: + config = {} + self.load_config(config) FunctionDoc.__init__(self, obj, doc=doc, config=config) + class SphinxClassDoc(SphinxDocString, ClassDoc): - def __init__(self, obj, doc=None, func_doc=None, config={}): - self.use_plots = config.get('use_plots', False) + def __init__(self, obj, doc=None, func_doc=None, config=None): + if config is None: + config = {} + self.load_config(config) ClassDoc.__init__(self, obj, doc=doc, func_doc=None, config=config) -class SphinxObjDoc(SphinxDocString): - def __init__(self, obj, doc=None, config={}): - self._f = obj - SphinxDocString.__init__(self, doc, config=config) -def get_doc_object(obj, what=None, doc=None, config={}): +class SphinxObjDoc(SphinxDocString, ObjDoc): + def __init__(self, obj, doc=None, config=None): + if config is None: + config = {} + self.load_config(config) + ObjDoc.__init__(self, obj, doc=doc, config=config) + + +# TODO: refactor to use docscrape.get_doc_object +def get_doc_object(obj, what=None, doc=None, config=None, builder=None): if what is None: if inspect.isclass(obj): - what = 'class' + what = "class" elif inspect.ismodule(obj): - what = 'module' - elif callable(obj): - what = 'function' + what = "module" + elif isinstance(obj, Callable): + what = "function" else: - what = 'object' - if what == 'class': - return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc, - config=config) - elif what in ('function', 'method'): + what = "object" + + if config is None: + config = {} + template_dirs = [os.path.join(os.path.dirname(__file__), "templates")] + if builder is not None: + template_loader = BuiltinTemplateLoader() + template_loader.init(builder, dirs=template_dirs) + else: + template_loader = FileSystemLoader(template_dirs) + template_env = SandboxedEnvironment(loader=template_loader) + config["template"] = template_env.get_template("numpydoc_docstring.rst") + + if what == "class": + return SphinxClassDoc(obj, func_doc=SphinxFunctionDoc, doc=doc, config=config) + elif what in ("function", "method"): return SphinxFunctionDoc(obj, doc=doc, config=config) else: if doc is None: diff --git a/docs/sphinxext/numpydoc.py b/docs/sphinxext/numpydoc.py index a5750603204..509f0533fdf 100644 --- a/docs/sphinxext/numpydoc.py +++ b/docs/sphinxext/numpydoc.py @@ -13,127 +13,329 @@ - Extract the signature from the docstring, if it can't be determined otherwise. -.. [1] https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt +.. [1] https://github.com/numpy/numpydoc """ -from __future__ import division, absolute_import, print_function - -import sys +from copy import deepcopy import re import pydoc -import sphinx import inspect -import collections +from collections.abc import Callable +import hashlib +import itertools -if sphinx.__version__ < '1.0.1': - raise RuntimeError("Sphinx 1.0.1 or newer is required") +from docutils.nodes import citation, Text, section, comment, reference +import sphinx +from sphinx.addnodes import pending_xref, desc_content +from sphinx.util import logging +from sphinx.errors import ExtensionError -from docscrape_sphinx import get_doc_object, SphinxDocString -from sphinx.util.compat import Directive +if sphinx.__version__ < "4.2": + raise RuntimeError("Sphinx 4.2 or newer is required") -if sys.version_info[0] >= 3: - sixu = lambda s: s -else: - sixu = lambda s: unicode(s, 'unicode_escape') +from .docscrape_sphinx import get_doc_object +from .validate import validate, ERROR_MSGS +from .xref import DEFAULT_LINKS +from . import __version__ +logger = logging.getLogger(__name__) -def mangle_docstrings(app, what, name, obj, options, lines, - reference_offset=[0]): +HASH_LEN = 12 - cfg = {'use_plots': app.config.numpydoc_use_plots, - 'show_class_members': app.config.numpydoc_show_class_members, - 'show_inherited_class_members': - app.config.numpydoc_show_inherited_class_members, - 'class_members_toctree': app.config.numpydoc_class_members_toctree} - u_NL = sixu('\n') - if what == 'module': - # Strip top title - pattern = '^\\s*[#*=]{4,}\\n[a-z0-9 -]+\\n[#*=]{4,}\\s*' - title_re = re.compile(sixu(pattern), re.I | re.S) - lines[:] = title_re.sub(sixu(''), u_NL.join(lines)).split(u_NL) - else: - doc = get_doc_object(obj, what, u_NL.join(lines), config=cfg) - if sys.version_info[0] >= 3: - doc = str(doc) - else: - doc = unicode(doc) - lines[:] = doc.split(u_NL) - - if (app.config.numpydoc_edit_link and hasattr(obj, '__name__') and - obj.__name__): - if hasattr(obj, '__module__'): - v = dict(full_name=sixu("%s.%s") % (obj.__module__, obj.__name__)) - else: - v = dict(full_name=obj.__name__) - lines += [sixu(''), sixu('.. htmlonly::'), sixu('')] - lines += [sixu(' %s') % x for x in - (app.config.numpydoc_edit_link % v).split("\n")] - - # replace reference numbers so that there are no duplicates - references = [] +def _traverse_or_findall(node, condition, **kwargs): + """Triage node.traverse (docutils <0.18.1) vs node.findall. + + TODO: This check can be removed when the minimum supported docutils version + for numpydoc is docutils>=0.18.1 + """ + return ( + node.findall(condition, **kwargs) + if hasattr(node, "findall") + else node.traverse(condition, **kwargs) + ) + + +def rename_references(app, what, name, obj, options, lines): + # decorate reference numbers so that there are no duplicates + # these are later undecorated in the doctree, in relabel_references + references = set() for line in lines: line = line.strip() - m = re.match(sixu('^.. \\[([a-z0-9_.-])\\]'), line, re.I) + m = re.match(r"^\.\. +\[(%s)\]" % app.config.numpydoc_citation_re, line, re.I) if m: - references.append(m.group(1)) + references.add(m.group(1)) - # start renaming from the longest string, to avoid overwriting parts - references.sort(key=lambda x: -len(x)) if references: - for i, line in enumerate(lines): - for r in references: - if re.match(sixu('^\\d+$'), r): - new_r = sixu("R%d") % (reference_offset[0] + int(r)) - else: - new_r = sixu("%s%d") % (r, reference_offset[0]) - lines[i] = lines[i].replace(sixu('[%s]_') % r, - sixu('[%s]_') % new_r) - lines[i] = lines[i].replace(sixu('.. [%s]') % r, - sixu('.. [%s]') % new_r) + # we use a hash to mangle the reference name to avoid invalid names + sha = hashlib.sha256() + sha.update(name.encode("utf8")) + prefix = "R" + sha.hexdigest()[:HASH_LEN] + + for r in references: + new_r = prefix + "-" + r + for i, line in enumerate(lines): + lines[i] = lines[i].replace(f"[{r}]_", f"[{new_r}]_") + lines[i] = lines[i].replace(f".. [{r}]", f".. [{new_r}]") + + +def _is_cite_in_numpydoc_docstring(citation_node): + # Find DEDUPLICATION_TAG in comment as last node of sibling section + + # XXX: I failed to use citation_node.traverse to do this: + section_node = citation_node.parent + + def is_docstring_section(node): + return isinstance(node, (section, desc_content)) + + while not is_docstring_section(section_node): + section_node = section_node.parent + if section_node is None: + return False + + sibling_sections = itertools.chain( + _traverse_or_findall( + section_node, + is_docstring_section, + include_self=True, + descend=False, + siblings=True, + ) + ) + for sibling_section in sibling_sections: + if not sibling_section.children: + continue + + for child in sibling_section.children[::-1]: + if not isinstance(child, comment): + continue + + if child.rawsource.strip() == DEDUPLICATION_TAG.strip(): + return True + + return False + + +def relabel_references(app, doc): + # Change 'hash-ref' to 'ref' in label text + for citation_node in _traverse_or_findall(doc, citation): + if not _is_cite_in_numpydoc_docstring(citation_node): + continue + label_node = citation_node[0] + prefix, _, new_label = label_node[0].astext().partition("-") + assert len(prefix) == HASH_LEN + 1 + new_text = Text(new_label) + label_node.replace(label_node[0], new_text) + + for id_ in citation_node["backrefs"]: + ref = doc.ids[id_] + ref_text = ref[0] + + # Sphinx has created pending_xref nodes with [reftext] text. + def matching_pending_xref(node): + return ( + isinstance(node, pending_xref) + and node[0].astext() == f"[{ref_text}]" + ) + + for xref_node in _traverse_or_findall(ref.parent, matching_pending_xref): + xref_node.replace(xref_node[0], Text(f"[{new_text}]")) + ref.replace(ref_text, new_text.copy()) + + +def clean_backrefs(app, doc, docname): + # only::latex directive has resulted in citation backrefs without reference + known_ref_ids = set() + for ref in _traverse_or_findall(doc, reference, descend=True): + for id_ in ref["ids"]: + known_ref_ids.add(id_) + for citation_node in _traverse_or_findall(doc, citation, descend=True): + # remove backrefs to non-existent refs + citation_node["backrefs"] = [ + id_ for id_ in citation_node["backrefs"] if id_ in known_ref_ids + ] + + +DEDUPLICATION_TAG = " !! processed by numpydoc !!" + + +def mangle_docstrings(app, what, name, obj, options, lines): + if DEDUPLICATION_TAG in lines: + return + show_inherited_class_members = app.config.numpydoc_show_inherited_class_members + if isinstance(show_inherited_class_members, dict): + try: + show_inherited_class_members = show_inherited_class_members[name] + except KeyError: + show_inherited_class_members = True + + cfg = { + "use_plots": app.config.numpydoc_use_plots, + "show_class_members": app.config.numpydoc_show_class_members, + "show_inherited_class_members": show_inherited_class_members, + "class_members_toctree": app.config.numpydoc_class_members_toctree, + "attributes_as_param_list": app.config.numpydoc_attributes_as_param_list, + "xref_param_type": app.config.numpydoc_xref_param_type, + "xref_aliases": app.config.numpydoc_xref_aliases_complete, + "xref_ignore": app.config.numpydoc_xref_ignore, + } - reference_offset[0] += len(references) + cfg.update(options or {}) + u_NL = "\n" + if what == "module": + # Strip top title + pattern = "^\\s*[#*=]{4,}\\n[a-z0-9 -]+\\n[#*=]{4,}\\s*" + title_re = re.compile(pattern, re.I | re.S) + lines[:] = title_re.sub("", u_NL.join(lines)).split(u_NL) + else: + try: + doc = get_doc_object( + obj, what, u_NL.join(lines), config=cfg, builder=app.builder + ) + lines[:] = str(doc).split(u_NL) + except Exception: + logger.error("[numpydoc] While processing docstring for %r", name) + raise + + if app.config.numpydoc_validation_checks: + # If the user has supplied patterns to ignore via the + # numpydoc_validation_exclude config option, skip validation for + # any objs whose name matches any of the patterns + excluder = app.config.numpydoc_validation_excluder + exclude_from_validation = excluder.search(name) if excluder else False + if not exclude_from_validation: + # TODO: Currently, all validation checks are run and only those + # selected via config are reported. It would be more efficient to + # only run the selected checks. + errors = validate(doc)["errors"] + if {err[0] for err in errors} & app.config.numpydoc_validation_checks: + msg = ( + f"[numpydoc] Validation warnings while processing " + f"docstring for {name!r}:\n" + ) + for err in errors: + if err[0] in app.config.numpydoc_validation_checks: + msg += f" {err[0]}: {err[1]}\n" + logger.warning(msg) + + # call function to replace reference numbers so that there are no + # duplicates + rename_references(app, what, name, obj, options, lines) + + lines += ["..", DEDUPLICATION_TAG] def mangle_signature(app, what, name, obj, options, sig, retann): # Do not try to inspect classes that don't define `__init__` - if (inspect.isclass(obj) and - (not hasattr(obj, '__init__') or - 'initializes x; see ' in pydoc.getdoc(obj.__init__))): - return '', '' + if inspect.isclass(obj) and ( + not hasattr(obj, "__init__") + or "initializes x; see " in pydoc.getdoc(obj.__init__) + ): + return "", "" - if not (isinstance(obj, collections.Callable) or - hasattr(obj, '__argspec_is_invalid_')): + if not (isinstance(obj, Callable) or hasattr(obj, "__argspec_is_invalid_")): return - if not hasattr(obj, '__doc__'): + if not hasattr(obj, "__doc__"): return - - doc = SphinxDocString(pydoc.getdoc(obj)) - if doc['Signature']: - sig = re.sub(sixu("^[^(]*"), sixu(""), doc['Signature']) - return sig, sixu('') + doc = get_doc_object(obj, config={"show_class_members": False}) + sig = doc["Signature"] or _clean_text_signature( + getattr(obj, "__text_signature__", None) + ) + if sig: + sig = re.sub("^[^(]*", "", sig) + return sig, "" + + +def _clean_text_signature(sig): + if sig is None: + return None + start_pattern = re.compile(r"^[^(]*\(") + start, end = start_pattern.search(sig).span() + start_sig = sig[start:end] + sig = sig[end:-1] + sig = re.sub(r"^\$(self|module|type)(,\s|$)", "", sig, count=1) + sig = re.sub(r"(^|(?<=,\s))/,\s\*", "*", sig, count=1) + return start_sig + sig + ")" def setup(app, get_doc_object_=get_doc_object): - if not hasattr(app, 'add_config_value'): + if not hasattr(app, "add_config_value"): return # probably called by nose, better bail out global get_doc_object get_doc_object = get_doc_object_ - app.connect('autodoc-process-docstring', mangle_docstrings) - app.connect('autodoc-process-signature', mangle_signature) - app.add_config_value('numpydoc_edit_link', None, False) - app.add_config_value('numpydoc_use_plots', None, False) - app.add_config_value('numpydoc_show_class_members', True, True) - app.add_config_value('numpydoc_show_inherited_class_members', True, True) - app.add_config_value('numpydoc_class_members_toctree', True, True) + app.setup_extension("sphinx.ext.autosummary") + app.connect("config-inited", update_config) + app.connect("autodoc-process-docstring", mangle_docstrings) + app.connect("autodoc-process-signature", mangle_signature) + app.connect("doctree-read", relabel_references) + app.connect("doctree-resolved", clean_backrefs) + app.add_config_value("numpydoc_use_plots", None, False) + app.add_config_value("numpydoc_show_class_members", True, True) + app.add_config_value( + "numpydoc_show_inherited_class_members", True, True, types=(bool, dict) + ) + app.add_config_value("numpydoc_class_members_toctree", True, True) + app.add_config_value("numpydoc_citation_re", "[a-z0-9_.-]+", True) + app.add_config_value("numpydoc_attributes_as_param_list", True, True) + app.add_config_value("numpydoc_xref_param_type", False, True) + app.add_config_value("numpydoc_xref_aliases", dict(), True) + app.add_config_value("numpydoc_xref_ignore", set(), True) + app.add_config_value("numpydoc_validation_checks", set(), True) + app.add_config_value("numpydoc_validation_exclude", set(), False) # Extra mangling domains app.add_domain(NumpyPythonDomain) app.add_domain(NumpyCDomain) + metadata = {"version": __version__, "parallel_read_safe": True} + return metadata + + +def update_config(app, config=None): + """Update the configuration with default values.""" + if config is None: # needed for testing and old Sphinx + config = app.config + # Do not simply overwrite the `app.config.numpydoc_xref_aliases` + # otherwise the next sphinx-build will compare the incoming values (without + # our additions) to the old values (with our additions) and trigger + # a full rebuild! + numpydoc_xref_aliases_complete = deepcopy(config.numpydoc_xref_aliases) + for key, value in DEFAULT_LINKS.items(): + if key not in numpydoc_xref_aliases_complete: + numpydoc_xref_aliases_complete[key] = value + config.numpydoc_xref_aliases_complete = numpydoc_xref_aliases_complete + + # Processing to determine whether numpydoc_validation_checks is treated + # as a blocklist or allowlist + valid_error_codes = set(ERROR_MSGS.keys()) + if "all" in config.numpydoc_validation_checks: + block = deepcopy(config.numpydoc_validation_checks) + config.numpydoc_validation_checks = valid_error_codes - block + # Ensure that the validation check set contains only valid error codes + invalid_error_codes = config.numpydoc_validation_checks - valid_error_codes + if invalid_error_codes: + raise ValueError( + f"Unrecognized validation code(s) in numpydoc_validation_checks " + f"config value: {invalid_error_codes}" + ) + + # Generate the regexp for docstrings to ignore during validation + if isinstance(config.numpydoc_validation_exclude, str): + raise ValueError( + f"numpydoc_validation_exclude must be a container of strings, " + f"e.g. [{config.numpydoc_validation_exclude!r}]." + ) + config.numpydoc_validation_excluder = None + if config.numpydoc_validation_exclude: + exclude_expr = re.compile( + r"|".join(exp for exp in config.numpydoc_validation_exclude) + ) + config.numpydoc_validation_excluder = exclude_expr + + # ------------------------------------------------------------------------------ # Docstring-mangling domains # ------------------------------------------------------------------------------ @@ -143,44 +345,101 @@ def setup(app, get_doc_object_=get_doc_object): from sphinx.domains.python import PythonDomain -class ManglingDomainBase(object): +class ManglingDomainBase: directive_mangling_map = {} def __init__(self, *a, **kw): - super(ManglingDomainBase, self).__init__(*a, **kw) + super().__init__(*a, **kw) self.wrap_mangling_directives() def wrap_mangling_directives(self): for name, objtype in list(self.directive_mangling_map.items()): self.directives[name] = wrap_mangling_directive( - self.directives[name], objtype) + self.directives[name], objtype + ) class NumpyPythonDomain(ManglingDomainBase, PythonDomain): - name = 'np' + name = "np" directive_mangling_map = { - 'function': 'function', - 'class': 'class', - 'exception': 'class', - 'method': 'function', - 'classmethod': 'function', - 'staticmethod': 'function', - 'attribute': 'attribute', + "function": "function", + "class": "class", + "exception": "class", + "method": "function", + "classmethod": "function", + "staticmethod": "function", + "attribute": "attribute", } indices = [] class NumpyCDomain(ManglingDomainBase, CDomain): - name = 'np-c' + name = "np-c" directive_mangling_map = { - 'function': 'function', - 'member': 'attribute', - 'macro': 'function', - 'type': 'class', - 'var': 'object', + "function": "function", + "member": "attribute", + "macro": "function", + "type": "class", + "var": "object", } +def match_items(lines, content_old): + """Create items for mangled lines. + + This function tries to match the lines in ``lines`` with the items (source + file references and line numbers) in ``content_old``. The + ``mangle_docstrings`` function changes the actual docstrings, but doesn't + keep track of where each line came from. The manging does many operations + on the original lines, which are hard to track afterwards. + + Many of the line changes come from deleting or inserting blank lines. This + function tries to match lines by ignoring blank lines. All other changes + (such as inserting figures or changes in the references) are completely + ignored, so the generated line numbers will be off if ``mangle_docstrings`` + does anything non-trivial. + + This is a best-effort function and the real fix would be to make + ``mangle_docstrings`` actually keep track of the ``items`` together with + the ``lines``. + + Examples + -------- + >>> lines = ['', 'A', '', 'B', ' ', '', 'C', 'D'] + >>> lines_old = ['a', '', '', 'b', '', 'c'] + >>> items_old = [('file1.py', 0), ('file1.py', 1), ('file1.py', 2), + ... ('file2.py', 0), ('file2.py', 1), ('file2.py', 2)] + >>> content_old = ViewList(lines_old, items=items_old) + >>> match_items(lines, content_old) # doctest: +NORMALIZE_WHITESPACE + [('file1.py', 0), ('file1.py', 0), ('file2.py', 0), ('file2.py', 0), + ('file2.py', 2), ('file2.py', 2), ('file2.py', 2), ('file2.py', 2)] + >>> # first 2 ``lines`` are matched to 'a', second 2 to 'b', rest to 'c' + >>> # actual content is completely ignored. + + Notes + ----- + The algorithm tries to match any line in ``lines`` with one in + ``lines_old``. It skips over all empty lines in ``lines_old`` and assigns + this line number to all lines in ``lines``, unless a non-empty line is + found in ``lines`` in which case it goes to the next line in ``lines_old``. + + """ + items_new = [] + lines_old = content_old.data + items_old = content_old.items + j = 0 + for i, line in enumerate(lines): + # go to next non-empty line in old: + # line.strip() checks whether the string is all whitespace + while j < len(lines_old) - 1 and not lines_old[j].strip(): + j += 1 + items_new.append(items_old[j]) + if line.strip() and j < len(lines_old) - 1: + j += 1 + assert len(items_new) == len(lines) + return items_new + + def wrap_mangling_directive(base_directive, objtype): class directive(base_directive): def run(self): @@ -188,7 +447,7 @@ def run(self): name = None if self.arguments: - m = re.match(r'^(.*\s+)?(.*?)(\(.*)?', self.arguments[0]) + m = re.match(r"^(.*\s+)?(.*?)(\(.*)?", self.arguments[0]) name = m.group(2).strip() if not name: @@ -196,7 +455,9 @@ def run(self): lines = list(self.content) mangle_docstrings(env.app, objtype, name, None, None, lines) - self.content = ViewList(lines, self.content.parent) + if self.content: + items = match_items(lines, self.content) + self.content = ViewList(lines, items=items, parent=self.content.parent) return base_directive.run(self)