diff --git a/AUTHORS.rst b/AUTHORS.rst index 9ecbd9d5a10..c92d21d1dc4 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -106,6 +106,7 @@ Contributors * Taku Shimizu -- epub3 builder * Thomas Lamb -- linkcheck builder * Thomas Waldmann -- apidoc module fixes +* Till Hoffmann -- doctest option to exit after first failed test * Tim Hoffmann -- theme improvements * Vince Salvino -- JavaScript search improvements * Will Maier -- directory HTML builder diff --git a/CHANGES.rst b/CHANGES.rst index 99e1c9bf89f..93faeb96c54 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,9 @@ Deprecated Features added -------------- +* #13332: Add :confval:`doctest_fail_fast` option to exit after the first failed + test. Patch by Till Hoffmann. + Bugs fixed ---------- diff --git a/doc/usage/extensions/doctest.rst b/doc/usage/extensions/doctest.rst index 60c67827967..f8518fabb7a 100644 --- a/doc/usage/extensions/doctest.rst +++ b/doc/usage/extensions/doctest.rst @@ -452,3 +452,11 @@ The doctest extension uses the following configuration values: Also, removal of ```` and ``# doctest:`` options only works in :rst:dir:`doctest` blocks, though you may set :confval:`trim_doctest_flags` to achieve that in all code blocks with Python console content. + +.. confval:: doctest_fail_fast + :type: :code-py:`bool` + :default: :code-py:`False` + + Exit when the first failure is encountered. + + .. versionadded:: 8.2.0 diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py index 105c50a6923..343534f10ce 100644 --- a/sphinx/ext/doctest.py +++ b/sphinx/ext/doctest.py @@ -358,10 +358,17 @@ def finish(self) -> None: def s(v: int) -> str: return 's' if v != 1 else '' + header = 'Doctest summary' + if self.total_failures or self.setup_failures or self.cleanup_failures: + self.app.statuscode = 1 + if self.config.doctest_fail_fast: + header = f'{header} (exiting after first failed test)' + underline = '=' * len(header) + self._out( f""" -Doctest summary -=============== +{header} +{underline} {self.total_tries:5} test{s(self.total_tries)} {self.total_failures:5} failure{s(self.total_failures)} in tests {self.setup_failures:5} failure{s(self.setup_failures)} in setup code @@ -370,15 +377,14 @@ def s(v: int) -> str: ) self.outfile.close() - if self.total_failures or self.setup_failures or self.cleanup_failures: - self.app.statuscode = 1 - def write_documents(self, docnames: Set[str]) -> None: logger.info(bold('running tests...')) for docname in sorted(docnames): # no need to resolve the doctree doctree = self.env.get_doctree(docname) - self.test_doc(docname, doctree) + success = self.test_doc(docname, doctree) + if not success and self.config.doctest_fail_fast: + break def get_filename_for_node(self, node: Node, docname: str) -> str: """Try to get the file which actually contains the doctest, not the @@ -419,7 +425,7 @@ def skipped(self, node: Element) -> bool: exec(self.config.doctest_global_cleanup, context) # NoQA: S102 return should_skip - def test_doc(self, docname: str, doctree: Node) -> None: + def test_doc(self, docname: str, doctree: Node) -> bool: groups: dict[str, TestGroup] = {} add_to_all_groups = [] self.setup_runner = SphinxDocTestRunner(verbose=False, optionflags=self.opt) @@ -496,13 +502,17 @@ def condition(node: Node) -> bool: for group in groups.values(): group.add_code(code) if not groups: - return + return True show_successes = self.config.doctest_show_successes if show_successes: self._out(f'\nDocument: {docname}\n----------{"-" * len(docname)}\n') + success = True for group in groups.values(): - self.test_group(group) + if not self.test_group(group): + success = False + if self.config.doctest_fail_fast: + break # Separately count results from setup code res_f, res_t = self.setup_runner.summarize(self._out, verbose=False) self.setup_failures += res_f @@ -517,13 +527,14 @@ def condition(node: Node) -> bool: ) self.cleanup_failures += res_f self.cleanup_tries += res_t + return success def compile( self, code: str, name: str, type: str, flags: Any, dont_inherit: bool ) -> Any: return compile(code, name, self.type, flags, dont_inherit) - def test_group(self, group: TestGroup) -> None: + def test_group(self, group: TestGroup) -> bool: ns: dict[str, Any] = {} def run_setup_cleanup( @@ -553,9 +564,10 @@ def run_setup_cleanup( # run the setup code if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'): # if setup failed, don't run the group - return + return False # run the tests + success = True for code in group.tests: if len(code) == 1: # ordinary doctests (code/output interleaved) @@ -608,11 +620,19 @@ def run_setup_cleanup( self.type = 'exec' # multiple statements again # DocTest.__init__ copies the globs namespace, which we don't want test.globs = ns + old_f = self.test_runner.failures # also don't clear the globs namespace after running the doctest self.test_runner.run(test, out=self._warn_out, clear_globs=False) + if self.test_runner.failures > old_f: + success = False + if self.config.doctest_fail_fast: + break # run the cleanup - run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup') + if not run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup'): + return False + + return success def setup(app: Sphinx) -> ExtensionMetadata: @@ -638,6 +658,7 @@ def setup(app: Sphinx) -> ExtensionMetadata: '', types=frozenset({int}), ) + app.add_config_value('doctest_fail_fast', False, '', types=frozenset({bool})) return { 'version': sphinx.__display_version__, 'parallel_read_safe': True, diff --git a/tests/roots/test-ext-doctest-fail-fast/conf.py b/tests/roots/test-ext-doctest-fail-fast/conf.py new file mode 100644 index 00000000000..227afbb2c95 --- /dev/null +++ b/tests/roots/test-ext-doctest-fail-fast/conf.py @@ -0,0 +1,11 @@ +extensions = ['sphinx.ext.doctest'] + +project = 'test project for doctest' +root_doc = 'fail-fast' +source_suffix = { + '.txt': 'restructuredtext', +} +exclude_patterns = ['_build'] + +# Set in tests. +# doctest_fail_fast = ... diff --git a/tests/roots/test-ext-doctest-fail-fast/fail-fast.txt b/tests/roots/test-ext-doctest-fail-fast/fail-fast.txt new file mode 100644 index 00000000000..70a05af487b --- /dev/null +++ b/tests/roots/test-ext-doctest-fail-fast/fail-fast.txt @@ -0,0 +1,11 @@ +Testing fast failure in the doctest extension +============================================= + +>>> 1 + 1 +2 + +>>> 1 + 1 +3 + +>>> 1 + 1 +3 diff --git a/tests/test_extensions/test_ext_doctest.py b/tests/test_extensions/test_ext_doctest.py index cb540fda7ec..810f8244ba8 100644 --- a/tests/test_extensions/test_ext_doctest.py +++ b/tests/test_extensions/test_ext_doctest.py @@ -147,3 +147,23 @@ def test_reporting_with_autodoc(app, capfd): assert 'File "dir/bar.py", line ?, in default' in failures assert 'File "foo.py", line ?, in default' in failures assert 'File "index.rst", line 4, in default' in failures + + +@pytest.mark.sphinx('doctest', testroot='ext-doctest-fail-fast') +@pytest.mark.parametrize('fail_fast', [False, True, None]) +def test_fail_fast(app, fail_fast, capsys): + if fail_fast is not None: + app.config.doctest_fail_fast = fail_fast + # Patch builder to get a copy of the output + written = [] + app.builder._out = written.append + app.build(force_all=True) + assert app.statuscode + + written = ''.join(written) + if fail_fast: + assert 'Doctest summary (exiting after first failed test)' in written + assert '1 failure in tests' in written + else: + assert 'Doctest summary\n' in written + assert '2 failures in tests' in written