Implement understanding on __tracebackhide__

__tracebackhide__ is an attribute that when set to a true value mean the
frame need to be skipped/hidden.

Implement this in both ultratb and un ipdb so that by default frames are
hidden, add switches to toggle the behavior.

We reimplement do_up/down but skipping hidden frames when enabled.
This also uses the special value `_ipython_bottom_` to hide all of the
ipython frames.

This also add completion to the debugger

IPython/core/debugger.py

@@ -280,26 +280,31 @@ def __init__(self, color_scheme=None, completekey=None,
 
         # Set the prompt - the default prompt is '(Pdb)'
         self.prompt = prompt
+        self.skip_hidden = True
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
         self.parser.style = scheme
 
+
+    def hidden_frames(self, stack):
+        """
+        Given an index in the stack return wether it should be skipped.
+
+        This is used in up/down and where to skip frames.
+        """
+        ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack]
+        ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"]
+        if ip_start:
+            ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)]
+        return ip_hide
+
     def interaction(self, frame, traceback):
         try:
             OldPdb.interaction(self, frame, traceback)
         except KeyboardInterrupt:
-            self.stdout.write('\n' + self.shell.get_exception_only())
-
-    def new_do_up(self, arg):
-        OldPdb.do_up(self, arg)
-    do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
-
-    def new_do_down(self, arg):
-        OldPdb.do_down(self, arg)
-
-    do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
+            self.stdout.write("\n" + self.shell.get_exception_only())
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
@@ -320,6 +325,8 @@ def new_do_restart(self, arg):
         return self.do_quit(arg)
 
     def print_stack_trace(self, context=None):
+        Colors = self.color_scheme_table.active_colors
+        ColorsNormal = Colors.Normal
         if context is None:
             context = self.context
         try:
@@ -329,8 +336,21 @@ def print_stack_trace(self, context=None):
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
         try:
-            for frame_lineno in self.stack:
+            skipped = 0
+            for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack):
+                if hidden and self.skip_hidden:
+                    skipped += 1
+                    continue
+                if skipped:
+                    print(
+                        f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
+                    )
+                    skipped = 0
                 self.print_stack_entry(frame_lineno, context=context)
+            if skipped:
+                print(
+                    f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
+                )
         except KeyboardInterrupt:
             pass
 
@@ -485,6 +505,16 @@ def print_list_lines(self, filename, first, last):
         except KeyboardInterrupt:
             pass
 
+    def do_skip_hidden(self, arg):
+        """
+        Change whether or not we should skip frames with the
+        __tracebackhide__ attribute.
+        """
+        if arg.strip().lower() in ("true", "yes"):
+            self.skip_hidden = True
+        elif arg.strip().lower() in ("false", "no"):
+            self.skip_hidden = False
+
     def do_list(self, arg):
         """Print lines of code from the current stack frame
         """
@@ -631,6 +661,109 @@ def do_where(self, arg):
 
     do_w = do_where
 
+    def stop_here(self, frame):
+        hidden = False
+        if self.skip_hidden:
+            hidden = frame.f_locals.get("__tracebackhide__", False)
+        if hidden:
+            Colors = self.color_scheme_table.active_colors
+            ColorsNormal = Colors.Normal
+            print(f"{Colors.excName}    [... skipped 1 hidden frame]{ColorsNormal}\n")
+
+        return super().stop_here(frame)
+
+    def do_up(self, arg):
+        """u(p) [count]
+        Move the current frame count (default one) levels up in the
+        stack trace (to an older frame).
+
+        Will skip hidden frames.
+        """
+        ## modified version of upstream that skips
+        # frames with __tracebackide__
+        if self.curindex == 0:
+            self.error("Oldest frame")
+            return
+        try:
+            count = int(arg or 1)
+        except ValueError:
+            self.error("Invalid frame count (%s)" % arg)
+            return
+        skipped = 0
+        if count < 0:
+            _newframe = 0
+        else:
+            _newindex = self.curindex
+            counter = 0
+            hidden_frames = self.hidden_frames(self.stack)
+            for i in range(self.curindex - 1, -1, -1):
+                frame = self.stack[i][0]
+                if hidden_frames[i] and self.skip_hidden:
+                    skipped += 1
+                    continue
+                counter += 1
+                if counter >= count:
+                    break
+            else:
+                # if no break occured.
+                self.error("all frames above hidden")
+                return
+
+            Colors = self.color_scheme_table.active_colors
+            ColorsNormal = Colors.Normal
+            _newframe = i
+        self._select_frame(_newframe)
+        if skipped:
+            print(
+                f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
+            )
+
+    def do_down(self, arg):
+        """d(own) [count]
+        Move the current frame count (default one) levels down in the
+        stack trace (to a newer frame).
+
+        Will skip hidden frames.
+        """
+        if self.curindex + 1 == len(self.stack):
+            self.error("Newest frame")
+            return
+        try:
+            count = int(arg or 1)
+        except ValueError:
+            self.error("Invalid frame count (%s)" % arg)
+            return
+        if count < 0:
+            _newframe = len(self.stack) - 1
+        else:
+            _newindex = self.curindex
+            counter = 0
+            skipped = 0
+            hidden_frames = self.hidden_frames(self.stack)
+            for i in range(self.curindex + 1, len(self.stack)):
+                frame = self.stack[i][0]
+                if hidden_frames[i] and self.skip_hidden:
+                    skipped += 1
+                    continue
+                counter += 1
+                if counter >= count:
+                    break
+            else:
+                self.error("all frames bellow hidden")
+                return
+
+            Colors = self.color_scheme_table.active_colors
+            ColorsNormal = Colors.Normal
+            if skipped:
+                print(
+                    f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
+                )
+            _newframe = i
+
+        self._select_frame(_newframe)
+
+    do_d = do_down
+    do_u = do_up
 
 class InterruptiblePdb(Pdb):
     """Version of debugger where KeyboardInterrupt exits the debugger altogether."""

IPython/core/interactiveshell.py

@@ -2211,11 +2211,20 @@ def complete(self, text, line=None, cursor_pos=None):
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
-    def set_custom_completer(self, completer, pos=0):
+    def set_custom_completer(self, completer, pos=0) -> None:
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
-        list where you want the completer to be inserted."""
+        list where you want the completer to be inserted.
+
+        `completer` should have the following signature::
+
+            def completion(self: Completer, text: string) -> List[str]:
+                raise NotImplementedError
+
+        It will be bound to the current Completer instance and pass some text
+        and return a list with current completions to suggest to the user.
+        """
 
         newcomp = types.MethodType(completer, self.Completer)
         self.Completer.custom_matchers.insert(pos,newcomp)
@@ -3310,6 +3319,9 @@ async def run_code(self, code_obj, result=None, *, async_=False):
         False : successful execution.
         True : an error occurred.
         """
+        # special value to say that anything above is IPython and should be
+        # hidden.
+        __tracebackhide__ = "__ipython_bottom__"
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook, sys.excepthook = sys.excepthook, self.excepthook

IPython/core/magics/basic.py

@@ -364,13 +364,25 @@ def xmode(self, parameter_s=''):
 
         Valid modes: Plain, Context, Verbose, and Minimal.
 
-        If called without arguments, acts as a toggle."""
+        If called without arguments, acts as a toggle.
+
+        When in verbose mode the value --show (and --hide) 
+        will respectively show (or hide) frames with ``__tracebackhide__ =
+        True`` value set.
+        """
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
+        if parameter_s.strip() == "--show":
+            shell.InteractiveTB.skip_hidden = False
+            return
+        if parameter_s.strip() == "--hide":
+            shell.InteractiveTB.skip_hidden = True
+            return
+
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)

IPython/core/tests/test_debugger.py

@@ -4,20 +4,24 @@
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import bdb
+import builtins
+import os
 import signal
+import subprocess
 import sys
 import time
 import warnings
+from subprocess import PIPE, CalledProcessError, check_output
 from tempfile import NamedTemporaryFile
-from subprocess import check_output, CalledProcessError, PIPE
-import subprocess
+from textwrap import dedent
 from unittest.mock import patch
-import builtins
-import bdb
 
 import nose.tools as nt
 
 from IPython.core import debugger
+from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE
+from IPython.testing.decorators import skip_win32
 
 #-----------------------------------------------------------------------------
 # Helper classes, from CPython's Pdb test suite
@@ -251,3 +255,69 @@ def raising_input(msg="", called=[0]):
         # implementation would involve a subprocess, but that adds issues with
         # interrupting subprocesses that are rather complex, so it's simpler
         # just to do it this way.
+
+@skip_win32
+def test_xmode_skip():
+    """that xmode skip frames
+
+    Not as a doctest as pytest does not run doctests.
+    """
+    import pexpect
+    env = os.environ.copy()
+    env["IPY_TEST_SIMPLE_PROMPT"] = "1"
+
+    child = pexpect.spawn(
+        sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env
+    )
+    child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE
+
+    child.expect("IPython")
+    child.expect("\n")
+    child.expect_exact("In [1]")
+
+    block = dedent(
+        """
+def f():
+    __tracebackhide__ = True
+    g()
+
+def g():
+    raise ValueError
+
+f()
+    """
+    )
+
+    for line in block.splitlines():
+        child.sendline(line)
+        child.expect_exact(line)
+    child.expect_exact("skipping")
+
+    block = dedent(
+        """
+def f():
+    __tracebackhide__ = True
+    g()
+
+def g():
+    from IPython.core.debugger import set_trace
+    set_trace()
+
+f()
+    """
+    )
+
+    for line in block.splitlines():
+        child.sendline(line)
+        child.expect_exact(line)
+
+    child.expect("ipdb>")
+    child.sendline("w")
+    child.expect("hidden")
+    child.expect("ipdb>")
+    child.sendline("skip_hidden false")
+    child.sendline("w")
+    child.expect("__traceba")
+    child.expect("ipdb>")
+
+    child.close()