Add support for Markdown in TUI, console, clipboard output

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[test,richtext]"
+          pip install -e ".[test,richtext,markdown]"
       - name: Run tests
         run: |
           pytest

setup.py

@@ -38,12 +38,16 @@
         "beautifulsoup4>=4.5.0,<5.0",
         "wcwidth>=0.1.7",
         "urwid>=2.0.0,<3.0",
-        "tomlkit>=0.10.0,<1.0"
+        "tomlkit>=0.10.0,<1.0",
     ],
     extras_require={
         # Required to display rich text in the TUI
         "richtext": [
-            "urwidgets>=0.1,<0.2"
+            "urwidgets>=0.1,<0.2",
+        ],
+        "markdown": [
+            "pypandoc>=1.12.0,<2.0",
+            "pypandoc-binary>=1.12.0,<2.0",
         ],
         "dev": [
             "coverage",

tests/test_console.py

@@ -588,8 +588,6 @@ def test_notifications(mock_get, capsys):
         "────────────────────────────────────────────────────────────────────────────────────────────────────",
         "",
     ])
-
-
 @mock.patch('toot.http.get')
 def test_notifications_empty(mock_get, capsys):
     mock_get.return_value = MockResponse([])

toot/commands.py

@@ -58,7 +58,8 @@ def timeline(app, user, args, generator=None):
             items = reversed(items)
 
         statuses = [from_dict(Status, item) for item in items]
-        print_timeline(statuses)
+
+        print_timeline(statuses, render_mode="markdown" if args.markdown else "plaintext")
 
         if args.once or not sys.stdout.isatty():
             break
@@ -71,7 +72,7 @@ def timeline(app, user, args, generator=None):
 def status(app, user, args):
     status = api.single_status(app, user, args.status_id)
     status = from_dict(Status, status)
-    print_status(status)
+    print_status(status, render_mode="markdown" if args.markdown else "plaintext")
 
 
 def thread(app, user, args):
@@ -87,7 +88,7 @@ def thread(app, user, args):
         thread.append(item)
 
     statuses = [from_dict(Status, s) for s in thread]
-    print_timeline(statuses)
+    print_timeline(statuses, render_mode="markdown" if args.markdown else "plaintext")
 
 
 def post(app, user, args):
@@ -572,7 +573,7 @@ def notifications(app, user, args):
         notifications = reversed(notifications)
 
     notifications = [from_dict(Notification, n) for n in notifications]
-    print_notifications(notifications)
+    print_notifications(notifications, render_mode="markdown" if args.markdown else "plaintext")
 
 
 def tui(app, user, args):

toot/console.py

@@ -238,6 +238,12 @@ def editor(value):
     "help": "print json instead of plaintext",
 })
 
+markdown_arg = (["-md", "--markdown"], {
+    "action": "store_true",
+    "help": "print status messages in markdown instead of plaintext",
+    "default": False,
+})
+
 # Arguments for selecting a timeline (see `toot.commands.get_timeline_generator`)
 common_timeline_args = [
     (["-p", "--public"], {
@@ -284,6 +290,7 @@ def editor(value):
         "default": False,
         "help": "Only show the first <count> toots, do not prompt to continue.",
     }),
+    markdown_arg,
 ]
 
 timeline_args = common_timeline_args + timeline_and_bookmark_args
@@ -426,7 +433,8 @@ def editor(value):
                 "action": "store_true",
                 "default": False,
                 "help": "Only print mentions",
-            })
+            }),
+            markdown_arg,
         ],
         require_auth=True,
     ),
@@ -464,6 +472,7 @@ def editor(value):
             (["status_id"], {
                 "help": "Show thread for toot.",
             }),
+            markdown_arg,
         ],
         require_auth=True,
     ),
@@ -474,6 +483,7 @@ def editor(value):
             (["status_id"], {
                 "help": "ID of the status to show.",
             }),
+            markdown_arg,
         ],
         require_auth=True,
     ),

toot/output.py

@@ -5,7 +5,8 @@
 
 from functools import lru_cache
 from toot import settings
-from toot.utils import get_text, html_to_paragraphs
+from toot.utils import get_text
+from toot.richtext import html_to_text
 from toot.entities import Account, Instance, Notification, Poll, Status
 from toot.wcstring import wc_wrap
 from typing import List
@@ -174,7 +175,6 @@ def print_account(account: Account):
     print_out(f"<green>@{account.acct}</green> {account.display_name}")
 
     if account.note:
-        print_out("")
         print_html(account.note)
 
     since = account.created_at.strftime('%Y-%m-%d')
@@ -277,7 +277,7 @@ def print_search_results(results):
         print_out("<yellow>Nothing found</yellow>")
 
 
-def print_status(status: Status, width: int = 80):
+def print_status(status: Status, width=80, render_mode=""):
     status_id = status.id
     in_reply_to_id = status.in_reply_to_id
     reblogged_by = status.account if status.reblog else None
@@ -299,8 +299,7 @@ def print_status(status: Status, width: int = 80):
         f"<yellow>{time}</yellow>",
     )
 
-    print_out("")
-    print_html(status.content, width)
+    print_html(status.content, width, render_mode=render_mode)
 
     if status.media_attachments:
         print_out("\nMedia:")
@@ -321,15 +320,10 @@ def print_status(status: Status, width: int = 80):
     )
 
 
-def print_html(text, width=80):
-    first = True
-    for paragraph in html_to_paragraphs(text):
-        if not first:
-            print_out("")
-        for line in paragraph:
-            for subline in wc_wrap(line, width):
-                print_out(highlight_hashtags(subline))
-        first = False
+def print_html(text, width=80, render_mode=""):
+    markdown = "\n".join(html_to_text(text, columns=width, render_mode=render_mode, highlight_tags=False))
+    print_out("")
+    print_out(markdown)
 
 
 def print_poll(poll: Poll):
@@ -358,10 +352,10 @@ def print_poll(poll: Poll):
     print_out(poll_footer)
 
 
-def print_timeline(items: List[Status], width=100):
+def print_timeline(items: List[Status], width=100, render_mode=""):
     print_out("─" * width)
     for item in items:
-        print_status(item, width)
+        print_status(item, width, render_mode=render_mode)
         print_out("─" * width)
 
 
@@ -373,7 +367,7 @@ def print_timeline(items: List[Status], width=100):
 }
 
 
-def print_notification(notification: Notification, width=100):
+def print_notification(notification: Notification, width=100, render_mode=""):
     account = f"{notification.account.display_name} @{notification.account.acct}"
     msg = notification_msgs.get(notification.type)
     if msg is None:
@@ -382,10 +376,10 @@ def print_notification(notification: Notification, width=100):
     print_out("─" * width)
     print_out(msg.format(account=account))
     if notification.status:
-        print_status(notification.status, width)
+        print_status(notification.status, width, render_mode=render_mode)
 
 
-def print_notifications(notifications: List[Notification], width=100):
+def print_notifications(notifications: List[Notification], render_mode="", width=100):
     for notification in notifications:
-        print_notification(notification)
+        print_notification(notification, render_mode=render_mode)
     print_out("─" * width)

toot/richtext/__init__.py

@@ -0,0 +1,15 @@
+from toot.exceptions import ConsoleError
+from toot.richtext.plaintext import html_to_plaintext
+from typing import List
+
+try:
+    # first preference, render markup with pypandoc
+    from .markdown import html_to_text
+
+except ImportError:
+    # Fallback to render in plaintext
+    def html_to_text(html: str, columns=80, render_mode: str = "", highlight_tags=False) -> List:
+        if render_mode == "markdown":
+            raise ConsoleError("Can't render as markdown because the pypandoc library is not available.")
+
+        return html_to_plaintext(html, columns, highlight_tags)

toot/richtext/markdown.py

@@ -0,0 +1,17 @@
+from toot.exceptions import ConsoleError
+from pypandoc import convert_text
+from toot.richtext.plaintext import html_to_plaintext
+from typing import List
+
+
+def html_to_text(html: str, columns=80, render_mode: str = "", highlight_tags=False) -> List:
+    if render_mode == "plaintext":
+        return html_to_plaintext(html, columns, highlight_tags)
+    elif render_mode == "markdown" or render_mode == "":
+        return [convert_text(
+            html,
+            format="html",
+            to="gfm-raw_html",
+            extra_args=["--wrap=auto", f"--columns={columns}"],
+        )]
+    raise ConsoleError("Unknown render mode; specify 'plaintext' or 'markdown'")

toot/richtext/plaintext.py

@@ -0,0 +1,20 @@
+from toot.utils import html_to_paragraphs
+from toot.wcstring import wc_wrap
+from toot.tui.utils import highlight_hashtags
+from typing import List
+
+
+def html_to_plaintext(html: str, columns=80, highlight_tags=False) -> List:
+    output = []
+    first = True
+    for paragraph in html_to_paragraphs(html):
+        if not first:
+            output.append("")
+        for line in paragraph:
+            for subline in wc_wrap(line, columns):
+                if highlight_tags:
+                    output.append(highlight_hashtags(subline))
+                else:
+                    output.append(subline)
+        first = False
+    return output

toot/tui/app.py

@@ -7,6 +7,8 @@
 from toot import api, config, __version__, settings
 from toot.console import get_default_visibility
 from toot.exceptions import ApiError
+from toot.richtext import html_to_text
+from toot.utils.datetime import parse_datetime
 
 from .compose import StatusComposer
 from .constants import PALETTE
@@ -654,9 +656,22 @@ def _done(loop):
         return self.run_in_thread(_delete, done_callback=_done)
 
     def copy_status(self, status):
-        # TODO: copy a better version of status content
-        # including URLs
-        copy_to_clipboard(self.screen, status.original.data["content"])
+
+        markdown = "\n".join(html_to_text(status.original.data["content"], columns=1024, highlight_tags=False))
+
+        time = parse_datetime(status.original.data['created_at'])
+        time = time.strftime('%Y-%m-%d %H:%M %Z')
+
+        text_status = (f"{status.original.data['url']}\n\n"
+            + (status.original.author.display_name or "")
+            + "\n"
+            + (status.original.author.account or "")
+            + "\n\n"
+            + markdown
+            + "\n\n"
+            + f"Created at: {time}")
+
+        copy_to_clipboard(self.screen, text_status)
         self.footer.set_message(f"Status {status.original.id} copied")
 
     # --- Overlay handling -----------------------------------------------------

toot/tui/richtext/__init__.py

@@ -1,18 +1,24 @@
 import urwid
-
 from toot.tui.utils import highlight_hashtags
 from toot.utils import format_content
 from typing import List
 
 try:
+    # our first preference is to render using urwidgets
     from .richtext import html_to_widgets, url_to_widget
+
 except ImportError:
-    # Fallback if urwidgets are not available
-    def html_to_widgets(html: str) -> List[urwid.Widget]:
-        return [
-            urwid.Text(highlight_hashtags(line))
-            for line in format_content(html)
-        ]
+    try:
+        # second preference, render markup with pypandoc
+        from .markdown import html_to_widgets, url_to_widget
+
+    except ImportError:
+        # Fallback to render in plaintext
+
+        def url_to_widget(url: str):
+            return urwid.Text(("link", url))
 
-    def url_to_widget(url: str):
-        return urwid.Text(("link", url))
+        def html_to_widgets(html: str) -> List[urwid.Widget]:
+            return [
+                urwid.Text(highlight_hashtags(line)) for line in format_content(html)
+            ]

toot/tui/richtext/markdown.py

@@ -0,0 +1,21 @@
+import urwid
+from pypandoc import convert_text
+
+from typing import List
+
+
+def url_to_widget(url: str):
+    return urwid.Text(("link", url))
+
+
+def html_to_widgets(html: str) -> List[urwid.Widget]:
+    return [
+        urwid.Text(
+            convert_text(
+                html,
+                format="html",
+                to="gfm-raw_html",
+                extra_args=["--wrap=none"],
+            )
+        )
+    ]