[doctools, lazylex] Add type annotations, using pyannotate

Fixed some of them manually, e.g. with IO[str]

devtools/types.sh

@@ -62,11 +62,15 @@ check-mycpp() {
 
 check-doctools() {
   local -a files=(
-    doctools/oils_doc.py lazylex/html.py
+    $(for x in doctools/*.py; do echo $x; done | grep -v '_test.py' )
+    lazylex/html.py
   )
-  #local -a flags=( --py2 --no-strict-optional --strict )
-  local -a flags=( --py2 --no-strict-optional )
+  # 777 errors before pyann
+  # 583 afterward
+  local -a flags=( --py2 --no-strict-optional --strict )
+  #local -a flags=( --py2 --no-strict-optional )
 
+  set -x
   mypy-check "${flags[@]}" "${files[@]}"
 }
 

doctools/cmark.py

@@ -7,6 +7,12 @@
 from __future__ import print_function
 
 import ctypes
+from typing import List
+from typing import Tuple
+from typing import Union
+from typing import Optional
+from typing import IO
+from typing import Dict
 try:
     from HTMLParser import HTMLParser
 except ImportError:
@@ -74,6 +80,7 @@ def log(msg, *args):
 
 
 def md2html(md):
+    # type: (str) -> str
     if sys.version_info.major == 2:
         md_bytes = md
     else:
@@ -104,6 +111,7 @@ class TocExtractor(HTMLParser):
     """
 
     def __init__(self):
+        # type: () -> None
         HTMLParser.__init__(self)
 
         # make targets for these, regardless of whether the TOC links to them.
@@ -122,6 +130,7 @@ def __init__(self):
         self.headings = []
 
     def handle_starttag(self, tag, attrs):
+        # type: (str, List[Tuple[str, str]]) -> None
         if tag == 'div':
             if attrs == [('id', 'toc')]:
                 log('%s> %s %s', self.indent * '  ', tag, attrs)
@@ -149,6 +158,7 @@ def handle_starttag(self, tag, attrs):
             self.capturing = True  # record the text inside <h2></h2> etc.
 
     def handle_endtag(self, tag):
+        # type: (str) -> None
         # Debug print
         if tag == 'div':
             self.indent -= 1
@@ -164,6 +174,7 @@ def handle_endtag(self, tag):
             self._AppendHtml('</%s>' % tag)
 
     def handle_entityref(self, data):
+        # type: (str) -> None
         """
         From Python docs:
         This method is called to process a named character reference of the form
@@ -174,6 +185,7 @@ def handle_entityref(self, data):
             self._AppendHtml('&%s;' % data)
 
     def handle_data(self, data):
+        # type: (str) -> None
         # Debug print
         if self.indent > 0:
             log('%s| %r', self.indent * '  ', data)
@@ -183,11 +195,13 @@ def handle_data(self, data):
             self._AppendText(data)
 
     def _AppendText(self, text):
+        # type: (str) -> None
         """Accumulate text of the last heading."""
         _, _, _, _, text_parts = self.headings[-1]
         text_parts.append(text)
 
     def _AppendHtml(self, html):
+        # type: (str) -> None
         """Accumulate HTML of the last heading."""
         _, _, _, html_parts, _ = self.headings[-1]
         html_parts.append(html)
@@ -201,8 +215,14 @@ def _AppendHtml(self, html):
 ANCHOR_FMT = '<a name="%s"></a>\n'
 
 
-def _MakeTocInsertions(opts, toc_tags, headings, toc_pos,
-                       preserve_anchor_case):
+def _MakeTocInsertions(
+        opts,  # type: Any
+        toc_tags,  # type: Union[List[str], Tuple[str, str]]
+        headings,  # type: List[Tuple[int, str, None, List[str], List[str]]]
+        toc_pos,  # type: int
+        preserve_anchor_case,  # type: bool
+):
+    # type: (...) -> List[Tuple[int, str]]
     """Given extract headings list and TOC position, return a list of insertions.
 
     The insertions <div> for the TOC itself, and <a name=""> for the targets.
@@ -266,7 +286,12 @@ def _MakeTocInsertions(opts, toc_tags, headings, toc_pos,
     return insertions
 
 
-def _MakeTocInsertionsDense(headings, toc_pos, preserve_anchor_case):
+def _MakeTocInsertionsDense(
+        headings,  # type: List[Tuple[int, str, Optional[str], List[str], List[str]]]
+        toc_pos,  # type: int
+        preserve_anchor_case,  # type: bool
+):
+    # type: (...) -> List[Tuple[int, str]]
     """For the dense-toc style with columns, used by doc/ref
 
     The style above is simpler: it outputs a div for every line:
@@ -360,6 +385,7 @@ def _MakeTocInsertionsDense(headings, toc_pos, preserve_anchor_case):
 
 
 def _ApplyInsertions(lines, insertions, out_file):
+    # type: (List[str], List[Tuple[int, str]], IO[str]) -> None
     assert insertions, "Should be at least one insertion"
     j = 0
     n = len(insertions)
@@ -376,7 +402,15 @@ def _ApplyInsertions(lines, insertions, out_file):
         out_file.write(line)
 
 
-def Render(opts, meta, in_file, out_file, use_fastlex=True, debug_out=None):
+def Render(
+        opts,  # type: Any
+        meta,  # type: Dict
+        in_file,  # type: IO[str]
+        out_file,  # type: IO[str]
+        use_fastlex=True,  # type: bool
+        debug_out=None,  # type: Optional[Any]
+):
+    # type: (...) -> None
     if debug_out is None:
         debug_out = []
 
@@ -456,6 +490,7 @@ def Render(opts, meta, in_file, out_file, use_fastlex=True, debug_out=None):
 
 
 def Options():
+    # type: () -> Any
     p = optparse.OptionParser('cmark.py [options]')
 
     p.add_option('--common-mark',

doctools/cmark_test.py

@@ -108,6 +108,7 @@ def f():
 class RenderTest(unittest.TestCase):
 
     def testRender(self):
+        # type: () -> None
         opts, _ = cmark.Options().parse_args([])
 
         out_file = cStringIO.StringIO()
@@ -119,6 +120,7 @@ def testRender(self):
         print(out_file.getvalue())
 
     def testNewRender(self):
+        # type: () -> None
         # New style of doc
 
         new_flags = ['--toc-tag', 'h2', '--toc-tag', 'h3']
@@ -132,6 +134,7 @@ def testNewRender(self):
         self.assert_('<div class="toclevel1"><a href="#one">' in h, h)
 
     def testNewPrettyHref(self):
+        # type: () -> None
         # New style of doc
 
         new_flags = ['--toc-tag', 'h2', '--toc-tag', 'h3', '--toc-pretty-href']
@@ -147,6 +150,7 @@ def testNewPrettyHref(self):
         print(h)
 
     def testExtractor(self):
+        # type: () -> None
         parser = cmark.TocExtractor()
         parser.feed(_HTML_1)
         self.assertEqual(5, parser.toc_begin_line)
@@ -187,6 +191,7 @@ def testExtractor(self):
         self.assertEqual('Two', ''.join(text))
 
     def testExtractorDense(self):
+        # type: () -> None
         parser = cmark.TocExtractor()
         parser.feed(_HTML_1.replace('"toc"', '"dense-toc"'))
 

doctools/help_gen.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python2
 from __future__ import print_function
+from typing import List
+from typing import Any
+from typing import Dict
+from typing import Iterator
 """help_gen.py
 
 Ideas for HTML -> ANSI converter:
@@ -85,13 +89,15 @@ def _StringToHref(s):
 class TopicHtmlRenderer(object):
 
     def __init__(self, chapter, debug_out, linkify_stop_col):
+        # type: (str, List, int) -> None
         self.chapter = chapter
         self.debug_out = debug_out
         self.linkify_stop_col = linkify_stop_col
 
         self.html_page = 'chap-%s.html' % chapter
 
     def _PrintTopic(self, m, out, line_info):
+        # type: (Any, html.Output, Dict[str, Any]) -> None
         # The X
         topic_impl = True
         if m.group(1):
@@ -111,6 +117,7 @@ def _PrintTopic(self, m, out, line_info):
         out.Print('</a>')
 
     def Render(self, line):
+        # type: (str) -> str
         """Convert a line of text to HTML.
 
         Topics are highlighted and X made red.
@@ -217,6 +224,7 @@ class Splitter(HTMLParser.HTMLParser):
     """
 
     def __init__(self, heading_tags, out):
+        # type: (List[str], List) -> None
         HTMLParser.HTMLParser.__init__(self)
         self.heading_tags = heading_tags
         self.out = out
@@ -271,6 +279,7 @@ def handle_data(self, data):
                 self.cur_group[3].append(data)
 
     def end(self):
+        # type: () -> None
         if self.cur_group:
             self.out.append(self.cur_group)
 
@@ -282,6 +291,7 @@ def end(self):
 
 
 def ExtractBody(s):
+    # type: (str) -> str
     """Extract what's in between <body></body>
 
     The splitter needs balanced tags, and what's in <head> isn't
@@ -316,6 +326,7 @@ def ExtractBody(s):
 
 
 def SplitIntoCards(heading_tags, contents):
+    # type: (List[str], str) -> Iterator
     contents = ExtractBody(contents)
 
     groups = []

doctools/help_gen_test.py

@@ -11,6 +11,7 @@
 class HelpGenTest(unittest.TestCase):
 
     def testTopicRe(self):
+        # type: () -> None
         CASES = [
             ('hello   ', True),
             ('X hello   ', True),
@@ -29,6 +30,7 @@ def testTopicRe(self):
             self.assertEqual(matched, bool(m))
 
     def testTopicHtml(self):
+        # type: () -> None
         os.environ['OILS_VERSION'] = '0.7.pre5'
 
         # Three spaces before
@@ -91,6 +93,7 @@ def testTopicHtml(self):
             print()
 
     def testSplitIntoCards(self):
+        # type: () -> None
         contents = """
 <h2>YSH Expression Language</h2>
 

doctools/html_head.py

@@ -12,7 +12,8 @@
 try:
     import html
 except ImportError:
-    import cgi as html  # only for cgi.escape -> html.escape
+    # only for cgi.escape -> html.escape
+    import cgi as html  # type: ignore
 try:
     import cStringIO
 except ImportError:

doctools/html_lib.py

@@ -9,9 +9,11 @@
 
 import cgi
 import re
+from typing import List
 
 
 def AttrsToString(attrs):
+    # type: (List) -> str
     if not attrs:
         return ''
 
@@ -21,6 +23,7 @@ def AttrsToString(attrs):
 
 
 def PrettyHref(s, preserve_anchor_case=False):
+    # type: (str, bool) -> str
     """Turn arbitrary heading text into href with no special characters.
 
     This is modeled after what github does.  It makes everything lower case.

doctools/html_lib_test.py

@@ -9,6 +9,7 @@
 class FunctionsTest(unittest.TestCase):
 
     def testPrettyHref(self):
+        # type: () -> None
         self.assertEqual('foo-bar', html_lib.PrettyHref('foo  bar', False))
         self.assertEqual('why-not', html_lib.PrettyHref('Why Not??', False))
         self.assertEqual('backslash-foo',

doctools/oils_doc.py

@@ -12,6 +12,10 @@
 from __future__ import print_function
 
 import cgi
+from typing import Iterator
+from typing import Any
+from typing import List
+from typing import Optional
 try:
     from cStringIO import StringIO
 except ImportError:
@@ -35,6 +39,7 @@ def __init__(self, fmt):
         self.fmt = fmt
 
     def __call__(self, value):
+        # type: (str) -> str
         return self.fmt % {'value': value}
 
 
@@ -100,6 +105,7 @@ def __call__(self, value):
 
 
 def ExpandLinks(s):
+    # type: (str) -> str
     """Expand $xref:bash and so forth."""
     f = StringIO()
     out = html.Output(s, f)
@@ -165,6 +171,7 @@ class _Plugin(object):
     """
 
     def __init__(self, s, start_pos, end_pos):
+        # type: (str, int, int) -> None
         self.s = s
         self.start_pos = start_pos
         self.end_pos = end_pos
@@ -201,6 +208,7 @@ def PrintHighlighted(self, out):
 
 
 def Lines(s, start_pos, end_pos):
+    # type: (str, int, int) -> Iterator[int]
     """Yields positions in s that end a line."""
     pos = start_pos
     while pos < end_pos:
@@ -218,6 +226,7 @@ class ShPromptPlugin(_Plugin):
     """Highlight shell prompts."""
 
     def PrintHighlighted(self, out):
+        # type: (html.Output) -> None
         pos = self.start_pos
         for line_end in Lines(self.s, self.start_pos, self.end_pos):
 
@@ -368,6 +377,7 @@ def SimpleHighlightCode(s):
 
 
 def HighlightCode(s, default_highlighter, debug_out=None):
+    # type: (str, Optional[Any], Optional[List]) -> str
     """
     Algorithm:
     1. Collect what's inside <pre><code> ...