From 283fb7cbbafce0979802c706051ea167d3e5c226 Mon Sep 17 00:00:00 2001 From: Andy C Date: Tue, 14 Jan 2025 16:40:09 -0500 Subject: [PATCH] [doctools] More type annotations with pyannotate This tool works nicely. --- devtools/pyann.sh | 196 ++------------------------------------- devtools/pyann_driver.py | 130 +++++++++++++++----------- devtools/types.sh | 2 +- doctools/help_gen.py | 22 ++++- doctools/ul_table.py | 15 +-- 5 files changed, 111 insertions(+), 254 deletions(-) diff --git a/devtools/pyann.sh b/devtools/pyann.sh index 75535a608..ffdc82668 100755 --- a/devtools/pyann.sh +++ b/devtools/pyann.sh @@ -11,142 +11,11 @@ source devtools/common.sh readonly PY_PATH='.:vendor/' # note: could consolidate with other scripts -deps() { - set -x - #pip install typing pyannotate - - # got error with 0.67.0 - #pip3 install 'mypy==0.660' - - # Without --upgrade, it won't install the latest version. - # In .travis.yaml we apparently install the latest version too (?) - pip3 install --user --upgrade 'mypy' -} - -checkable-files() { - # syntax_abbrev.py is "included" in _devbuild/gen/syntax_asdl.py; it's not a standalone module - metrics/source-code.sh osh-files | grep -v syntax_abbrev.py - metrics/source-code.sh oil-lang-files -} - -need-typechecking() { - # This command is useful to find files to annotate and add to - # $MORE_OIL_MANIFEST. - # It shows all the files that are not included in - # $MORE_OIL_MANIFEST or $OSH_PARSE_MANIFEST, and thus are not yet - # typechecked by typecheck-more-oil here or - # `types/oil-slice.sh soil-run`. - - build/dynamic-deps.sh osh-eval - echo - - comm -2 -3 \ - <(checkable-files | sort | grep '.py$') \ - <({ more-oil-manifest; cat _build/NINJA/osh_eval/typecheck.txt; } | sort) \ - | xargs wc -l | sort -n -} - -readonly -a COMMON_TYPE_MODULES=(_devbuild/gen/runtime_asdl.py _devbuild/gen/syntax_asdl.py) - -add-imports() { - # Temporary helper to add missing class imports to the 'if - # TYPE_CHECKING:' block of a single module, if the relevant - # classes are found in one of COMMON_TYPE_MODULES - - # Also, this saves the typechecking output to the file named by - # $typecheck_out, to make it possible to avoid having to run two - # redundant (and slow) typechecking commands. You can just cat that - # file after running this function. - local module=$1 - export PYTHONPATH=$PY_PATH - readonly module_tmp=_tmp/add-imports-module.tmp - readonly typecheck_out=_tmp/add-imports-typecheck-output - set +o pipefail - # unbuffer is just to preserve colorization (it tricks the command - # into thinking it's writing to a pty instead of a pipe) - unbuffer types/run.sh typecheck-files "$module" | tee "$typecheck_out" | \ - grep 'Name.*is not defined' | sed -r 's/.*'\''(\w+)'\''.*/\1/' | \ - sort -u | python devtools/findclassdefs.py "${COMMON_TYPE_MODULES[@]}" | \ - xargs python devtools/typeimports.py "$module" > "$module_tmp" - set -o pipefail - - if ! diff -q "$module_tmp" "$module" > /dev/null - then - cp $module "_tmp/add-imports.$(basename $module).bak" - mv "$module_tmp" "$module" - echo "Updated $module" - fi -} - # # PyAnnotate # -# This has a bug -#pyannotate() { ~/.local/bin/pyannotate "$@"; } - -readonly PYANN_REPO=~/git/oils-for-unix/pyannotate/ - -VENV=_tmp/pyann-venv - -make-venv() { - python3 -m venv $VENV -} - -install-deps() { - . $VENV/bin/activate - python3 -m pip install -r $PYANN_REPO/requirements.txt -} - -pyann-patched() { - . $VENV/bin/activate - local tool=$PYANN_REPO/pyannotate_tools/annotations - #export PYTHONPATH=$PYANN_REPO:vendor - - # --dump can help - python3 $tool "$@" -} - - -# -# Second try -# - -VENV2=_tmp/pyann-venv2 - -make-venv2() { - python3 -m venv $VENV2 -} - -install2() { - . $VENV2/bin/activate - python3 -m pip install pyannotate -} - -tool-demo2() { - . $VENV2/bin/activate - python3 -m pyannotate_tools.annotations -} - -lib-demo2() { - . $VENV2/bin/activate - #echo $PYTHONPATH - - # DOES NOT WORK - this is Python 2 code!!! - python3 devtools/pyann_driver.py "$@" - - ls -l type_info.json - wc -l type_info.json -} - -# -# Third try - the problem is python2 -# - -deps3() { - # Gah my custom python2 build doesn't have pip or venv! - python2 -m pip install -r $PYANN_REPO/requirements.txt -} +# 2025-01: These old versions could go in vendor/ ! They're still useful. # September 2019 PYANN_URL='https://files.pythonhosted.org/packages/0d/26/2f68c02fae0b88d9cefdbc632edad190d61621b5660c72c920be1e52631e/pyannotate-1.2.0.tar.gz' @@ -162,6 +31,15 @@ download-tarballs() { $PYANN_URL $MYPY_EXT_URL $SIX_URL } +extract-tarballs() { + pushd _tmp + for t in pyannotate-*.gz mypy_extensions-*.gz six-*.gz; do + echo "=== $t" + tar -x -z < $t + done + popd +} + PY_PATH_2='.:vendor:_tmp/pyannotate-1.2.0:_tmp/mypy_extensions-0.4.3:_tmp/six-1.17.0' collect-types() { @@ -182,30 +60,6 @@ collect-types() { wc -l type_info.json } -osh-pyann() { - export PYTHONPATH=".:$PYANN_REPO" - PYANN_OUT='a1.json' bin/oil.py osh "$@" -} - -pyann-demo() { - rm -f -v *.json - osh-pyann -c 'pushd /; echo hi; popd' - ls -l *.json -} - -pyann-interactive() { - osh-pyann --rcfile /dev/null "$@" -} - -pyann-spec-demo() { - local dir=_tmp/pyann-spec - mkdir -p $dir - export OSH_LIST=bin/osh-pyann - test/spec.sh assign --pyann-out-dir $dir "$@" - - ls -l $dir -} - peek-type-info() { grep path type_info.json | sort | uniq -c | sort -n } @@ -226,34 +80,4 @@ apply-types() { #pyann-patched --type-info $json "${files[@]}" "$@" } -apply-many() { - for j in _tmp/pyann-spec/*.json; do - apply-types $j -w - done -} - -sub() { - local f=$1 - types/refactor.py sub < $f > _tmp/sub.txt - diff -u _tmp/sub.txt $f -} - -audit-hacks() { - # I used a trailing _ in a couple places to indicates hacks - # A MyPy upgrade might fix this? - #egrep -n --context 1 '[a-z]+_ ' osh/*_parse.py - - # spids on base class issue - egrep --color -n --context 1 '_temp' osh/*_parse.py - - echo --- - - # a few casts because Id ; is TokenWord. - egrep --color -w 'cast' {osh,core,frontend}/*.py - - echo --- - - egrep --color -w 'type: ignore' {osh,core,frontend}/*.py -} - "$@" diff --git a/devtools/pyann_driver.py b/devtools/pyann_driver.py index 0b3ae69dd..c2bdf9e90 100755 --- a/devtools/pyann_driver.py +++ b/devtools/pyann_driver.py @@ -16,80 +16,100 @@ import glob + def TopLevel(): - """Copy some metaprogramming that only happens at the top level.""" - # from core/meta.py - from core.meta import ( - _ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES, TEST_UNARY_LOOKUP, - TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP, - types_asdl - ) - from core import id_kind_def - - ID_SPEC = id_kind_def.IdSpec(_ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES) - - id_kind_def.AddKinds(ID_SPEC) - id_kind_def.AddBoolKinds(ID_SPEC, types_asdl.bool_arg_type_e) # must come second - id_kind_def.SetupTestBuiltin(ID_SPEC, + """Copy some metaprogramming that only happens at the top level.""" + # from core/meta.py + from core.meta import (_ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES, TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP, - TEST_OTHER_LOOKUP, - types_asdl.bool_arg_type_e) + TEST_OTHER_LOOKUP, types_asdl) + from core import id_kind_def + + ID_SPEC = id_kind_def.IdSpec(_ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES) + + id_kind_def.AddKinds(ID_SPEC) + id_kind_def.AddBoolKinds(ID_SPEC, + types_asdl.bool_arg_type_e) # must come second + id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP, + TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP, + types_asdl.bool_arg_type_e) - from osh import arith_parse - spec = arith_parse.MakeShellSpec() + from osh import arith_parse + spec = arith_parse.MakeShellSpec() def Match(): - from frontend.match import _MatchOshToken_Slow, _MatchTokenSlow - from frontend import lexer_def - MATCHER = _MatchOshToken_Slow(lexer_def.LEXER_DEF) - ECHO_MATCHER = _MatchTokenSlow(lexer_def.ECHO_E_DEF) - GLOB_MATCHER = _MatchTokenSlow(lexer_def.GLOB_DEF) - PS1_MATCHER = _MatchTokenSlow(lexer_def.PS1_DEF) - HISTORY_MATCHER = _MatchTokenSlow(lexer_def.HISTORY_DEF) + from frontend.match import _MatchOshToken_Slow, _MatchTokenSlow + from frontend import lexer_def + MATCHER = _MatchOshToken_Slow(lexer_def.LEXER_DEF) + ECHO_MATCHER = _MatchTokenSlow(lexer_def.ECHO_E_DEF) + GLOB_MATCHER = _MatchTokenSlow(lexer_def.GLOB_DEF) + PS1_MATCHER = _MatchTokenSlow(lexer_def.PS1_DEF) + HISTORY_MATCHER = _MatchTokenSlow(lexer_def.HISTORY_DEF) def Arith(): - from osh.arith_parse import MakeShellSpec - SPEC = MakeShellSpec() + from osh.arith_parse import MakeShellSpec + SPEC = MakeShellSpec() -def main(): - loader = unittest.TestLoader() +def UnitTests(): + loader = unittest.TestLoader() + + g = glob.glob + py = g('lazylex/*_test.py') + g('doctools/*_test.py') + #py = g('frontend/*_test.py') + g('osh/*_test.py') + g('core/*_test.py') + g('') + # hangs + #py.remove('core/process_test.py') + + modules = [] + for p in py: + mod_name = p[:-3].replace('/', '.') + print(mod_name) + modules.append(__import__(mod_name, fromlist=['.'])) - g = glob.glob - py = g('lazylex/*_test.py') + g('doctools/*_test.py') - #py = g('frontend/*_test.py') + g('osh/*_test.py') + g('core/*_test.py') + g('') - # hangs - #py.remove('core/process_test.py') + for m in modules: + print(m) - modules = [] - for p in py: - mod_name = p[:-3].replace('/', '.') - print(mod_name) - modules.append(__import__(mod_name, fromlist=['.'])) + suites = [loader.loadTestsFromModule(m) for m in modules] - for m in modules: - print(m) + suite = unittest.TestSuite() + for s in suites: + suite.addTest(s) - suites = [loader.loadTestsFromModule(m) for m in modules] + runner = unittest.TextTestRunner() - suite = unittest.TestSuite() - for s in suites: - suite.addTest(s) + collect_types.init_types_collection() + with collect_types.collect(): + runner.run(suite) + if 0: + TopLevel() + Match() + Arith() - runner = unittest.TextTestRunner() + collect_types.dump_stats('type_info.json') - collect_types.init_types_collection() - with collect_types.collect(): - runner.run(suite) - if 0: - TopLevel() - Match() - Arith() - collect_types.dump_stats('type_info.json') +def Doctools(): + from doctools import help_gen + from doctools import oils_doc + + collect_types.init_types_collection() + with collect_types.collect(): + help_gen.main([ + '', 'cards-from-chapters', '_devbuild/help', + '_tmp/code-blocks/help_meta.py', '_gen/frontend/help_meta', + '_release/VERSION/doc/ref/chap-front-end.html' + ]) + #oils_doc.main([]) + + collect_types.dump_stats('type_info.json') + + +def main(): + #UnitTests() + Doctools() if __name__ == '__main__': - main() + main() diff --git a/devtools/types.sh b/devtools/types.sh index a807ba706..e196c70bf 100755 --- a/devtools/types.sh +++ b/devtools/types.sh @@ -61,7 +61,7 @@ check-mycpp() { } check-doctools() { - if false; then + if true; then local -a files=( $(for x in doctools/*.py; do echo $x; done | grep -v '_test.py' ) lazylex/html.py diff --git a/doctools/help_gen.py b/doctools/help_gen.py index 1fa069d6b..02c16b495 100755 --- a/doctools/help_gen.py +++ b/doctools/help_gen.py @@ -1,9 +1,6 @@ #!/usr/bin/env python2 from __future__ import print_function -from typing import List -from typing import Any -from typing import Dict -from typing import Iterator +from typing import List, Any, Dict, Iterator, Tuple, Optional, IO """help_gen.py Ideas for HTML -> ANSI converter: @@ -236,11 +233,13 @@ def __init__(self, heading_tags, out): self.indent = 0 def log(self, msg, *args): + # type: (str, *Any) -> None ind = self.indent * ' ' if 0: log(ind + msg, *args) def handle_starttag(self, tag, attrs): + # type: (str, List[Tuple[str, str]]) -> None if tag in self.heading_tags: self.in_heading = True if self.cur_group: @@ -252,6 +251,7 @@ def handle_starttag(self, tag, attrs): self.indent += 1 def handle_endtag(self, tag): + # type: (str) -> None if tag in self.heading_tags: self.in_heading = False @@ -259,6 +259,7 @@ def handle_endtag(self, tag): self.indent -= 1 def handle_entityref(self, name): + # type: (str) -> None """ From Python docs: This method is called to process a named character reference of the form @@ -272,6 +273,7 @@ def handle_entityref(self, name): self.cur_group[3].append(c) def handle_data(self, data): + # type: (str) -> None self.log('data %r', data) if self.in_heading: self.cur_group[2].append(data) @@ -407,6 +409,7 @@ class DocNode(object): """To visualize doc structure.""" def __init__(self, name, attrs=None, text=None): + # type: (str, Optional[List[Tuple[str, str]]], Optional[str]) -> None self.name = name self.attrs = attrs # for h2 and h3 links self.text = text @@ -437,7 +440,12 @@ def CardsFromIndex(sh, out_prefix): out_prefix) -def CardsFromChapters(out_dir, tag_level, paths): +def CardsFromChapters( + out_dir, # type: str + tag_level, # type: str + paths, # type: List[str] +): + # type: (...) -> Tuple[Dict[str, Optional[str]], DocNode] """ Args: paths: list of chap-*.html to read @@ -511,11 +519,13 @@ def CardsFromChapters(out_dir, tag_level, paths): class StrPool(object): def __init__(self): + # type: () -> None self.var_names = {} self.global_strs = [] self.unique_id = 1 def Add(self, s): + # type: (str) -> None if s in self.var_names: return @@ -531,6 +541,7 @@ def Add(self, s): def WriteTopicDict(topic_dict, header_f, cc_f): + # type: (Dict[str, Optional[str]], IO[bytes], IO[bytes]) -> None header_f.write(''' #include "mycpp/runtime.h" @@ -577,6 +588,7 @@ def WriteTopicDict(topic_dict, header_f, cc_f): def main(argv): + # type: (List[str]) -> None action = argv[1] if action == 'cards-from-index': diff --git a/doctools/ul_table.py b/doctools/ul_table.py index efa2de05a..2cb30d630 100755 --- a/doctools/ul_table.py +++ b/doctools/ul_table.py @@ -15,7 +15,6 @@ from typing import List from typing import Optional from typing import Tuple -from typing import Union from typing import Any from typing import Dict @@ -48,6 +47,8 @@ def RemoveComments(s): _WHITESPACE_RE = re.compile(r'\s*') +TdAttrs = List[Tuple[str, str]] + class UlTableParser(object): @@ -160,7 +161,7 @@ def FindUlTable(self): return -1 def _ListItem(self): - # type: () -> Tuple[Optional[List[Tuple[str, str]]], Optional[str]] + # type: () -> Tuple[Optional[TdAttrs], Optional[str]] """Parse a list item nested below thead or tr. Returns: @@ -244,7 +245,7 @@ def _ListItem(self): return td_attrs, inner_html def _ParseTHead(self): - # type: () -> Union[List[Tuple[List[Tuple[str, str]], str]], List[Tuple[Optional[List[Tuple[str, str]]], str]]] + # type: () -> List[Tuple[Optional[TdAttrs], str]] """ Assume we're looking at the first