[doctools] More type annotations with pyannotate

This tool works nicely.
oils-for-unix · Jan 14, 2025 · 283fb7c · 283fb7c
1 parent e7e02b5
commit 283fb7c
Show file tree

Hide file tree

Showing 5 changed files with 111 additions and 254 deletions.
diff --git a/devtools/pyann.sh b/devtools/pyann.sh
@@ -11,142 +11,11 @@ source devtools/common.sh
 
 readonly PY_PATH='.:vendor/'  # note: could consolidate with other scripts
 
-deps() {
-  set -x
-  #pip install typing pyannotate
-
-  # got error with 0.67.0
-  #pip3 install 'mypy==0.660'
-
-  # Without --upgrade, it won't install the latest version.
-  # In .travis.yaml we apparently install the latest version too (?)
-  pip3 install --user --upgrade 'mypy'
-}
-
-checkable-files() {
-  # syntax_abbrev.py is "included" in _devbuild/gen/syntax_asdl.py; it's not a standalone module
-  metrics/source-code.sh osh-files | grep -v syntax_abbrev.py
-  metrics/source-code.sh oil-lang-files
-}
-
-need-typechecking() {
-  # This command is useful to find files to annotate and add to
-  # $MORE_OIL_MANIFEST.
-  # It shows all the files that are not included in
-  # $MORE_OIL_MANIFEST or $OSH_PARSE_MANIFEST, and thus are not yet
-  # typechecked by typecheck-more-oil here or
-  # `types/oil-slice.sh soil-run`.
-
-  build/dynamic-deps.sh osh-eval
-  echo
-
-  comm -2 -3 \
-    <(checkable-files | sort | grep '.py$') \
-    <({ more-oil-manifest; cat _build/NINJA/osh_eval/typecheck.txt; } | sort) \
-    | xargs wc -l | sort -n
-}
-
-readonly -a COMMON_TYPE_MODULES=(_devbuild/gen/runtime_asdl.py _devbuild/gen/syntax_asdl.py)
-
-add-imports() {
-  # Temporary helper to add missing class imports to the 'if
-  # TYPE_CHECKING:' block of a single module, if the relevant
-  # classes are found in one of COMMON_TYPE_MODULES
-
-  # Also, this saves the typechecking output to the file named by
-  # $typecheck_out, to make it possible to avoid having to run two
-  # redundant (and slow) typechecking commands.  You can just cat that
-  # file after running this function.
-  local module=$1
-  export PYTHONPATH=$PY_PATH
-  readonly module_tmp=_tmp/add-imports-module.tmp
-  readonly typecheck_out=_tmp/add-imports-typecheck-output
-  set +o pipefail
-  # unbuffer is just to preserve colorization (it tricks the command
-  # into thinking it's writing to a pty instead of a pipe)
-  unbuffer types/run.sh typecheck-files "$module" | tee "$typecheck_out" | \
-    grep 'Name.*is not defined' | sed -r 's/.*'\''(\w+)'\''.*/\1/' | \
-    sort -u | python devtools/findclassdefs.py "${COMMON_TYPE_MODULES[@]}" | \
-    xargs python devtools/typeimports.py "$module" > "$module_tmp"
-  set -o pipefail
-
-  if ! diff -q "$module_tmp" "$module" > /dev/null
-  then
-    cp $module "_tmp/add-imports.$(basename $module).bak"
-    mv "$module_tmp" "$module"
-	echo "Updated $module"
-  fi
-}
-
 #
 # PyAnnotate
 #
 
-# This has a bug
-#pyannotate() { ~/.local/bin/pyannotate "$@"; }
-
-readonly PYANN_REPO=~/git/oils-for-unix/pyannotate/
-
-VENV=_tmp/pyann-venv
-
-make-venv() {
-  python3 -m venv $VENV
-}
-
-install-deps() {
-  . $VENV/bin/activate
-  python3 -m pip install -r $PYANN_REPO/requirements.txt
-}
-
-pyann-patched() {
-  . $VENV/bin/activate
-  local tool=$PYANN_REPO/pyannotate_tools/annotations
-  #export PYTHONPATH=$PYANN_REPO:vendor
-
-  # --dump can help
-  python3 $tool "$@"
-}
-
-
-#
-# Second try
-#
-
-VENV2=_tmp/pyann-venv2
-
-make-venv2() {
-  python3 -m venv $VENV2
-}
-
-install2() {
-  . $VENV2/bin/activate
-  python3 -m pip install pyannotate
-}
-
-tool-demo2() {
-  . $VENV2/bin/activate
-  python3 -m pyannotate_tools.annotations
-}
-
-lib-demo2() {
-  . $VENV2/bin/activate
-  #echo $PYTHONPATH
-
-  # DOES NOT WORK - this is Python 2 code!!!
-  python3 devtools/pyann_driver.py "$@"
-
-  ls -l type_info.json
-  wc -l type_info.json
-}
-
-#
-# Third try - the problem is python2
-#
-
-deps3() {
-  # Gah my custom python2 build doesn't have pip or venv!
-  python2 -m pip install -r $PYANN_REPO/requirements.txt
-}
+# 2025-01: These old versions could go in vendor/ !  They're still useful.
 
 # September 2019
 PYANN_URL='https://files.pythonhosted.org/packages/0d/26/2f68c02fae0b88d9cefdbc632edad190d61621b5660c72c920be1e52631e/pyannotate-1.2.0.tar.gz'
@@ -162,6 +31,15 @@ download-tarballs() {
     $PYANN_URL $MYPY_EXT_URL $SIX_URL
 }
 
+extract-tarballs() {
+  pushd _tmp
+  for t in pyannotate-*.gz mypy_extensions-*.gz six-*.gz; do
+    echo "=== $t"
+    tar -x -z < $t
+  done
+  popd
+}
+
 PY_PATH_2='.:vendor:_tmp/pyannotate-1.2.0:_tmp/mypy_extensions-0.4.3:_tmp/six-1.17.0'
 
 collect-types() {
@@ -182,30 +60,6 @@ collect-types() {
   wc -l type_info.json
 }
 
-osh-pyann() {
-  export PYTHONPATH=".:$PYANN_REPO"
-  PYANN_OUT='a1.json' bin/oil.py osh "$@"
-}
-
-pyann-demo() {
-  rm -f -v *.json
-  osh-pyann -c 'pushd /; echo hi; popd'
-  ls -l *.json
-}
-
-pyann-interactive() {
-  osh-pyann --rcfile /dev/null "$@"
-}
-
-pyann-spec-demo() {
-  local dir=_tmp/pyann-spec
-  mkdir -p $dir
-  export OSH_LIST=bin/osh-pyann
-  test/spec.sh assign --pyann-out-dir $dir "$@"
-
-  ls -l $dir
-}
-
 peek-type-info() {
   grep path type_info.json | sort | uniq -c | sort -n
 }
@@ -226,34 +80,4 @@ apply-types() {
   #pyann-patched --type-info $json "${files[@]}" "$@"
 }
 
-apply-many() {
-  for j in _tmp/pyann-spec/*.json; do
-    apply-types $j -w
-  done
-}
-
-sub() {
-  local f=$1
-  types/refactor.py sub < $f > _tmp/sub.txt
-  diff -u _tmp/sub.txt $f
-}
-
-audit-hacks() {
-  # I used a trailing _ in a couple places to indicates hacks
-  # A MyPy upgrade might fix this?
-  #egrep -n --context 1 '[a-z]+_ ' osh/*_parse.py
-
-  # spids on base class issue
-  egrep --color -n --context 1 '_temp' osh/*_parse.py
-
-  echo ---
-
-  # a few casts because Id ; is TokenWord.
-  egrep --color -w 'cast' {osh,core,frontend}/*.py
-
-  echo ---
-
-  egrep --color -w 'type: ignore' {osh,core,frontend}/*.py
-}
-
 "$@"
diff --git a/devtools/pyann_driver.py b/devtools/pyann_driver.py
@@ -16,80 +16,100 @@
 
 import glob
 
+
 def TopLevel():
-  """Copy some metaprogramming that only happens at the top level."""
-  # from core/meta.py
-  from core.meta import (
-      _ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES, TEST_UNARY_LOOKUP,
-      TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP,
-      types_asdl
-  )
-  from core import id_kind_def
-
-  ID_SPEC = id_kind_def.IdSpec(_ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES)
-
-  id_kind_def.AddKinds(ID_SPEC)
-  id_kind_def.AddBoolKinds(ID_SPEC, types_asdl.bool_arg_type_e)  # must come second
-  id_kind_def.SetupTestBuiltin(ID_SPEC,
+    """Copy some metaprogramming that only happens at the top level."""
+    # from core/meta.py
+    from core.meta import (_ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES,
                            TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP,
-                           TEST_OTHER_LOOKUP,
-                           types_asdl.bool_arg_type_e)
+                           TEST_OTHER_LOOKUP, types_asdl)
+    from core import id_kind_def
+
+    ID_SPEC = id_kind_def.IdSpec(_ID_TO_KIND_INTEGERS, BOOL_ARG_TYPES)
+
+    id_kind_def.AddKinds(ID_SPEC)
+    id_kind_def.AddBoolKinds(ID_SPEC,
+                             types_asdl.bool_arg_type_e)  # must come second
+    id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP,
+                                 TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP,
+                                 types_asdl.bool_arg_type_e)
 
-  from osh import arith_parse
-  spec = arith_parse.MakeShellSpec()
+    from osh import arith_parse
+    spec = arith_parse.MakeShellSpec()
 
 
 def Match():
-  from frontend.match import _MatchOshToken_Slow, _MatchTokenSlow
-  from frontend import lexer_def
-  MATCHER = _MatchOshToken_Slow(lexer_def.LEXER_DEF)
-  ECHO_MATCHER = _MatchTokenSlow(lexer_def.ECHO_E_DEF)
-  GLOB_MATCHER = _MatchTokenSlow(lexer_def.GLOB_DEF)
-  PS1_MATCHER = _MatchTokenSlow(lexer_def.PS1_DEF)
-  HISTORY_MATCHER = _MatchTokenSlow(lexer_def.HISTORY_DEF)
+    from frontend.match import _MatchOshToken_Slow, _MatchTokenSlow
+    from frontend import lexer_def
+    MATCHER = _MatchOshToken_Slow(lexer_def.LEXER_DEF)
+    ECHO_MATCHER = _MatchTokenSlow(lexer_def.ECHO_E_DEF)
+    GLOB_MATCHER = _MatchTokenSlow(lexer_def.GLOB_DEF)
+    PS1_MATCHER = _MatchTokenSlow(lexer_def.PS1_DEF)
+    HISTORY_MATCHER = _MatchTokenSlow(lexer_def.HISTORY_DEF)
 
 
 def Arith():
-  from osh.arith_parse import MakeShellSpec
-  SPEC = MakeShellSpec()
+    from osh.arith_parse import MakeShellSpec
+    SPEC = MakeShellSpec()
 
 
-def main():
-  loader = unittest.TestLoader()
+def UnitTests():
+    loader = unittest.TestLoader()
+
+    g = glob.glob
+    py = g('lazylex/*_test.py') + g('doctools/*_test.py')
+    #py = g('frontend/*_test.py') + g('osh/*_test.py') + g('core/*_test.py') + g('')
+    # hangs
+    #py.remove('core/process_test.py')
+
+    modules = []
+    for p in py:
+        mod_name = p[:-3].replace('/', '.')
+        print(mod_name)
+        modules.append(__import__(mod_name, fromlist=['.']))
 
-  g = glob.glob
-  py = g('lazylex/*_test.py') + g('doctools/*_test.py')
-  #py = g('frontend/*_test.py') + g('osh/*_test.py') + g('core/*_test.py') + g('')
-  # hangs
-  #py.remove('core/process_test.py')
+    for m in modules:
+        print(m)
 
-  modules = []
-  for p in py:
-    mod_name = p[:-3].replace('/', '.')
-    print(mod_name)
-    modules.append(__import__(mod_name, fromlist=['.']))
+    suites = [loader.loadTestsFromModule(m) for m in modules]
 
-  for m in modules:
-    print(m)
+    suite = unittest.TestSuite()
+    for s in suites:
+        suite.addTest(s)
 
-  suites = [loader.loadTestsFromModule(m) for m in modules]
+    runner = unittest.TextTestRunner()
 
-  suite = unittest.TestSuite()
-  for s in suites:
-    suite.addTest(s)
+    collect_types.init_types_collection()
+    with collect_types.collect():
+        runner.run(suite)
+        if 0:
+            TopLevel()
+            Match()
+            Arith()
 
-  runner = unittest.TextTestRunner()
+    collect_types.dump_stats('type_info.json')
 
-  collect_types.init_types_collection()
-  with collect_types.collect():
-    runner.run(suite)
-    if 0:
-      TopLevel()
-      Match()
-      Arith()
 
-  collect_types.dump_stats('type_info.json')
+def Doctools():
+    from doctools import help_gen
+    from doctools import oils_doc
+
+    collect_types.init_types_collection()
+    with collect_types.collect():
+        help_gen.main([
+            '', 'cards-from-chapters', '_devbuild/help',
+            '_tmp/code-blocks/help_meta.py', '_gen/frontend/help_meta',
+            '_release/VERSION/doc/ref/chap-front-end.html'
+        ])
+        #oils_doc.main([])
+
+    collect_types.dump_stats('type_info.json')
+
+
+def main():
+    #UnitTests()
+    Doctools()
 
 
 if __name__ == '__main__':
-  main()
+    main()
diff --git a/devtools/types.sh b/devtools/types.sh
@@ -61,7 +61,7 @@ check-mycpp() {
 }
 
 check-doctools() {
-  if false; then
+  if true; then
     local -a files=(
       $(for x in doctools/*.py; do echo $x; done | grep -v '_test.py' )
       lazylex/html.py