Skip to content

Commit

Permalink
Merge branch 'simplify_createstubs' of https://github.com/Josverl/mic…
Browse files Browse the repository at this point in the history 
…ropython-stubber into simplify_createstubs

Signed-off-by: Jos Verlinde <[email protected]>
  • Loading branch information
@Josverl
Josverl committed Jan 24, 2024
2 parents c53d1fe + 14f055c commit c03da53
Show file tree
Hide file tree
Showing 35 changed files with 2,685 additions and 1,293 deletions.
183 changes: 120 additions & 63 deletions mip/v5/createstubs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import logging
import os
import sys
from time import sleep

try:
from ujson import dumps
Expand All @@ -23,11 +24,26 @@
except ImportError:
from ucollections import OrderedDict # type: ignore

try:
from nope_machine import WDT

wdt = WDT()

except ImportError:

class _WDT:
def feed(self):
pass

wdt = _WDT()


wdt.feed()

__version__ = "v1.16.2"
ENOENT = 2
_MAX_CLASS_LEVEL = 2 # Max class nesting
LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"]
from time import sleep


class Stubber:
Expand All @@ -45,11 +61,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore
self.log.info("Port: {}".format(self.info["port"]))
self.log.info("Board: {}".format(self.info["board"]))
gc.collect()
wdt.feed()
if firmware_id:
self._fwid = firmware_id.lower()
else:
if self.info["family"] == "micropython":
self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info)
self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-")
else:
self._fwid = "{family}-v{version}-{port}".format(**self.info)
self._start_free = gc.mem_free() # type: ignore
Expand Down Expand Up @@ -98,6 +115,7 @@ def get_obj_attributes(self, item_instance: object):
try:
val = getattr(item_instance, name)
# name , item_repr(value) , type as text, item_instance, order
self.log.debug("attribute {}:{}".format(name, val))
try:
type_text = repr(type(val)).split("'")[1]
except IndexError:
Expand Down Expand Up @@ -137,6 +155,7 @@ def create_all_stubs(self):
self.log.info("Finally done")

def create_one_stub(self, module_name: str):
wdt.feed()
if module_name in self.problematic:
self.log.warning("Skip module: {:<25} : Known problematic".format(module_name))
return False
Expand Down Expand Up @@ -191,7 +210,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool:
module_name, self._fwid, info_, __version__
)
fp.write(s)
fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n")
fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n")
self.write_object_stub(fp, new_module, module_name, "")

self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/")))
Expand All @@ -202,10 +221,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool:
del new_module
except (OSError, KeyError): # lgtm [py/unreachable-statement]
self.log.warning("could not del new_module")
try:
del sys.modules[module_name]
except KeyError:
self.log.debug("could not del sys.modules[{}]".format(module_name))
# lets not try - most times it does not work anyway
# try:
# del sys.modules[module_name]
# except KeyError:
# self.log.warning("could not del sys.modules[{}]".format(module_name))
gc.collect()
return True

Expand All @@ -231,8 +251,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str,
self.log.warning("NameError: invalid name {}".format(item_name))
continue
# Class expansion only on first 3 levels (bit of a hack)
if item_type_txt == "<class 'type'>" and len(indent) <= _MAX_CLASS_LEVEL * 4:
self.log.debug("{0}class {1}:".format(indent, item_name))
if (
item_type_txt == "<class 'type'>"
and len(indent) <= _MAX_CLASS_LEVEL * 4
# and not obj_name.endswith(".Pin")
# avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms
):
self.log.info("{0}class {1}:".format(indent, item_name))
superclass = ""
is_exception = (
item_name.endswith("Exception")
Expand Down Expand Up @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str,
s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t)
else:
# something else
if t not in ["object", "set", "frozenset"]:
# Possibly default others to item_instance object ?
if t in ["object", "set", "frozenset", "Pin", "FileIO"]:
# https://docs.python.org/3/tutorial/classes.html#item_instance-objects
# use these types for the attribute
s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
else:
# Requires Python 3.6 syntax, which is OK for the stubs/pyi
t = "Incomplete"
# Requires Python 3.6 syntax, which is OK for the stubs/pyi
s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr)
fp.write(s)
self.log.debug("\n" + s)
else:
Expand All @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str,

fp.write(indent + item_name + " # type: Incomplete\n")

del items
del errors
try:
del item_name, item_repr, item_type_txt, item_instance # type: ignore
except (OSError, KeyError, NameError):
pass
# del items
# del errors
# try:
# del item_name, item_repr, item_type_txt, item_instance # type: ignore
# except (OSError, KeyError, NameError):
# pass

@property
def flat_fwid(self):
Expand All @@ -340,6 +367,7 @@ def flat_fwid(self):

def clean(self, path: str = None): # type: ignore
"Remove all files from the stub folder"
wdt.feed()
if path is None:
path = self.path
self.log.info("Clean/remove files in folder: {}".format(path))
Expand All @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore

def report(self, filename: str = "modules.json"):
"create json with list of exported modules"
wdt.feed()
self.log.info(
"Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)
)
Expand Down Expand Up @@ -423,13 +452,19 @@ def ensure_folder(path: str):

def _build(s):
# extract build from sys.version or os.uname().version if available
# 'v1.13-103-gb137d064e'
# 'MicroPython v1.23.0-preview.6.g3d0b6276f'
# sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f'
# sys.implementation.version: 'v1.13-103-gb137d064e'
if not s:
return ""
s = s.split(" on ", 1)[0] if " on " in s else s
s = s.split("; ", 1)[1] if "; " in s else s
b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1]
if s.startswith("v"):
if not "-" in s:
return ""
b = s.split("-")[1]
return b
if not "-preview" in s:
return ""
b = s.split("-preview")[1].split(".")[1]
return b


Expand Down Expand Up @@ -565,56 +600,78 @@ def version_str(version: tuple): # -> str:


def read_boardname(info, desc: str = ""):
info["board"] = info["board"].replace(" ", "_")
found = False
for filename in [d + "/board_info.csv" for d in LIBS]:
for filename in [d + "/board_name.txt" for d in LIBS]:
wdt.feed()
# print("look up the board name in the file", filename)
if file_exists(filename):
descr = desc or info["board"].strip()
pos = descr.rfind(" with")
if pos != -1:
short_descr = descr[:pos].strip()
else:
short_descr = ""
print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr))
if find_board(info, descr, filename, short_descr):
with open(filename, "r") as file:
data = file.read()
if data:
info["board"] = data.strip()
found = True
break
if not found:
print("Board not found, guessing board name")
descr = desc or info["board"].strip()
if "with " + info["cpu"].upper() in descr:
# remove the with cpu part
descr = descr.split("with " + info["cpu"].upper())[0].strip()
descr = ""
# descr = desc or info["board"].strip()
# if "with " + info["cpu"].upper() in descr:
# # remove the with cpu part
# descr = descr.split("with " + info["cpu"].upper())[0].strip()
info["board"] = descr
info["board"] = info["board"].replace(" ", "_")
gc.collect()


def find_board(info: dict, descr: str, filename: str, short_descr: str):
"Find the board in the provided board_info.csv file"
short_hit = ""
with open(filename, "r") as file:
# ugly code to make testable in python and micropython
# TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file)
while 1:
line = file.readline()
if not line:
break
descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip()
if descr_ == descr:
info["board"] = board_
return True
elif short_descr and descr_ == short_descr:
if "with" in short_descr:
# Good enough - no need to trawl the entire file
info["board"] = board_
return True
# good enough if not found in the rest of the file (but slow)
short_hit = board_
if short_hit:
info["board"] = short_hit
return True
return False
# def read_boardname(info, desc: str = ""):
# wdt.feed()
# # print("look up the board name in the file", filename)
# if file_exists(filename):
# descr = desc or info["board"].strip()
# pos = descr.rfind(" with")
# if pos != -1:
# short_descr = descr[:pos].strip()
# else:
# short_descr = ""
# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr))
# if find_board(info, descr, filename, short_descr):
# found = True
# break
# if not found:
# print("Board not found, guessing board name")
# descr = desc or info["board"].strip()
# if "with " + info["cpu"].upper() in descr:
# # remove the with cpu part
# descr = descr.split("with " + info["cpu"].upper())[0].strip()
# info["board"] = descr
# info["board"] = info["board"].replace(" ", "_")
# gc.collect()


# def find_board(info: dict, descr: str, filename: str, short_descr: str):
# "Find the board in the provided board_info.csv file"
# short_hit = ""
# with open(filename, "r") as file:
# # ugly code to make testable in python and micropython
# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file)
# while 1:
# line = file.readline()
# if not line:
# break
# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip()
# if descr_ == descr:
# info["board"] = board_
# return True
# elif short_descr and descr_ == short_descr:
# if "with" in short_descr:
# # Good enough - no need to trawl the entire file
# info["board"] = board_
# return True
# # good enough if not found in the rest of the file (but slow)
# short_hit = board_
# if short_hit:
# info["board"] = short_hit
# return True
# return False


def get_root() -> str: # sourcery skip: use-assigned-variable
Expand Down
Loading

0 comments on commit c03da53

Please sign in to comment.