diff --git a/mip/v5/createstubs.py b/mip/v5/createstubs.py index 3ae80a57..71c910a5 100644 --- a/mip/v5/createstubs.py +++ b/mip/v5/createstubs.py @@ -7,6 +7,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -23,11 +24,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -45,11 +61,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -98,6 +115,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -137,6 +155,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -191,7 +210,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: module_name, self._fwid, info_, __version__ ) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -202,10 +221,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -231,8 +251,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info( "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path) ) @@ -423,13 +452,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -565,56 +600,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_db.py b/mip/v5/createstubs_db.py index 9ea3a8d1..6904d4ca 100644 --- a/mip/v5/createstubs_db.py +++ b/mip/v5/createstubs_db.py @@ -26,6 +26,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -42,11 +43,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -64,11 +80,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -117,6 +134,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -156,6 +174,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -208,7 +227,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -219,10 +238,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -248,8 +268,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -321,12 +346,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -336,12 +363,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -355,6 +382,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -377,6 +405,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -436,13 +465,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -576,56 +611,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_db_min.py b/mip/v5/createstubs_db_min.py index 1c40ecbe..05817f71 100644 --- a/mip/v5/createstubs_db_min.py +++ b/mip/v5/createstubs_db_min.py @@ -26,6 +26,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -42,11 +43,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -64,11 +80,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -117,6 +134,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -156,6 +174,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -208,7 +227,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -219,10 +238,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -248,8 +268,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -321,12 +346,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -336,12 +363,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -355,6 +382,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -377,6 +405,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -436,13 +465,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -576,56 +611,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_db_mpy.mpy b/mip/v5/createstubs_db_mpy.mpy index 877627e9..78198f80 100644 Binary files a/mip/v5/createstubs_db_mpy.mpy and b/mip/v5/createstubs_db_mpy.mpy differ diff --git a/mip/v5/createstubs_lvgl.py b/mip/v5/createstubs_lvgl.py index 52a2375e..58b23e70 100644 --- a/mip/v5/createstubs_lvgl.py +++ b/mip/v5/createstubs_lvgl.py @@ -11,6 +11,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -27,11 +28,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -49,11 +65,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -102,6 +119,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -141,6 +159,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -193,7 +212,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -204,10 +223,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -233,8 +253,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -421,13 +450,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -561,56 +596,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_lvgl_min.py b/mip/v5/createstubs_lvgl_min.py index d3b5468e..cdab9a5a 100644 --- a/mip/v5/createstubs_lvgl_min.py +++ b/mip/v5/createstubs_lvgl_min.py @@ -11,6 +11,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -27,11 +28,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -49,11 +65,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -102,6 +119,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -141,6 +159,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -193,7 +212,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -204,10 +223,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -233,8 +253,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -421,13 +450,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -561,56 +596,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_lvgl_mpy.mpy b/mip/v5/createstubs_lvgl_mpy.mpy index 76d37d58..120b0c08 100644 Binary files a/mip/v5/createstubs_lvgl_mpy.mpy and b/mip/v5/createstubs_lvgl_mpy.mpy differ diff --git a/mip/v5/createstubs_mem.py b/mip/v5/createstubs_mem.py index 34cd9f06..81164f7d 100644 --- a/mip/v5/createstubs_mem.py +++ b/mip/v5/createstubs_mem.py @@ -17,6 +17,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -33,11 +34,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -55,11 +71,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -108,6 +125,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -147,6 +165,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -199,7 +218,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -210,10 +229,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -239,8 +259,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -312,12 +337,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -327,12 +354,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -346,6 +373,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -368,6 +396,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -427,13 +456,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -567,56 +602,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_mem_min.py b/mip/v5/createstubs_mem_min.py index f7fb6c09..d610d52b 100644 --- a/mip/v5/createstubs_mem_min.py +++ b/mip/v5/createstubs_mem_min.py @@ -17,6 +17,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -33,11 +34,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -55,11 +71,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -108,6 +125,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -147,6 +165,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -199,7 +218,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -210,10 +229,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -239,8 +259,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -312,12 +337,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -327,12 +354,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -346,6 +373,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -368,6 +396,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -427,13 +456,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -567,56 +602,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_mem_mpy.mpy b/mip/v5/createstubs_mem_mpy.mpy index 8a60fad9..14b67b80 100644 Binary files a/mip/v5/createstubs_mem_mpy.mpy and b/mip/v5/createstubs_mem_mpy.mpy differ diff --git a/mip/v5/createstubs_min.py b/mip/v5/createstubs_min.py index f2fd2cf6..7b134cbf 100644 --- a/mip/v5/createstubs_min.py +++ b/mip/v5/createstubs_min.py @@ -7,6 +7,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -23,11 +24,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -45,11 +61,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -98,6 +115,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -137,6 +155,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -191,7 +210,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: module_name, self._fwid, info_, __version__ ) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -202,10 +221,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -231,8 +251,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info( # "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path) # ) @@ -423,13 +452,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -565,56 +600,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v5/createstubs_mpy.mpy b/mip/v5/createstubs_mpy.mpy index b778346b..0bd6b3d2 100644 Binary files a/mip/v5/createstubs_mpy.mpy and b/mip/v5/createstubs_mpy.mpy differ diff --git a/mip/v6/createstubs.py b/mip/v6/createstubs.py index 3ae80a57..71c910a5 100644 --- a/mip/v6/createstubs.py +++ b/mip/v6/createstubs.py @@ -7,6 +7,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -23,11 +24,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -45,11 +61,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -98,6 +115,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -137,6 +155,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -191,7 +210,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: module_name, self._fwid, info_, __version__ ) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -202,10 +221,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -231,8 +251,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info( "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path) ) @@ -423,13 +452,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -565,56 +600,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_db.py b/mip/v6/createstubs_db.py index 9ea3a8d1..6904d4ca 100644 --- a/mip/v6/createstubs_db.py +++ b/mip/v6/createstubs_db.py @@ -26,6 +26,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -42,11 +43,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -64,11 +80,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -117,6 +134,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -156,6 +174,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -208,7 +227,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -219,10 +238,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -248,8 +268,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -321,12 +346,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -336,12 +363,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -355,6 +382,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -377,6 +405,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -436,13 +465,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -576,56 +611,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_db_min.py b/mip/v6/createstubs_db_min.py index 1c40ecbe..05817f71 100644 --- a/mip/v6/createstubs_db_min.py +++ b/mip/v6/createstubs_db_min.py @@ -26,6 +26,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -42,11 +43,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -64,11 +80,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -117,6 +134,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -156,6 +174,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -208,7 +227,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -219,10 +238,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -248,8 +268,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -321,12 +346,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -336,12 +363,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -355,6 +382,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -377,6 +405,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -436,13 +465,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -576,56 +611,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_db_mpy.mpy b/mip/v6/createstubs_db_mpy.mpy index bcb0e208..c1d3ab0c 100644 Binary files a/mip/v6/createstubs_db_mpy.mpy and b/mip/v6/createstubs_db_mpy.mpy differ diff --git a/mip/v6/createstubs_lvgl.py b/mip/v6/createstubs_lvgl.py index 52a2375e..58b23e70 100644 --- a/mip/v6/createstubs_lvgl.py +++ b/mip/v6/createstubs_lvgl.py @@ -11,6 +11,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -27,11 +28,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -49,11 +65,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -102,6 +119,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -141,6 +159,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -193,7 +212,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -204,10 +223,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -233,8 +253,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -421,13 +450,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -561,56 +596,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_lvgl_min.py b/mip/v6/createstubs_lvgl_min.py index d3b5468e..cdab9a5a 100644 --- a/mip/v6/createstubs_lvgl_min.py +++ b/mip/v6/createstubs_lvgl_min.py @@ -11,6 +11,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -27,11 +28,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -49,11 +65,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -102,6 +119,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -141,6 +159,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -193,7 +212,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -204,10 +223,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -233,8 +253,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -421,13 +450,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -561,56 +596,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_lvgl_mpy.mpy b/mip/v6/createstubs_lvgl_mpy.mpy index a97a8ac8..ec7b3c33 100644 Binary files a/mip/v6/createstubs_lvgl_mpy.mpy and b/mip/v6/createstubs_lvgl_mpy.mpy differ diff --git a/mip/v6/createstubs_mem.py b/mip/v6/createstubs_mem.py index 34cd9f06..81164f7d 100644 --- a/mip/v6/createstubs_mem.py +++ b/mip/v6/createstubs_mem.py @@ -17,6 +17,7 @@ import logging import os import sys +from time import sleep try: from ujson import dumps @@ -33,11 +34,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -55,11 +71,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -108,6 +125,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -147,6 +165,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -199,7 +218,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -210,10 +229,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] self.log.warning("could not del new_module") - try: - del sys.modules[module_name] - except KeyError: - self.log.debug("could not del sys.modules[{}]".format(module_name)) + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + # self.log.warning("could not del sys.modules[{}]".format(module_name)) gc.collect() return True @@ -239,8 +259,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -312,12 +337,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) self.log.debug("\n" + s) else: @@ -327,12 +354,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -346,6 +373,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -368,6 +396,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -427,13 +456,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -567,56 +602,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_mem_min.py b/mip/v6/createstubs_mem_min.py index f7fb6c09..d610d52b 100644 --- a/mip/v6/createstubs_mem_min.py +++ b/mip/v6/createstubs_mem_min.py @@ -17,6 +17,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -33,11 +34,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -55,11 +71,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -108,6 +125,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -147,6 +165,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -199,7 +218,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -210,10 +229,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -239,8 +259,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -312,12 +337,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -327,12 +354,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -346,6 +373,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -368,6 +396,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -427,13 +456,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -567,56 +602,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_mem_mpy.mpy b/mip/v6/createstubs_mem_mpy.mpy index d35e79d5..520ab82c 100644 Binary files a/mip/v6/createstubs_mem_mpy.mpy and b/mip/v6/createstubs_mem_mpy.mpy differ diff --git a/mip/v6/createstubs_min.py b/mip/v6/createstubs_min.py index f2fd2cf6..7b134cbf 100644 --- a/mip/v6/createstubs_min.py +++ b/mip/v6/createstubs_min.py @@ -7,6 +7,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -23,11 +24,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -45,11 +61,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -98,6 +115,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -137,6 +155,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -191,7 +210,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: module_name, self._fwid, info_, __version__ ) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -202,10 +221,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -231,8 +251,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info( # "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path) # ) @@ -423,13 +452,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -565,56 +600,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/mip/v6/createstubs_mpy.mpy b/mip/v6/createstubs_mpy.mpy index d59f272e..db5b7968 100644 Binary files a/mip/v6/createstubs_mpy.mpy and b/mip/v6/createstubs_mpy.mpy differ diff --git a/src/stubber/board/createstubs_db.py b/src/stubber/board/createstubs_db.py index 96557273..ee508f84 100644 --- a/src/stubber/board/createstubs_db.py +++ b/src/stubber/board/createstubs_db.py @@ -43,6 +43,21 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() __version__ = "v1.16.2" ENOENT = 2 @@ -65,6 +80,7 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: @@ -158,6 +174,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -365,6 +382,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -387,6 +405,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -589,16 +608,79 @@ def version_str(version: tuple): # -> str: return v_str -def get_boardname() -> str: - "Read the board name from the boardname.py file that may have been created upfront" - try: - from boardname import BOARDNAME # type: ignore - - log.info("Found BOARDNAME: {}".format(BOARDNAME)) - except ImportError: - log.info("BOARDNAME not found") - BOARDNAME = "" - return BOARDNAME +def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") + found = False + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() + # print("look up the board name in the file", filename) + if file_exists(filename): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() + found = True + break + if not found: + print("Board not found, guessing board name") + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() + info["board"] = descr + + +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/src/stubber/board/createstubs_db_min.py b/src/stubber/board/createstubs_db_min.py index 1c40ecbe..05817f71 100644 --- a/src/stubber/board/createstubs_db_min.py +++ b/src/stubber/board/createstubs_db_min.py @@ -26,6 +26,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -42,11 +43,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -64,11 +80,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -117,6 +134,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -156,6 +174,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -208,7 +227,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -219,10 +238,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -248,8 +268,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -321,12 +346,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -336,12 +363,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -355,6 +382,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -377,6 +405,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -436,13 +465,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -576,56 +611,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/src/stubber/board/createstubs_db_mpy.mpy b/src/stubber/board/createstubs_db_mpy.mpy index 5a662774..029c6ff3 100644 Binary files a/src/stubber/board/createstubs_db_mpy.mpy and b/src/stubber/board/createstubs_db_mpy.mpy differ diff --git a/src/stubber/board/createstubs_lvgl.py b/src/stubber/board/createstubs_lvgl.py index 8b631992..0f5624b5 100644 --- a/src/stubber/board/createstubs_lvgl.py +++ b/src/stubber/board/createstubs_lvgl.py @@ -28,6 +28,21 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() __version__ = "v1.16.2" ENOENT = 2 @@ -50,6 +65,7 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: @@ -143,6 +159,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -193,7 +210,9 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: with open(file_name, "w") as fp: # todo: improve header info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") - s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) + s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format( + module_name, self._fwid, info_, __version__ + ) fp.write(s) fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") @@ -290,7 +309,9 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, first = "self, " # class method - add function decoration if "bound_method" in item_type_txt or "bound_method" in item_repr: - s = "{}@classmethod\n".format(indent) + "{}def {}(cls, *args, **kwargs) -> {}:\n".format(indent, item_name, ret) + s = "{}@classmethod\n".format(indent) + "{}def {}(cls, *args, **kwargs) -> {}:\n".format( + indent, item_name, ret + ) else: s = "{}def {}({}*args, **kwargs) -> {}:\n".format(indent, item_name, first, ret) s += indent + " ...\n\n" @@ -350,6 +371,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -372,7 +394,10 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" - self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) + wdt.feed() + self.log.info( + "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path) + ) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) gc.collect() @@ -574,16 +599,79 @@ def version_str(version: tuple): # -> str: return v_str -def get_boardname() -> str: - "Read the board name from the boardname.py file that may have been created upfront" - try: - from boardname import BOARDNAME # type: ignore - - log.info("Found BOARDNAME: {}".format(BOARDNAME)) - except ImportError: - log.info("BOARDNAME not found") - BOARDNAME = "" - return BOARDNAME +def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") + found = False + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() + # print("look up the board name in the file", filename) + if file_exists(filename): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() + found = True + break + if not found: + print("Board not found, guessing board name") + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() + info["board"] = descr + + +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/src/stubber/board/createstubs_lvgl_min.py b/src/stubber/board/createstubs_lvgl_min.py index d3b5468e..cdab9a5a 100644 --- a/src/stubber/board/createstubs_lvgl_min.py +++ b/src/stubber/board/createstubs_lvgl_min.py @@ -11,6 +11,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -27,11 +28,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -49,11 +65,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -102,6 +119,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -141,6 +159,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -193,7 +212,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -204,10 +223,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -233,8 +253,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -421,13 +450,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -561,56 +596,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/src/stubber/board/createstubs_lvgl_mpy.mpy b/src/stubber/board/createstubs_lvgl_mpy.mpy index ee3c7c62..0022ec0e 100644 Binary files a/src/stubber/board/createstubs_lvgl_mpy.mpy and b/src/stubber/board/createstubs_lvgl_mpy.mpy differ diff --git a/src/stubber/board/createstubs_mem.py b/src/stubber/board/createstubs_mem.py index f21a4dfb..cbdb970d 100644 --- a/src/stubber/board/createstubs_mem.py +++ b/src/stubber/board/createstubs_mem.py @@ -34,6 +34,21 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() __version__ = "v1.16.2" ENOENT = 2 @@ -56,6 +71,7 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore self.log.info("Port: {}".format(self.info["port"])) self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: @@ -149,6 +165,7 @@ def create_all_stubs(self): self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -356,6 +373,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path self.log.info("Clean/remove files in folder: {}".format(path)) @@ -378,6 +396,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) self.log.info("Report file: {}".format(f_name)) @@ -580,16 +599,79 @@ def version_str(version: tuple): # -> str: return v_str -def get_boardname() -> str: - "Read the board name from the boardname.py file that may have been created upfront" - try: - from boardname import BOARDNAME # type: ignore - - log.info("Found BOARDNAME: {}".format(BOARDNAME)) - except ImportError: - log.info("BOARDNAME not found") - BOARDNAME = "" - return BOARDNAME +def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") + found = False + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() + # print("look up the board name in the file", filename) + if file_exists(filename): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() + found = True + break + if not found: + print("Board not found, guessing board name") + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() + info["board"] = descr + + +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/src/stubber/board/createstubs_mem_min.py b/src/stubber/board/createstubs_mem_min.py index f7fb6c09..d610d52b 100644 --- a/src/stubber/board/createstubs_mem_min.py +++ b/src/stubber/board/createstubs_mem_min.py @@ -17,6 +17,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -33,11 +34,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -55,11 +71,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -108,6 +125,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -147,6 +165,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -199,7 +218,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: info_ = str(self.info).replace("OrderedDict(", "").replace("})", "}") s = '"""\nModule: \'{0}\' on {1}\n"""\n# MCU: {2}\n# Stubber: {3}\n'.format(module_name, self._fwid, info_, __version__) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -210,10 +229,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -239,8 +259,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -312,12 +337,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -327,12 +354,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -346,6 +373,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -368,6 +396,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info("Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path)) f_name = "{}/{}".format(self.path, filename) # self.log.info("Report file: {}".format(f_name)) @@ -427,13 +456,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -567,56 +602,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/src/stubber/board/createstubs_mem_mpy.mpy b/src/stubber/board/createstubs_mem_mpy.mpy index 3b020972..e3d7d717 100644 Binary files a/src/stubber/board/createstubs_mem_mpy.mpy and b/src/stubber/board/createstubs_mem_mpy.mpy differ diff --git a/src/stubber/board/createstubs_min.py b/src/stubber/board/createstubs_min.py index f2fd2cf6..7b134cbf 100644 --- a/src/stubber/board/createstubs_min.py +++ b/src/stubber/board/createstubs_min.py @@ -7,6 +7,7 @@ # import logging import os import sys +from time import sleep try: from ujson import dumps @@ -23,11 +24,26 @@ except ImportError: from ucollections import OrderedDict # type: ignore +try: + from nope_machine import WDT + + wdt = WDT() + +except ImportError: + + class _WDT: + def feed(self): + pass + + wdt = _WDT() + + +wdt.feed() + __version__ = "v1.16.2" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = [".", "/lib", "/sd/lib", "/flash/lib", "lib"] -from time import sleep class Stubber: @@ -45,11 +61,12 @@ def __init__(self, path: str = None, firmware_id: str = None): # type: ignore # self.log.info("Port: {}".format(self.info["port"])) # self.log.info("Board: {}".format(self.info["board"])) gc.collect() + wdt.feed() if firmware_id: self._fwid = firmware_id.lower() else: if self.info["family"] == "micropython": - self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info) + self._fwid = "{family}-v{version}-{port}-{board}".format(**self.info).rstrip("-") else: self._fwid = "{family}-v{version}-{port}".format(**self.info) self._start_free = gc.mem_free() # type: ignore @@ -98,6 +115,7 @@ def get_obj_attributes(self, item_instance: object): try: val = getattr(item_instance, name) # name , item_repr(value) , type as text, item_instance, order + # self.log.debug("attribute {}:{}".format(name, val)) try: type_text = repr(type(val)).split("'")[1] except IndexError: @@ -137,6 +155,7 @@ def create_all_stubs(self): # self.log.info("Finally done") def create_one_stub(self, module_name: str): + wdt.feed() if module_name in self.problematic: # self.log.warning("Skip module: {:<25} : Known problematic".format(module_name)) return False @@ -191,7 +210,7 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: module_name, self._fwid, info_, __version__ ) fp.write(s) - fp.write("from typing import Any\nfrom _typeshed import Incomplete\n\n") + fp.write("from __future__ import annotations\nfrom typing import Any\nfrom _typeshed import Incomplete\n\n") self.write_object_stub(fp, new_module, module_name, "") self._report.append('{{"module": "{}", "file": "{}"}}'.format(module_name, file_name.replace("\\", "/"))) @@ -202,10 +221,11 @@ def create_module_stub(self, module_name: str, file_name: str = None) -> bool: del new_module except (OSError, KeyError): # lgtm [py/unreachable-statement] pass - try: - del sys.modules[module_name] - except KeyError: - pass + # lets not try - most times it does not work anyway + # try: + # del sys.modules[module_name] + # except KeyError: + pass gc.collect() return True @@ -231,8 +251,13 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, # self.log.warning("NameError: invalid name {}".format(item_name)) continue # Class expansion only on first 3 levels (bit of a hack) - if item_type_txt == "" and len(indent) <= _MAX_CLASS_LEVEL * 4: - # self.log.debug("{0}class {1}:".format(indent, item_name)) + if ( + item_type_txt == "" + and len(indent) <= _MAX_CLASS_LEVEL * 4 + # and not obj_name.endswith(".Pin") + # avoid expansion of Pin.cpu / Pin.board to avoid crashes on most platforms + ): + # self.log.info("{0}class {1}:".format(indent, item_name)) superclass = "" is_exception = ( item_name.endswith("Exception") @@ -306,12 +331,14 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, s = "{0}{1} = {2} # type: {3}\n".format(indent, item_name, ev[t], t) else: # something else - if t not in ["object", "set", "frozenset"]: - # Possibly default others to item_instance object ? + if t in ["object", "set", "frozenset", "Pin", "FileIO"]: # https://docs.python.org/3/tutorial/classes.html#item_instance-objects + # use these types for the attribute + s = "{0}{1} : {2} ## = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + else: + # Requires Python 3.6 syntax, which is OK for the stubs/pyi t = "Incomplete" - # Requires Python 3.6 syntax, which is OK for the stubs/pyi - s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) + s = "{0}{1} : {2} ## {3} = {4}\n".format(indent, item_name, t, item_type_txt, item_repr) fp.write(s) # self.log.debug("\n" + s) else: @@ -321,12 +348,12 @@ def write_object_stub(self, fp, object_expr: object, obj_name: str, indent: str, fp.write(indent + item_name + " # type: Incomplete\n") - del items - del errors - try: - del item_name, item_repr, item_type_txt, item_instance # type: ignore - except (OSError, KeyError, NameError): - pass + # del items + # del errors + # try: + # del item_name, item_repr, item_type_txt, item_instance # type: ignore + # except (OSError, KeyError, NameError): + # pass @property def flat_fwid(self): @@ -340,6 +367,7 @@ def flat_fwid(self): def clean(self, path: str = None): # type: ignore "Remove all files from the stub folder" + wdt.feed() if path is None: path = self.path # self.log.info("Clean/remove files in folder: {}".format(path)) @@ -362,6 +390,7 @@ def clean(self, path: str = None): # type: ignore def report(self, filename: str = "modules.json"): "create json with list of exported modules" + wdt.feed() # self.log.info( # "Created stubs for {} modules on board {}\nPath: {}".format(len(self._report), self._fwid, self.path) # ) @@ -423,13 +452,19 @@ def ensure_folder(path: str): def _build(s): # extract build from sys.version or os.uname().version if available - # 'v1.13-103-gb137d064e' - # 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.version: 'MicroPython v1.23.0-preview.6.g3d0b6276f' + # sys.implementation.version: 'v1.13-103-gb137d064e' if not s: return "" s = s.split(" on ", 1)[0] if " on " in s else s - s = s.split("; ", 1)[1] if "; " in s else s - b = s.split("-")[1] if s.startswith("v") else s.split("-", 1)[-1].split(".")[1] + if s.startswith("v"): + if not "-" in s: + return "" + b = s.split("-")[1] + return b + if not "-preview" in s: + return "" + b = s.split("-preview")[1].split(".")[1] return b @@ -565,56 +600,78 @@ def version_str(version: tuple): # -> str: def read_boardname(info, desc: str = ""): + info["board"] = info["board"].replace(" ", "_") found = False - for filename in [d + "/board_info.csv" for d in LIBS]: + for filename in [d + "/board_name.txt" for d in LIBS]: + wdt.feed() # # print("look up the board name in the file", filename) if file_exists(filename): - descr = desc or info["board"].strip() - pos = descr.rfind(" with") - if pos != -1: - short_descr = descr[:pos].strip() - else: - short_descr = "" - # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) - if find_board(info, descr, filename, short_descr): + with open(filename, "r") as file: + data = file.read() + if data: + info["board"] = data.strip() found = True break if not found: # print("Board not found, guessing board name") - descr = desc or info["board"].strip() - if "with " + info["cpu"].upper() in descr: - # remove the with cpu part - descr = descr.split("with " + info["cpu"].upper())[0].strip() + descr = "" + # descr = desc or info["board"].strip() + # if "with " + info["cpu"].upper() in descr: + # # remove the with cpu part + # descr = descr.split("with " + info["cpu"].upper())[0].strip() info["board"] = descr - info["board"] = info["board"].replace(" ", "_") - gc.collect() -def find_board(info: dict, descr: str, filename: str, short_descr: str): - "Find the board in the provided board_info.csv file" - short_hit = "" - with open(filename, "r") as file: - # ugly code to make testable in python and micropython - # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) - while 1: - line = file.readline() - if not line: - break - descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() - if descr_ == descr: - info["board"] = board_ - return True - elif short_descr and descr_ == short_descr: - if "with" in short_descr: - # Good enough - no need to trawl the entire file - info["board"] = board_ - return True - # good enough if not found in the rest of the file (but slow) - short_hit = board_ - if short_hit: - info["board"] = short_hit - return True - return False +# def read_boardname(info, desc: str = ""): +# wdt.feed() +# # # print("look up the board name in the file", filename) +# if file_exists(filename): +# descr = desc or info["board"].strip() +# pos = descr.rfind(" with") +# if pos != -1: +# short_descr = descr[:pos].strip() +# else: +# short_descr = "" +# # print("searching info file: {} for: '{}' or '{}'".format(filename, descr, short_descr)) +# if find_board(info, descr, filename, short_descr): +# found = True +# break +# if not found: +# # print("Board not found, guessing board name") +# descr = desc or info["board"].strip() +# if "with " + info["cpu"].upper() in descr: +# # remove the with cpu part +# descr = descr.split("with " + info["cpu"].upper())[0].strip() +# info["board"] = descr +# info["board"] = info["board"].replace(" ", "_") +# gc.collect() + + +# def find_board(info: dict, descr: str, filename: str, short_descr: str): +# "Find the board in the provided board_info.csv file" +# short_hit = "" +# with open(filename, "r") as file: +# # ugly code to make testable in python and micropython +# # TODO: This is VERY slow on micropython whith MPREMOTE mount on esp32 (2-3 minutes to read file) +# while 1: +# line = file.readline() +# if not line: +# break +# descr_, board_ = line.split(",")[0].strip(), line.split(",")[1].strip() +# if descr_ == descr: +# info["board"] = board_ +# return True +# elif short_descr and descr_ == short_descr: +# if "with" in short_descr: +# # Good enough - no need to trawl the entire file +# info["board"] = board_ +# return True +# # good enough if not found in the rest of the file (but slow) +# short_hit = board_ +# if short_hit: +# info["board"] = short_hit +# return True +# return False def get_root() -> str: # sourcery skip: use-assigned-variable diff --git a/src/stubber/board/createstubs_mpy.mpy b/src/stubber/board/createstubs_mpy.mpy index e75d7e7d..1f7e3a6a 100644 Binary files a/src/stubber/board/createstubs_mpy.mpy and b/src/stubber/board/createstubs_mpy.mpy differ