Merge branch 'dev' of gh:visualDust/neetbox

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 dist/
 poetry.lock
 test/optional
+build_and_reinstall_wheel.cmd
 
 # log files
 log

.pre-commit-config.yaml

@@ -18,6 +18,7 @@ repos:
     hooks:
     -   id: black
         files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
+        args: [--line-length=100]
 -   repo: https://github.com/pycqa/isort
     rev: 5.11.5
     hooks:

neetbox/cli/parse.py

@@ -3,7 +3,9 @@
 import click
 
 import neetbox
-from neetbox.daemon._apis import get_status_of
+from neetbox.daemon._client_apis import get_status_of
+from neetbox.logging.formatting import LogStyle
+from neetbox.logging.logger import Logger
 from neetbox.logging.formatting import LogStyle
 from neetbox.logging.logger import Logger
 

neetbox/config/__init__.py

@@ -4,24 +4,22 @@
 # URL:    https://gong.host
 # Date:   20230413
 
+
 import inspect
-from typing import Optional, Union
 
-from neetbox.config._config import DEFAULT_CONFIG as default
+from neetbox.config._config import DEFAULT_WORKSPACE_CONFIG as default
 from neetbox.config._config import get_current
-from neetbox.utils.framing import *
+from neetbox.utils.framing import get_frame_module_traceback
 
 
 def get_module_level_config(module=None):
     try:
         module = (
-            module or get_frame_module_traceback(traceback=2).__name__
+            module or get_frame_module_traceback(traceback=2).__name__  # type: ignore
         )  # try to trace if module not given
-        if (
-            type(module) is not str
-        ):  # try to trace the belonging module of the given object
-            module = inspect.getmodule(module).__name__
-    except:
+        if type(module) is not str:  # try to trace the belonging module of the given object
+            module = inspect.getmodule(module).__name__  # type: ignore
+    except Exception:
         module = "@"  # faild to trace, returning all configs
     the_config = get_current()
     sub_module_names = module.split(".")
@@ -36,4 +34,4 @@ def get_module_level_config(module=None):
     return the_config
 
 
-__all__ = ["config", "get_module_level_config", "default"]
+__all__ = ["get_module_level_config", "default"]

neetbox/config/_config.py

@@ -4,12 +4,22 @@
 # URL:    https://gong.host
 # Date:   20230413
 
-import collections
-from multiprocessing import current_process
 
-from neetbox.utils.mvc import patch
+DEFAULT_GLOBAL_CONFIG = {
+    "daemon": {
+        "enable": True,
+        "allowIpython": False,
+        "servers": [
+            {"host": "localhost", "port": "20202"},
+        ],
+        "mode": "detached",
+        "displayName": None,
+        "uploadInterval": 10,
+        "mute": True,
+    },
+}
 
-DEFAULT_CONFIG = {
+DEFAULT_WORKSPACE_CONFIG = {
     "name": None,
     "version": None,
     "logging": {"logdir": None},
@@ -36,7 +46,7 @@
         },
     },
 }
-WORKSPACE_CONFIG: dict = DEFAULT_CONFIG.copy()
+WORKSPACE_CONFIG: dict = DEFAULT_WORKSPACE_CONFIG.copy()
 
 
 def update_with(cfg: dict):

neetbox/core/registry.py

@@ -48,7 +48,7 @@ def __new__(cls, name: str) -> "Registry":
 
     # not compatible with python below 3.8
     def __init__(self, name, *args, **kwargs) -> None:
-        if not "initialized" in self:
+        if "initialized" not in self:
             self["initialized"] = True
             self.name = name
 

neetbox/daemon/__init__.py

@@ -5,16 +5,13 @@
 # Date:   20230414
 
 import json
-import os
-import platform
 import subprocess
 import time
 
 from neetbox.daemon._daemon_client import connect_daemon
 from neetbox.daemon.daemonable_process import DaemonableProcess
 from neetbox.logging import logger
 from neetbox.pipeline import listen, watch
-from neetbox.utils import pkg
 
 
 def __attach_daemon(daemon_config):
@@ -29,11 +26,17 @@ def __attach_daemon(daemon_config):
                 "ipython, try to set 'allowIpython' to True."
             )
             return False  # ignore if debugging in ipython
-    pkg.is_installed("flask", try_install_if_not=True)
     _online_status = connect_daemon(daemon_config)  # try to connect daemon
     logger.log("daemon connection status: " + str(_online_status))
     if not _online_status:  # if no daemon online
-        logger.log(
+        if daemon_config["server"] not in ["localhost", "127.0.0.1", "0.0.0.0"]:
+            # daemon not running on localhost
+            logger.err(
+                f"No daemon running at {daemon_config['server']}:{daemon_config['port']}, daemon will not be attached. Continue anyway."
+            )
+            return False
+
+        logger.warn(
             f"No daemon running at {daemon_config['server']}:{daemon_config['port']}, trying to create daemon..."
         )
 
@@ -57,9 +60,7 @@ def __attach_daemon(daemon_config):
             else:
                 exit_code = popen.poll()
                 if exit_code is not None:
-                    logger.err(
-                        f"Daemon process exited unexpectedly with exit code {exit_code}."
-                    )
+                    logger.err(f"Daemon process exited unexpectedly with exit code {exit_code}.")
                     return False
 
                 time.sleep(0.5)

neetbox/daemon/_apis.py → neetbox/daemon/_client_apis.py

@@ -5,21 +5,16 @@
 # Date:   20230414
 
 
+from neetbox.config import get_module_level_config
 from neetbox.daemon._local_http_client import _local_http_client
+from neetbox.logging import logger
 from neetbox.utils import pkg
 from neetbox.utils.framing import get_frame_module_traceback
 
-module_name = get_frame_module_traceback().__name__
+module_name = get_frame_module_traceback().__name__  # type: ignore
 assert pkg.is_installed(
     "httpx", try_install_if_not=True
 ), f"{module_name} requires httpx which is not installed"
-import json
-import time
-
-import httpx
-
-from neetbox.config import get_module_level_config
-from neetbox.logging import logger
 
 logger = logger("NEETBOX DAEMON API")
 

neetbox/daemon/_daemon_client.py

@@ -7,6 +7,7 @@
 import json
 import time
 from threading import Thread
+from typing import Union
 
 from neetbox.config import get_module_level_config
 from neetbox.daemon._local_http_client import _local_http_client
@@ -15,14 +16,13 @@
 
 __TIME_UNIT_SEC = 0.1
 
-__upload_thread: Thread = None
+__upload_thread: Union[Thread, None] = None
 
 
 def _upload_thread(daemon_config, base_addr, display_name):
     _ctr = 0
     _api_name = "sync"
     _api_addr = f"{base_addr}/{_api_name}/{display_name}"
-    global _update_value_dict
     _disconnect_flag = False
     _disconnect_retries = 10
     while True:
@@ -31,17 +31,18 @@ def _upload_thread(daemon_config, base_addr, display_name):
         time.sleep(__TIME_UNIT_SEC)
         if _ctr % _upload_interval:  # not zero
             continue
-        # upload data
+        # dump status as json
         _data = json.dumps(_update_value_dict, default=str)
         _headers = {"Content-Type": "application/json"}
         try:
-            resp = _local_http_client.post(_api_addr, data=_data, headers=_headers)
-            if resp.is_error:
+            # upload data
+            resp = _local_http_client.post(_api_addr, json=_data, headers=_headers)
+            if resp.is_error:  # upload failed
                 raise IOError(f"Failed to upload data to daemon. ({resp.status_code})")
         except Exception as e:
-            if _disconnect_flag:
+            if _disconnect_flag:  # already in retries
                 _disconnect_retries -= 1
-                if not _disconnect_retries:
+                if not _disconnect_retries:  # retry count down exceeded
                     logger.err(
                         "Failed to reconnect to daemon after {10} retries, Trying to launch new daemon..."
                     )
@@ -50,51 +51,48 @@ def _upload_thread(daemon_config, base_addr, display_name):
                     _try_attach_daemon()
                     time.sleep(__TIME_UNIT_SEC)
                 continue
-            logger.warn(
-                f"Failed to upload data to daemon cause {e}. Waiting for reconnect..."
-            )
+            logger.warn(f"Failed to upload data to daemon cause {e}. Waiting for reconnect...")
             _disconnect_flag = True
         else:
             if not _disconnect_flag:
                 continue
-            logger.ok(f"Successfully reconnected to daemon.")
+            logger.ok("Successfully reconnected to daemon.")
             _disconnect_flag = False
             _disconnect_retries = 10
 
 
-def connect_daemon(daemon_config):
+def connect_daemon(daemon_config, launch_upload_thread=True):
     _display_name = get_module_level_config()["displayName"]
     _launch_config = get_module_level_config("@")
     _display_name = _display_name or _launch_config["name"]
 
-    logger.log(
-        f"Connecting to daemon at {daemon_config['server']}:{daemon_config['port']} ..."
-    )
+    logger.log(f"Connecting to daemon at {daemon_config['server']}:{daemon_config['port']} ...")
     _daemon_address = f"{daemon_config['server']}:{daemon_config['port']}"
-    base_addr = f"http://{_daemon_address}"
+    _base_addr = f"http://{_daemon_address}"
 
     # check if daemon is alive
-    def _check_daemon_alive():
+    def _check_daemon_alive(_api_addr):
         _api_name = "hello"
-        _api_addr = f"{base_addr}/{_api_name}"
+        _api_addr = f"{_api_addr}/{_api_name}"
         r = _local_http_client.get(_api_addr)
         if r.is_error:
             raise IOError(f"Daemon at {_api_addr} is not alive. ({r.status_code})")
-        logger.log(f"daemon response from {_api_addr} is {r} ({r.status_code})")
 
     try:
-        _check_daemon_alive()
+        _check_daemon_alive(_base_addr)
+        logger.ok(f"daemon alive at {_base_addr}")
     except Exception as e:
         logger.err(e)
         return False
 
-    global __upload_thread
-    if __upload_thread is None or not __upload_thread.is_alive():
-        __upload_thread = Thread(
-            target=_upload_thread,
-            daemon=True,
-            args=[daemon_config, base_addr, _display_name],
-        )
-        __upload_thread.start()
+    if launch_upload_thread:
+        global __upload_thread
+        if __upload_thread is None or not __upload_thread.is_alive():
+            __upload_thread = Thread(
+                target=_upload_thread,
+                daemon=True,
+                args=[daemon_config, _base_addr, _display_name],
+            )
+            __upload_thread.start()
 
     return True

neetbox/daemon/_daemon_launcher.py

@@ -1,16 +1,14 @@
-print("========= Daemon Launcher =========")
-
 import argparse
 import json
-import os
 import sys
 
-from neetbox.daemon._daemon import daemon_process
+from neetbox.daemon._flask_server import daemon_process
 
 # sys.stdout=open(r'D:\Projects\ML\neetbox\logdir\daemon.log', 'a+')
 
 
 # from neetbox.daemon._local_http_client import _local_http_client
+print("========= Daemon Launcher =========")
 
 
 def run():
@@ -27,6 +25,5 @@ def run():
 
 
 print("_daemon_launcher is starting with __name__ =", __name__, " and args =", sys.argv)
-
 run()
 print("_daemon_launcher: exiting")