Merge pull request #64 from visualDust/dev

Dev

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 dist/
 poetry.lock
 test/optional
+build_and_reinstall_wheel.cmd
 
 # log files
 log

.pre-commit-config.yaml

@@ -18,6 +18,7 @@ repos:
     hooks:
     -   id: black
         files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
+        args: [--line-length=100]
 -   repo: https://github.com/pycqa/isort
     rev: 5.11.5
     hooks:

neetbox.toml

@@ -1,11 +1,22 @@
-name = "FakeConfig"
+name = "neetbox"
 
 [logging]
+level = "DEBUG"
 
 [pipeline]
-
-[integrations]
+updateInterval = 10
 
 [daemon]
 enable = true
-allowIpython = true
+host = "localhost"
+port = 20202
+allowIpython = false
+mute = true
+mode = "detached"
+uploadInterval = 10
+
+[integrations.environment.hardware]
+monit = "true"
+
+[integrations.environment.platform]
+monit = "true"

neetbox/__init__.py

@@ -36,9 +36,7 @@ def init(path=None, load=False, **kwargs) -> bool:
                     if "name" in kwargs and kwargs["name"]:  # using given name
                         _config["name"] = kwargs["name"]
                     else:  # using the folder name
-                        _config["name"] = os.path.basename(
-                            os.path.normpath(os.getcwd())
-                        )
+                        _config["name"] = os.path.basename(os.path.normpath(os.getcwd()))
                     toml.dump(_config, config_file)
                 logger.ok(f"Workspace config created as {config_file_path}.")
                 return True
@@ -75,9 +73,10 @@ def init(path=None, load=False, **kwargs) -> bool:
 
 
 is_in_daemon_process = (
-    "NEETBOX_DAEMON_PROCESS" in os.environ
-    and os.environ["NEETBOX_DAEMON_PROCESS"] == "1"
+    "NEETBOX_DAEMON_PROCESS" in os.environ and os.environ["NEETBOX_DAEMON_PROCESS"] == "1"
 )
 # print('prevent_daemon_loading =', is_in_daemon_process)
 if os.path.isfile(config_file_name) and not is_in_daemon_process:  # if in a workspace
-    init(load=True)
+    success = init(load=True)  # init from config file
+    if not success:
+        os._exit(255)

neetbox/cli/parse.py

@@ -3,7 +3,7 @@
 import click
 
 import neetbox
-from neetbox.daemon._apis import get_status_of
+from neetbox.daemon.client._client_apis import get_status_of
 from neetbox.logging.formatting import LogStyle
 from neetbox.logging.logger import Logger
 

neetbox/config/__init__.py

@@ -4,24 +4,22 @@
 # URL:    https://gong.host
 # Date:   20230413
 
+
 import inspect
-from typing import Optional, Union
 
-from neetbox.config._config import DEFAULT_CONFIG as default
+from neetbox.config._config import DEFAULT_WORKSPACE_CONFIG as default
 from neetbox.config._config import get_current
-from neetbox.utils.framing import *
+from neetbox.utils.framing import get_frame_module_traceback
 
 
 def get_module_level_config(module=None):
     try:
         module = (
-            module or get_frame_module_traceback(traceback=2).__name__
+            module or get_frame_module_traceback(traceback=2).__name__  # type: ignore
         )  # try to trace if module not given
-        if (
-            type(module) is not str
-        ):  # try to trace the belonging module of the given object
-            module = inspect.getmodule(module).__name__
-    except:
+        if type(module) is not str:  # try to trace the belonging module of the given object
+            module = inspect.getmodule(module).__name__  # type: ignore
+    except Exception:
         module = "@"  # faild to trace, returning all configs
     the_config = get_current()
     sub_module_names = module.split(".")
@@ -36,4 +34,4 @@ def get_module_level_config(module=None):
     return the_config
 
 
-__all__ = ["config", "get_module_level_config", "default"]
+__all__ = ["get_module_level_config", "default"]

neetbox/config/_config.py

@@ -4,39 +4,28 @@
 # URL:    https://gong.host
 # Date:   20230413
 
-import collections
-from multiprocessing import current_process
-
-from neetbox.utils.mvc import patch
-
-DEFAULT_CONFIG = {
+DEFAULT_WORKSPACE_CONFIG = {
     "name": None,
     "version": None,
-    "logging": {"logdir": None},
+    "logging": {"level": "INFO", "logdir": None},
     "pipeline": {
         "updateInterval": 10,
     },
     "integrations": {
-        "environment": {
-            "gpus": "auto",
-        },
-        "datasets": [],
+        "environment": {"hardware": {"monit": "true"}, "platform": {"monit": "true"}},
     },
     "daemon": {
         "enable": True,
-        "allowIpython": False,
-        "server": "localhost",
+        "host": "localhost",
         "port": 20202,
-        "mode": "detached",
         "displayName": None,
-        "uploadInterval": 10,
+        "allowIpython": False,
         "mute": True,
-        "launcher": {
-            "port": 20202,
-        },
+        "mode": "detached",
+        "uploadInterval": 10,
     },
 }
-WORKSPACE_CONFIG: dict = DEFAULT_CONFIG.copy()
+WORKSPACE_CONFIG: dict = DEFAULT_WORKSPACE_CONFIG.copy()
 
 
 def update_with(cfg: dict):

neetbox/core/__init__.py

@@ -1,3 +1,7 @@
+from rich.console import Console
+
 from neetbox.core.registry import Registry
 
-__all__ = ["Registry"]
+console = Console()
+
+__all__ = ["Registry", "console"]

neetbox/core/registry.py

@@ -9,7 +9,7 @@
 from typing import Any, Dict, List, Optional, Sequence, Union
 
 from neetbox.logging import logger
-from neetbox.utils.format import *
+from neetbox.utils.formatting import *
 
 
 class _RegEndpoint:
@@ -48,7 +48,7 @@ def __new__(cls, name: str) -> "Registry":
 
     # not compatible with python below 3.8
     def __init__(self, name, *args, **kwargs) -> None:
-        if not "initialized" in self:
+        if "initialized" not in self:
             self["initialized"] = True
             self.name = name
 
@@ -71,9 +71,7 @@ def _register(
                     f"{name} already exists in Registry:{self.name}. If you want to overwrite, try to register with 'force=True'"
                 )
             else:
-                logger.warn(
-                    f"Overwritting existing '{name}' in Registry '{self.name}'."
-                )
+                logger.warn(f"Overwritting existing '{name}' in Registry '{self.name}'.")
                 self[name] = _endp
         else:
             self[name] = _endp
@@ -96,9 +94,7 @@ def find(
     ):
         if not name and not tags:
             logger.err(
-                ValueError(
-                    "Please provide at least the name or the tags you want to find."
-                ),
+                ValueError("Please provide at least the name or the tags you want to find."),
                 reraise=True,
             )
         results = []
@@ -122,11 +118,7 @@ def _tags_match(f_tags, s_tags) -> bool:
                     return False
             return True
 
-        results = {
-            _name: _endp.what
-            for _name, _endp in results
-            if _tags_match(tags, _endp.tags)
-        }
+        results = {_name: _endp.what for _name, _endp in results if _tags_match(tags, _endp.tags)}
         return results
 
     def __getitem__(self, __key: str) -> Any:
@@ -152,27 +144,15 @@ def update(self, *args, **kwargs):
         return self.__dict__.update(*args, **kwargs)
 
     def keys(self):
-        return [
-            _item[0]
-            for _item in self.__dict__.items()
-            if type(_item[1]) is _RegEndpoint
-        ]
+        return [_item[0] for _item in self.__dict__.items() if type(_item[1]) is _RegEndpoint]
 
     def values(self):
-        return [
-            _item[1].what
-            for _item in self.__dict__.items()
-            if type(_item[1]) is _RegEndpoint
-        ]
+        return [_item[1].what for _item in self.__dict__.items() if type(_item[1]) is _RegEndpoint]
 
     def items(self, _real_type=True):
-        _legal_items = [
-            _item for _item in self.__dict__.items() if type(_item[1]) is _RegEndpoint
-        ]
+        _legal_items = [_item for _item in self.__dict__.items() if type(_item[1]) is _RegEndpoint]
         if _real_type:
-            _legal_items = [
-                (_k, _v.what) for _k, _v in _legal_items if type(_v) is _RegEndpoint
-            ]
+            _legal_items = [(_k, _v.what) for _k, _v in _legal_items if type(_v) is _RegEndpoint]
         return _legal_items
 
     def pop(self, *args):

neetbox/daemon/__init__.py

@@ -5,16 +5,16 @@
 # Date:   20230414
 
 import json
-import os
-import platform
 import subprocess
 import time
 
-from neetbox.daemon._daemon_client import connect_daemon
-from neetbox.daemon.daemonable_process import DaemonableProcess
+import neetbox
+from neetbox.config import get_module_level_config
+from neetbox.daemon.client._action_agent import _NeetActionManager as NeetActionManager
+from neetbox.daemon.client._daemon_client import connect_daemon
+from neetbox.daemon.server.daemonable_process import DaemonableProcess
 from neetbox.logging import logger
 from neetbox.pipeline import listen, watch
-from neetbox.utils import pkg
 
 
 def __attach_daemon(daemon_config):
@@ -29,17 +29,24 @@ def __attach_daemon(daemon_config):
                 "ipython, try to set 'allowIpython' to True."
             )
             return False  # ignore if debugging in ipython
-    pkg.is_installed("flask", try_install_if_not=True)
-    _online_status = connect_daemon(daemon_config)  # try to connect daemon
-    logger.log("daemon connection status: " + str(_online_status))
-    if not _online_status:  # if no daemon online
-        logger.log(
-            f"No daemon running at {daemon_config['server']}:{daemon_config['port']}, trying to create daemon..."
+    _is_daemon_server_online = connect_daemon()  # try to connect daemon
+    logger.log("daemon connection status: " + str(_is_daemon_server_online))
+    if not _is_daemon_server_online:  # if no daemon online
+        # check if possible to launch
+        if daemon_config["host"] not in ["localhost", "127.0.0.1", "0.0.0.0"]:
+            # daemon not running on localhost
+            logger.err(
+                f"No daemon running at {daemon_config['host']}:{daemon_config['port']}, daemon will not be attached. Continue anyway."
+            )
+            return False
+
+        logger.warn(
+            f"No daemon running at {daemon_config['host']}:{daemon_config['port']}, trying to create daemon..."
         )
 
         popen = DaemonableProcess(
-            target="neetbox.daemon._daemon_launcher",
-            args=["--config", json.dumps(daemon_config["launcher"])],
+            target="neetbox.daemon.server._daemon_launcher",
+            args=["--config", json.dumps(daemon_config)],
             mode=daemon_config["mode"],
             redirect_stdout=subprocess.DEVNULL if daemon_config["mute"] else None,
             env_append={"NEETBOX_DAEMON_PROCESS": "1"},
@@ -52,14 +59,12 @@ def __attach_daemon(daemon_config):
 
         logger.log("Created daemon process, trying to connect to daemon...")
         while time.perf_counter() - _time_begin < 10:  # try connect daemon
-            if connect_daemon(daemon_config):
+            if connect_daemon():
                 return True
             else:
                 exit_code = popen.poll()
                 if exit_code is not None:
-                    logger.err(
-                        f"Daemon process exited unexpectedly with exit code {exit_code}."
-                    )
+                    logger.err(f"Daemon process exited unexpectedly with exit code {exit_code}.")
                     return False
 
                 time.sleep(0.5)
@@ -78,4 +83,5 @@ def _try_attach_daemon():
         __attach_daemon(_cfg)
 
 
-__all__ = ["watch", "listen", "_try_attach_daemon"]
+action = NeetActionManager.register
+__all__ = ["watch", "listen", "action", "NeetActionManager", "_try_attach_daemon"]