From 4a37929f3fccf1c5ae14c0cd24b1ebb28475c385 Mon Sep 17 00:00:00 2001 From: PuQing Date: Sat, 25 Nov 2023 12:03:30 +0800 Subject: [PATCH 01/54] Refactor LogWriter to be an abstract base class --- neetbox/logging/_writer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/neetbox/logging/_writer.py b/neetbox/logging/_writer.py index 8f6c69b9..0be94a89 100644 --- a/neetbox/logging/_writer.py +++ b/neetbox/logging/_writer.py @@ -3,6 +3,7 @@ import json import os import pathlib +from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import date, datetime from typing import Any, Callable, Iterable, Optional, Union @@ -13,8 +14,8 @@ from neetbox.utils import formatting -# Log writer interface -class LogWriter: +class LogWriter(ABC): + @abstractmethod def write(self, raw_log): pass From a3e2a7d68c4084b37d9325cacd5e314211e71a72 Mon Sep 17 00:00:00 2001 From: PuQing Date: Sat, 25 Nov 2023 13:41:23 +0800 Subject: [PATCH 02/54] Refactor PackedAction class and NeetActionManager methods --- neetbox/daemon/client/_action_agent.py | 32 ++++++++++++++------------ 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index 25c21eb8..735634fb 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -10,7 +10,7 @@ from neetbox.utils.mvc import Singleton -class PackedAction(Callable): +class PackedAction: def __init__(self, function: Callable, name=None, blocking=False, **kwargs): super().__init__(**kwargs) self.function = function @@ -29,22 +29,22 @@ def eval_call(self, params: dict): class _NeetActionManager(metaclass=Singleton): __ACTION_POOL: Registry = Registry("__NEET_ACTIONS") + @staticmethod def get_action_names(): - action_names = _NeetActionManager.__ACTION_POOL.keys() - actions = {} - for n in action_names: - actions[n] = _NeetActionManager.__ACTION_POOL[n].argspec - return actions + return { + n: _NeetActionManager.__ACTION_POOL[n].argspec + for n in _NeetActionManager.__ACTION_POOL.keys() + } + @staticmethod def get_action_dict(): - action_dict = {} - action_names = _NeetActionManager.__ACTION_POOL.keys() - for name in action_names: - action = _NeetActionManager.__ACTION_POOL[name] - action_dict[name] = action.argspec.args - return action_dict - - def eval_call(name: str, params: dict, callback: None): + return { + name: _NeetActionManager.__ACTION_POOL[name].argspec.args + for name in _NeetActionManager.__ACTION_POOL.keys() + } + + @staticmethod + def eval_call(name: str, params: dict, callback: Optional[Callable] = None): if name not in _NeetActionManager.__ACTION_POOL: logger.err(f"Could not find action with name {name}, action stopped.") return False @@ -73,10 +73,12 @@ def _update_action_dict(): # for status updater return _NeetActionManager.get_action_dict() + @staticmethod def register(name: Optional[str] = None, blocking: bool = False): return functools.partial(_NeetActionManager._register, name=name, blocking=blocking) - def _register(function: Callable, name: str = None, blocking: bool = False): + @staticmethod + def _register(function: Callable, name: Optional[str] = None, blocking: bool = False): packed = PackedAction(function=function, name=name, blocking=blocking) _NeetActionManager.__ACTION_POOL._register(what=packed, name=packed.name, force=True) _NeetActionManager._update_action_dict() # update for sync From e670ac068428399d2d065661cd806519a5493c5c Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sat, 25 Nov 2023 15:09:30 +0800 Subject: [PATCH 03/54] patch fix --- .gitignore | 1 - neetbox/daemon/server/_server.py | 4 ++++ tests/client/neetbox.toml | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 tests/client/neetbox.toml diff --git a/.gitignore b/.gitignore index 769cc16e..1df1b08e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .vscode .pytest_cache -neetbox.toml __pycache__/ dist/ poetry.lock diff --git a/neetbox/daemon/server/_server.py b/neetbox/daemon/server/_server.py index d73ad774..fe5c2348 100644 --- a/neetbox/daemon/server/_server.py +++ b/neetbox/daemon/server/_server.py @@ -200,10 +200,14 @@ def handle_ws_message(client, server: WebsocketServer, message): if _project_name not in __BRIDGES: # project name must exist # drop anyway if not exist + if debug: + print(f"handle log. {_project_name} not found.") return else: # forward to frontends _target_bridge = __BRIDGES[_project_name] + if debug: + print(f"forward log to frontends on{_project_name}.") for web_ws in _target_bridge.web_ws_list: server.send_message( client=web_ws, msg=message diff --git a/tests/client/neetbox.toml b/tests/client/neetbox.toml new file mode 100644 index 00000000..f7267388 --- /dev/null +++ b/tests/client/neetbox.toml @@ -0,0 +1,22 @@ +name = "neet_test" + +[logging] +level = "INFO" + +[pipeline] +updateInterval = 10 + +[daemon] +enable = true +host = "localhost" +port = 5000 +allowIpython = false +mute = true +mode = "detached" +uploadInterval = 10 + +[integrations.environment.hardware] +monit = "true" + +[integrations.environment.platform] +monit = "true" From 6cc056983cd8ea9397a59fbbf744cbd32c3a91b3 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sat, 25 Nov 2023 15:51:08 +0800 Subject: [PATCH 04/54] now server forwards neet actions --- neetbox/daemon/readme.md | 12 ++++ neetbox/daemon/server/_server.py | 107 ++++++++++++++++++++++--------- 2 files changed, 90 insertions(+), 29 deletions(-) diff --git a/neetbox/daemon/readme.md b/neetbox/daemon/readme.md index 37cd60d7..7f077b32 100644 --- a/neetbox/daemon/readme.md +++ b/neetbox/daemon/readme.md @@ -28,14 +28,22 @@ script above should launch a simple case of neetbox project with some logs and s websocke messages are described in json. There is a dataclass representing websocket message: ```python +EVENT_TYPE_NAME_KEY = "event-type" +EVENT_ID_NAME_KEY = "event-id" +NAME_NAME_KEY = "name" +PAYLOAD_NAME_KEY = "payload" + + @dataclass class WsMsg: + name: str event_type: str payload: Any event_id: int = -1 def json(self): return { + NAME_NAME_KEY: self.name, EVENT_TYPE_NAME_KEY: self.event_type, EVENT_ID_NAME_KEY: self.event_id, PAYLOAD_NAME_KEY: self.payload, @@ -44,14 +52,18 @@ class WsMsg: ```json { + "name" : ..., "event-type" : ..., "payload" : ..., "event-id" : ... } ``` +an simple websocket message should include: + | key | value type | description | | :--------: | :--------: | :----------------------------------------------------: | +| name | string | project name | | event-type | string | indicate type of data in payload | | payload | string | actual data | | event-id | int | for events who need ack. default -1 means no event id. | diff --git a/neetbox/daemon/server/_server.py b/neetbox/daemon/server/_server.py index fe5c2348..1140bc22 100644 --- a/neetbox/daemon/server/_server.py +++ b/neetbox/daemon/server/_server.py @@ -5,12 +5,20 @@ # Date: 20230414 import os + +from rich.console import Console + +console = Console() +import logging import sys import time from dataclasses import dataclass from threading import Thread from typing import Any, Dict, Tuple +werkzeug_log = logging.getLogger("werkzeug") +werkzeug_log.setLevel(logging.ERROR) # disable flask http call logs + if __name__ == "__main__": import ultraimport # if run server solely, sssssssuse relative import, do not trigger neetbox init @@ -18,6 +26,7 @@ from _protocol import * else: from neetbox.daemon._protocol import * + import setproctitle from flask import abort, json, request from websocket_server import WebsocketServer @@ -52,18 +61,16 @@ def ws_send(self): # =============================================================== if debug: - print("Running with debug, using APIFlask") + console.log(f"Running with debug, using APIFlask") from apiflask import APIFlask app = APIFlask(__PROC_NAME) else: - print("Running in production mode, escaping APIFlask") + console.log(f"Running in production mode, using Flask") from flask import Flask app = Flask(__PROC_NAME) - # app = APIFlask(__PROC_NAME) - # websocket server ws_server = WebsocketServer(port=cfg["port"] + 1) __BRIDGES = {} # manage connections @@ -119,9 +126,11 @@ def ws_send(self): """ def handle_ws_connect(client, server): - print(f"client {client} connected. waiting for assigning...") + console.log(f"client {client} connected. waiting for handshake...") def handle_ws_disconnect(client, server): + if client["id"] not in connected_clients: + return # client disconnected before handshake, returning anyway _project_name, _who = connected_clients[client["id"]] if _who == "cli": # remove client from Bridge __BRIDGES[_project_name].cli_ws = None @@ -131,20 +140,25 @@ def handle_ws_disconnect(client, server): ] __BRIDGES[_project_name].web_ws_list = _new_web_ws_list del connected_clients[client["id"]] - print(f"a {_who} disconnected with id {client['id']}") + console.log(f"a {_who} disconnected with id {client['id']}") # logger.info(f"Websocket ({conn_type}) for {name} disconnected") def handle_ws_message(client, server: WebsocketServer, message): - message = json.loads(message) + message_dict = json.loads(message) # handle event-type - _event_type = message[EVENT_TYPE_NAME_KEY] - _payload = message[PAYLOAD_NAME_KEY] - _event_id = message[EVENT_ID_NAME_KEY] - _project_name = message[NAME_NAME_KEY] + _event_type = message_dict[EVENT_TYPE_NAME_KEY] + _payload = message_dict[PAYLOAD_NAME_KEY] + _event_id = message_dict[EVENT_ID_NAME_KEY] + _project_name = message_dict[NAME_NAME_KEY] if _event_type == "handshake": # handle handshake + if client["id"] in connected_clients: + # !!! cli/web could change their project name by handshake twice, this is a legal behavior + handle_ws_disconnect( + client=client, server=server + ) # perform "software disconnect" before "software connect" again # assign this client to a Bridge _who = _payload["who"] - print(f"handling handshake for {_who} with name {_project_name}") + console.log(f"handling handshake for {_who} with name {_project_name}") if _who == "web": # new connection from frontend # check if Bridge with name exist @@ -194,31 +208,63 @@ def handle_ws_message(client, server: WebsocketServer, message): ).json() ), ) + return # return after handling handshake + + if client["id"] not in connected_clients: + return # !!! not handling messages from cli/web without handshake. handshake is a special case and should be handled anyway before this check. + + _, _who = connected_clients[client["id"]] # check if is web or cli + + def send_to_frontends_of_name(name, message): + if name not in __BRIDGES: + if debug: + console.log( + f"cannot broadcast message to frontends under name {name}: name not found." + ) + return # no such bridge + _target_bridge = __BRIDGES[name] + for web_ws in _target_bridge.web_ws_list: + server.send_message( + client=web_ws, msg=message + ) # forward original message to frontend + return - elif _event_type == "log": # handle log - # forward log to frontend + def send_to_client_of_name(name, message): + if name not in __BRIDGES: + if debug: + console.log( + f"cannot forward message to client under name {name}: name not found." + ) + return # no such bridge + _target_bridge = __BRIDGES[name] + _client = _target_bridge.cli_ws + server.send_message(client=_client, msg=message) # forward original message to client + return + + if _event_type == "log": # handle log + # forward log to frontend. logs should only be sent by cli and only be received by frontends if _project_name not in __BRIDGES: # project name must exist # drop anyway if not exist if debug: - print(f"handle log. {_project_name} not found.") + console.log(f"handle log. {_project_name} not found.") return else: - # forward to frontends - _target_bridge = __BRIDGES[_project_name] - if debug: - print(f"forward log to frontends on{_project_name}.") - for web_ws in _target_bridge.web_ws_list: - server.send_message( - client=web_ws, msg=message - ) # forward original message to frontend - - elif _event_type == "action": - # todo forward action query to cli - pass - elif _event_type == "ack": + send_to_frontends_of_name( + name=_project_name, message=message + ) # forward to frontends + return # return after handling log forwarding + + if _event_type == "action": + if _who == "web": # frontend send action query to client + send_to_frontends_of_name(_project_name, message=message) + else: # _who == 'cli', client send action result back to frontend(s) + send_to_client_of_name(_project_name, message=message) + return # return after handling action forwarding + + if _event_type == "ack": # todo forward ack to waiting acks - pass + return # return after handling ack ws_server.set_fn_new_client(handle_ws_connect) ws_server.set_fn_client_left(handle_ws_disconnect) @@ -280,6 +326,7 @@ def __sleep_and_shutdown(secs=3): os._exit(0) Thread(target=__sleep_and_shutdown).start() # shutdown after 3 seconds + console.log(f"BYE.") return f"shutdown in {3} seconds." def _count_down_thread(): @@ -293,8 +340,10 @@ def _count_down_thread(): count_down_thread = Thread(target=_count_down_thread, daemon=True) count_down_thread.start() + console.log(f"launching websocket server...") ws_server.run_forever(threaded=True) + console.log(f"launching flask server...") app.run(host="0.0.0.0", port=cfg["port"]) From c7b46f8a5e6c2459d18135f574d5a9e152e465f9 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sat, 25 Nov 2023 17:30:11 +0800 Subject: [PATCH 05/54] fixed colored log prefix does not skip ws writer --- neetbox/logging/logger.py | 10 +++++----- tests/client/test.py | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index 92faae4c..cbe22203 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -128,7 +128,7 @@ def ok(self, *message, flag="OK"): self.log( *message, prefix=f"[{colored_text(flag, 'green')}]", - skip_writers=["file"], + skip_writers=["file", "ws"], traceback=3, ) self.log(*message, prefix=flag, skip_writers=["stdout"], traceback=3) @@ -139,7 +139,7 @@ def debug(self, *message, flag="DEBUG"): self.log( *message, prefix=f"[{colored_text(flag, 'cyan')}]", - skip_writers=["file"], + skip_writers=["file", "ws"], traceback=3, ) self.log(*message, prefix=flag, skip_writers=["stdout"], traceback=3) @@ -150,7 +150,7 @@ def info(self, *message, flag="INFO"): self.log( *message, prefix=f"[{colored_text(flag, 'white')}]", - skip_writers=["file"], + skip_writers=["file", "ws"], traceback=3, ) self.log(*message, prefix=flag, skip_writers=["stdout"], traceback=3) @@ -161,7 +161,7 @@ def warn(self, *message, flag="WARNING"): self.log( *message, prefix=f"[{colored_text(flag, 'yellow')}]", - skip_writers=["file"], + skip_writers=["file", "ws"], traceback=3, ) self.log(*message, prefix=flag, skip_writers=["stdout"], traceback=3) @@ -174,7 +174,7 @@ def err(self, err, flag="ERROR", reraise=False): self.log( str(err), prefix=f"[{colored_text(flag,'red')}]", - skip_writers=["file"], + skip_writers=["file", "ws"], traceback=3, ) self.log(str(err), prefix=flag, skip_writers=["stdout"], traceback=3) diff --git a/tests/client/test.py b/tests/client/test.py index b121f2b7..edcc3321 100644 --- a/tests/client/test.py +++ b/tests/client/test.py @@ -19,6 +19,15 @@ def print_to_console(metrix): logger.log(f"metrix from train: {metrix}") +@watch("log-some-prefix", freq=50) +def log_with_some_prefix(): + logger.ok("some ok") + logger.info("some info") + logger.debug("some debug") + logger.warn("some warn") + logger.err("some error") + + @action(name="action-1") def action_1(text): logger.log(f"action 1 triggered. text = {text}") From 49c296071607a43291859bb9264f16973a61a319 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sat, 25 Nov 2023 22:40:38 +0800 Subject: [PATCH 06/54] now clients only uploads important status via https, ignoring user level @watch --- neetbox/core/registry.py | 56 +++++-- neetbox/daemon/client/_action_agent.py | 5 +- neetbox/daemon/client/_client.py | 2 +- neetbox/daemon/client/_update_thread.py | 4 +- neetbox/integrations/environment/hardware.py | 27 +++- neetbox/integrations/environment/platform.py | 3 +- neetbox/pipeline/_signal_and_slot.py | 156 +++++++++++-------- tests/client/test.py | 2 +- tests/test_core.py | 1 + 9 files changed, 168 insertions(+), 88 deletions(-) diff --git a/neetbox/core/registry.py b/neetbox/core/registry.py index c14daf9f..b2744218 100644 --- a/neetbox/core/registry.py +++ b/neetbox/core/registry.py @@ -27,6 +27,16 @@ def __str__(self) -> str: return f"{self.what} with tags: {self.tags}" +def _tags_match(search_tags, in_tags) -> bool: + # check if all tags in f_tags are listed in s_tags + if type(search_tags) is not list: + search_tags = [search_tags] + for _t in search_tags: + if _t not in in_tags: + return False + return True + + class Registry(dict): """Register Helper Class @@ -39,6 +49,7 @@ class Registry(dict): def __new__(cls, name: str) -> "Registry": assert is_pure_ansi(name), "Registry name should not contain non-ansi char." + name = name.replace(" ", "-") if name in cls._registry_pool: return cls._registry_pool[name] # logger.log(f"Creating Registry for '{name}'") @@ -56,7 +67,7 @@ def _register( self, what: Any, name: Optional[str] = None, - force: bool = True, + overwrite: bool = True, tags: Optional[Union[str, Sequence[str]]] = None, ): # if not (inspect.isfunction(what) or inspect.isclass(what)): @@ -66,12 +77,12 @@ def _register( tags = [tags] _endp = _RegEndpoint(what, tags) if name in self.keys(): - if not force: + if not overwrite: logger.warn( - f"{name} already exists in Registry:{self.name}. If you want to overwrite, try to register with 'force=True'" + f"{name} already exists in Registry:{self.name}. If you want to overwrite, try to register with 'overwrite=True'" ) else: - logger.warn(f"Overwritting existing '{name}' in Registry '{self.name}'.") + # logger.warn(f"Overwritting existing '{name}' in Registry '{self.name}'.") self[name] = _endp else: self[name] = _endp @@ -81,10 +92,10 @@ def register( self, *, name: Optional[str] = None, - force: bool = True, + overwrite: bool = True, tags: Optional[Union[str, Sequence[str]]] = None, ): - return functools.partial(self._register, name=name, force=force, tags=tags) + return functools.partial(self._register, name=name, overwrite=overwrite, tags=tags) @classmethod def find( @@ -108,25 +119,36 @@ def find( results.append((name, reg[name])) # filter tags - if type(tags) is str: + if type(tags) is not list: tags = [tags] - def _tags_match(f_tags, s_tags) -> bool: - # check if all tags in f_tags are listed in s_tags - for _t in f_tags: - if _t not in s_tags: - return False - return True - results = {_name: _endp.what for _name, _endp in results if _tags_match(tags, _endp.tags)} return results + def filter(self, tags: Optional[Union[str, Sequence[str]]] = None): + results = { + _name: _endp.what for _name, _endp in self._items() if _tags_match(tags, _endp.tags) + } + return results + def __getitem__(self, __key: str) -> Any: _v = self.__dict__[__key] if type(_v) is _RegEndpoint: _v = _v.what return _v + def get(self, key: str, **kwargs): + if key in self.__dict__: + _v = self.__dict__[key] + if type(_v) is _RegEndpoint: + _v = _v.what + return _v + else: + if "default" in kwargs: + return kwargs["default"] + else: + raise RuntimeError(f"key {key} not found") + def __setitem__(self, k, v) -> None: assert is_pure_ansi(k), "Only ANSI chars are allowed for registering things." self.__dict__[k] = v @@ -155,6 +177,12 @@ def items(self, _real_type=True): _legal_items = [(_k, _v.what) for _k, _v in _legal_items if type(_v) is _RegEndpoint] return _legal_items + def _items(self, _real_type=True): + _legal_items = [_item for _item in self.__dict__.items() if type(_item[1]) is _RegEndpoint] + if _real_type: + _legal_items = [(_k, _v) for _k, _v in _legal_items if type(_v) is _RegEndpoint] + return _legal_items + def pop(self, *args): return self.__dict__.pop(*args) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index 8a257a3c..aea1da6d 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -9,6 +9,7 @@ from neetbox.daemon.client._client import connection from neetbox.logging import logger from neetbox.pipeline import watch +from neetbox.pipeline._signal_and_slot import SYSTEM_CHANNEL from neetbox.utils.mvc import Singleton @@ -70,7 +71,7 @@ def run_and_callback(target_action, params, callback): else: # blocking run return target_action.eval_call(params) - @watch(initiative=True) + @watch(name="__action", initiative=True, _channel=SYSTEM_CHANNEL) def _update_action_dict(): # for status updater return _NeetActionManager.get_action_dict() @@ -80,7 +81,7 @@ def register(name: Optional[str] = None, blocking: bool = False): def _register(function: Callable, name: str = None, blocking: bool = False): packed = PackedAction(function=function, name=name, blocking=blocking) - _NeetActionManager.__ACTION_POOL._register(what=packed, name=packed.name, force=True) + _NeetActionManager.__ACTION_POOL._register(what=packed, name=packed.name, overwrite=True) _NeetActionManager._update_action_dict() # update for sync return function diff --git a/neetbox/daemon/client/_client.py b/neetbox/daemon/client/_client.py index ca3597d5..31ade744 100644 --- a/neetbox/daemon/client/_client.py +++ b/neetbox/daemon/client/_client.py @@ -88,7 +88,7 @@ def __on_ws_open(ws: websocket.WebSocketApp): ws.send( # send handshake request json.dumps( { - NAME_NAME_KEY: {_display_name}, + NAME_NAME_KEY: _display_name, EVENT_TYPE_NAME_KEY: "handshake", PAYLOAD_NAME_KEY: {"who": "cli"}, EVENT_ID_NAME_KEY: 0, # todo how does ack work diff --git a/neetbox/daemon/client/_update_thread.py b/neetbox/daemon/client/_update_thread.py index 228c4d71..8a36b7c0 100644 --- a/neetbox/daemon/client/_update_thread.py +++ b/neetbox/daemon/client/_update_thread.py @@ -13,7 +13,7 @@ from neetbox.daemon.client._client import connection from neetbox.daemon.server._server import CLIENT_API_ROOT from neetbox.logging import logger -from neetbox.pipeline._signal_and_slot import _update_value_dict +from neetbox.pipeline._signal_and_slot import _UPDATE_VALUE_DICT, SYSTEM_CHANNEL __TIME_UNIT_SEC = 0.1 @@ -33,7 +33,7 @@ def _upload_thread(daemon_config, base_addr, display_name): if _ctr % _upload_interval: # not zero continue # dump status as json - _data = json.dumps(_update_value_dict, default=str) + _data = json.dumps(_UPDATE_VALUE_DICT[SYSTEM_CHANNEL], default=str) _headers = {"Content-Type": "application/json"} try: # upload data diff --git a/neetbox/integrations/environment/hardware.py b/neetbox/integrations/environment/hardware.py index c9ce5223..594b7059 100644 --- a/neetbox/integrations/environment/hardware.py +++ b/neetbox/integrations/environment/hardware.py @@ -13,6 +13,7 @@ from GPUtil import GPU from neetbox.pipeline import watch +from neetbox.pipeline._signal_and_slot import SYSTEM_CHANNEL from neetbox.utils import pkg from neetbox.utils.framing import get_frame_module_traceback from neetbox.utils.mvc import Singleton @@ -46,6 +47,7 @@ def parse(cls, gpu: GPU): _instance = _GPU_STAT() _instance["id"] = gpu.id _instance["name"] = gpu.name + _instance["load"] = gpu.load _instance["memoryUtil"] = gpu.memoryUtil _instance["memoryTotal"] = gpu.memoryTotal _instance["memoryFree"] = gpu.memoryFree @@ -72,10 +74,19 @@ def __init__(self) -> None: self["cpus"] = [_CPU_STAT() for _ in range(psutil.cpu_count(logical=True))] self["gpus"] = [_GPU_STAT.parse(_gpu) for _gpu in GPUtil.getGPUs()] self._with_gpu = False if len(self["gpus"]) == 0 else True - + ram_stat = psutil.virtual_memory() + self["ram"] = { + "total": ram_stat[0] / 1e9, + "available": ram_stat[1] / 1e9, + "used": ram_stat[3] / 1e9, + "free": ram_stat[4] / 1e9, + } # the environment shoube be imported in the __init__.py of the outer module. And the watcher thread should be auto started self.set_update_intervel() + def json(self): + return {"cpus": self["cpus"], "ram": self["ram"], "gpus": self["gpus"]} + def set_update_intervel(self, intervel=1.0) -> None: if intervel < 1.0: self._do_watch = False @@ -86,6 +97,7 @@ def set_update_intervel(self, intervel=1.0) -> None: def watcher_fun(env_instance: _Hardware, do_update_gpus: bool): while env_instance._do_watch: + # update cpu usage cpu_percent = psutil.cpu_percent(percpu=True) cpu_freq = psutil.cpu_freq(percpu=True) if len(cpu_freq) == 1: @@ -96,6 +108,15 @@ def watcher_fun(env_instance: _Hardware, do_update_gpus: bool): percent=cpu_percent[index], freq=cpu_freq[index], ) + # update memory usage + ram_stat = psutil.virtual_memory() + self["ram"] = { + "total": ram_stat[0] / 1e9, + "available": ram_stat[1] / 1e9, + "used": ram_stat[3] / 1e9, + "free": ram_stat[4] / 1e9, + } + # update gpu usage if do_update_gpus: env_instance["gpus"] = [_GPU_STAT.parse(_gpu) for _gpu in GPUtil.getGPUs()] env_instance[""] = psutil.cpu_stats() @@ -110,6 +131,6 @@ def watcher_fun(env_instance: _Hardware, do_update_gpus: bool): # watch updates in daemon -@watch(name="hardware") +@watch(name="hardware", _channel=SYSTEM_CHANNEL) def update_env_stat(): - return dict(hardware) + return hardware.json() diff --git a/neetbox/integrations/environment/platform.py b/neetbox/integrations/environment/platform.py index f05d980e..8913454e 100644 --- a/neetbox/integrations/environment/platform.py +++ b/neetbox/integrations/environment/platform.py @@ -10,6 +10,7 @@ import subprocess from neetbox.pipeline import watch +from neetbox.pipeline._signal_and_slot import SYSTEM_CHANNEL from neetbox.utils.mvc import Singleton @@ -55,7 +56,7 @@ def exec(self, command): # watch updates in daemon -@watch(name="platform", initiative=True) +@watch(name="platform", initiative=True, _channel=SYSTEM_CHANNEL) def update_env_stat(): return dict(platform) diff --git a/neetbox/pipeline/_signal_and_slot.py b/neetbox/pipeline/_signal_and_slot.py index 9f76f6ab..757021ea 100644 --- a/neetbox/pipeline/_signal_and_slot.py +++ b/neetbox/pipeline/_signal_and_slot.py @@ -6,95 +6,104 @@ import collections import time +from dataclasses import dataclass from datetime import datetime from functools import partial from threading import Thread from typing import Any, Callable, Optional, Union +from uuid import uuid4 from neetbox.config import get_module_level_config from neetbox.core import Registry from neetbox.logging import logger -_watch_queue_dict = Registry("__pipeline_watch") +__TIME_CTR_MAX_CYCLE = 9999999 +__TIME_UNIT_SEC = 0.1 +_WATCH_QUERY_DICT = Registry("__pipeline_watch") +_LISTEN_QUERY_DICT = collections.defaultdict(lambda: {}) +_UPDATE_VALUE_DICT = collections.defaultdict(lambda: {}) -def __default_empty_dict(): - return {} - - -_listen_queue_dict = collections.defaultdict(__default_empty_dict) -__TIME_UNIT_SEC = 0.1 -__TIME_CTR_MAX_CYCLE = 9999999 -_update_value_dict = {} +_DEFAULT_CHANNEL = str( + uuid4() +) # default watch and listen channel. users use this channel by default. default channel name varies on each start +SYSTEM_CHANNEL = "__system" # values on this channel will upload via http client +@dataclass class _WatchConfig(dict): - def __init__(self, name, freq, initiative=False): - self["name"] = name - self["freq"] = freq - self["initiative"] = initiative + name: str + interval: int + initiative: bool + channel: str = _DEFAULT_CHANNEL # use channel to distinct those values to upload via http class _WatchedFun: - def __init__(self, func, watch_cfg) -> None: + def __init__(self, func: Callable, cfg: _WatchConfig) -> None: self.func = func - self.others = watch_cfg + self.cfg = cfg + + def __call__(self, *args, **kwargs) -> Any: + return self.func(*args, **kwargs) - def __call__(self, *args: Any, **kwds: Any) -> Any: - return self.func(*args, **kwds) + def __repr__(self) -> str: + return f"{self.func.__name__}, {self.cfg}" -def __get(name): - _the_value = _update_value_dict.get(name, None) +def __get(name: str, channel): + _the_value = _UPDATE_VALUE_DICT[channel].get(name, default=None) if _the_value and "value" in _the_value: _the_value = _the_value["value"] return _the_value -def __update_and_get(name, *args, **kwargs): - global _update_value_dict - _watched_fun: _WatchedFun = _watch_queue_dict[name] - _watch_config = _watched_fun.others +def __update_and_get(name: str, *args, **kwargs): + global _UPDATE_VALUE_DICT + _watched_fun: _WatchedFun = _WATCH_QUERY_DICT[name] + _watch_config = _watched_fun.cfg + _channel = _watch_config.channel _the_value = _watched_fun(*args, **kwargs) - _update_value_dict[name] = { + _UPDATE_VALUE_DICT[_channel][name] = { "value": _the_value, "timestamp": datetime.timestamp(datetime.now()), - "interval": (_watch_config["freq"] * __TIME_UNIT_SEC), + "interval": (_watch_config.interval * __TIME_UNIT_SEC), } - def _so_update_and_ping_listen(_name, _value, _watch_config): + def __call_listeners(_name: str, _value, _cfg: _WatchConfig): t0 = time.perf_counter() - for _listener_name, _listener_func in _listen_queue_dict[_name].items(): + for _listener_name, _listener_func in _LISTEN_QUERY_DICT[_name].items(): _listener_func(_value) t1 = time.perf_counter() delta_t = t1 - t0 - _update_freq = _watch_config["freq"] - expected_time_limit = _update_freq * __TIME_UNIT_SEC - if not _watch_config["initiative"] >= 0 and delta_t > expected_time_limit: + _update_interval = _cfg.interval + expected_time_limit = _update_interval * __TIME_UNIT_SEC + if not _cfg.initiative >= 0 and delta_t > expected_time_limit: logger.warn( f"Watched value {_name} takes longer time({delta_t:.8f}s) to update than it was expected({expected_time_limit}s)." ) - Thread( - target=_so_update_and_ping_listen, args=(name, _the_value, _watch_config), daemon=True - ).start() + Thread(target=__call_listeners, args=(name, _the_value, _watch_config), daemon=True).start() return _the_value -def _watch(func: Callable, name: Optional[str], freq: float, initiative=False, force=False): - """Function decorator to let the daemon watch a value of the function - - Args: - func (function): A function returns a tuple '(name,value)'. 'name' represents the name of the value. - """ +def _watch( + func: Callable, + name: Optional[str], + interval: float, + initiative: bool = False, + overwrite: bool = False, + _channel: str = None, +): + _channel = _channel or _DEFAULT_CHANNEL name = name or func.__name__ - _watch_queue_dict._register( + _WATCH_QUERY_DICT._register( name=name, what=_WatchedFun( func=func, - watch_cfg=_WatchConfig(name, freq=freq, initiative=initiative), + cfg=_WatchConfig(name, interval=interval, initiative=initiative, channel=_channel), ), - force=force, + overwrite=overwrite, + tags=_channel, ) if initiative: # initiatively update the value dict when the function was called manually logger.debug( @@ -103,45 +112,64 @@ def _watch(func: Callable, name: Optional[str], freq: float, initiative=False, f return partial(__update_and_get, name) else: logger.debug( - f"added {name} to daemon monitor. It will update every {freq*__TIME_UNIT_SEC} second(s)." + f"added {name} to daemon monitor. It will update every {interval*__TIME_UNIT_SEC} second(s)." ) - return partial(__get, name) + return partial(__get, name, _channel) -def watch(name=None, freq=None, initiative=False, force=False): +def watch( + name: str = None, + interval: float = None, + initiative: bool = False, + overwrite: bool = False, + _channel: str = None, +): if not initiative: # passively update - freq = freq or get_module_level_config()["updateInterval"] + interval = interval or get_module_level_config()["updateInterval"] else: - freq = -1 - return partial(_watch, name=name, freq=freq, initiative=initiative, force=force) + interval = -1 + return partial( + _watch, + name=name, + interval=interval, + initiative=initiative, + overwrite=overwrite, + _channel=_channel, + ) -def _listen(func: Callable, target: Union[str, Callable], name: Optional[str] = None, force=False): - name = name or func.__name__ +def _listen( + func: Callable, # the listener itself + target: Union[str, Callable], # user may pass a Callable or name to watch as target + listener_name: Optional[str] = None, # or user may pass a name to watch + overwrite: bool = False, +): + listener_name = listener_name or func.__name__ if not isinstance(target, str): - if type(target) is partial: + if type(target) is partial: # solve target is a function with @watch if target.func in [__update_and_get, __get]: target = target.args[0] else: target = target.__name__ - if name in _listen_queue_dict[target]: - if not force: + if listener_name in _LISTEN_QUERY_DICT[target]: + if not overwrite: logger.warn( - f"There is already a listener called '{name}' lisiting '{target}' If you want to overwrite, try to register with 'force=True'" + f"There is already a listener called '{listener_name}' lisiting '{target}' If you want to overwrite, try to listen with 'overwrite=True'" ) return func else: - logger.warn( - f"There is already a listener called '{name}' lisiting '{target}', overwriting." - ) - _listen_queue_dict[target][name] = func - logger.debug(f"{name} is now lisiting to {target}.") + # logger.warn( + # f"There is already a listener called '{listener_name}' lisiting '{target}', overwriting." + # ) + pass + _LISTEN_QUERY_DICT[target][listener_name] = func + logger.debug(f"{listener_name} is now lisiting to {target}.") return func -def listen(target, name: Optional[str] = None, force=False): - return partial(_listen, target=target, name=name, force=force) +def listen(target: Union[str, Any], listener_name: Optional[str] = None, overwrite: bool = False): + return partial(_listen, target=target, listener_name=listener_name, overwrite=overwrite) def _update_thread(): @@ -150,9 +178,9 @@ def _update_thread(): while True: _ctr = (_ctr + 1) % __TIME_CTR_MAX_CYCLE time.sleep(__TIME_UNIT_SEC) - for _vname, _watched_fun in _watch_queue_dict.items(): - _watch_config = _watched_fun.others - if not _watch_config["initiative"] and _ctr % _watch_config["freq"] == 0: # do update + for _vname, _watched_fun in _WATCH_QUERY_DICT.items(): + _watch_config = _watched_fun.cfg + if not _watch_config.initiative and _ctr % _watch_config.interval == 0: # do update _ = __update_and_get(_vname) diff --git a/tests/client/test.py b/tests/client/test.py index edcc3321..116439ab 100644 --- a/tests/client/test.py +++ b/tests/client/test.py @@ -19,7 +19,7 @@ def print_to_console(metrix): logger.log(f"metrix from train: {metrix}") -@watch("log-some-prefix", freq=50) +@watch("log-some-prefix", interval=50) def log_with_some_prefix(): logger.ok("some ok") logger.info("some info") diff --git a/tests/test_core.py b/tests/test_core.py index 9e0e186f..15758216 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -21,5 +21,6 @@ class C: pass print(f"Things in reg1: {reg1}") + print(f"Get not exist in reg1: {reg1.get('bbbbb', default='wow default value')}") print(f"Things in reg2: {reg2}") print(f"Finding functions: {Registry.find(tags='function')}") From 0e5fc7080a1797015e1111464f410bbb568af05f Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 26 Nov 2023 03:51:49 +0800 Subject: [PATCH 07/54] added description to neet action --- neetbox/daemon/client/_action_agent.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index aea1da6d..e610a957 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -14,10 +14,18 @@ class PackedAction(Callable): - def __init__(self, function: Callable, name=None, blocking=False, **kwargs): + def __init__( + self, + function: Callable, + name: str = None, + description: str = None, + blocking: bool = False, + **kwargs, + ): super().__init__(**kwargs) self.function = function self.name = name if name else function.__name__ + self.description = description self.argspec = inspect.getfullargspec(self.function) self.blocking = blocking @@ -43,8 +51,12 @@ def get_action_dict(): action_dict = {} action_names = _NeetActionManager.__ACTION_POOL.keys() for name in action_names: - action = _NeetActionManager.__ACTION_POOL[name] - action_dict[name] = {"args": action.argspec.args, "blocking": action.blocking} + action: PackedAction = _NeetActionManager.__ACTION_POOL[name] + action_dict[name] = { + "description": action.description, + "args": action.argspec.args, + "blocking": action.blocking, + } return action_dict def eval_call(name: str, params: dict, callback: None): From 9b368581ad9d48195f59768bad2288e271173c56 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 26 Nov 2023 04:02:40 +0800 Subject: [PATCH 08/54] patch fix --- neetbox/daemon/client/_action_agent.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index e610a957..1b8fa7d4 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -88,11 +88,18 @@ def _update_action_dict(): # for status updater return _NeetActionManager.get_action_dict() - def register(name: Optional[str] = None, blocking: bool = False): - return functools.partial(_NeetActionManager._register, name=name, blocking=blocking) + def register(name: Optional[str] = None, description: str = None, blocking: bool = False): + return functools.partial( + _NeetActionManager._register, name=name, description=description, blocking=blocking + ) - def _register(function: Callable, name: str = None, blocking: bool = False): - packed = PackedAction(function=function, name=name, blocking=blocking) + def _register( + function: Callable, name: str = None, description: str = None, blocking: bool = False + ): + description = description or function.__doc__ + packed = PackedAction( + function=function, name=name, description=description, blocking=blocking + ) _NeetActionManager.__ACTION_POOL._register(what=packed, name=packed.name, overwrite=True) _NeetActionManager._update_action_dict() # update for sync return function From 59fe774086fd0e9e5ecb9001802c162de2f21a73 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 26 Nov 2023 04:32:06 +0800 Subject: [PATCH 09/54] whatever' --- neetbox/daemon/client/_action_agent.py | 19 ++++++++++++++++++- tests/client/test.py | 5 +++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index 1b8fa7d4..e6d35dc2 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -96,7 +96,24 @@ def register(name: Optional[str] = None, description: str = None, blocking: bool def _register( function: Callable, name: str = None, description: str = None, blocking: bool = False ): - description = description or function.__doc__ + if ( + description is None and function.__doc__ is not None + ): # parse function doc as description + description = function.__doc__ + if description: + _description_lines = [] + for _line in description.split("\n"): + if len(_line): # remove empty lines + _description_lines.append(_line) + # find shortest lstrip + min_lstrip = 99999 + for _line in _description_lines[1:]: # skip first line + min_lstrip = min(len(_line) - len(_line.lstrip()), min_lstrip) + _parsed_description = _description_lines[0] + "\n" + for _line in _description_lines[1:]: + _parsed_description += _line[min_lstrip:] + "\n" + description = _parsed_description + packed = PackedAction( function=function, name=name, description=description, blocking=blocking ) diff --git a/tests/client/test.py b/tests/client/test.py index 116439ab..702f45db 100644 --- a/tests/client/test.py +++ b/tests/client/test.py @@ -30,6 +30,11 @@ def log_with_some_prefix(): @action(name="action-1") def action_1(text): + """take action 1 + + Args: + text (str): text to print + """ logger.log(f"action 1 triggered. text = {text}") From 2871ad40ce9a81cc87eb444f21f5c858bb948eff Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 26 Nov 2023 05:02:59 +0800 Subject: [PATCH 10/54] fixed action query forward direction error --- neetbox/daemon/server/_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neetbox/daemon/server/_server.py b/neetbox/daemon/server/_server.py index 1140bc22..4da0747f 100644 --- a/neetbox/daemon/server/_server.py +++ b/neetbox/daemon/server/_server.py @@ -257,9 +257,9 @@ def send_to_client_of_name(name, message): if _event_type == "action": if _who == "web": # frontend send action query to client - send_to_frontends_of_name(_project_name, message=message) - else: # _who == 'cli', client send action result back to frontend(s) send_to_client_of_name(_project_name, message=message) + else: # _who == 'cli', client send action result back to frontend(s) + send_to_frontends_of_name(_project_name, message=message) return # return after handling action forwarding if _event_type == "ack": From b9216f56ebb8b4aa91f9f2ac3ca165e06027c782 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 26 Nov 2023 05:45:26 +0800 Subject: [PATCH 11/54] added param annotations for action dict sent to frontend --- neetbox/daemon/client/_action_agent.py | 20 +++++++++++++++----- tests/client/test.py | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index e6d35dc2..67dad213 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -29,6 +29,20 @@ def __init__( self.argspec = inspect.getfullargspec(self.function) self.blocking = blocking + def get_props_dict(self): + # _arg_dict = { + # _arg_name: self.argspec.annotations.get(_arg_name, None) + # for _arg_name in self.argspec.args + # } + _arg_anno_dict = self.function.__annotations__ + _args = self.argspec.args + __arg_dict = {_arg_name: _arg_anno_dict.get(_arg_name, any).__name__ for _arg_name in _args} + return { + "description": self.description, + "args": __arg_dict, + "blocking": self.blocking, + } + def __call__(self, **argv): self.function(argv) # ignore blocking @@ -52,11 +66,7 @@ def get_action_dict(): action_names = _NeetActionManager.__ACTION_POOL.keys() for name in action_names: action: PackedAction = _NeetActionManager.__ACTION_POOL[name] - action_dict[name] = { - "description": action.description, - "args": action.argspec.args, - "blocking": action.blocking, - } + action_dict[name] = action.get_props_dict() return action_dict def eval_call(name: str, params: dict, callback: None): diff --git a/tests/client/test.py b/tests/client/test.py index 702f45db..0fd9eedf 100644 --- a/tests/client/test.py +++ b/tests/client/test.py @@ -29,7 +29,7 @@ def log_with_some_prefix(): @action(name="action-1") -def action_1(text): +def action_1(text: str): """take action 1 Args: From 5649804264a19c8cd3ea7823a5fe488955e59b1a Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 26 Nov 2023 05:49:28 +0800 Subject: [PATCH 12/54] fixed callback not working on actions with blocking=True --- neetbox/daemon/client/_action_agent.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index 67dad213..0e10388f 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -77,21 +77,21 @@ def eval_call(name: str, params: dict, callback: None): logger.log( f"Agent runs function '{target_action.name}', blocking = {target_action.blocking}" ) - if not target_action.blocking: # non-blocking run in thread - def run_and_callback(target_action, params, callback): - returned_data = target_action.eval_call(params) - if callback: - callback(returned_data) + def run_and_callback(): + returned_data = target_action.eval_call(params) + if callback: + callback(returned_data) + if not target_action.blocking: # non-blocking run in thread Thread( target=run_and_callback, - kwargs={"target_action": target_action, "params": params, "callback": callback}, daemon=True, ).start() - return None + return else: # blocking run - return target_action.eval_call(params) + run_and_callback() + return @watch(name="__action", initiative=True, _channel=SYSTEM_CHANNEL) def _update_action_dict(): From 9ce1de6223a1b3f2c7f0001a09343f272e77a4ae Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 26 Nov 2023 05:59:46 +0800 Subject: [PATCH 13/54] fixed unexpected arg event_id in ws_send --- neetbox/daemon/client/_action_agent.py | 2 +- neetbox/daemon/client/_client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/neetbox/daemon/client/_action_agent.py b/neetbox/daemon/client/_action_agent.py index 0e10388f..10562d10 100644 --- a/neetbox/daemon/client/_action_agent.py +++ b/neetbox/daemon/client/_action_agent.py @@ -142,7 +142,7 @@ def __listen_to_actions(msg): name=_action_name, params=_action_args, callback=lambda x: connection.ws_send( - event_type="action", payload={"name": _action_name, "result": x}, _event_id=_event_id + event_type="action", payload={"name": _action_name, "result": x}, event_id=_event_id ), ) diff --git a/neetbox/daemon/client/_client.py b/neetbox/daemon/client/_client.py index 31ade744..e1353e77 100644 --- a/neetbox/daemon/client/_client.py +++ b/neetbox/daemon/client/_client.py @@ -138,7 +138,7 @@ def __on_ws_message(ws: websocket.WebSocketApp, msg): f"Subscriber {name} crashed on message event {event_type_name}, ignoring." ) - def ws_send(event_type: str, payload): + def ws_send(event_type: str, payload, event_id=-1): logger.debug(f"ws sending {payload}") if ClientConn.__ws_client: # if ws client exist ClientConn.__ws_client.send( @@ -147,7 +147,7 @@ def ws_send(event_type: str, payload): NAME_NAME_KEY: ClientConn._display_name, EVENT_TYPE_NAME_KEY: event_type, PAYLOAD_NAME_KEY: payload, - EVENT_ID_NAME_KEY: -1, # todo how does ack work + EVENT_ID_NAME_KEY: event_id, } ) ) From f228004f2fd10b18859b3175aa374214a7977dd9 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Thu, 23 Nov 2023 21:34:26 +0800 Subject: [PATCH 14/54] feat: frontend --- neetbox/frontend/.eslintrc.cjs | 21 + neetbox/frontend/.gitignore | 24 + neetbox/frontend/README.md | 30 + neetbox/frontend/index.html | 13 + neetbox/frontend/package.json | 37 + neetbox/frontend/public/logo.svg | 1 + neetbox/frontend/public/vite.svg | 1 + neetbox/frontend/src/App.tsx | 61 + neetbox/frontend/src/components/Footer.tsx | 14 + .../dashboard/project/platformProps.tsx | 68 + neetbox/frontend/src/components/echarts.tsx | 30 + .../frontend/src/components/logo.module.css | 15 + neetbox/frontend/src/components/logo.tsx | 30 + .../frontend/src/components/themeSwitcher.tsx | 16 + neetbox/frontend/src/hooks/useAPI.ts | 21 + neetbox/frontend/src/index.css | 16 + neetbox/frontend/src/main.tsx | 31 + neetbox/frontend/src/pages/console/index.tsx | 33 + .../frontend/src/pages/console/overview.tsx | 10 + .../src/pages/console/proejctDashboard.tsx | 110 + .../frontend/src/pages/console/sidebar.tsx | 42 + .../frontend/src/pages/login/index.module.css | 77 + neetbox/frontend/src/pages/login/index.tsx | 40 + neetbox/frontend/src/styles/global.css | 16 + neetbox/frontend/src/vite-env.d.ts | 1 + neetbox/frontend/tests/backend/test.py | 32 + neetbox/frontend/tsconfig.json | 25 + neetbox/frontend/tsconfig.node.json | 10 + neetbox/frontend/vite.config.ts | 19 + neetbox/frontend/yarn.lock | 2694 +++++++++++++++++ pyproject.toml | 3 + scripts/release.linux.sh | 12 + 32 files changed, 3553 insertions(+) create mode 100644 neetbox/frontend/.eslintrc.cjs create mode 100644 neetbox/frontend/.gitignore create mode 100644 neetbox/frontend/README.md create mode 100644 neetbox/frontend/index.html create mode 100644 neetbox/frontend/package.json create mode 100644 neetbox/frontend/public/logo.svg create mode 100644 neetbox/frontend/public/vite.svg create mode 100644 neetbox/frontend/src/App.tsx create mode 100644 neetbox/frontend/src/components/Footer.tsx create mode 100644 neetbox/frontend/src/components/dashboard/project/platformProps.tsx create mode 100644 neetbox/frontend/src/components/echarts.tsx create mode 100644 neetbox/frontend/src/components/logo.module.css create mode 100644 neetbox/frontend/src/components/logo.tsx create mode 100644 neetbox/frontend/src/components/themeSwitcher.tsx create mode 100644 neetbox/frontend/src/hooks/useAPI.ts create mode 100644 neetbox/frontend/src/index.css create mode 100644 neetbox/frontend/src/main.tsx create mode 100644 neetbox/frontend/src/pages/console/index.tsx create mode 100644 neetbox/frontend/src/pages/console/overview.tsx create mode 100644 neetbox/frontend/src/pages/console/proejctDashboard.tsx create mode 100644 neetbox/frontend/src/pages/console/sidebar.tsx create mode 100644 neetbox/frontend/src/pages/login/index.module.css create mode 100644 neetbox/frontend/src/pages/login/index.tsx create mode 100644 neetbox/frontend/src/styles/global.css create mode 100644 neetbox/frontend/src/vite-env.d.ts create mode 100644 neetbox/frontend/tests/backend/test.py create mode 100644 neetbox/frontend/tsconfig.json create mode 100644 neetbox/frontend/tsconfig.node.json create mode 100644 neetbox/frontend/vite.config.ts create mode 100644 neetbox/frontend/yarn.lock create mode 100755 scripts/release.linux.sh diff --git a/neetbox/frontend/.eslintrc.cjs b/neetbox/frontend/.eslintrc.cjs new file mode 100644 index 00000000..1ecfadbe --- /dev/null +++ b/neetbox/frontend/.eslintrc.cjs @@ -0,0 +1,21 @@ +module.exports = { + root: true, + env: { browser: true, es2020: true }, + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + 'plugin:react-hooks/recommended', + 'plugin:import/recommended', + 'plugin:import/typescript', + ], + ignorePatterns: ['dist', '.eslintrc.cjs'], + parser: '@typescript-eslint/parser', + plugins: ['react-refresh'], + rules: { + 'react-refresh/only-export-components': [ + 'warn', + { allowConstantExport: true }, + ], + 'import/order': 'warn' + }, +} diff --git a/neetbox/frontend/.gitignore b/neetbox/frontend/.gitignore new file mode 100644 index 00000000..a547bf36 --- /dev/null +++ b/neetbox/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/neetbox/frontend/README.md b/neetbox/frontend/README.md new file mode 100644 index 00000000..0d6babed --- /dev/null +++ b/neetbox/frontend/README.md @@ -0,0 +1,30 @@ +# React + TypeScript + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh + +## Expanding the ESLint configuration + +If you are developing a production application, we recommend updating the configuration to enable type aware lint rules: + +- Configure the top-level `parserOptions` property like this: + +```js +export default { + // other rules... + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + project: ['./tsconfig.json', './tsconfig.node.json'], + tsconfigRootDir: __dirname, + }, +} +``` + +- Replace `plugin:@typescript-eslint/recommended` to `plugin:@typescript-eslint/recommended-type-checked` or `plugin:@typescript-eslint/strict-type-checked` +- Optionally add `plugin:@typescript-eslint/stylistic-type-checked` +- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and add `plugin:react/recommended` & `plugin:react/jsx-runtime` to the `extends` list diff --git a/neetbox/frontend/index.html b/neetbox/frontend/index.html new file mode 100644 index 00000000..96ac4b63 --- /dev/null +++ b/neetbox/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + Neetbox + + +
+ + + diff --git a/neetbox/frontend/package.json b/neetbox/frontend/package.json new file mode 100644 index 00000000..606443c5 --- /dev/null +++ b/neetbox/frontend/package.json @@ -0,0 +1,37 @@ +{ + "name": "neetcenter", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "build-with-tsc": "tsc && vite build", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "preview": "vite preview" + }, + "dependencies": { + "@douyinfe/semi-ui": "^2.47.0", + "@semi-bot/semi-theme-nyx-c": "^1.0.8", + "echarts": "^5.4.3", + "jotai": "^2.5.1", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "react-router-dom": "^6.19.0", + "swr": "^2.2.4", + "vite-plugin-semi-theme": "^0.5.0" + }, + "devDependencies": { + "@types/react": "^18.2.37", + "@types/react-dom": "^18.2.15", + "@typescript-eslint/eslint-plugin": "^6.10.0", + "@typescript-eslint/parser": "^6.10.0", + "@vitejs/plugin-react": "^4.2.0", + "eslint": "^8.53.0", + "eslint-plugin-import": "^2.29.0", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.4.4", + "typescript": "^5.2.2", + "vite": "^5.0.0" + } +} diff --git a/neetbox/frontend/public/logo.svg b/neetbox/frontend/public/logo.svg new file mode 100644 index 00000000..ae1f8b10 --- /dev/null +++ b/neetbox/frontend/public/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/neetbox/frontend/public/vite.svg b/neetbox/frontend/public/vite.svg new file mode 100644 index 00000000..e7b8dfb1 --- /dev/null +++ b/neetbox/frontend/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/neetbox/frontend/src/App.tsx b/neetbox/frontend/src/App.tsx new file mode 100644 index 00000000..11c0e5c5 --- /dev/null +++ b/neetbox/frontend/src/App.tsx @@ -0,0 +1,61 @@ +import { + Layout, + Button, + Space, + Toast, + Typography, + Nav, + Avatar, + Form, + Checkbox, +} from "@douyinfe/semi-ui"; +import React from "react"; +import SwitchColorMode from "./components/themeSwitcher"; +import { Link, Outlet } from "react-router-dom"; +import "./styles/global.css"; +import FooterContent from "./components/Footer"; + +const { Header, Footer, Content } = Layout; + +export default function AppLayout() { + return ( + +
+ + NEET Center + +
+ + + + + + +
+
+ + + + + +