From 54b72dde4f121b37b3aed4b76218cc774212a3ca Mon Sep 17 00:00:00 2001 From: visualDust Date: Wed, 13 Dec 2023 20:05:39 +0800 Subject: [PATCH 01/20] feat: show hyperparameter --- frontend/package.json | 1 + .../dashboard/project/hyperParams.tsx | 40 +++++++++++++++++++ .../components/dashboard/project/images.tsx | 4 +- .../components/dashboard/project/jsonView.tsx | 15 +++++++ .../dashboard/project/runSelect.tsx | 3 ++ .../components/dashboard/project/scatters.tsx | 4 +- frontend/src/components/loading.tsx | 4 +- frontend/src/hooks/useProject.ts | 6 +-- frontend/src/services/types.ts | 6 +++ frontend/yarn.lock | 12 ++++++ 10 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 frontend/src/components/dashboard/project/hyperParams.tsx create mode 100644 frontend/src/components/dashboard/project/jsonView.tsx diff --git a/frontend/package.json b/frontend/package.json index bf10be30..aac15070 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,6 +20,7 @@ "@douyinfe/semi-icons": "^2.47.1", "@douyinfe/semi-ui": "^2.47.1", "@semi-bot/semi-theme-nyx-c": "^1.0.8", + "@uiw/react-json-view": "^2.0.0-alpha.10", "echarts": "^5.4.3", "jotai": "^2.5.1", "nanoid": "^5.0.3", diff --git a/frontend/src/components/dashboard/project/hyperParams.tsx b/frontend/src/components/dashboard/project/hyperParams.tsx new file mode 100644 index 00000000..31594867 --- /dev/null +++ b/frontend/src/components/dashboard/project/hyperParams.tsx @@ -0,0 +1,40 @@ +import { memo } from "react"; +import { Popover, Space, Typography } from "@douyinfe/semi-ui"; +import { IconInfoCircle } from "@douyinfe/semi-icons"; +import { useProjectRunStatus } from "../../../hooks/useProject"; +import Loading from "../../loading"; +import { JsonViewThemed } from "./jsonView"; + +export const HyperParams = memo(({ projectId, runId, trigger = "click", position }: any) => { + return ( + } + > + + + ); +}); + +const HyperParamsContent = memo(({ projectId, runId }: any) => { + const runStatus = useProjectRunStatus(projectId, runId); + const value = runStatus?.hyperparameters; + return ( + + Hyperparameter + {!runStatus ? ( + + ) : value == null ? ( + N/A + ) : ( + + )} + + ); +}); diff --git a/frontend/src/components/dashboard/project/images.tsx b/frontend/src/components/dashboard/project/images.tsx index 7246317d..7614da96 100644 --- a/frontend/src/components/dashboard/project/images.tsx +++ b/frontend/src/components/dashboard/project/images.tsx @@ -17,7 +17,9 @@ export const Images = memo(() => { export const AllImageViewers = memo(() => { const { projectId, runId } = useCurrentProject()!; const series = useProjectSeries(projectId, runId!, "image"); - return series?.map((s) => ) ?? ; + return ( + series?.map((s) => ) ?? + ); }); const SeriesViewer = memo(({ series }: { series: string }) => { diff --git a/frontend/src/components/dashboard/project/jsonView.tsx b/frontend/src/components/dashboard/project/jsonView.tsx new file mode 100644 index 00000000..be3c1446 --- /dev/null +++ b/frontend/src/components/dashboard/project/jsonView.tsx @@ -0,0 +1,15 @@ +import JsonView from "@uiw/react-json-view"; +// eslint-disable-next-line import/no-unresolved +import { darkTheme } from "@uiw/react-json-view/dark"; +import { memo } from "react"; +import { useTheme } from "../../../hooks/useTheme"; + +export const JsonViewThemed = memo((props: any) => { + const { darkMode } = useTheme(); + return ( + + ); +}); diff --git a/frontend/src/components/dashboard/project/runSelect.tsx b/frontend/src/components/dashboard/project/runSelect.tsx index 0a935b26..c13d5ecb 100644 --- a/frontend/src/components/dashboard/project/runSelect.tsx +++ b/frontend/src/components/dashboard/project/runSelect.tsx @@ -5,6 +5,7 @@ import { FormApi } from "@douyinfe/semi-ui/lib/es/form"; import Loading from "../../loading"; import { useCurrentProject, useProjectRunIds } from "../../../hooks/useProject"; import { fetcher } from "../../../services/api"; +import { HyperParams } from "./hyperParams"; export const RunSelect = memo((props: any) => { const { setRunId, runIds, mutateRunIds, projectId, runId, isOnlineRun } = props; @@ -103,6 +104,7 @@ export const RunSelect = memo((props: any) => { /> )} {item.online ? Online : Offline} + ); })} @@ -110,6 +112,7 @@ export const RunSelect = memo((props: any) => { ) : ( )} + {changing && } { export const AllScatterViewers = memo(() => { const { projectId, runId } = useCurrentProject(); const series = useProjectSeries(projectId, runId!, "scalar"); - return series?.map((s) => ) ?? ; + return ( + series?.map((s) => ) ?? + ); }); export const ScatterViewer = memo(({ series }: { series: string }) => { diff --git a/frontend/src/components/loading.tsx b/frontend/src/components/loading.tsx index 6c9861c1..2aa16a03 100644 --- a/frontend/src/components/loading.tsx +++ b/frontend/src/components/loading.tsx @@ -7,11 +7,13 @@ export default function Loading({ height = "100px", size = "middle", text, + vertical, }: { width?: string; height?: string; size?: SpinSize; text?: ReactNode; + vertical?: boolean; }) { return (
{text ? ( - + {text} diff --git a/frontend/src/hooks/useProject.ts b/frontend/src/hooks/useProject.ts index def11b9d..2f25937c 100644 --- a/frontend/src/hooks/useProject.ts +++ b/frontend/src/hooks/useProject.ts @@ -4,7 +4,7 @@ import { getProject } from "../services/projects"; import { WsMsg } from "../services/projectWebsocket"; import { fetcher, useAPI } from "../services/api"; import { Condition, createCondition } from "../utils/condition"; -import { ActionInfo, PlatformInfo } from "../services/types"; +import { ActionInfo, PlatformInfo, RunStatus } from "../services/types"; import { slideWindow } from "../utils/array"; import { IdleTimer } from "../utils/timer"; @@ -28,9 +28,9 @@ export function useProjectRunIds(id: string) { return { data: data?.runids, mutate }; } -export function useProjectRunStatus(id: string, runId?: string) { +export function useProjectRunStatus(id: string, runId?: string): RunStatus | undefined { const { data } = useAPI(`/project/${id}/run/${runId}`, { refreshInterval: 5000 }); - return !runId ? undefined : (data as { action: ActionInfo; platform: PlatformInfo }); + return !runId ? undefined : data; } export function useProjectWebSocketReady(id: string) { diff --git a/frontend/src/services/types.ts b/frontend/src/services/types.ts index e20afe56..2412b921 100644 --- a/frontend/src/services/types.ts +++ b/frontend/src/services/types.ts @@ -60,3 +60,9 @@ export interface ImageMetadata { series: string; }; } + +export interface RunStatus { + action: ActionInfo; + platform: PlatformInfo; + hyperparameters: any; +} diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 67381aa5..325a79fc 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1397,6 +1397,17 @@ __metadata: languageName: node linkType: hard +"@uiw/react-json-view@npm:^2.0.0-alpha.10": + version: 2.0.0-alpha.10 + resolution: "@uiw/react-json-view@npm:2.0.0-alpha.10" + peerDependencies: + "@babel/runtime": ">=7.10.0" + react: ">=18.0.0" + react-dom: ">=18.0.0" + checksum: d1278e92320251b1b61ecacd5c701989bd732ce4155b14a00b674d151c5b0da1b4a043b8696d23675006bf345559e56492960e33a9c0f40bc3c18eed031a136d + languageName: node + linkType: hard + "@ungap/structured-clone@npm:^1.2.0": version: 1.2.0 resolution: "@ungap/structured-clone@npm:1.2.0" @@ -4299,6 +4310,7 @@ __metadata: "@types/react-dom": "npm:^18.2.15" "@typescript-eslint/eslint-plugin": "npm:^6.10.0" "@typescript-eslint/parser": "npm:^6.10.0" + "@uiw/react-json-view": "npm:^2.0.0-alpha.10" "@vitejs/plugin-react": "npm:^4.2.0" cypress: "npm:^13.6.1" echarts: "npm:^5.4.3" From 217667182770dae369c79db19e18f3efbc90a02f Mon Sep 17 00:00:00 2001 From: visualDust Date: Thu, 14 Dec 2023 00:13:10 +0800 Subject: [PATCH 02/20] feat: json view for hardware and platform data --- .../dashboard/project/hardware/cpugraph.tsx | 6 ++- .../dashboard/project/hardware/gpugraph.tsx | 8 +++- .../project/hardware/graphWrapper.tsx | 18 +++++++ .../dashboard/project/hardware/ramgraph.tsx | 6 ++- .../dashboard/project/hardware/utils.ts | 2 +- .../components/dashboard/project/jsonView.css | 8 ++++ .../components/dashboard/project/jsonView.tsx | 48 +++++++++++++++++-- .../dashboard/project/platformProps.tsx | 7 +++ frontend/src/components/sectionTitle.tsx | 3 +- .../src/pages/console/projectDashboard.tsx | 19 ++++---- frontend/src/utils/timeDataMapper.ts | 4 ++ 11 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 frontend/src/components/dashboard/project/hardware/graphWrapper.tsx create mode 100644 frontend/src/components/dashboard/project/jsonView.css diff --git a/frontend/src/components/dashboard/project/hardware/cpugraph.tsx b/frontend/src/components/dashboard/project/hardware/cpugraph.tsx index d3f53992..d67e5cb0 100644 --- a/frontend/src/components/dashboard/project/hardware/cpugraph.tsx +++ b/frontend/src/components/dashboard/project/hardware/cpugraph.tsx @@ -3,6 +3,7 @@ import { ECharts } from "../../../echarts"; import { CpuInfo } from "../../../../services/types"; import { TimeDataMapper } from "../../../../utils/timeDataMapper"; import { getTimeAxisOptions } from "./utils"; +import { GraphWrapper } from "./graphWrapper"; export const CPUGraph = ({ data }: { data: TimeDataMapper }) => { const cpus = data.getValue(0); @@ -18,6 +19,7 @@ export const CPUGraph = ({ data }: { data: TimeDataMapper }) => { bottom: 30, }, title: { + left: 20, text: `CPU (${cpus.length} threads)`, textStyle: { fontSize: 12, @@ -56,6 +58,8 @@ export const CPUGraph = ({ data }: { data: TimeDataMapper }) => { }, [cpus, data]); return ( - + + + ); }; diff --git a/frontend/src/components/dashboard/project/hardware/gpugraph.tsx b/frontend/src/components/dashboard/project/hardware/gpugraph.tsx index bd5d6393..ba2a5d7e 100644 --- a/frontend/src/components/dashboard/project/hardware/gpugraph.tsx +++ b/frontend/src/components/dashboard/project/hardware/gpugraph.tsx @@ -3,10 +3,11 @@ import { ECharts } from "../../../echarts"; import { GpuInfo } from "../../../../services/types"; import { TimeDataMapper } from "../../../../utils/timeDataMapper"; import { getTimeAxisOptions } from "./utils"; +import { GraphWrapper } from "./graphWrapper"; export const GPUGraph = ({ data }: { data: TimeDataMapper }) => { const initialOption = () => { - const gpu = data.mapValue((x) => x)[0]; + const gpu = data.getValue(0); return { backgroundColor: "transparent", animation: false, @@ -18,6 +19,7 @@ export const GPUGraph = ({ data }: { data: TimeDataMapper }) => { bottom: 30, }, title: { + left: 20, text: `GPU${gpu.id}: ${gpu.name}`, textStyle: { fontSize: 12, @@ -76,6 +78,8 @@ export const GPUGraph = ({ data }: { data: TimeDataMapper }) => { }, [data]); return ( - + + + ); }; diff --git a/frontend/src/components/dashboard/project/hardware/graphWrapper.tsx b/frontend/src/components/dashboard/project/hardware/graphWrapper.tsx new file mode 100644 index 00000000..3fd1a926 --- /dev/null +++ b/frontend/src/components/dashboard/project/hardware/graphWrapper.tsx @@ -0,0 +1,18 @@ +import { PropsWithChildren, memo } from "react"; +import { JsonPopover } from "../jsonView"; + +export const GraphWrapper = memo( + ({ title, lastValue, children }: PropsWithChildren<{ title: string; lastValue: any }>) => { + return ( +
+ {children} + +
+ ); + }, +); diff --git a/frontend/src/components/dashboard/project/hardware/ramgraph.tsx b/frontend/src/components/dashboard/project/hardware/ramgraph.tsx index 4c3cbec3..2fac9831 100644 --- a/frontend/src/components/dashboard/project/hardware/ramgraph.tsx +++ b/frontend/src/components/dashboard/project/hardware/ramgraph.tsx @@ -3,6 +3,7 @@ import { ECharts } from "../../../echarts"; import { RamInfo } from "../../../../services/types"; import { TimeDataMapper } from "../../../../utils/timeDataMapper"; import { getTimeAxisOptions } from "./utils"; +import { GraphWrapper } from "./graphWrapper"; export const RAMGraph = ({ data }: { data: TimeDataMapper }) => { const initialOption = () => { @@ -13,6 +14,7 @@ export const RAMGraph = ({ data }: { data: TimeDataMapper }) => { trigger: "axis", }, title: { + left: 20, text: `RAM`, textStyle: { fontSize: 12, @@ -59,6 +61,8 @@ export const RAMGraph = ({ data }: { data: TimeDataMapper }) => { }, [data]); return ( - + + + ); }; diff --git a/frontend/src/components/dashboard/project/hardware/utils.ts b/frontend/src/components/dashboard/project/hardware/utils.ts index a949a196..ff461dfd 100644 --- a/frontend/src/components/dashboard/project/hardware/utils.ts +++ b/frontend/src/components/dashboard/project/hardware/utils.ts @@ -2,7 +2,7 @@ import { TimeDataMapper } from "../../../../utils/timeDataMapper"; const viewRangeSeconds = 300; const dataInterval = 2; -export const fetchDataCount = Math.ceil(viewRangeSeconds / dataInterval); +export const fetchDataCount = Math.ceil((viewRangeSeconds / dataInterval) * 1.1); export function getTimeAxisOptions(mapper: TimeDataMapper) { const latestTime = new Date(mapper.data[mapper.data.length - 1].timestamp).getTime(); diff --git a/frontend/src/components/dashboard/project/jsonView.css b/frontend/src/components/dashboard/project/jsonView.css new file mode 100644 index 00000000..351bbebd --- /dev/null +++ b/frontend/src/components/dashboard/project/jsonView.css @@ -0,0 +1,8 @@ +.w-json-view-container { + padding-left: 1em; +} + +span[style="display: inline-flex; align-items: center;"] { + /* position: absolute; */ + transform: translate(-1em, 0); +} diff --git a/frontend/src/components/dashboard/project/jsonView.tsx b/frontend/src/components/dashboard/project/jsonView.tsx index be3c1446..14698fd7 100644 --- a/frontend/src/components/dashboard/project/jsonView.tsx +++ b/frontend/src/components/dashboard/project/jsonView.tsx @@ -1,15 +1,57 @@ import JsonView from "@uiw/react-json-view"; // eslint-disable-next-line import/no-unresolved -import { darkTheme } from "@uiw/react-json-view/dark"; -import { memo } from "react"; +import { githubDarkTheme } from "@uiw/react-json-view/githubDark"; +import { CSSProperties, memo } from "react"; +import { Popover, Space, Typography } from "@douyinfe/semi-ui"; +import { IconCode, IconCodeStroked } from "@douyinfe/semi-icons"; +import { Position } from "@douyinfe/semi-ui/lib/es/tooltip"; import { useTheme } from "../../../hooks/useTheme"; +import "./jsonView.css"; export const JsonViewThemed = memo((props: any) => { const { darkMode } = useTheme(); return ( ); }); + +export const JsonPopover = memo( + ({ + value, + title, + position, + style, + }: { + value: any; + title?: string; + position?: Position; + style?: CSSProperties; + }) => { + return ( + + {title && {title}} + + + } + > + + + ); + }, +); diff --git a/frontend/src/components/dashboard/project/platformProps.tsx b/frontend/src/components/dashboard/project/platformProps.tsx index 77e35fc9..37f67283 100644 --- a/frontend/src/components/dashboard/project/platformProps.tsx +++ b/frontend/src/components/dashboard/project/platformProps.tsx @@ -4,6 +4,7 @@ import { IconCopy } from "@douyinfe/semi-icons"; import { useMemoJSON } from "../../../hooks/useMemoJSON"; import { useCurrentProject, useProjectRunStatus } from "../../../hooks/useProject"; import { PlatformInfo } from "../../../services/types"; +import { JsonPopover } from "./jsonView"; const PropCard = memo(({ propName, propValue }: { propName: string; propValue: PlatformInfo[string] }) => { const { Text } = Typography; @@ -73,3 +74,9 @@ export default function PlatformProps(): React.JSX.Element {
); } + +export function PlatformTitleJson() { + const { projectId, runId } = useCurrentProject(); + const runStatus = useProjectRunStatus(projectId, runId); + return ; +} diff --git a/frontend/src/components/sectionTitle.tsx b/frontend/src/components/sectionTitle.tsx index 11f17669..adaf9a21 100644 --- a/frontend/src/components/sectionTitle.tsx +++ b/frontend/src/components/sectionTitle.tsx @@ -1,7 +1,8 @@ import { Typography } from "@douyinfe/semi-ui"; +import { ReactNode } from "react"; interface Props { - title: string; + title: ReactNode; } export function SectionTitle(props: Props) { diff --git a/frontend/src/pages/console/projectDashboard.tsx b/frontend/src/pages/console/projectDashboard.tsx index bf1b3192..d74e22a0 100644 --- a/frontend/src/pages/console/projectDashboard.tsx +++ b/frontend/src/pages/console/projectDashboard.tsx @@ -1,7 +1,7 @@ import { useParams, useSearchParams } from "react-router-dom"; import { useEffect, useMemo } from "react"; -import { Divider } from "@douyinfe/semi-ui"; -import { ProjectContext, useProjectRunIds, useProjectStatus } from "../../hooks/useProject"; +import { Divider, Space } from "@douyinfe/semi-ui"; +import { ProjectContext, useProjectStatus } from "../../hooks/useProject"; import { Logs } from "../../components/dashboard/project/logs/logs"; import { Actions } from "../../components/dashboard/project/actions"; import { Hardware } from "../../components/dashboard/project/hardware"; @@ -10,7 +10,7 @@ import { AppTitle } from "../../components/appTitle"; import { ImagesAndScatters } from "../../components/dashboard/project/imagesAndScatters"; import { getProject } from "../../services/projects"; import { RunSelect } from "../../components/dashboard/project/runSelect"; -import PlatformProps from "../../components/dashboard/project/platformProps"; +import PlatformProps, { PlatformTitleJson } from "../../components/dashboard/project/platformProps"; import Loading from "../../components/loading"; import { addNotice } from "../../utils/notification"; @@ -92,15 +92,18 @@ function ProjectDashboard() { - {/* - */} - {/* - */} - + + Platform + + + } + /> ) : ( diff --git a/frontend/src/utils/timeDataMapper.ts b/frontend/src/utils/timeDataMapper.ts index 5b81ea9b..f1296942 100644 --- a/frontend/src/utils/timeDataMapper.ts +++ b/frontend/src/utils/timeDataMapper.ts @@ -4,6 +4,10 @@ export class TimeDataMapper TTarget, ) {} + get length() { + return this.data.length; + } + chain(chainMapper: (value: TTarget) => TNewTarget) { return new TimeDataMapper(this.data, (x) => chainMapper(this.mapper(x))); } From bbdff2dc69972e52b80a56d81a4b7c38550004ae Mon Sep 17 00:00:00 2001 From: VisualDust Date: Thu, 14 Dec 2023 12:28:47 +0800 Subject: [PATCH 03/20] fix TypeError post got an unexpected keyword argument root --- neetbox/_daemon/client/_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/neetbox/_daemon/client/_client.py b/neetbox/_daemon/client/_client.py index 3ac9ff40..94942210 100644 --- a/neetbox/_daemon/client/_client.py +++ b/neetbox/_daemon/client/_client.py @@ -48,19 +48,19 @@ class ClientConn(metaclass=Singleton): # http client http: httpx.Client = _load_http_client() - def post(api: str, *args, **kwargs): + def post(api: str, root: str = None, *args, **kwargs): url = addr_of_api(api) return ClientConn.http.post(url, *args, **kwargs) - def get(api: str, *args, **kwargs): + def get(api: str, root: str = None, *args, **kwargs): url = addr_of_api(api) return ClientConn.http.get(url, *args, **kwargs) - def put(api: str, *args, **kwargs): + def put(api: str, root: str = None, *args, **kwargs): url = addr_of_api(api) return ClientConn.http.put(url, *args, **kwargs) - def delete(api: str, *args, **kwargs): + def delete(api: str, root: str = None, *args, **kwargs): url = addr_of_api(api) return ClientConn.http.delete(url, *args, **kwargs) From d2ca11fe889c2a57af77e7a5a816d4de8edc013f Mon Sep 17 00:00:00 2001 From: VisualDust Date: Thu, 14 Dec 2023 17:26:39 +0800 Subject: [PATCH 04/20] fix poetry not including frontend_dist sub folders --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f49897f0..871176d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Topic :: Utilities", ] -include = ["neetbox/frontend_dist/*"] +include = ["neetbox/frontend_dist/**/*"] [tool.poetry.dependencies] python = ">=3.9,<3.13" From 4e668353686f3fc52b6af958216d1ba9357e967d Mon Sep 17 00:00:00 2001 From: VisualDust Date: Thu, 14 Dec 2023 23:52:58 +0800 Subject: [PATCH 05/20] delete project id when run id list is empty --- neetbox/_daemon/server/_bridge.py | 7 ++++ neetbox/_daemon/server/_flask_server.py | 4 ++ neetbox/_daemon/server/_websocket_server.py | 2 + neetbox/_daemon/server/history/_db.py | 41 ++++++++++++++++++++- 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/neetbox/_daemon/server/_bridge.py b/neetbox/_daemon/server/_bridge.py index 3e9c0acd..c4504acd 100644 --- a/neetbox/_daemon/server/_bridge.py +++ b/neetbox/_daemon/server/_bridge.py @@ -64,6 +64,13 @@ def __new__(cls, project_id: str, **kwargs) -> None: logger.info(f"created new Bridge for project id '{project_id}'") return cls._id2bridge[project_id] + def __del__(self): # on delete + logger.info(f"bridge project id {self.project_id} handling on delete...") + if 0 == len(self.get_run_ids()): # if there is no active run id + self.historyDB.finialize() + del self.historyDB # delete history db + logger.info(f"bridge of project id {self.project_id} deleted.") + @classmethod def items(cls): return cls._id2bridge.items() diff --git a/neetbox/_daemon/server/_flask_server.py b/neetbox/_daemon/server/_flask_server.py index e7b00c4b..ba7e5d47 100644 --- a/neetbox/_daemon/server/_flask_server.py +++ b/neetbox/_daemon/server/_flask_server.py @@ -63,6 +63,8 @@ def get_status_of_all_proejcts(): @app.route(f"{FRONTEND_API_ROOT}/project/", methods=["GET"]) def get_status_of_project_id(project_id: str): + if not Bridge.has(project_id): + return abort(404) # project id must exist bridge = Bridge.of_id(project_id) return _project_status_from_bridge(bridge) @@ -137,6 +139,8 @@ def delete_run_id(project_id: str, run_id: str): if bridge.is_online(run_id): # cannot delete running projects abort(400, {ERROR_KEY: "can only delete history run id."}) bridge.historyDB.delete_run_id(run_id) + if 0 == len(bridge.get_run_ids()): # check if all the run ids are deleted + del Bridge._id2bridge[project_id] # delete the empty bridge return {RESULT_KEY: "success"} @app.route(f"/image/", methods=["POST"]) diff --git a/neetbox/_daemon/server/_websocket_server.py b/neetbox/_daemon/server/_websocket_server.py index 93c55727..645b497f 100644 --- a/neetbox/_daemon/server/_websocket_server.py +++ b/neetbox/_daemon/server/_websocket_server.py @@ -40,6 +40,8 @@ def handle_ws_disconnect(client, server): return # client disconnected before handshake, returning anyway project_id, run_id, who = connected_clients[ws_client_id] bridge = Bridge.of_id(project_id) + if not bridge: + return # do nothing if bridge has been deleted if who == IdentityType.WEB: # is web ws, remove from bridge's web ws list _new_web_ws_list = [ c for c in Bridge.of_id(project_id).web_ws_list if c["id"] != ws_client_id diff --git a/neetbox/_daemon/server/history/_db.py b/neetbox/_daemon/server/history/_db.py index 8d7bd9e6..2e332ebe 100644 --- a/neetbox/_daemon/server/history/_db.py +++ b/neetbox/_daemon/server/history/_db.py @@ -1,5 +1,6 @@ import collections import json +import os import sqlite3 from datetime import datetime from random import random @@ -158,10 +159,14 @@ class DBConnection: _id2dbc = {} # not static. instance level vars - connection: sqlite3.Connection + project_id: str # of which project id + file_path: str # where is the db file + connection: sqlite3.Connection # the db connection _inited_tables: collections.defaultdict def __new__(cls, project_id: str = None, path: str = None, **kwargs) -> "DBConnection": + if path is None and project_id is None: + raise RuntimeError(f"please provide at least project id or path when creating db") if path is None: # make path from project id path = f"{HISTORY_FILE_ROOT}/{project_id}.{HISTORY_FILE_TYPE_NAME}" if path in cls._path2dbc: @@ -170,6 +175,7 @@ def __new__(cls, project_id: str = None, path: str = None, **kwargs) -> "DBConne return cls._id2dbc[project_id] new_dbc = super().__new__(cls, **kwargs) # connect to sqlite + new_dbc.file_path = path new_dbc.connection = sqlite3.connect(path, check_same_thread=False, isolation_level=None) new_dbc.connection.execute("pragma journal_mode=wal") # set journal mode WAL new_dbc.connection.execute("PRAGMA foreign_keys = ON") # enable foreign keys features @@ -188,9 +194,42 @@ def __new__(cls, project_id: str = None, path: str = None, **kwargs) -> "DBConne ) cls._path2dbc[path] = new_dbc cls._id2dbc[project_id] = new_dbc + new_dbc.project_id = project_id logger.ok(f"History file(version={_db_file_version}) for project id '{project_id}' loaded.") return new_dbc + def finialize(self): + if self.project_id not in DBConnection._id2dbc: + logger.err( + RuntimeError(f"could not find db to delete with project id {self.project_id}") + ) + del DBConnection._id2dbc[self.project_id] + del DBConnection._path2dbc[self.file_path] + logger.info(f"deleting history DB for project id {self.project_id}...") + if self.connection: + try: + self.connection.close() + except Exception as e: + logger.err( + RuntimeError( + f"failed to close DB connection for project id {self.project_id} because {e}" + ), + reraise=True, + ) + try: + for suffix in ["", "-shm", "-wal"]: # remove db files + _path = f"{self.file_path}{suffix}" + if os.path.exists(_path): + os.remove(_path) + except Exception as e: + logger.err( + RuntimeError( + f"failed to delete DB file for project id {self.project_id} because {e}" + ), + reraise=True, + ) + logger.info(f"History db for project id {self.project_id} has been deleted.") + @classmethod def items(cls): return cls._id2dbc.items() From 7c7c634f646b5bf01b7986744987f1dca196e026 Mon Sep 17 00:00:00 2001 From: lideming Date: Fri, 15 Dec 2023 00:23:45 +0800 Subject: [PATCH 06/20] fix: error after deleting current run --- .../components/dashboard/project/runSelect.tsx | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/dashboard/project/runSelect.tsx b/frontend/src/components/dashboard/project/runSelect.tsx index c13d5ecb..ee91d4aa 100644 --- a/frontend/src/components/dashboard/project/runSelect.tsx +++ b/frontend/src/components/dashboard/project/runSelect.tsx @@ -2,13 +2,15 @@ import { memo, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Space, Select, Tag, Button, Modal, Form, Toast } from "@douyinfe/semi-ui"; import { IconDelete, IconEdit } from "@douyinfe/semi-icons"; import { FormApi } from "@douyinfe/semi-ui/lib/es/form"; +import { useNavigate } from "react-router-dom"; import Loading from "../../loading"; -import { useCurrentProject, useProjectRunIds } from "../../../hooks/useProject"; +import { useCurrentProject } from "../../../hooks/useProject"; import { fetcher } from "../../../services/api"; import { HyperParams } from "./hyperParams"; export const RunSelect = memo((props: any) => { const { setRunId, runIds, mutateRunIds, projectId, runId, isOnlineRun } = props; + const navigate = useNavigate(); const items = useMemo( () => [...(runIds ?? [])] @@ -51,7 +53,7 @@ export const RunSelect = memo((props: any) => { ) : ( Offline )} - {items.find((x) => x.runid == p.value).timestamp} + {items.find((x) => x.runid == p.value)?.timestamp} )} > @@ -92,7 +94,14 @@ export const RunSelect = memo((props: any) => { content: `Deleting run ${item.timestamp} (${item.runid})`, centered: true, onOk: async () => { - // await new Promise((r) => setTimeout(r, 1000)); + if (item.runid === runId) { + const existedId = runIds.find((x) => x.runid !== runId); + if (existedId) { + setRunId(existedId); + } else { + navigate("/"); + } + } await fetcher(`/project/${projectId}/run/${item.runid}`, { method: "DELETE" }); mutateRunIds(); Toast.success({ From 7b852bbf4c6ac22afd1a1fe208f87ab71312d08a Mon Sep 17 00:00:00 2001 From: lideming Date: Fri, 15 Dec 2023 00:30:27 +0800 Subject: [PATCH 07/20] fix: "no actions" text --- .../components/dashboard/project/actions.tsx | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/frontend/src/components/dashboard/project/actions.tsx b/frontend/src/components/dashboard/project/actions.tsx index b6e39beb..ce10a89c 100644 --- a/frontend/src/components/dashboard/project/actions.tsx +++ b/frontend/src/components/dashboard/project/actions.tsx @@ -11,32 +11,30 @@ export function Actions() { const { projectId, runId } = useCurrentProject(); const status = useProjectRunStatus(projectId, runId); const actions = status?.action as ActionInfo; - const actionList = Object.entries(useMemoJSON(actions ?? {})); + const actionListMemo = Object.entries(useMemoJSON(actions ?? {})); const [blocking, setBlocking] = useState(false); - return actions ? ( + return !status ? ( + + ) : !status.action ? ( + + No actions ( + + docs + + ) + + ) : ( - {actionList.length ? ( - actionList.map(([actionName, actionOptions]) => ( - - )) - ) : ( - - No actions ( - - docs - - ) - - )} + {actionListMemo.map(([actionName, actionOptions]) => ( + + ))} - ) : ( - ); } From 04d4e3cbcd0476172f6e54a43c972886686fcbed Mon Sep 17 00:00:00 2001 From: VisualDust Date: Fri, 15 Dec 2023 22:46:59 +0800 Subject: [PATCH 08/20] minor fix --- neetbox/_daemon/__init__.py | 4 +--- neetbox/_daemon/client/_client.py | 8 +++++--- neetbox/_daemon/server/_flask_server.py | 5 +++-- neetbox/_daemon/server/_server.py | 12 ++++++------ neetbox/_daemon/server/_websocket_server.py | 13 +++++++++---- neetbox/_daemon/server/history/_db.py | 6 +++--- neetbox/cli/parse.py | 2 +- neetbox/logging/_writer.py | 13 ++++--------- neetbox/logging/formatting.py | 2 +- neetbox/logging/logger.py | 4 +++- tests/client/neetbox.toml | 2 +- 11 files changed, 37 insertions(+), 34 deletions(-) diff --git a/neetbox/_daemon/__init__.py b/neetbox/_daemon/__init__.py index db097837..4bee8de2 100644 --- a/neetbox/_daemon/__init__.py +++ b/neetbox/_daemon/__init__.py @@ -31,9 +31,7 @@ def connect(): "ipython, try to set 'allowIpython' to True." ) return False # ignore if debugging in ipython - _is_daemon_server_online = check_server_connectivity() # try to connect daemon - logger.debug("daemon connection status: " + str(_is_daemon_server_online)) - if not _is_daemon_server_online: # if no daemon online + if not check_server_connectivity(): # if no daemon online # check if possible to launch if daemon_config["host"] not in ["localhost", "127.0.0.1", "0.0.0.0"]: # daemon not running on localhost diff --git a/neetbox/_daemon/client/_client.py b/neetbox/_daemon/client/_client.py index 94942210..a8754236 100644 --- a/neetbox/_daemon/client/_client.py +++ b/neetbox/_daemon/client/_client.py @@ -93,12 +93,14 @@ def _init_ws(cls): cfg = get_module_level_config() # ws server url - cls.ws_server_addr = f"ws://{cfg['host']}:{cfg['port'] + 1}" + host = cfg["host"] + port = cfg["port"] + 1 + cls.ws_server_addr = f"ws://{host}:{port}" # create websocket app logger.log(f"creating websocket connection to {cls.ws_server_addr}", skip_writers=["ws"]) cls.wsApp = websocket.WebSocketApp( - cls.ws_server_addr, + url=cls.ws_server_addr, on_open=cls.__on_ws_open, on_message=cls.__on_ws_message, on_error=cls.__on_ws_err, @@ -106,7 +108,7 @@ def _init_ws(cls): ) Thread( - target=cls.wsApp.run_forever, kwargs={"reconnect": True}, daemon=True + target=cls.wsApp.run_forever, kwargs={"reconnect": 1}, daemon=True ).start() # initialize and start ws thread _ws_initialized = True diff --git a/neetbox/_daemon/server/_flask_server.py b/neetbox/_daemon/server/_flask_server.py index ba7e5d47..bef82d48 100644 --- a/neetbox/_daemon/server/_flask_server.py +++ b/neetbox/_daemon/server/_flask_server.py @@ -24,9 +24,10 @@ def get_flask_server(debug=False): __PROC_NAME = "NEETBOX" - from neetbox.logging import LogStyle, logger + from neetbox.logging import LogStyle + from neetbox.logging.logger import Logger - logger = logger(__PROC_NAME, LogStyle(skip_writers=["ws"])) + logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) if debug: logger.log(f"Running with debug, using APIFlask") diff --git a/neetbox/_daemon/server/_server.py b/neetbox/_daemon/server/_server.py index a4c762b0..2a720219 100644 --- a/neetbox/_daemon/server/_server.py +++ b/neetbox/_daemon/server/_server.py @@ -15,10 +15,10 @@ def server_process(cfg, debug=False): setproctitle.setproctitle("NEETBOX SERVER") - from neetbox.logging import LogStyle, logger - - logger = logger("NEETBOX", LogStyle(skip_writers=["ws"])) + from neetbox.logging import LogStyle + from neetbox.logging.logger import Logger + logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) # load bridges Bridge.load_histories() # load history files @@ -33,9 +33,9 @@ def server_process(cfg, debug=False): logger.log(f"launching websocket server...") ws_server.run_forever(threaded=True) - _port = cfg["port"] - logger.log(f"visit frontend at http://localhost:{_port}") - flask_server.run(host="0.0.0.0", port=_port) + port = cfg["port"] + logger.log(f"launching flask server on port {port}") + flask_server.run(host="0.0.0.0", port=port) if __name__ == "__main__": diff --git a/neetbox/_daemon/server/_websocket_server.py b/neetbox/_daemon/server/_websocket_server.py index 645b497f..0957665f 100644 --- a/neetbox/_daemon/server/_websocket_server.py +++ b/neetbox/_daemon/server/_websocket_server.py @@ -19,11 +19,15 @@ def get_web_socket_server(config, debug=False): from websocket_server import WebsocketServer from neetbox._daemon.server._bridge import Bridge - from neetbox.logging import LogStyle, logger + from neetbox.logging import LogStyle + from neetbox.logging.logger import Logger - logger = logger("NEETBOX", LogStyle(skip_writers=["ws"])) console = Console() - ws_server = WebsocketServer(host="0.0.0.0", port=config["port"] + 1) + logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) + + port = config["port"] + 1 + logger.info(f"creating web socket server on port {port}") + ws_server = WebsocketServer(host="0.0.0.0", port=port) connected_clients: Dict( int, Tuple(str, str, IdentityType) ) = {} # {cid:(project_id, run_id,type)} store connection only @@ -55,7 +59,6 @@ def handle_ws_disconnect(client, server): logger.info( f"a {who}(ws client id {ws_client_id}) disconnected from project '{project_id}'" ) - # logger.info(f"Websocket ({conn_type}) for {name} disconnected") def on_event_type_handshake(client, server, message: EventMsg): ws_client_id = client["id"] @@ -128,6 +131,8 @@ def on_event_type_handshake(client, server, message: EventMsg): msg=EventMsg.merge(message, merge_msg).dumps(), ) + logger.ok(f"client(id={client['id']}) on {client['address']} handshake succeed.") + if debug: table = Table( title="Connected Websockets", box=box.MINIMAL_DOUBLE_HEAD, show_lines=True diff --git a/neetbox/_daemon/server/history/_db.py b/neetbox/_daemon/server/history/_db.py index 2e332ebe..03917c03 100644 --- a/neetbox/_daemon/server/history/_db.py +++ b/neetbox/_daemon/server/history/_db.py @@ -7,10 +7,10 @@ from typing import Dict, Tuple, Union from neetbox._daemon._protocol import * -from neetbox.logging import logger -from neetbox.logging.formatting import LogStyle +from neetbox.logging import LogStyle +from neetbox.logging.logger import Logger -logger = logger("NEETBOX", LogStyle(skip_writers=["ws", "file"])) +logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) class QueryCondition: diff --git a/neetbox/cli/parse.py b/neetbox/cli/parse.py index a43f918f..8a2903a0 100644 --- a/neetbox/cli/parse.py +++ b/neetbox/cli/parse.py @@ -21,7 +21,7 @@ console = Console() -logger = Logger("NEETBOX", style=LogStyle(with_datetime=False, skip_writers=["ws"])) +logger = Logger("NEETBOX", style=LogStyle(with_datetime=False, skip_writers=["ws", "file"])) def get_daemon_config(): diff --git a/neetbox/logging/_writer.py b/neetbox/logging/_writer.py index 2b3ee4e1..ed0eb1bc 100644 --- a/neetbox/logging/_writer.py +++ b/neetbox/logging/_writer.py @@ -169,16 +169,11 @@ def __init__(self, path) -> None: def write(self, raw_log: RawLog): _msg_dict = raw_log.json() _style = raw_log.style - text_msg = str( - _msg_dict["series"] - + _msg_dict["timestamp"] - + _style.split_char_txt * min(len(_msg_dict["timestamp"]), 1) - + _msg_dict["whom"] - + _style.split_char_txt * min(len(_msg_dict["whom"]), 1) - + _msg_dict["message"] - + "\n" + _series_text = f"[{_msg_dict['series']}]" if _msg_dict["series"] else "" + text_msg = _style.split_char_txt.join( + [_msg_dict["timestamp"], _series_text, _msg_dict["whom"], _msg_dict["message"]] ) - self.file_writer.write(text_msg) + self.file_writer.write(text_msg + "\n") # ================== WS LOG WRITER ===================== diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index 04296f60..ef41bab8 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -22,7 +22,7 @@ class LogStyle: with_datetime: bool = True split_char_cmd = " > " split_char_identity = "/" - split_char_txt = " | " + split_char_txt = " " skip_writers: list[str] = None @classmethod diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index 70d321a0..86bb8a3c 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -70,8 +70,10 @@ def __init__(self, whom=None, style: Optional[LogStyle] = None): self.ws_writer = webSocketLogWriter self.file_writer = None _cfg = get_module_level_config() - self.set_log_dir(_cfg["logdir"]) self.set_log_level(_cfg["level"]) + if style: + if style.skip_writers and "file" not in style.skip_writers: + self.set_log_dir(_cfg["logdir"]) def __call__(self, whom: Any = None, style: Optional[LogStyle] = None) -> "Logger": """Attention! do not call this logger instance unless you know what are you doing. Users should use the default logger by import logger from neetbox.logging. diff --git a/tests/client/neetbox.toml b/tests/client/neetbox.toml index 349915e3..81ec2cab 100644 --- a/tests/client/neetbox.toml +++ b/tests/client/neetbox.toml @@ -14,7 +14,7 @@ autoload = true [daemon] enable = true host = "127.0.0.1" -port = 5000 +port = 20202 allowIpython = false mute = true mode = "detached" From 3a25b343d912689e437a82dd1797485da34bab56 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sat, 16 Dec 2023 19:58:42 +0800 Subject: [PATCH 09/20] vvvvery heavy refactor --- neetbox/__init__.py | 32 +- neetbox/_daemon/__init__.py | 79 ----- neetbox/_daemon/client/_client.py | 239 -------------- neetbox/_daemon/readme.md | 163 ---------- neetbox/{_daemon => }/_protocol.py | 0 neetbox/cli/parse.py | 14 +- neetbox/client/__init__.py | 11 +- ...lient_action_agent.py => _action_agent.py} | 5 +- neetbox/client/_client.py | 293 ++++++++++++++++++ neetbox/{cli => client}/_client_web_apis.py | 6 +- neetbox/client/{_writers => }/_image.py | 5 +- neetbox/client/{_writers => }/_plot.py | 11 +- neetbox/client/_signal_and_slot.py | 2 +- neetbox/config/__init__.py | 2 +- neetbox/config/_workspace.py | 51 ++- neetbox/core/registry.py | 2 +- neetbox/extension/__init__.py | 4 +- neetbox/extension/environment/hardware.py | 10 +- neetbox/extension/environment/platform.py | 6 +- neetbox/extension/torch/arch/cnn.py | 2 +- neetbox/extension/torch/profile.py | 2 +- neetbox/logging/formatting.py | 2 +- neetbox/logging/logger.py | 2 +- neetbox/{_daemon => }/server/_bridge.py | 5 +- .../{_daemon => }/server/_daemon_launcher.py | 2 +- .../server/_daemonable_process.py | 0 .../{_daemon => }/server/_fastapi_server.py | 0 neetbox/{_daemon => }/server/_flask_server.py | 6 +- neetbox/{_daemon => }/server/_server.py | 12 +- .../{_daemon => }/server/_websocket_server.py | 6 +- .../{_daemon => }/server/history/__init__.py | 12 - neetbox/{_daemon => }/server/history/_db.py | 3 +- neetbox/utils/__init__.py | 2 +- neetbox/utils/_resource.py | 2 +- neetbox/utils/formatting.py | 2 +- neetbox/utils/framing.py | 2 +- neetbox/utils/massive.py | 24 ++ neetbox/utils/mvc.py | 2 +- tests/client/neetbox.toml | 17 +- tests/snake/neetbox.toml | 8 +- 40 files changed, 434 insertions(+), 614 deletions(-) delete mode 100644 neetbox/_daemon/__init__.py delete mode 100644 neetbox/_daemon/client/_client.py delete mode 100644 neetbox/_daemon/readme.md rename neetbox/{_daemon => }/_protocol.py (100%) rename neetbox/client/{_client_action_agent.py => _action_agent.py} (98%) create mode 100644 neetbox/client/_client.py rename neetbox/{cli => client}/_client_web_apis.py (86%) rename neetbox/client/{_writers => }/_image.py (98%) rename neetbox/client/{_writers => }/_plot.py (85%) rename neetbox/{_daemon => }/server/_bridge.py (98%) rename neetbox/{_daemon => }/server/_daemon_launcher.py (92%) rename neetbox/{_daemon => }/server/_daemonable_process.py (100%) rename neetbox/{_daemon => }/server/_fastapi_server.py (100%) rename neetbox/{_daemon => }/server/_flask_server.py (98%) rename neetbox/{_daemon => }/server/_server.py (76%) rename neetbox/{_daemon => }/server/_websocket_server.py (98%) rename neetbox/{_daemon => }/server/history/__init__.py (82%) rename neetbox/{_daemon => }/server/history/_db.py (99%) diff --git a/neetbox/__init__.py b/neetbox/__init__.py index b0980068..02eb089e 100644 --- a/neetbox/__init__.py +++ b/neetbox/__init__.py @@ -1,28 +1,10 @@ -import os -import sys +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# URL: https://github.com/visualDust +# Date: 20231216 - -def _load_workspace(connect_daemon=True): - from neetbox.config import get_module_level_config - - get_module_level_config() # run things after init workspace - if connect_daemon: - import neetbox._daemon as _daemon - - _daemon.connect() - - -is_in_daemon_process = ( - "NEETBOX_DAEMON_PROCESS" in os.environ and os.environ["NEETBOX_DAEMON_PROCESS"] == "1" -) - -if len(sys.argv) > 0 and sys.argv[0].endswith("neet") or is_in_daemon_process: - # running in cli or daemon process, do not load workspace - pass -else: - _load_workspace(connect_daemon=True) - -from neetbox.client import ( +from .client import ( action, add_figure, add_hyperparams, @@ -31,7 +13,7 @@ def _load_workspace(connect_daemon=True): listen, watch, ) -from neetbox.logging import logger +from .logging import logger __all__ = [ "add_image", diff --git a/neetbox/_daemon/__init__.py b/neetbox/_daemon/__init__.py deleted file mode 100644 index 4bee8de2..00000000 --- a/neetbox/_daemon/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Author: GavinGong aka VisualDust -# URL: https://gong.host -# Date: 20230414 - -import json -import subprocess -import time - -from neetbox._daemon.server._daemonable_process import DaemonableProcess -from neetbox.config import get_module_level_config -from neetbox.logging.formatting import LogStyle -from neetbox.logging.logger import Logger - -from .client._client import check_server_connectivity, connection - -logger = Logger(style=LogStyle(with_datetime=False, skip_writers=["ws"])) - - -def connect(): - daemon_config = get_module_level_config() - if not daemon_config["allowIpython"]: - try: - eval("__IPYTHON__") - except NameError: - pass - else: - logger.log( - "NEETBOX DAEMON won't start when debugging in ipython console. If you want to allow daemon run in " - "ipython, try to set 'allowIpython' to True." - ) - return False # ignore if debugging in ipython - if not check_server_connectivity(): # if no daemon online - # check if possible to launch - if daemon_config["host"] not in ["localhost", "127.0.0.1", "0.0.0.0"]: - # daemon not running on localhost - logger.err( - f"No daemon running at {daemon_config['host']}:{daemon_config['port']}, daemon will not be attached. Continue anyway." - ) - return False - - logger.log( - f"No daemon running at {daemon_config['host']}:{daemon_config['port']}, trying to create daemon..." - ) - - popen = DaemonableProcess( - target="neetbox._daemon.server._daemon_launcher", - args=["--config", json.dumps(daemon_config)], - mode=daemon_config["mode"], - redirect_stdout=subprocess.DEVNULL if daemon_config["mute"] else None, - env_append={"NEETBOX_DAEMON_PROCESS": "1"}, - ).start() - - time.sleep(1) - - _retry_timeout = 10 - _time_begin = time.perf_counter() - - logger.log("Created daemon process, trying to connect to daemon...") - while time.perf_counter() - _time_begin < 10: # try connect daemon - if check_server_connectivity(): - return True - else: - exit_code = popen.poll() - if exit_code is not None: - logger.err(f"Daemon process exited unexpectedly with exit code {exit_code}.") - return False - - time.sleep(0.5) - - logger.err( - f"Failed to connect to daemon after {_retry_timeout}s, daemon connector won't start." - ) - return False - - -ws_subscribe = connection.ws_subscribe -__all__ = ["ws_subscribe"] diff --git a/neetbox/_daemon/client/_client.py b/neetbox/_daemon/client/_client.py deleted file mode 100644 index a8754236..00000000 --- a/neetbox/_daemon/client/_client.py +++ /dev/null @@ -1,239 +0,0 @@ -import functools -import json -import logging -import time -from collections import defaultdict -from threading import Thread -from typing import Callable - -import httpx -import websocket - -from neetbox._daemon._protocol import * -from neetbox.config import get_module_level_config, get_project_id, get_run_id -from neetbox.logging.formatting import LogStyle -from neetbox.logging.logger import Logger -from neetbox.utils.mvc import Singleton - -logger = Logger(whom=None, style=LogStyle(skip_writers=["ws"])) - -httpx_logger = logging.getLogger("httpx") -httpx_logger.setLevel(logging.ERROR) - -_ws_initialized = False # indicate whether websocket has been initialized - - -def _load_http_client(): - __local_http_client = httpx.Client( - proxies={ - "http://": None, - "https://": None, - } - ) # type: ignore - return __local_http_client - - -def addr_of_api(api, root=None): - if not root: - _cfg = get_module_level_config() - _daemon_server_address = f"{_cfg['host']}:{_cfg['port']}" - root = f"http://{_daemon_server_address}" - if not api.startswith("/"): - api = f"/{api}" - return f"{root}{api}" - - -# singleton -class ClientConn(metaclass=Singleton): - # http client - http: httpx.Client = _load_http_client() - - def post(api: str, root: str = None, *args, **kwargs): - url = addr_of_api(api) - return ClientConn.http.post(url, *args, **kwargs) - - def get(api: str, root: str = None, *args, **kwargs): - url = addr_of_api(api) - return ClientConn.http.get(url, *args, **kwargs) - - def put(api: str, root: str = None, *args, **kwargs): - url = addr_of_api(api) - return ClientConn.http.put(url, *args, **kwargs) - - def delete(api: str, root: str = None, *args, **kwargs): - url = addr_of_api(api) - return ClientConn.http.delete(url, *args, **kwargs) - - __ws_client: websocket.WebSocketApp = None # _websocket_client - __ws_subscription = defaultdict(lambda: {}) # default to no subscribers - - def ws_subscribe(event_type_name: str, name: str = None): - """let a function subscribe to ws messages with event type name. - !!! dfor inner APIs only, do not use this in your code! - !!! developers should contorl blocking on their own functions - - Args: - function (Callable): who is subscribing the event type - event_type_name (str, optional): Which event to listen. Defaults to None. - """ - return functools.partial( - ClientConn._ws_subscribe, event_type_name=event_type_name, name=name - ) - - def _ws_subscribe(function: Callable, event_type_name: str, name=None): - name = name or function.__name__ - ClientConn.__ws_subscription[event_type_name][name] = function - # logger.info(f"ws: {name} subscribed to '{event_type_name}'") - - @classmethod - def _init_ws(cls): - global _ws_initialized - if _ws_initialized: - return - - cfg = get_module_level_config() - # ws server url - host = cfg["host"] - port = cfg["port"] + 1 - cls.ws_server_addr = f"ws://{host}:{port}" - - # create websocket app - logger.log(f"creating websocket connection to {cls.ws_server_addr}", skip_writers=["ws"]) - cls.wsApp = websocket.WebSocketApp( - url=cls.ws_server_addr, - on_open=cls.__on_ws_open, - on_message=cls.__on_ws_message, - on_error=cls.__on_ws_err, - on_close=cls.__on_ws_close, - ) - - Thread( - target=cls.wsApp.run_forever, kwargs={"reconnect": 1}, daemon=True - ).start() # initialize and start ws thread - - _ws_initialized = True - - def __on_ws_open(ws: websocket.WebSocketApp): - project_id = get_project_id() - logger.ok(f"client websocket connected. sending handshake as '{project_id}'...") - handshake_msg = EventMsg( # handshake request message - project_id=project_id, - run_id=get_run_id(), - event_type=EVENT_TYPE_NAME_HANDSHAKE, - who=IdentityType.CLI, - event_id=0, - ).dumps() - ws.send(handshake_msg) - - def __on_ws_err(ws: websocket.WebSocketApp, msg): - logger.err(f"client websocket encountered {msg}") - - def __on_ws_close(ws: websocket.WebSocketApp, close_status_code, close_msg): - logger.warn(f"client websocket closed") - if close_status_code or close_msg: - logger.warn(f"ws close status code: {close_status_code}") - logger.warn("ws close message: {close_msg}") - ClientConn.__ws_client = None - - def __on_ws_message(ws: websocket.WebSocketApp, message): - message = EventMsg.loads(message) # message should be json - if message.event_type == EVENT_TYPE_NAME_HANDSHAKE: - assert message.payload["result"] == 200 - logger.ok(f"handshake succeed.") - ws.send( - EventMsg( - project_id=get_project_id(), - event_id=message.event_id, - event_type=EVENT_TYPE_NAME_STATUS, - series="config", - run_id=get_run_id(), - payload=get_module_level_config("@"), - ).dumps() - ) - ClientConn.__ws_client = ws - # return # DO NOT return! - if message.event_type not in ClientConn.__ws_subscription: - logger.warn( - f"Client received a(n) {message.event_type} event but nobody subscribes it. Ignoring anyway." - ) - for name, subscriber in ClientConn.__ws_subscription[message.event_type].items(): - try: - subscriber(message) # pass payload message into subscriber - except Exception as e: - # subscriber throws error - logger.err( - f"Subscriber {name} crashed on message event {message.event_type}, ignoring." - ) - - @classmethod - def _ws_send(cls, message: EventMsg): - cls.__ws_client.send(message.dumps()) - - _ws_message_query = [] - - @classmethod - def ws_send( - cls, - event_type: str, - payload: dict, - series=None, - timestamp: str = None, - event_id=-1, - _history_len=-1, - ): - message = EventMsg( - project_id=get_project_id(), - run_id=get_run_id(), - event_type=event_type, - event_id=event_id, - who=IdentityType.CLI, - series=series, - payload=payload, - timestamp=timestamp or get_timestamp(), - history_len=_history_len, - ) - cls._ws_message_query.append(message) - send_result = "" - if cls.__ws_client is not None: # if ws client exist - try: - while len(cls._ws_message_query): - cls._ws_send(message=cls._ws_message_query[0]) - cls._ws_message_query.pop(0) - return - except Exception as e: - send_result = f"{e}" - else: - send_result = "web socket not connected" - logger.warn(f"websocket send fialed: {send_result}, message appended to query.") - - -# assign this connection to websocket log writer -from neetbox.logging._writer import _assign_connection_to_WebSocketLogWriter - -_assign_connection_to_WebSocketLogWriter(ClientConn) -connection = ClientConn - - -def check_server_connectivity(cfg=None): - _cfg = cfg or get_module_level_config() - logger.log(f"Connecting to daemon at {_cfg['host']}:{_cfg['port']} ...") - _daemon_server_address = f"{_cfg['host']}:{_cfg['port']}" - _base_addr = f"http://{_daemon_server_address}" - - # check if daemon is alive - def _check_daemon_alive(_api_addr): - _api_name = "hello" - _api_addr = f"{_api_addr}/{_api_name}" - r = connection.http.get(_api_addr) - if r.is_error: - raise IOError(f"Daemon at {_api_addr} is not alive. ({r.status_code})") - - try: - _check_daemon_alive(_base_addr) - logger.ok(f"daemon alive at {_base_addr}") - except Exception as e: - logger.err(e) - return False - - connection._init_ws() - return True diff --git a/neetbox/_daemon/readme.md b/neetbox/_daemon/readme.md deleted file mode 100644 index 7f077b32..00000000 --- a/neetbox/_daemon/readme.md +++ /dev/null @@ -1,163 +0,0 @@ -# Testing DAEMON - -NEETBOX daemon consists of client side and server side. While client side syncs status of running project and platform information including hardware, server side provides apis for status monitoring and websocket forcasting between client and frontends. - -Basically neetbox will also launch a backend on localhost when a project launching configured with daemon server address at localhost. The server will run in background without any output, and you may want to run a server with output for debug purposes. - -## How to test neetbox server - -at neetbox project root: - -```bash -python neetbox/daemon/server/_server.py -``` - -script above should launch a server in debug mode on `localhost:5000`, it wont read the port in `neetbox.toml`. a swegger UI is provided at [localhost:5000/docs](http://127.0.0.1:5000/docs) in debug mode. websocket server should run on port `5001`. - -If you want to simulate a basic neetbox client sending message to server, at neetbox project root: - -```bash -cd tests/client -python test.py -``` - -script above should launch a simple case of neetbox project with some logs and status sending to server. - -## Websocket message standard - -websocke messages are described in json. There is a dataclass representing websocket message: - -```python -EVENT_TYPE_NAME_KEY = "event-type" -EVENT_ID_NAME_KEY = "event-id" -NAME_NAME_KEY = "name" -PAYLOAD_NAME_KEY = "payload" - - -@dataclass -class WsMsg: - name: str - event_type: str - payload: Any - event_id: int = -1 - - def json(self): - return { - NAME_NAME_KEY: self.name, - EVENT_TYPE_NAME_KEY: self.event_type, - EVENT_ID_NAME_KEY: self.event_id, - PAYLOAD_NAME_KEY: self.payload, - } -``` - -```json -{ - "name" : ..., - "event-type" : ..., - "payload" : ..., - "event-id" : ... -} -``` - -an simple websocket message should include: - -| key | value type | description | -| :--------: | :--------: | :----------------------------------------------------: | -| name | string | project name | -| event-type | string | indicate type of data in payload | -| payload | string | actual data | -| event-id | int | for events who need ack. default -1 means no event id. | - -## Event types - -the table is increasing. a frequent check would keep you up to date. - -| event-type | accepting direction | means | -| :--------: | :---------------------------: | :----------------------------------------------------------: | -| handshake | cli <--> server <--> frontend | string in `payload` indicate connection type ('cli'/'web') | -| log | cli -> server -> frontend | `payload` contains log data | -| action | cli <- server <- frontend | `payload` contains action trigger | -| ack | cli <--> server <--> frontend | `payload` contains ack, and `event-id` should be a valid key | - -## Examples of websocket data - -### handshake - -for instance, frontend connected to server. frontend should report connection type immediately by sending: - -```json -{ - "event-type": "handshake", - "name": "project name", - "payload": { - "who": "web" - }, - "event-id": X -} -``` - -where `event-id` is used to send ack to the starter of the connection, it should be a random int value. - -### cli sending log to frontend - -cli sents log(s) via websocket, server will receives and broadcast this message to related frontends. cli should send: - -```json -{ - "event-type": "log", - "name": "project name", - "payload": { - "log" : {...json representing log data...} - }, - "event-id": -1 -} -``` - -where `event-id` is a useless segment, leave it default. it's okay if nobody receives log. - -### frontend(s) querys action to cli - -frontend send action request to server, and server will forwards the message to cli. frontend should send: - -```json -{ - "event-type" : "action", - "name": "project name", - "payload" : { - "name" : , - "args" : {...arg names and values...} - }, - "event-id" : x -} -``` - -front may want to know the result of action. for example, whether the action was invoked successfully. therefore, `event-id` is necessary for cli to shape a ack response. - -### cli acks frontend action query - -cli execute action query(s) from frontend, and gives response by sending ack: - -```json -{ - "event-type" : "action", - "name": "project name", - "payload" : { - "name" : , - "result" : - }, - "event-id" : x -} -``` - -> CAUTION ! -> -> - frontend should look for list of actions via `/status` api instead of websocket. -> - when **frontend** receive websocket message with `event-type` = `action`, it must be the action result returned from client. -> - when **client** receive websocket message with `event-type` = `action`, it must be the action queried by frontend. -> - only actions with `blocking` = `true` could return result to frontend. - -where `event-id` is same as received action query. - ---- - -Those are only examples. use them wisely. diff --git a/neetbox/_daemon/_protocol.py b/neetbox/_protocol.py similarity index 100% rename from neetbox/_daemon/_protocol.py rename to neetbox/_protocol.py diff --git a/neetbox/cli/parse.py b/neetbox/cli/parse.py index 8a2903a0..fec74bf3 100644 --- a/neetbox/cli/parse.py +++ b/neetbox/cli/parse.py @@ -5,9 +5,8 @@ from rich.console import Console from rich.table import Table -import neetbox._daemon as daemon_module import neetbox.config._cli_user_config as cliconfig -from neetbox._daemon.server._server import server_process +from neetbox.client._client_web_apis import * from neetbox.config._workspace import ( _get_module_level_config, _init_workspace, @@ -15,17 +14,16 @@ ) from neetbox.logging.formatting import LogStyle from neetbox.logging.logger import Logger +from neetbox.server._server import server_process from neetbox.utils.massive import check_read_toml -from ._client_web_apis import * - console = Console() logger = Logger("NEETBOX", style=LogStyle(with_datetime=False, skip_writers=["ws", "file"])) -def get_daemon_config(): - return _get_module_level_config(daemon_module) +def get_connection_config(): + return _get_module_level_config("client") # def get_base_addr(port=0): @@ -107,7 +105,7 @@ def status_command(name): def serve(port, debug): """serve neetbox server in attached mode""" _try_load_workspace_if_applicable() - _daemon_config = get_daemon_config() + _daemon_config = get_connection_config() try: logger.log(f"Launching server using config: {_daemon_config}") if port: @@ -131,7 +129,7 @@ def shutdown_server(port): """shutdown neetbox server on specific port""" _try_load_workspace_if_applicable() try: - daemon_config = get_daemon_config() + daemon_config = get_connection_config() if port: daemon_config["port"] = port _response = shutdown() diff --git a/neetbox/client/__init__.py b/neetbox/client/__init__.py index cd3a11e2..31f2c431 100644 --- a/neetbox/client/__init__.py +++ b/neetbox/client/__init__.py @@ -1,21 +1,24 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20231206 -from ._client_action_agent import _NeetActionManager as NeetActionManager +from ._action_agent import _NeetActionManager as NeetActionManager +from ._client import connection +from ._image import add_figure, add_image +from ._plot import add_hyperparams, add_scalar from ._signal_and_slot import listen, watch -from ._writers._image import add_figure, add_image -from ._writers._plot import add_hyperparams, add_scalar action = NeetActionManager.register +ws_subscribe = connection.ws_subscribe __all__ = [ "add_image", "add_scalar", "add_figure", "add_hyperparams", + "ws_subscribe", "action", "NeetActionManager", "watch", diff --git a/neetbox/client/_client_action_agent.py b/neetbox/client/_action_agent.py similarity index 98% rename from neetbox/client/_client_action_agent.py rename to neetbox/client/_action_agent.py index 336ae010..6295d702 100644 --- a/neetbox/client/_client_action_agent.py +++ b/neetbox/client/_action_agent.py @@ -4,12 +4,13 @@ from threading import Thread from typing import Callable, Optional -from neetbox._daemon._protocol import * -from neetbox._daemon.client._client import connection +from neetbox._protocol import * from neetbox.core import Registry from neetbox.logging import logger from neetbox.utils.mvc import Singleton +from ._client import Connection as connection + class PackedAction(Callable): def __init__( diff --git a/neetbox/client/_client.py b/neetbox/client/_client.py new file mode 100644 index 00000000..16649694 --- /dev/null +++ b/neetbox/client/_client.py @@ -0,0 +1,293 @@ +import functools +import json +import logging +import subprocess +import time +from collections import defaultdict +from threading import Thread +from typing import Callable + +import httpx +import websocket + +from neetbox._protocol import * +from neetbox.config import get_module_level_config, get_project_id, get_run_id +from neetbox.logging.formatting import LogStyle +from neetbox.logging.logger import Logger +from neetbox.server._daemonable_process import DaemonableProcess +from neetbox.utils.massive import is_loopback +from neetbox.utils.mvc import Singleton + +logger = Logger(whom=None, style=LogStyle(skip_writers=["ws"])) + +httpx_logger = logging.getLogger("httpx") +httpx_logger.setLevel(logging.ERROR) + + +def addr_of_api(api, http_root=None): + if not http_root: + config = get_module_level_config() + daemon_server_address = f"{config['host']}:{config['port']}" + http_root = f"http://{daemon_server_address}" + if not api.startswith("/"): + api = f"/{api}" + return f"{http_root}{api}" + + +class Connection(metaclass=Singleton): # singleton + # statics + online_mode: bool = None + httpxClient: httpx.Client = httpx.Client( # httpx client + proxies={ + "http://": None, + "https://": None, + } + ) # type: ignore + wsApp: websocket.WebSocketApp = None # websocket client app + __initialized: bool = False + is_ws_connected: bool = False + ws_message_query = [] # websocket message query + ws_subscribers = defaultdict(list) # default to no subscribers + + @classmethod + def post(cls, api: str, root: str = None, *args, **kwargs): + url = addr_of_api(api, http_root=root) + return Connection.httpxClient.post(url, *args, **kwargs) + + @classmethod + def get(cls, api: str, root: str = None, *args, **kwargs): + url = addr_of_api(api, http_root=root) + return Connection.httpxClient.get(url, *args, **kwargs) + + @classmethod + def put(cls, api: str, root: str = None, *args, **kwargs): + url = addr_of_api(api, http_root=root) + return Connection.httpxClient.put(url, *args, **kwargs) + + @classmethod + def delete(cls, api: str, root: str = None, *args, **kwargs): + url = addr_of_api(api, http_root=root) + return Connection.httpxClient.delete(url, *args, **kwargs) + + @classmethod + def ws_subscribe(cls, event_type_name: str): + """let a function subscribe to ws messages with event type name. + !!! dfor inner APIs only, do not use this in your code! + !!! developers should contorl blocking on their own functions + + Args: + function (Callable): who is subscribing the event type + event_type_name (str, optional): Which event to listen. Defaults to None. + """ + + def _ws_subscribe(function: Callable): + Connection.ws_subscribers[event_type_name].append(function) + # logger.info(f"ws: {name} subscribed to '{event_type_name}'") + return function + + return _ws_subscribe + + @classmethod + def check_server_connectivity(cls, config=None): + config = config or get_module_level_config() + logger.log(f"Connecting to daemon at {config['host']}:{config ['port']} ...") + daemon_server_address = f"{config['host']}:{config['port']}" + http_root = f"http://{daemon_server_address}" + + # check if daemon is alive + def fetch_hello(root): + try: + r = cls.get(api="hello", root=root) + assert r.json()["hello"] == "hello" + except: + raise IOError(f"Daemon at {root} is not alive. ({r. status_code})") + + try: + fetch_hello(http_root) + logger.ok(f"daemon alive at {http_root}") + return True + except Exception as e: + logger.err(e) + return False + + @classmethod + def _connect(cls, config=None): + if cls.__initialized: + return # if already initialized, do nothing + + config = config or get_module_level_config() + if not config["enable"]: # check if enable + cls.online_mode = False + cls.__initialized = True + return + + if not config["allowIpython"]: # check if allow ipython + try: + eval("__IPYTHON__") # check if in ipython + except NameError: # not in ipython + pass + else: # in ipython + logger.info( + "NEETBOX DAEMON won't start when debugging in ipython console. If you want to allow daemon run in " + "ipython, try to set 'allowIpython' to True." + ) + cls.online_mode = False + cls.__initialized = True + return # ignore if debugging in ipython + + server_host = config["host"] + server_port = config["port"] + if not cls.check_server_connectivity(): # if daemon not online + if not is_loopback(server_host): # daemon not running on localhost + logger.err( + RuntimeError( + f"No daemon running at {server_host}:{server_port}, daemon will not be attached, stopping..." + ), + reraise=True, + ) + # connecting localhost but no server alive, create one + logger.log( + f"No daemon running on {server_host}:{server_port}, trying to create daemon..." + ) + popen = DaemonableProcess( # server daemon + target="neetbox.server._daemon_launcher", + args=["--config", json.dumps(config)], + mode=config["mode"], + redirect_stdout=subprocess.DEVNULL if config["mute"] else None, + env_append={"NEETBOX_DAEMON_PROCESS": "1"}, + ).start() + time.sleep(1) + _retry_timeout = 10 + _time_begin = time.perf_counter() + logger.log("Created daemon process, trying to connect to daemon...") + + while time.perf_counter() - _time_begin < 10: # try connect daemon + if cls.check_server_connectivity(): + break + else: + exit_code = popen.poll() + if exit_code is not None: + logger.err( + f"Daemon process exited unexpectedly with exit code {exit_code}." + ) + return False + time.sleep(0.5) + logger.err( + RuntimeError(f"Failed to connect to daemon after {_retry_timeout}s, stopping..."), + reraise=True, + ) + + cls.online_mode = True # enable online mode + cls.ws_server_url = f"ws://{server_host}:{server_port + 1}" # ws server url + + logger.log(f"creating websocket connection to {cls.ws_server_url}") + cls.wsApp = websocket.WebSocketApp( # create websocket client + url=cls.ws_server_url, + on_open=cls.__on_ws_open, + on_message=cls.__on_ws_message, + on_error=cls.__on_ws_err, + on_close=cls.__on_ws_close, + ) + + Thread( + target=cls.wsApp.run_forever, kwargs={"reconnect": 1}, daemon=True + ).start() # initialize and start ws thread + + cls.__initialized = True + + def __on_ws_open(ws: websocket.WebSocketApp): + project_id = get_project_id() + logger.ok(f"client websocket connected. sending handshake as '{project_id}'...") + handshake_msg = EventMsg( # handshake request message + project_id=project_id, + run_id=get_run_id(), + event_type=EVENT_TYPE_NAME_HANDSHAKE, + who=IdentityType.CLI, + event_id=0, + ).dumps() + ws.send(handshake_msg) + + def __on_ws_err(ws: websocket.WebSocketApp, msg): + logger.err(f"client websocket encountered {msg}") + + def __on_ws_close(ws: websocket.WebSocketApp, close_status_code, close_msg): + logger.warn(f"client websocket closed") + if close_status_code or close_msg: + logger.warn(f"ws close status code: {close_status_code}") + logger.warn("ws close message: {close_msg}") + Connection.is_ws_connected = False + + def __on_ws_message(ws: websocket.WebSocketApp, message): + message = EventMsg.loads(message) # message should be json + if message.event_type == EVENT_TYPE_NAME_HANDSHAKE: + assert message.payload["result"] == 200 + logger.ok(f"handshake succeed.") + ws.send( # send immediately without querying + EventMsg( + project_id=get_project_id(), + event_id=message.event_id, + event_type=EVENT_TYPE_NAME_STATUS, + series="config", + run_id=get_run_id(), + payload=get_module_level_config("@"), + ).dumps() + ) + Connection.is_ws_connected = True + # return # DO NOT return! + if message.event_type not in Connection.ws_subscribers: + logger.warn( + f"Client received a(n) {message.event_type} event but nobody subscribes it. Ignoring anyway." + ) + for subscriber in Connection.ws_subscribers[message.event_type]: + try: + subscriber(message) # pass payload message into subscriber + except Exception as e: + # subscriber throws error + logger.err( + f"Subscriber {subscriber.__name__} crashed on message event {message.event_type}, ignoring." + ) + + @classmethod + def ws_send( + cls, + event_type: str, + payload: dict, + series=None, + timestamp: str = None, + event_id=-1, + _history_len=-1, + ): + if not cls.__initialized: + cls._connect() + if cls.__initialized and cls.online_mode == False: # if online mode is false + while len(cls.ws_message_query): + cls.ws_message_query.pop() + return + message = EventMsg( + project_id=get_project_id(), + run_id=get_run_id(), + event_type=event_type, + event_id=event_id, + who=IdentityType.CLI, + series=series, + payload=payload, + timestamp=timestamp or get_timestamp(), + history_len=_history_len, + ) + cls.ws_message_query.append(message) + if cls.is_ws_connected: # if ws client exist + try: + while len(cls.ws_message_query): + cls.wsApp.send(cls.ws_message_query[0].dumps()) + cls.ws_message_query.pop(0) + return + except Exception as e: + pass # todo what to do + + +connection = Connection + +# assign this connection to websocket log writer +from neetbox.logging._writer import _assign_connection_to_WebSocketLogWriter + +_assign_connection_to_WebSocketLogWriter(Connection) diff --git a/neetbox/cli/_client_web_apis.py b/neetbox/client/_client_web_apis.py similarity index 86% rename from neetbox/cli/_client_web_apis.py rename to neetbox/client/_client_web_apis.py index 46ae81af..ed698e24 100644 --- a/neetbox/cli/_client_web_apis.py +++ b/neetbox/client/_client_web_apis.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230414 -from neetbox._daemon._protocol import * -from neetbox._daemon.client._client import connection +from neetbox._protocol import * +from neetbox.client import connection def _get(api, root=None): diff --git a/neetbox/client/_writers/_image.py b/neetbox/client/_image.py similarity index 98% rename from neetbox/client/_writers/_image.py rename to neetbox/client/_image.py index 2ae0a683..f34d61f5 100644 --- a/neetbox/client/_writers/_image.py +++ b/neetbox/client/_image.py @@ -11,12 +11,13 @@ import numpy as np from PIL import Image -from neetbox._daemon._protocol import * -from neetbox._daemon.client._client import connection +from neetbox._protocol import * from neetbox.config import get_project_id, get_run_id from neetbox.logging import logger from neetbox.utils.x2numpy import * +from ._client import connection + # ===================== IMAGE things ===================== # diff --git a/neetbox/client/_writers/_plot.py b/neetbox/client/_plot.py similarity index 85% rename from neetbox/client/_writers/_plot.py rename to neetbox/client/_plot.py index c9fbb50f..bea022f2 100644 --- a/neetbox/client/_writers/_plot.py +++ b/neetbox/client/_plot.py @@ -1,17 +1,14 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20231211 -from typing import Optional - -import numpy as np - -from neetbox._daemon._protocol import * -from neetbox._daemon.client._client import connection +from neetbox._protocol import * from neetbox.utils.x2numpy import * +from ._client import connection + # ===================== PLOTTING things ===================== # diff --git a/neetbox/client/_signal_and_slot.py b/neetbox/client/_signal_and_slot.py index d0ea8426..83da8287 100644 --- a/neetbox/client/_signal_and_slot.py +++ b/neetbox/client/_signal_and_slot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230417 import collections diff --git a/neetbox/config/__init__.py b/neetbox/config/__init__.py index d424032a..afa5a898 100644 --- a/neetbox/config/__init__.py +++ b/neetbox/config/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230413 diff --git a/neetbox/config/_workspace.py b/neetbox/config/_workspace.py index 9ad1f253..da5c1e84 100644 --- a/neetbox/config/_workspace.py +++ b/neetbox/config/_workspace.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230413 @@ -11,6 +11,7 @@ import types import uuid from importlib.metadata import version +from threading import Thread from typing import Union from uuid import uuid4 @@ -30,14 +31,14 @@ "extension": { "autoload": True, }, - "daemon": { + "client": { "enable": True, "host": "127.0.0.1", "port": 20202, "allowIpython": False, "mute": True, "mode": "detached", - "uploadInterval": 0.5, + "uploadInterval": 1, }, } @@ -55,9 +56,23 @@ def _obtain_new_run_id(): _QUERY_ADD_EXTENSION_DEFAULT_CONFIG = [] +_IS_WORKSPACE_LOADED = False +_QUERY_AFTER_CONFIG_LOAD = [] + + +def on_config_loaded(func): + if _IS_WORKSPACE_LOADED: # if workspace already loaded + Thread(target=func, daemon=True).start() # run target + else: # not loaded yet, append to query + global _QUERY_AFTER_CONFIG_LOAD + _QUERY_AFTER_CONFIG_LOAD.append(func) + return func + def export_default_config(func): global _QUERY_ADD_EXTENSION_DEFAULT_CONFIG + if _IS_WORKSPACE_LOADED: + raise RuntimeError("should not add default config after workspace loaded") _QUERY_ADD_EXTENSION_DEFAULT_CONFIG.append(func) return func @@ -166,11 +181,8 @@ def _check_if_workspace_config_valid(path=None) -> bool: return False -_IS_EXTENSION_INITED = False - - def _load_workspace_config(folder="."): - global _IS_EXTENSION_INITED + global _IS_WORKSPACE_LOADED config_file_path = _check_if_workspace_config_valid( path=folder ) # check if config file is valid @@ -181,16 +193,23 @@ def _load_workspace_config(folder="."): extension._scan_sub_modules() _update_default_config_from_config_register() # load custom config into default config _obtain_new_run_id() # obtain new run id - _update_default_workspace_config_with(toml.load(config_file_path)) # load config file in - if not _IS_EXTENSION_INITED: - is_in_daemon_process = ( - "NEETBOX_DAEMON_PROCESS" in os.environ and os.environ["NEETBOX_DAEMON_PROCESS"] == "1" + config_from_file = toml.load(config_file_path) + if "version" not in config_from_file or config_from_file["version"] != NEETBOX_VERSION: + raise RuntimeError( + f"config file version not match: using neetbox version {NEETBOX_VERSION} but got config from version {config_from_file['version']}" ) - if len(sys.argv) > 0 and sys.argv[0].endswith("neet") or is_in_daemon_process: - pass - else: - extension._init_extensions() - _IS_EXTENSION_INITED = True + _update_default_workspace_config_with(config_from_file) # load config file in + + if ( + len(sys.argv) > 0 + and sys.argv[0].endswith("neet") + or ("NEETBOX_DAEMON_PROCESS" in os.environ and os.environ["NEETBOX_DAEMON_PROCESS"] == "1") + ): + pass + else: # on workspace loaded + _IS_WORKSPACE_LOADED = True + for func in _QUERY_AFTER_CONFIG_LOAD: + Thread(target=func, daemon=True).start() def _create_load_workspace(folder="."): diff --git a/neetbox/core/registry.py b/neetbox/core/registry.py index 3e5be4d9..14c933f2 100644 --- a/neetbox/core/registry.py +++ b/neetbox/core/registry.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230413 import functools diff --git a/neetbox/extension/__init__.py b/neetbox/extension/__init__.py index 3c5dc215..52c48562 100644 --- a/neetbox/extension/__init__.py +++ b/neetbox/extension/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230417 import importlib @@ -10,6 +10,7 @@ from neetbox.config._workspace import ( _get_module_level_config as get_module_level_config, ) +from neetbox.config._workspace import on_config_loaded from neetbox.core import Registry from neetbox.utils.framing import get_frame_module_traceback @@ -23,6 +24,7 @@ def _scan_sub_modules(): importlib.import_module(f"{__THIS_MODULE.__name__}.{sub_module_info.name}") +@on_config_loaded def _init_extensions(): """ DO NOT call before workspace config load diff --git a/neetbox/extension/environment/hardware.py b/neetbox/extension/environment/hardware.py index 09cd12f2..9da5a982 100644 --- a/neetbox/extension/environment/hardware.py +++ b/neetbox/extension/environment/hardware.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230413 @@ -94,8 +94,8 @@ def set_update_intervel(self, intervel=1.0) -> None: self._do_watch = True self._update_interval = intervel if not self._watcher or not self._watcher.is_alive(): - from neetbox._daemon._protocol import EVENT_TYPE_NAME_HARDWARE - from neetbox._daemon.client._client import connection + from neetbox._protocol import EVENT_TYPE_NAME_HARDWARE + from neetbox.client import connection def watcher_fun(env_instance: _Hardware, do_update_gpus: bool): while env_instance._do_watch: @@ -139,12 +139,12 @@ def watcher_fun(env_instance: _Hardware, do_update_gpus: bool): @export_default_config def return_default_config() -> dict: - return {"monit": True, "interval": 2} + return {"interval": 2} # watch updates in daemon @on_workspace_loaded(name="hardware-monit") def load_monit_hardware(): cfg = get_module_level_config() - if cfg["monit"]: # if do monit hardware + if cfg["interval"] > 0: # if do monit hardware hardware.set_update_intervel(cfg["interval"]) diff --git a/neetbox/extension/environment/platform.py b/neetbox/extension/environment/platform.py index a35809c5..0ffdf6ed 100644 --- a/neetbox/extension/environment/platform.py +++ b/neetbox/extension/environment/platform.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230417 import getpass @@ -53,12 +53,12 @@ def exec(self, command): # watch updates in daemon @on_workspace_loaded(name="show-platform-information") def load_send_platform_info(): - from neetbox._daemon._protocol import ( + from neetbox._protocol import ( EVENT_TYPE_NAME_HANDSHAKE, EVENT_TYPE_NAME_STATUS, EventMsg, ) - from neetbox._daemon.client._client import connection + from neetbox.client import connection platform = __Platform() diff --git a/neetbox/extension/torch/arch/cnn.py b/neetbox/extension/torch/arch/cnn.py index 430d7cac..6ace19be 100644 --- a/neetbox/extension/torch/arch/cnn.py +++ b/neetbox/extension/torch/arch/cnn.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230315 import torch.nn as nn diff --git a/neetbox/extension/torch/profile.py b/neetbox/extension/torch/profile.py index 45ad7404..ee6db097 100644 --- a/neetbox/extension/torch/profile.py +++ b/neetbox/extension/torch/profile.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230315 import time diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index ef41bab8..064abec1 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230318 from dataclasses import dataclass diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index 86bb8a3c..7a139deb 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230315 import functools diff --git a/neetbox/_daemon/server/_bridge.py b/neetbox/server/_bridge.py similarity index 98% rename from neetbox/_daemon/server/_bridge.py rename to neetbox/server/_bridge.py index c4504acd..2eccddb1 100644 --- a/neetbox/_daemon/server/_bridge.py +++ b/neetbox/server/_bridge.py @@ -1,15 +1,14 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20231204 -import json from typing import Dict from websocket_server import WebsocketServer -from neetbox._daemon._protocol import * +from neetbox._protocol import * from neetbox.logging import LogStyle, logger from .history import * diff --git a/neetbox/_daemon/server/_daemon_launcher.py b/neetbox/server/_daemon_launcher.py similarity index 92% rename from neetbox/_daemon/server/_daemon_launcher.py rename to neetbox/server/_daemon_launcher.py index bee0bcd3..8b3f4b80 100644 --- a/neetbox/_daemon/server/_daemon_launcher.py +++ b/neetbox/server/_daemon_launcher.py @@ -2,7 +2,7 @@ import json import sys -from neetbox._daemon.server._server import server_process +from neetbox.server._server import server_process # sys.stdout=open(r'D:\Projects\ML\neetbox\logdir\daemon.log', 'a+') diff --git a/neetbox/_daemon/server/_daemonable_process.py b/neetbox/server/_daemonable_process.py similarity index 100% rename from neetbox/_daemon/server/_daemonable_process.py rename to neetbox/server/_daemonable_process.py diff --git a/neetbox/_daemon/server/_fastapi_server.py b/neetbox/server/_fastapi_server.py similarity index 100% rename from neetbox/_daemon/server/_fastapi_server.py rename to neetbox/server/_fastapi_server.py diff --git a/neetbox/_daemon/server/_flask_server.py b/neetbox/server/_flask_server.py similarity index 98% rename from neetbox/_daemon/server/_flask_server.py rename to neetbox/server/_flask_server.py index bef82d48..b675a8ad 100644 --- a/neetbox/_daemon/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20231204 import logging @@ -13,8 +13,8 @@ import werkzeug from flask import Response, abort, json, redirect, request, send_from_directory -from neetbox._daemon._protocol import * -from neetbox._daemon.server._bridge import Bridge +from neetbox._protocol import * +from neetbox.server._bridge import Bridge from .history import * diff --git a/neetbox/_daemon/server/_server.py b/neetbox/server/_server.py similarity index 76% rename from neetbox/_daemon/server/_server.py rename to neetbox/server/_server.py index 2a720219..d922a22c 100644 --- a/neetbox/_daemon/server/_server.py +++ b/neetbox/server/_server.py @@ -1,16 +1,16 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230414 import setproctitle -from neetbox._daemon._protocol import * -from neetbox._daemon.server._bridge import Bridge -from neetbox._daemon.server._flask_server import get_flask_server -from neetbox._daemon.server._websocket_server import get_web_socket_server -from neetbox._daemon.server.history import * +from neetbox._protocol import * +from neetbox.server._bridge import Bridge +from neetbox.server._flask_server import get_flask_server +from neetbox.server._websocket_server import get_web_socket_server +from neetbox.server.history import * def server_process(cfg, debug=False): diff --git a/neetbox/_daemon/server/_websocket_server.py b/neetbox/server/_websocket_server.py similarity index 98% rename from neetbox/_daemon/server/_websocket_server.py rename to neetbox/server/_websocket_server.py index 0957665f..1c4360ab 100644 --- a/neetbox/_daemon/server/_websocket_server.py +++ b/neetbox/server/_websocket_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20231204 from typing import Dict, Tuple @@ -10,7 +10,7 @@ from rich.console import Console from rich.table import Table -from neetbox._daemon._protocol import * +from neetbox._protocol import * from .history import * @@ -18,9 +18,9 @@ def get_web_socket_server(config, debug=False): from websocket_server import WebsocketServer - from neetbox._daemon.server._bridge import Bridge from neetbox.logging import LogStyle from neetbox.logging.logger import Logger + from neetbox.server._bridge import Bridge console = Console() logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) diff --git a/neetbox/_daemon/server/history/__init__.py b/neetbox/server/history/__init__.py similarity index 82% rename from neetbox/_daemon/server/history/__init__.py rename to neetbox/server/history/__init__.py index 33ec0b74..55770d4c 100644 --- a/neetbox/_daemon/server/history/__init__.py +++ b/neetbox/server/history/__init__.py @@ -3,14 +3,8 @@ from neetbox.utils import ResourceLoader from ._db import ( - BLOB_COLUMN_NAME, HISTORY_FILE_ROOT, HISTORY_FILE_TYPE_NAME, - ID_COLUMN_NAME, - JSON_COLUMN_NAME, - METADATA_COLUMN_NAME, - SERIES_COLUMN_NAME, - TIMESTAMP_COLUMN_NAME, DBConnection, DbQueryFetchType, DbQuerySortType, @@ -59,10 +53,4 @@ def get_db_of_id(project_id, rescan: bool = True): "load_db_of_path", "HISTORY_FILE_ROOT", "HISTORY_FILE_TYPE_NAME", - "ID_COLUMN_NAME", - "TIMESTAMP_COLUMN_NAME", - "SERIES_COLUMN_NAME", - "JSON_COLUMN_NAME", - "METADATA_COLUMN_NAME", - "BLOB_COLUMN_NAME", ] diff --git a/neetbox/_daemon/server/history/_db.py b/neetbox/server/history/_db.py similarity index 99% rename from neetbox/_daemon/server/history/_db.py rename to neetbox/server/history/_db.py index 03917c03..54c42553 100644 --- a/neetbox/_daemon/server/history/_db.py +++ b/neetbox/server/history/_db.py @@ -3,10 +3,9 @@ import os import sqlite3 from datetime import datetime -from random import random from typing import Dict, Tuple, Union -from neetbox._daemon._protocol import * +from neetbox._protocol import * from neetbox.logging import LogStyle from neetbox.logging.logger import Logger diff --git a/neetbox/utils/__init__.py b/neetbox/utils/__init__.py index cf0fbfa8..37becd62 100644 --- a/neetbox/utils/__init__.py +++ b/neetbox/utils/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20231206 from ._package import pipPackageHealper as pkg diff --git a/neetbox/utils/_resource.py b/neetbox/utils/_resource.py index 9993b0f9..0dbd873f 100644 --- a/neetbox/utils/_resource.py +++ b/neetbox/utils/_resource.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230315 import os diff --git a/neetbox/utils/formatting.py b/neetbox/utils/formatting.py index 728d6e1b..4a9e5263 100644 --- a/neetbox/utils/formatting.py +++ b/neetbox/utils/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230319 import json diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index 4e4999aa..549d4927 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230319 import inspect diff --git a/neetbox/utils/massive.py b/neetbox/utils/massive.py index 8023b49a..5cc61d5b 100644 --- a/neetbox/utils/massive.py +++ b/neetbox/utils/massive.py @@ -1,11 +1,35 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# URL: https://github.com/visualDust +# Date: 20231216 + import functools import os +import socket +import struct import time from concurrent.futures import ThreadPoolExecutor _ThreadPoolExecutor = ThreadPoolExecutor() +def is_loopback(host): + loopback_checker = { + socket.AF_INET: lambda x: struct.unpack("!I", socket.inet_aton(x))[0] >> (32 - 8) == 127, + socket.AF_INET6: lambda x: x == "::1", + } + for family in (socket.AF_INET, socket.AF_INET6): + try: + r = socket.getaddrinfo(host, None, family, socket.SOCK_STREAM) + except socket.gaierror: + return False + for family, _, _, _, sockaddr in r: + if not loopback_checker[family](sockaddr[0]): + return False + return True + + def nonblocking(func): @functools.wraps(func) def _use_thread_pool(*args, **kwargs): diff --git a/neetbox/utils/mvc.py b/neetbox/utils/mvc.py index 2502c4b5..69a4cb53 100644 --- a/neetbox/utils/mvc.py +++ b/neetbox/utils/mvc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# URL: https://github.com/visualDust # Date: 20230416 diff --git a/tests/client/neetbox.toml b/tests/client/neetbox.toml index 81ec2cab..b7661bde 100644 --- a/tests/client/neetbox.toml +++ b/tests/client/neetbox.toml @@ -1,25 +1,22 @@ name = "client-test-script" -projectid = "a7a5572f-9bd8-4f3d-9dfc-78b01bd805ec" -version = "0.2.6" +version = "0.3.1" +projectid = "2757e5c6-7ae2-420c-91ed-85778a3cfd59" [logging] -level = "INFO" - -[pipeline] -updateInterval = 0.5 +level = "DEBUG" +logdir = "logs" [extension] autoload = true -[daemon] +[client] enable = true host = "127.0.0.1" -port = 20202 +port = 5000 allowIpython = false mute = true mode = "detached" -uploadInterval = 0.5 +uploadInterval = 1 [extension.environment.hardware] -monit = true interval = 2 diff --git a/tests/snake/neetbox.toml b/tests/snake/neetbox.toml index 7a642ff1..6958c00c 100644 --- a/tests/snake/neetbox.toml +++ b/tests/snake/neetbox.toml @@ -1,5 +1,6 @@ name = "snake" projectid = "cfaf7fd6-a82f-4de7-aa74-013a6e826bef" +version = "0.2.6" [logging] level = "INFO" @@ -10,7 +11,7 @@ updateInterval = 0.5 [extension] autoload = true -[daemon] +[client] enable = true host = "127.0.0.1" port = 5000 @@ -21,7 +22,4 @@ uploadInterval = 0.5 [extension.environment.hardware] monit = true -interval = 0.5 - -[extension.environment.platform] -monit = true +interval = 2 From 9e8309a2d1f92ebc69b0edc05472bb1d2c6e7257 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sat, 16 Dec 2023 21:38:30 +0800 Subject: [PATCH 10/20] fix wrong flask server static folder --- neetbox/server/_flask_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neetbox/server/_flask_server.py b/neetbox/server/_flask_server.py index b675a8ad..00ab8586 100644 --- a/neetbox/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -40,7 +40,7 @@ def get_flask_server(debug=False): app = Flask(__PROC_NAME, static_folder=None) - front_end_dist_path = os.path.join(os.path.dirname(__file__), "../../frontend_dist") + front_end_dist_path = os.path.join(os.path.dirname(__file__), "../frontend_dist") @app.route("/") def static_serve_root(): From ecb9f24b9a1a16f291621472feb8423800257525 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sat, 16 Dec 2023 22:50:01 +0800 Subject: [PATCH 11/20] patch progress --- neetbox/_protocol.py | 2 ++ neetbox/client/_client.py | 13 +++++-- neetbox/client/_progress.py | 67 +++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 neetbox/client/_progress.py diff --git a/neetbox/_protocol.py b/neetbox/_protocol.py index e18ba4ef..44909bbc 100644 --- a/neetbox/_protocol.py +++ b/neetbox/_protocol.py @@ -102,6 +102,7 @@ def merge(cls, x: Union["EventMsg", dict], y: Union["EventMsg", dict]): # known event names EVENT_TYPE_NAME_HANDSHAKE = "handshake" +EVENT_TYPE_NAME_WAVEHANDS = "wavehands" EVENT_TYPE_NAME_LOG = "log" EVENT_TYPE_NAME_ACTION = "action" EVENT_TYPE_NAME_SCALAR = "scalar" @@ -111,6 +112,7 @@ def merge(cls, x: Union["EventMsg", dict], y: Union["EventMsg", dict]): EVENT_TYPE_NAME_HPARAMS = "hyperparameters" EVENT_TYPE_NAME_STATUS = "status" EVENT_TYPE_NAME_HARDWARE = "hardware" +EVENT_TYPE_NAME_PROGRESS = "progress" # ===================== HTTP things ===================== diff --git a/neetbox/client/_client.py b/neetbox/client/_client.py index 16649694..00ad95c2 100644 --- a/neetbox/client/_client.py +++ b/neetbox/client/_client.py @@ -1,3 +1,4 @@ +import atexit import functools import json import logging @@ -18,11 +19,9 @@ from neetbox.utils.massive import is_loopback from neetbox.utils.mvc import Singleton +logging.getLogger("httpx").setLevel(logging.ERROR) logger = Logger(whom=None, style=LogStyle(skip_writers=["ws"])) -httpx_logger = logging.getLogger("httpx") -httpx_logger.setLevel(logging.ERROR) - def addr_of_api(api, http_root=None): if not http_root: @@ -287,6 +286,14 @@ def ws_send( connection = Connection + +def on_exit(): + if connection.wsApp is not None: + connection.wsApp.close() + logger.info("client disconnecting...", with_datetime=False) + + +atexit.register(on_exit) # assign this connection to websocket log writer from neetbox.logging._writer import _assign_connection_to_WebSocketLogWriter diff --git a/neetbox/client/_progress.py b/neetbox/client/_progress.py new file mode 100644 index 00000000..72a0e3af --- /dev/null +++ b/neetbox/client/_progress.py @@ -0,0 +1,67 @@ +from time import time + +from neetbox._protocol import * +from neetbox.utils.framing import get_frame_module_traceback + +from ._client import connection + + +class Progress: + def __init__(self, input, name=None): + if isinstance(input, int): + self.total = input + self.iterator = iter(range(input)) + else: + self.iterable = input + self.iterator = iter(input) + self.total = len(input) + if name is None: + name = get_frame_module_traceback().__name__ + self.name = name + self.done = 0 + self.start_time = time() # Track the start time + + def __enter__(self): + # print(f"entering {self.name}") + return self + + def __exit__(self, type, value, traceback): + # print(f"leaving {self.name}") + return + + def __iter__(self): + return self + + def __next__(self): + if self.done < self.total: + self.done += 1 + elapsed_time = time() - self.start_time + rate = ( + self.done / elapsed_time if elapsed_time > 0 else 0 + ) # Calculate the iteration rate + connection.ws_send( + event_type=EVENT_TYPE_NAME_PROGRESS, + series=self.name, + payload={"done": self.done, "total": self.total, "rate": rate}, + _history_len=1, + ) + print(f"{self.done}/{self.total}, {rate}/sec") + return next(self.iterator) + else: + raise StopIteration + + def __len__(self): + return self.total + + +# if __name__ == "__main__": +# from tqdm import tqdm +# from time import sleep + +# with Progress(10) as p: +# for i in p: +# sleep(0.1) # Simulate some work + +# with tqdm(Progress(range(10))) as p: +# for i in p: +# sleep(0.1) # Simulate some work From c8467ad817c0eb7ca7a840730e6a5a24f4f6a687 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 00:02:29 +0800 Subject: [PATCH 12/20] patch: progress --- neetbox/__init__.py | 2 + neetbox/client/__init__.py | 2 + neetbox/client/_progress.py | 21 +++++---- neetbox/utils/framing.py | 86 +++++++++++++++++++++---------------- tests/client/neetbox.toml | 2 +- tests/client/test.py | 9 ++-- 6 files changed, 68 insertions(+), 54 deletions(-) diff --git a/neetbox/__init__.py b/neetbox/__init__.py index 02eb089e..099abc57 100644 --- a/neetbox/__init__.py +++ b/neetbox/__init__.py @@ -12,6 +12,7 @@ add_scalar, listen, watch, + progress ) from .logging import logger @@ -24,4 +25,5 @@ "logger", "watch", "listen", + "progress" ] diff --git a/neetbox/client/__init__.py b/neetbox/client/__init__.py index 31f2c431..91b2f5c0 100644 --- a/neetbox/client/__init__.py +++ b/neetbox/client/__init__.py @@ -8,6 +8,7 @@ from ._client import connection from ._image import add_figure, add_image from ._plot import add_hyperparams, add_scalar +from ._progress import Progress as progress from ._signal_and_slot import listen, watch action = NeetActionManager.register @@ -23,4 +24,5 @@ "NeetActionManager", "watch", "listen", + "progress", ] diff --git a/neetbox/client/_progress.py b/neetbox/client/_progress.py index 72a0e3af..33df7d24 100644 --- a/neetbox/client/_progress.py +++ b/neetbox/client/_progress.py @@ -1,13 +1,13 @@ from time import time -from neetbox._protocol import * -from neetbox.utils.framing import get_frame_module_traceback - from ._client import connection +from neetbox._protocol import * +from neetbox.utils.framing import get_caller_identity_traceback + class Progress: - def __init__(self, input, name=None): + def __init__(self, input): if isinstance(input, int): self.total = input self.iterator = iter(range(input)) @@ -15,18 +15,17 @@ def __init__(self, input, name=None): self.iterable = input self.iterator = iter(input) self.total = len(input) - if name is None: - name = get_frame_module_traceback().__name__ - self.name = name + self.caller_identity = get_caller_identity_traceback(traceback=2) self.done = 0 self.start_time = time() # Track the start time + self.timestamp = get_timestamp() def __enter__(self): - # print(f"entering {self.name}") + # print(f"entering {self.caller_identity}") return self def __exit__(self, type, value, traceback): - # print(f"leaving {self.name}") + # print(f"leaving {self.caller_identity}") return def __iter__(self): @@ -41,11 +40,11 @@ def __next__(self): ) # Calculate the iteration rate connection.ws_send( event_type=EVENT_TYPE_NAME_PROGRESS, - series=self.name, + series=self.caller_identity.func_name, payload={"done": self.done, "total": self.total, "rate": rate}, + timestamp=self.timestamp, _history_len=1, ) - print(f"{self.done}/{self.total}, {rate}/sec") return next(self.iterator) else: raise StopIteration diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index 549d4927..9a07f95d 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -4,6 +4,7 @@ # URL: https://github.com/visualDust # Date: 20230319 +from dataclasses import dataclass import inspect import types from os import path @@ -41,49 +42,58 @@ def get_frame_filepath_traceback(traceback=1): return frame.filename -class TracedIdentity: - def __init__(self, frame=None) -> None: - self.frame = None - self.func_name = None - self.locals = None - self.class_obj = None - self.class_name = None - self.module = None - self.module_name = None - self.filepath = None - self.filename = None - if frame: - self.parse(frame) - - def parse(self, frame): - self.frame = frame - self.func_name = frame.function - self.locals = frame[0].f_locals - if "self" in self.locals: - self.class_obj = self.locals["self"].__class__ - if self.class_obj: - self.class_name = self.class_obj.__name__ +@dataclass +class TracebackIdentity: + frame = None + func_name = None + locals = None + class_obj = None + class_name = None + module = None + module_name = None + filepath = None + filename = None + + @classmethod + def parse(cls, frame) -> "TracebackIdentity": + instance = TracebackIdentity() + instance.frame = frame + instance.func_name = frame.function if frame.function != "" else None + instance.locals = frame[0].f_locals + if "self" in instance.locals: + instance.class_obj = instance.locals["self"].__class__ + if instance.class_obj: + instance.class_name = instance.class_obj.__name__ module: Union[types.ModuleType, None] = inspect.getmodule(frame[0]) - self.module = module - self.module_name = module.__name__ if module else None - self.filepath = path.abspath(frame.filename) - self.filename = path.basename(frame.filename) - return self - - def __str__(self) -> str: + instance.module = module + instance.module_name = module.__name__ if module else None + instance.filepath = path.abspath(frame.filename) + instance.filename = path.basename(frame.filename) + return instance + + def __eq__(self, __value: object) -> bool: + assert isinstance( + __value, TracebackIdentity + ), f"cannot compare {__value} as a {TracebackIdentity}" return ( - str(self.func_name) - + "," - + str(self.class_name) - + "," - + str(self.module_name) - + "," - + str(self.filepath) - + "," + self.filepath == __value.filepath + and self.module_name == __value.module_name + and self.class_name == __value.class_name + and self.func_name == __value.func_name + ) + + def __repr__(self) -> str: + return "\n".join( + [ + f"in file: \t {self.filepath}", + f"in modlue: \t {self.module_name}", + f"in class: \t {self.class_name}", + f"in function: \t {self.func_name}", + ] ) def get_caller_identity_traceback(traceback=1): frame = get_frame_traceback(traceback + 1) - traced_identity = TracedIdentity().parse(frame) + traced_identity = TracebackIdentity.parse(frame) return traced_identity diff --git a/tests/client/neetbox.toml b/tests/client/neetbox.toml index b7661bde..36cb2651 100644 --- a/tests/client/neetbox.toml +++ b/tests/client/neetbox.toml @@ -1,5 +1,5 @@ name = "client-test-script" -version = "0.3.1" +version = "0.3.2" projectid = "2757e5c6-7ae2-420c-91ed-85778a3cfd59" [logging] diff --git a/tests/client/test.py b/tests/client/test.py index 90cc8ff7..d04f4a2c 100644 --- a/tests/client/test.py +++ b/tests/client/test.py @@ -132,10 +132,11 @@ def log_with_some_prefix(): logger.err("some error") -train_config = {"epoch": 99999} +train_config = {"epoch": 9999} if __name__ == "__main__": neetbox.add_hyperparams(train_config) - for i in range(train_config["epoch"]): - sleep(1) - train(i) + with neetbox.progress(train_config["epoch"]) as epochs: + for i in epochs: + sleep(1) + train(i) From 4c6c3016a9f2023234ef7e9e4e0a89ac59c5c692 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 00:30:55 +0800 Subject: [PATCH 13/20] patch: progress --- neetbox/_protocol.py | 7 +++--- neetbox/client/_progress.py | 2 +- neetbox/extension/environment/hardware.py | 3 ++- neetbox/logging/_writer.py | 9 ++++---- neetbox/logging/logger.py | 2 +- neetbox/utils/framing.py | 27 ++++++++++++++--------- 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/neetbox/_protocol.py b/neetbox/_protocol.py index 44909bbc..36bf4ea9 100644 --- a/neetbox/_protocol.py +++ b/neetbox/_protocol.py @@ -55,6 +55,7 @@ class EventMsg: history_len: int = -1 id: int = None # id in database + @property def json(self): return { PROJECT_ID_KEY: self.project_id, @@ -70,7 +71,7 @@ def json(self): } def dumps(self): - return json.dumps(self.json(), default=str) + return json.dumps(self.json, default=str) @classmethod def loads(cls, src): @@ -91,8 +92,8 @@ def loads(cls, src): @classmethod def merge(cls, x: Union["EventMsg", dict], y: Union["EventMsg", dict]): - _x = x if isinstance(x, dict) else x.json() - _y = y if isinstance(y, dict) else y.json() + _x = x if isinstance(x, dict) else x.json + _y = y if isinstance(y, dict) else y.json for _k, _v in _y.items(): _x[_k] = _v return cls.loads(_x) diff --git a/neetbox/client/_progress.py b/neetbox/client/_progress.py index 33df7d24..44674324 100644 --- a/neetbox/client/_progress.py +++ b/neetbox/client/_progress.py @@ -40,7 +40,7 @@ def __next__(self): ) # Calculate the iteration rate connection.ws_send( event_type=EVENT_TYPE_NAME_PROGRESS, - series=self.caller_identity.func_name, + series=self.caller_identity.last_traceable, payload={"done": self.done, "total": self.total, "rate": rate}, timestamp=self.timestamp, _history_len=1, diff --git a/neetbox/extension/environment/hardware.py b/neetbox/extension/environment/hardware.py index 9da5a982..66b04006 100644 --- a/neetbox/extension/environment/hardware.py +++ b/neetbox/extension/environment/hardware.py @@ -84,6 +84,7 @@ def __init__(self) -> None: # the environment shoube be imported in the __init__.py of the outer module. And the watcher thread should be auto started # self.set_update_intervel() # do not watch by default + @property def json(self): return {"cpus": self["cpus"], "ram": self["ram"], "gpus": self["gpus"]} @@ -125,7 +126,7 @@ def watcher_fun(env_instance: _Hardware, do_update_gpus: bool): time.sleep(env_instance._update_interval) connection.ws_send( event_type=EVENT_TYPE_NAME_HARDWARE, - payload=env_instance.json(), + payload=env_instance.json, _history_len=1000, ) diff --git a/neetbox/logging/_writer.py b/neetbox/logging/_writer.py index ed0eb1bc..73222d64 100644 --- a/neetbox/logging/_writer.py +++ b/neetbox/logging/_writer.py @@ -45,6 +45,7 @@ def write_by(self, writer: LogWriter) -> bool: writer.write(self) return False + @property def json(self) -> dict: _default_style = self.style # prefix @@ -89,7 +90,7 @@ def json(self) -> dict: return {"series": prefix, "timestamp": timestamp, "whom": whom, "message": self.rich_msg} def __repr__(self) -> str: - return json.dumps(self.json(), default=str) + return json.dumps(self.json, default=str) # ================== CONSOLE LOG WRITER ===================== @@ -117,7 +118,7 @@ class __ConsoleLogWriter(LogWriter): _console = Console() def write(self, raw_log: RawLog): - _msg_dict = raw_log.json() + _msg_dict = raw_log.json _style = raw_log.style _prefix = _msg_dict["series"] _prefix = _console_prefix_2_colored_text[_prefix] + " " if _prefix else _prefix @@ -167,7 +168,7 @@ def __init__(self, path) -> None: self.file_writer = open(path, "a", encoding="utf-8", buffering=1) def write(self, raw_log: RawLog): - _msg_dict = raw_log.json() + _msg_dict = raw_log.json _style = raw_log.style _series_text = f"[{_msg_dict['series']}]" if _msg_dict["series"] else "" text_msg = _style.split_char_txt.join( @@ -184,7 +185,7 @@ class _WebSocketLogWriter(LogWriter): connection = None # connection should be assigned by neetbox._daemon.client._client to avoid recursive import def write(self, raw_log: RawLog): - json_data = raw_log.json() + json_data = raw_log.json payload = {"whom": json_data["whom"], "message": json_data["message"]} if _WebSocketLogWriter.connection: _WebSocketLogWriter.connection.ws_send( diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index 7a139deb..73046516 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -265,7 +265,7 @@ def err( with_identifier=with_identifier, with_datetime=with_datetime, ) - if type(err) is Exception: + if isinstance(err, Exception): if reraise: raise err elif self.log_level >= LogLevel.DEBUG: diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index 9a07f95d..162322c7 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -71,6 +71,21 @@ def parse(cls, frame) -> "TracebackIdentity": instance.filename = path.basename(frame.filename) return instance + @property + def last_traceable(self): + for identity in [self.func_name, self.class_name, self.module_name, self.filepath, None]: + if identity is not None: + return identity + + @property + def json(self): + return { + "file": f"{self.filepath}", + "modlue": f"{self.module_name}", + "class": f"{self.class_name}", + "function": f"{self.func_name}", + } + def __eq__(self, __value: object) -> bool: assert isinstance( __value, TracebackIdentity @@ -83,17 +98,9 @@ def __eq__(self, __value: object) -> bool: ) def __repr__(self) -> str: - return "\n".join( - [ - f"in file: \t {self.filepath}", - f"in modlue: \t {self.module_name}", - f"in class: \t {self.class_name}", - f"in function: \t {self.func_name}", - ] - ) + return "\n".join([f"{k}:\t{v}" for k, v in self.json.items()]) def get_caller_identity_traceback(traceback=1): frame = get_frame_traceback(traceback + 1) - traced_identity = TracebackIdentity.parse(frame) - return traced_identity + return TracebackIdentity.parse(frame) From 91e16b99f5adb6bc89c7dc8e755894312352505c Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 00:55:10 +0800 Subject: [PATCH 14/20] changed nothing --- neetbox/__init__.py | 2 +- neetbox/client/__init__.py | 2 +- neetbox/client/_client_web_apis.py | 2 +- neetbox/client/_plot.py | 2 +- neetbox/client/_progress.py | 18 ++++++++++++++++-- neetbox/client/_signal_and_slot.py | 2 +- neetbox/config/__init__.py | 2 +- neetbox/config/_workspace.py | 2 +- neetbox/core/registry.py | 2 +- neetbox/extension/__init__.py | 2 +- neetbox/extension/environment/hardware.py | 2 +- neetbox/extension/environment/platform.py | 2 +- neetbox/extension/torch/arch/cnn.py | 2 +- neetbox/extension/torch/profile.py | 2 +- neetbox/logging/formatting.py | 2 +- neetbox/logging/logger.py | 9 +++------ neetbox/server/_bridge.py | 2 +- neetbox/server/_flask_server.py | 2 +- neetbox/server/_server.py | 2 +- neetbox/server/_websocket_server.py | 2 +- neetbox/utils/__init__.py | 2 +- neetbox/utils/_resource.py | 2 +- neetbox/utils/formatting.py | 2 +- neetbox/utils/framing.py | 2 +- neetbox/utils/massive.py | 2 +- neetbox/utils/mvc.py | 2 +- 26 files changed, 43 insertions(+), 32 deletions(-) diff --git a/neetbox/__init__.py b/neetbox/__init__.py index 099abc57..a86bae2f 100644 --- a/neetbox/__init__.py +++ b/neetbox/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231216 from .client import ( diff --git a/neetbox/client/__init__.py b/neetbox/client/__init__.py index 91b2f5c0..ce3a45a7 100644 --- a/neetbox/client/__init__.py +++ b/neetbox/client/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231206 from ._action_agent import _NeetActionManager as NeetActionManager diff --git a/neetbox/client/_client_web_apis.py b/neetbox/client/_client_web_apis.py index ed698e24..550526a7 100644 --- a/neetbox/client/_client_web_apis.py +++ b/neetbox/client/_client_web_apis.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230414 diff --git a/neetbox/client/_plot.py b/neetbox/client/_plot.py index bea022f2..57c497e7 100644 --- a/neetbox/client/_plot.py +++ b/neetbox/client/_plot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231211 from neetbox._protocol import * diff --git a/neetbox/client/_progress.py b/neetbox/client/_progress.py index 44674324..ddb8ad04 100644 --- a/neetbox/client/_progress.py +++ b/neetbox/client/_progress.py @@ -1,4 +1,11 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# GITHUB: github.com/visualDust +# Date: 20231217 + from time import time +from typing import Callable from ._client import connection @@ -38,14 +45,21 @@ def __next__(self): rate = ( self.done / elapsed_time if elapsed_time > 0 else 0 ) # Calculate the iteration rate + iter_next = next(self.iterator) + current_name = f"{iter_next.__name__ if isinstance(iter_next, Callable) else iter_next}" connection.ws_send( event_type=EVENT_TYPE_NAME_PROGRESS, series=self.caller_identity.last_traceable, - payload={"done": self.done, "total": self.total, "rate": rate}, + payload={ + "step": self.done, + "current": current_name, + "total": self.total, + "rate": rate, + }, timestamp=self.timestamp, _history_len=1, ) - return next(self.iterator) + return iter_next else: raise StopIteration diff --git a/neetbox/client/_signal_and_slot.py b/neetbox/client/_signal_and_slot.py index 83da8287..e284a689 100644 --- a/neetbox/client/_signal_and_slot.py +++ b/neetbox/client/_signal_and_slot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230417 import collections diff --git a/neetbox/config/__init__.py b/neetbox/config/__init__.py index afa5a898..635caa28 100644 --- a/neetbox/config/__init__.py +++ b/neetbox/config/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230413 diff --git a/neetbox/config/_workspace.py b/neetbox/config/_workspace.py index da5c1e84..bce0d73b 100644 --- a/neetbox/config/_workspace.py +++ b/neetbox/config/_workspace.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230413 diff --git a/neetbox/core/registry.py b/neetbox/core/registry.py index 14c933f2..339eb325 100644 --- a/neetbox/core/registry.py +++ b/neetbox/core/registry.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230413 import functools diff --git a/neetbox/extension/__init__.py b/neetbox/extension/__init__.py index 52c48562..d1960d3d 100644 --- a/neetbox/extension/__init__.py +++ b/neetbox/extension/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230417 import importlib diff --git a/neetbox/extension/environment/hardware.py b/neetbox/extension/environment/hardware.py index 66b04006..4191a3e9 100644 --- a/neetbox/extension/environment/hardware.py +++ b/neetbox/extension/environment/hardware.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230413 diff --git a/neetbox/extension/environment/platform.py b/neetbox/extension/environment/platform.py index 0ffdf6ed..2b4802db 100644 --- a/neetbox/extension/environment/platform.py +++ b/neetbox/extension/environment/platform.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230417 import getpass diff --git a/neetbox/extension/torch/arch/cnn.py b/neetbox/extension/torch/arch/cnn.py index 6ace19be..b9336684 100644 --- a/neetbox/extension/torch/arch/cnn.py +++ b/neetbox/extension/torch/arch/cnn.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230315 import torch.nn as nn diff --git a/neetbox/extension/torch/profile.py b/neetbox/extension/torch/profile.py index ee6db097..9b8ec2bd 100644 --- a/neetbox/extension/torch/profile.py +++ b/neetbox/extension/torch/profile.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230315 import time diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index 064abec1..73b3cb91 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230318 from dataclasses import dataclass diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index 73046516..7cb0ed98 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230315 import functools @@ -265,11 +265,8 @@ def err( with_identifier=with_identifier, with_datetime=with_datetime, ) - if isinstance(err, Exception): - if reraise: - raise err - elif self.log_level >= LogLevel.DEBUG: - Logger._console.print_exception(err) + if reraise: + raise err return self def mention( diff --git a/neetbox/server/_bridge.py b/neetbox/server/_bridge.py index 2eccddb1..745c1369 100644 --- a/neetbox/server/_bridge.py +++ b/neetbox/server/_bridge.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231204 from typing import Dict diff --git a/neetbox/server/_flask_server.py b/neetbox/server/_flask_server.py index 00ab8586..84522971 100644 --- a/neetbox/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231204 import logging diff --git a/neetbox/server/_server.py b/neetbox/server/_server.py index d922a22c..23c75c7d 100644 --- a/neetbox/server/_server.py +++ b/neetbox/server/_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230414 import setproctitle diff --git a/neetbox/server/_websocket_server.py b/neetbox/server/_websocket_server.py index 1c4360ab..92b944ed 100644 --- a/neetbox/server/_websocket_server.py +++ b/neetbox/server/_websocket_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231204 from typing import Dict, Tuple diff --git a/neetbox/utils/__init__.py b/neetbox/utils/__init__.py index 37becd62..e17020ab 100644 --- a/neetbox/utils/__init__.py +++ b/neetbox/utils/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231206 from ._package import pipPackageHealper as pkg diff --git a/neetbox/utils/_resource.py b/neetbox/utils/_resource.py index 0dbd873f..69907960 100644 --- a/neetbox/utils/_resource.py +++ b/neetbox/utils/_resource.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230315 import os diff --git a/neetbox/utils/formatting.py b/neetbox/utils/formatting.py index 4a9e5263..993fbf0c 100644 --- a/neetbox/utils/formatting.py +++ b/neetbox/utils/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230319 import json diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index 162322c7..7c7aef75 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230319 from dataclasses import dataclass diff --git a/neetbox/utils/massive.py b/neetbox/utils/massive.py index 5cc61d5b..6ce24a32 100644 --- a/neetbox/utils/massive.py +++ b/neetbox/utils/massive.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20231216 import functools diff --git a/neetbox/utils/mvc.py b/neetbox/utils/mvc.py index 69a4cb53..a8b29fb9 100644 --- a/neetbox/utils/mvc.py +++ b/neetbox/utils/mvc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# GITHUB: github.com/visualDust # Date: 20230416 From 77ebfd993d6da21f443004f751a72c0a2fa2f9e0 Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 00:55:10 +0800 Subject: [PATCH 15/20] changed nothing --- neetbox/__init__.py | 2 +- neetbox/client/__init__.py | 2 +- neetbox/client/_client_web_apis.py | 2 +- neetbox/client/_plot.py | 2 +- neetbox/client/_progress.py | 18 ++++++++++++++++-- neetbox/client/_signal_and_slot.py | 2 +- neetbox/config/__init__.py | 2 +- neetbox/config/_workspace.py | 2 +- neetbox/core/registry.py | 2 +- neetbox/extension/__init__.py | 2 +- neetbox/extension/environment/hardware.py | 2 +- neetbox/extension/environment/platform.py | 2 +- neetbox/extension/torch/arch/cnn.py | 2 +- neetbox/extension/torch/profile.py | 2 +- neetbox/logging/formatting.py | 2 +- neetbox/logging/logger.py | 9 +++------ neetbox/server/_bridge.py | 2 +- neetbox/server/_flask_server.py | 2 +- neetbox/server/_server.py | 2 +- neetbox/server/_websocket_server.py | 2 +- neetbox/utils/__init__.py | 2 +- neetbox/utils/_resource.py | 2 +- neetbox/utils/formatting.py | 2 +- neetbox/utils/framing.py | 2 +- neetbox/utils/massive.py | 2 +- neetbox/utils/mvc.py | 2 +- 26 files changed, 43 insertions(+), 32 deletions(-) diff --git a/neetbox/__init__.py b/neetbox/__init__.py index 099abc57..c39e99e8 100644 --- a/neetbox/__init__.py +++ b/neetbox/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231216 from .client import ( diff --git a/neetbox/client/__init__.py b/neetbox/client/__init__.py index 91b2f5c0..1eda42a1 100644 --- a/neetbox/client/__init__.py +++ b/neetbox/client/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231206 from ._action_agent import _NeetActionManager as NeetActionManager diff --git a/neetbox/client/_client_web_apis.py b/neetbox/client/_client_web_apis.py index ed698e24..4a7e81b1 100644 --- a/neetbox/client/_client_web_apis.py +++ b/neetbox/client/_client_web_apis.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230414 diff --git a/neetbox/client/_plot.py b/neetbox/client/_plot.py index bea022f2..d2e72fd8 100644 --- a/neetbox/client/_plot.py +++ b/neetbox/client/_plot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231211 from neetbox._protocol import * diff --git a/neetbox/client/_progress.py b/neetbox/client/_progress.py index 44674324..d4142072 100644 --- a/neetbox/client/_progress.py +++ b/neetbox/client/_progress.py @@ -1,4 +1,11 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20231217 + from time import time +from typing import Callable from ._client import connection @@ -38,14 +45,21 @@ def __next__(self): rate = ( self.done / elapsed_time if elapsed_time > 0 else 0 ) # Calculate the iteration rate + iter_next = next(self.iterator) + current_name = f"{iter_next.__name__ if isinstance(iter_next, Callable) else iter_next}" connection.ws_send( event_type=EVENT_TYPE_NAME_PROGRESS, series=self.caller_identity.last_traceable, - payload={"done": self.done, "total": self.total, "rate": rate}, + payload={ + "step": self.done, + "current": current_name, + "total": self.total, + "rate": rate, + }, timestamp=self.timestamp, _history_len=1, ) - return next(self.iterator) + return iter_next else: raise StopIteration diff --git a/neetbox/client/_signal_and_slot.py b/neetbox/client/_signal_and_slot.py index 83da8287..2fecc0a0 100644 --- a/neetbox/client/_signal_and_slot.py +++ b/neetbox/client/_signal_and_slot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230417 import collections diff --git a/neetbox/config/__init__.py b/neetbox/config/__init__.py index afa5a898..fade4e59 100644 --- a/neetbox/config/__init__.py +++ b/neetbox/config/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 diff --git a/neetbox/config/_workspace.py b/neetbox/config/_workspace.py index da5c1e84..412b4b51 100644 --- a/neetbox/config/_workspace.py +++ b/neetbox/config/_workspace.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 diff --git a/neetbox/core/registry.py b/neetbox/core/registry.py index 14c933f2..9f7066b7 100644 --- a/neetbox/core/registry.py +++ b/neetbox/core/registry.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 import functools diff --git a/neetbox/extension/__init__.py b/neetbox/extension/__init__.py index 52c48562..eb19c224 100644 --- a/neetbox/extension/__init__.py +++ b/neetbox/extension/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230417 import importlib diff --git a/neetbox/extension/environment/hardware.py b/neetbox/extension/environment/hardware.py index 66b04006..9da2a47f 100644 --- a/neetbox/extension/environment/hardware.py +++ b/neetbox/extension/environment/hardware.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 diff --git a/neetbox/extension/environment/platform.py b/neetbox/extension/environment/platform.py index 0ffdf6ed..9d4d6396 100644 --- a/neetbox/extension/environment/platform.py +++ b/neetbox/extension/environment/platform.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230417 import getpass diff --git a/neetbox/extension/torch/arch/cnn.py b/neetbox/extension/torch/arch/cnn.py index 6ace19be..bb967ebc 100644 --- a/neetbox/extension/torch/arch/cnn.py +++ b/neetbox/extension/torch/arch/cnn.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import torch.nn as nn diff --git a/neetbox/extension/torch/profile.py b/neetbox/extension/torch/profile.py index ee6db097..729d8116 100644 --- a/neetbox/extension/torch/profile.py +++ b/neetbox/extension/torch/profile.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import time diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index 064abec1..3455c1b5 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230318 from dataclasses import dataclass diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index 73046516..a5a7e722 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import functools @@ -265,11 +265,8 @@ def err( with_identifier=with_identifier, with_datetime=with_datetime, ) - if isinstance(err, Exception): - if reraise: - raise err - elif self.log_level >= LogLevel.DEBUG: - Logger._console.print_exception(err) + if reraise: + raise err return self def mention( diff --git a/neetbox/server/_bridge.py b/neetbox/server/_bridge.py index 2eccddb1..afa45bb7 100644 --- a/neetbox/server/_bridge.py +++ b/neetbox/server/_bridge.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231204 from typing import Dict diff --git a/neetbox/server/_flask_server.py b/neetbox/server/_flask_server.py index 00ab8586..6aef5783 100644 --- a/neetbox/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231204 import logging diff --git a/neetbox/server/_server.py b/neetbox/server/_server.py index d922a22c..f5091f0f 100644 --- a/neetbox/server/_server.py +++ b/neetbox/server/_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230414 import setproctitle diff --git a/neetbox/server/_websocket_server.py b/neetbox/server/_websocket_server.py index 1c4360ab..a7110b80 100644 --- a/neetbox/server/_websocket_server.py +++ b/neetbox/server/_websocket_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231204 from typing import Dict, Tuple diff --git a/neetbox/utils/__init__.py b/neetbox/utils/__init__.py index 37becd62..536f7fcc 100644 --- a/neetbox/utils/__init__.py +++ b/neetbox/utils/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231206 from ._package import pipPackageHealper as pkg diff --git a/neetbox/utils/_resource.py b/neetbox/utils/_resource.py index 0dbd873f..72afb615 100644 --- a/neetbox/utils/_resource.py +++ b/neetbox/utils/_resource.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import os diff --git a/neetbox/utils/formatting.py b/neetbox/utils/formatting.py index 4a9e5263..963e1a33 100644 --- a/neetbox/utils/formatting.py +++ b/neetbox/utils/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230319 import json diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index 162322c7..8686df27 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230319 from dataclasses import dataclass diff --git a/neetbox/utils/massive.py b/neetbox/utils/massive.py index 5cc61d5b..bc451af5 100644 --- a/neetbox/utils/massive.py +++ b/neetbox/utils/massive.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20231216 import functools diff --git a/neetbox/utils/mvc.py b/neetbox/utils/mvc.py index 69a4cb53..afe47a71 100644 --- a/neetbox/utils/mvc.py +++ b/neetbox/utils/mvc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://github.com/visualDust +# Github: github.com/visualDust # Date: 20230416 From 61cb797eb3f8c058827b6f3bf38ca3d2265e99aa Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 00:58:55 +0800 Subject: [PATCH 16/20] Revert "changed nothing" This reverts commit 77ebfd993d6da21f443004f751a72c0a2fa2f9e0. --- neetbox/__init__.py | 2 +- neetbox/client/__init__.py | 2 +- neetbox/client/_client_web_apis.py | 2 +- neetbox/client/_plot.py | 2 +- neetbox/client/_progress.py | 18 ++---------------- neetbox/client/_signal_and_slot.py | 2 +- neetbox/config/__init__.py | 2 +- neetbox/config/_workspace.py | 2 +- neetbox/core/registry.py | 2 +- neetbox/extension/__init__.py | 2 +- neetbox/extension/environment/hardware.py | 2 +- neetbox/extension/environment/platform.py | 2 +- neetbox/extension/torch/arch/cnn.py | 2 +- neetbox/extension/torch/profile.py | 2 +- neetbox/logging/formatting.py | 2 +- neetbox/logging/logger.py | 9 ++++++--- neetbox/server/_bridge.py | 2 +- neetbox/server/_flask_server.py | 2 +- neetbox/server/_server.py | 2 +- neetbox/server/_websocket_server.py | 2 +- neetbox/utils/__init__.py | 2 +- neetbox/utils/_resource.py | 2 +- neetbox/utils/formatting.py | 2 +- neetbox/utils/framing.py | 2 +- neetbox/utils/massive.py | 2 +- neetbox/utils/mvc.py | 2 +- 26 files changed, 32 insertions(+), 43 deletions(-) diff --git a/neetbox/__init__.py b/neetbox/__init__.py index c39e99e8..099abc57 100644 --- a/neetbox/__init__.py +++ b/neetbox/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231216 from .client import ( diff --git a/neetbox/client/__init__.py b/neetbox/client/__init__.py index 1eda42a1..91b2f5c0 100644 --- a/neetbox/client/__init__.py +++ b/neetbox/client/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231206 from ._action_agent import _NeetActionManager as NeetActionManager diff --git a/neetbox/client/_client_web_apis.py b/neetbox/client/_client_web_apis.py index 4a7e81b1..ed698e24 100644 --- a/neetbox/client/_client_web_apis.py +++ b/neetbox/client/_client_web_apis.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230414 diff --git a/neetbox/client/_plot.py b/neetbox/client/_plot.py index d2e72fd8..bea022f2 100644 --- a/neetbox/client/_plot.py +++ b/neetbox/client/_plot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231211 from neetbox._protocol import * diff --git a/neetbox/client/_progress.py b/neetbox/client/_progress.py index d4142072..44674324 100644 --- a/neetbox/client/_progress.py +++ b/neetbox/client/_progress.py @@ -1,11 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Author: GavinGong aka VisualDust -# Github: github.com/visualDust -# Date: 20231217 - from time import time -from typing import Callable from ._client import connection @@ -45,21 +38,14 @@ def __next__(self): rate = ( self.done / elapsed_time if elapsed_time > 0 else 0 ) # Calculate the iteration rate - iter_next = next(self.iterator) - current_name = f"{iter_next.__name__ if isinstance(iter_next, Callable) else iter_next}" connection.ws_send( event_type=EVENT_TYPE_NAME_PROGRESS, series=self.caller_identity.last_traceable, - payload={ - "step": self.done, - "current": current_name, - "total": self.total, - "rate": rate, - }, + payload={"done": self.done, "total": self.total, "rate": rate}, timestamp=self.timestamp, _history_len=1, ) - return iter_next + return next(self.iterator) else: raise StopIteration diff --git a/neetbox/client/_signal_and_slot.py b/neetbox/client/_signal_and_slot.py index 2fecc0a0..83da8287 100644 --- a/neetbox/client/_signal_and_slot.py +++ b/neetbox/client/_signal_and_slot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230417 import collections diff --git a/neetbox/config/__init__.py b/neetbox/config/__init__.py index fade4e59..afa5a898 100644 --- a/neetbox/config/__init__.py +++ b/neetbox/config/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230413 diff --git a/neetbox/config/_workspace.py b/neetbox/config/_workspace.py index 412b4b51..da5c1e84 100644 --- a/neetbox/config/_workspace.py +++ b/neetbox/config/_workspace.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230413 diff --git a/neetbox/core/registry.py b/neetbox/core/registry.py index 9f7066b7..14c933f2 100644 --- a/neetbox/core/registry.py +++ b/neetbox/core/registry.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230413 import functools diff --git a/neetbox/extension/__init__.py b/neetbox/extension/__init__.py index eb19c224..52c48562 100644 --- a/neetbox/extension/__init__.py +++ b/neetbox/extension/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230417 import importlib diff --git a/neetbox/extension/environment/hardware.py b/neetbox/extension/environment/hardware.py index 9da2a47f..66b04006 100644 --- a/neetbox/extension/environment/hardware.py +++ b/neetbox/extension/environment/hardware.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230413 diff --git a/neetbox/extension/environment/platform.py b/neetbox/extension/environment/platform.py index 9d4d6396..0ffdf6ed 100644 --- a/neetbox/extension/environment/platform.py +++ b/neetbox/extension/environment/platform.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230417 import getpass diff --git a/neetbox/extension/torch/arch/cnn.py b/neetbox/extension/torch/arch/cnn.py index bb967ebc..6ace19be 100644 --- a/neetbox/extension/torch/arch/cnn.py +++ b/neetbox/extension/torch/arch/cnn.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230315 import torch.nn as nn diff --git a/neetbox/extension/torch/profile.py b/neetbox/extension/torch/profile.py index 729d8116..ee6db097 100644 --- a/neetbox/extension/torch/profile.py +++ b/neetbox/extension/torch/profile.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230315 import time diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index 3455c1b5..064abec1 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230318 from dataclasses import dataclass diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index a5a7e722..73046516 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230315 import functools @@ -265,8 +265,11 @@ def err( with_identifier=with_identifier, with_datetime=with_datetime, ) - if reraise: - raise err + if isinstance(err, Exception): + if reraise: + raise err + elif self.log_level >= LogLevel.DEBUG: + Logger._console.print_exception(err) return self def mention( diff --git a/neetbox/server/_bridge.py b/neetbox/server/_bridge.py index afa45bb7..2eccddb1 100644 --- a/neetbox/server/_bridge.py +++ b/neetbox/server/_bridge.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231204 from typing import Dict diff --git a/neetbox/server/_flask_server.py b/neetbox/server/_flask_server.py index 6aef5783..00ab8586 100644 --- a/neetbox/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231204 import logging diff --git a/neetbox/server/_server.py b/neetbox/server/_server.py index f5091f0f..d922a22c 100644 --- a/neetbox/server/_server.py +++ b/neetbox/server/_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230414 import setproctitle diff --git a/neetbox/server/_websocket_server.py b/neetbox/server/_websocket_server.py index a7110b80..1c4360ab 100644 --- a/neetbox/server/_websocket_server.py +++ b/neetbox/server/_websocket_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231204 from typing import Dict, Tuple diff --git a/neetbox/utils/__init__.py b/neetbox/utils/__init__.py index 536f7fcc..37becd62 100644 --- a/neetbox/utils/__init__.py +++ b/neetbox/utils/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231206 from ._package import pipPackageHealper as pkg diff --git a/neetbox/utils/_resource.py b/neetbox/utils/_resource.py index 72afb615..0dbd873f 100644 --- a/neetbox/utils/_resource.py +++ b/neetbox/utils/_resource.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230315 import os diff --git a/neetbox/utils/formatting.py b/neetbox/utils/formatting.py index 963e1a33..4a9e5263 100644 --- a/neetbox/utils/formatting.py +++ b/neetbox/utils/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230319 import json diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index 8686df27..162322c7 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230319 from dataclasses import dataclass diff --git a/neetbox/utils/massive.py b/neetbox/utils/massive.py index bc451af5..5cc61d5b 100644 --- a/neetbox/utils/massive.py +++ b/neetbox/utils/massive.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20231216 import functools diff --git a/neetbox/utils/mvc.py b/neetbox/utils/mvc.py index afe47a71..69a4cb53 100644 --- a/neetbox/utils/mvc.py +++ b/neetbox/utils/mvc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# Github: github.com/visualDust +# URL: https://github.com/visualDust # Date: 20230416 From 4f45a4df52fe2a018a63ef7a196a153f5ee9318f Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 01:03:12 +0800 Subject: [PATCH 17/20] changed nothing --- neetbox/__init__.py | 2 +- neetbox/_protocol.py | 6 ++++++ neetbox/cli/parse.py | 6 ++++++ neetbox/client/__init__.py | 2 +- neetbox/client/_action_agent.py | 6 ++++++ neetbox/client/_client.py | 6 ++++++ neetbox/client/_client_web_apis.py | 2 +- neetbox/client/_image.py | 2 +- neetbox/client/_plot.py | 2 +- neetbox/client/_progress.py | 2 +- neetbox/client/_signal_and_slot.py | 2 +- neetbox/config/__init__.py | 2 +- neetbox/config/_cli_user_config.py | 6 ++++++ neetbox/config/_workspace.py | 2 +- neetbox/core/registry.py | 2 +- neetbox/extension/__init__.py | 2 +- neetbox/extension/environment/hardware.py | 2 +- neetbox/extension/environment/platform.py | 2 +- neetbox/extension/torch/arch/cnn.py | 2 +- neetbox/extension/torch/profile.py | 2 +- neetbox/logging/_writer.py | 6 ++++++ neetbox/logging/formatting.py | 2 +- neetbox/logging/logger.py | 2 +- neetbox/server/_bridge.py | 2 +- neetbox/server/_fastapi_server.py | 6 ++++++ neetbox/server/_flask_server.py | 2 +- neetbox/server/_server.py | 2 +- neetbox/server/_websocket_server.py | 2 +- neetbox/server/history/__init__.py | 6 ++++++ neetbox/server/history/_db.py | 6 ++++++ neetbox/utils/__init__.py | 2 +- neetbox/utils/_resource.py | 2 +- neetbox/utils/formatting.py | 2 +- neetbox/utils/framing.py | 2 +- neetbox/utils/massive.py | 2 +- neetbox/utils/mvc.py | 2 +- 36 files changed, 81 insertions(+), 27 deletions(-) diff --git a/neetbox/__init__.py b/neetbox/__init__.py index a86bae2f..c39e99e8 100644 --- a/neetbox/__init__.py +++ b/neetbox/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231216 from .client import ( diff --git a/neetbox/_protocol.py b/neetbox/_protocol.py index 36bf4ea9..fe9c01fb 100644 --- a/neetbox/_protocol.py +++ b/neetbox/_protocol.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20231201 + import json from dataclasses import dataclass from datetime import datetime as dt diff --git a/neetbox/cli/parse.py b/neetbox/cli/parse.py index fec74bf3..5d4571ea 100644 --- a/neetbox/cli/parse.py +++ b/neetbox/cli/parse.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20231013 + import json import os diff --git a/neetbox/client/__init__.py b/neetbox/client/__init__.py index ce3a45a7..1eda42a1 100644 --- a/neetbox/client/__init__.py +++ b/neetbox/client/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231206 from ._action_agent import _NeetActionManager as NeetActionManager diff --git a/neetbox/client/_action_agent.py b/neetbox/client/_action_agent.py index 6295d702..3f57c146 100644 --- a/neetbox/client/_action_agent.py +++ b/neetbox/client/_action_agent.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20231122 + import functools import inspect from ast import literal_eval diff --git a/neetbox/client/_client.py b/neetbox/client/_client.py index 00ad95c2..acd59295 100644 --- a/neetbox/client/_client.py +++ b/neetbox/client/_client.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 2023102 + import atexit import functools import json diff --git a/neetbox/client/_client_web_apis.py b/neetbox/client/_client_web_apis.py index 550526a7..4a7e81b1 100644 --- a/neetbox/client/_client_web_apis.py +++ b/neetbox/client/_client_web_apis.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230414 diff --git a/neetbox/client/_image.py b/neetbox/client/_image.py index f34d61f5..eb0cb591 100644 --- a/neetbox/client/_image.py +++ b/neetbox/client/_image.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# URL: https://gong.host +# Github: github.com/visualDust # Date: 20231211 import io diff --git a/neetbox/client/_plot.py b/neetbox/client/_plot.py index 57c497e7..d2e72fd8 100644 --- a/neetbox/client/_plot.py +++ b/neetbox/client/_plot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231211 from neetbox._protocol import * diff --git a/neetbox/client/_progress.py b/neetbox/client/_progress.py index ddb8ad04..d4142072 100644 --- a/neetbox/client/_progress.py +++ b/neetbox/client/_progress.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231217 from time import time diff --git a/neetbox/client/_signal_and_slot.py b/neetbox/client/_signal_and_slot.py index e284a689..2fecc0a0 100644 --- a/neetbox/client/_signal_and_slot.py +++ b/neetbox/client/_signal_and_slot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230417 import collections diff --git a/neetbox/config/__init__.py b/neetbox/config/__init__.py index 635caa28..fade4e59 100644 --- a/neetbox/config/__init__.py +++ b/neetbox/config/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 diff --git a/neetbox/config/_cli_user_config.py b/neetbox/config/_cli_user_config.py index 68849bcc..f7eb5c94 100644 --- a/neetbox/config/_cli_user_config.py +++ b/neetbox/config/_cli_user_config.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20231203 + import os from importlib.metadata import version diff --git a/neetbox/config/_workspace.py b/neetbox/config/_workspace.py index bce0d73b..412b4b51 100644 --- a/neetbox/config/_workspace.py +++ b/neetbox/config/_workspace.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 diff --git a/neetbox/core/registry.py b/neetbox/core/registry.py index 339eb325..9f7066b7 100644 --- a/neetbox/core/registry.py +++ b/neetbox/core/registry.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 import functools diff --git a/neetbox/extension/__init__.py b/neetbox/extension/__init__.py index d1960d3d..eb19c224 100644 --- a/neetbox/extension/__init__.py +++ b/neetbox/extension/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230417 import importlib diff --git a/neetbox/extension/environment/hardware.py b/neetbox/extension/environment/hardware.py index 4191a3e9..9da2a47f 100644 --- a/neetbox/extension/environment/hardware.py +++ b/neetbox/extension/environment/hardware.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230413 diff --git a/neetbox/extension/environment/platform.py b/neetbox/extension/environment/platform.py index 2b4802db..9d4d6396 100644 --- a/neetbox/extension/environment/platform.py +++ b/neetbox/extension/environment/platform.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230417 import getpass diff --git a/neetbox/extension/torch/arch/cnn.py b/neetbox/extension/torch/arch/cnn.py index b9336684..bb967ebc 100644 --- a/neetbox/extension/torch/arch/cnn.py +++ b/neetbox/extension/torch/arch/cnn.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import torch.nn as nn diff --git a/neetbox/extension/torch/profile.py b/neetbox/extension/torch/profile.py index 9b8ec2bd..729d8116 100644 --- a/neetbox/extension/torch/profile.py +++ b/neetbox/extension/torch/profile.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import time diff --git a/neetbox/logging/_writer.py b/neetbox/logging/_writer.py index 73222d64..bd4c9c63 100644 --- a/neetbox/logging/_writer.py +++ b/neetbox/logging/_writer.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20230413 + import inspect import io import json diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index 73b3cb91..3455c1b5 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230318 from dataclasses import dataclass diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index 7cb0ed98..a5a7e722 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import functools diff --git a/neetbox/server/_bridge.py b/neetbox/server/_bridge.py index 745c1369..afa45bb7 100644 --- a/neetbox/server/_bridge.py +++ b/neetbox/server/_bridge.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231204 from typing import Dict diff --git a/neetbox/server/_fastapi_server.py b/neetbox/server/_fastapi_server.py index 45081031..632ec7b6 100644 --- a/neetbox/server/_fastapi_server.py +++ b/neetbox/server/_fastapi_server.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: PuQing +# Github: github.com/andPuQing +# Date: 20231103 + from typing import Any, Dict, List import uvicorn diff --git a/neetbox/server/_flask_server.py b/neetbox/server/_flask_server.py index 84522971..6aef5783 100644 --- a/neetbox/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231204 import logging diff --git a/neetbox/server/_server.py b/neetbox/server/_server.py index 23c75c7d..f5091f0f 100644 --- a/neetbox/server/_server.py +++ b/neetbox/server/_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230414 import setproctitle diff --git a/neetbox/server/_websocket_server.py b/neetbox/server/_websocket_server.py index 92b944ed..a7110b80 100644 --- a/neetbox/server/_websocket_server.py +++ b/neetbox/server/_websocket_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231204 from typing import Dict, Tuple diff --git a/neetbox/server/history/__init__.py b/neetbox/server/history/__init__.py index 55770d4c..5c79986a 100644 --- a/neetbox/server/history/__init__.py +++ b/neetbox/server/history/__init__.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20231201 + import os from neetbox.utils import ResourceLoader diff --git a/neetbox/server/history/_db.py b/neetbox/server/history/_db.py index 54c42553..5c2a2190 100644 --- a/neetbox/server/history/_db.py +++ b/neetbox/server/history/_db.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# +# Author: GavinGong aka VisualDust +# Github: github.com/visualDust +# Date: 20231201 + import collections import json import os diff --git a/neetbox/utils/__init__.py b/neetbox/utils/__init__.py index e17020ab..536f7fcc 100644 --- a/neetbox/utils/__init__.py +++ b/neetbox/utils/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231206 from ._package import pipPackageHealper as pkg diff --git a/neetbox/utils/_resource.py b/neetbox/utils/_resource.py index 69907960..72afb615 100644 --- a/neetbox/utils/_resource.py +++ b/neetbox/utils/_resource.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230315 import os diff --git a/neetbox/utils/formatting.py b/neetbox/utils/formatting.py index 993fbf0c..963e1a33 100644 --- a/neetbox/utils/formatting.py +++ b/neetbox/utils/formatting.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230319 import json diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index 7c7aef75..8686df27 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230319 from dataclasses import dataclass diff --git a/neetbox/utils/massive.py b/neetbox/utils/massive.py index 6ce24a32..bc451af5 100644 --- a/neetbox/utils/massive.py +++ b/neetbox/utils/massive.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20231216 import functools diff --git a/neetbox/utils/mvc.py b/neetbox/utils/mvc.py index a8b29fb9..afe47a71 100644 --- a/neetbox/utils/mvc.py +++ b/neetbox/utils/mvc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Author: GavinGong aka VisualDust -# GITHUB: github.com/visualDust +# Github: github.com/visualDust # Date: 20230416 From 6a41da7292565a1b178691009da6248010df8088 Mon Sep 17 00:00:00 2001 From: visualdust Date: Sun, 17 Dec 2023 13:05:26 +0800 Subject: [PATCH 18/20] minor changes --- neetbox.toml | 23 +++++++++-------------- neetbox/__init__.py | 5 +++-- neetbox/_protocol.py | 3 +++ neetbox/config/_workspace.py | 2 +- neetbox/logging/formatting.py | 2 +- neetbox/server/_flask_server.py | 3 ++- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/neetbox.toml b/neetbox.toml index 06960969..f8eda894 100644 --- a/neetbox.toml +++ b/neetbox.toml @@ -1,27 +1,22 @@ name = "neetbox" -projectid = "b42c4749-1fa8-4a9d-ae30-80e2c69309f8" +version = "0.3.2" +projectid = "fd9ba05f-5f33-4afb-9156-08caae9ab37a" [logging] level = "INFO" - -[pipeline] -updateInterval = 0.5 +logdir = "logs" [extension] -autoload = false +autoload = true -[daemon] +[client] enable = true host = "127.0.0.1" port = 20202 allowIpython = false -mute = false -mode = "attached" -uploadInterval = 0.5 +mute = true +mode = "detached" +uploadInterval = 1 [extension.environment.hardware] -monit = true -interval = 0.5 - -[extension.environment.platform] -monit = true +interval = 2 diff --git a/neetbox/__init__.py b/neetbox/__init__.py index c39e99e8..971ac290 100644 --- a/neetbox/__init__.py +++ b/neetbox/__init__.py @@ -12,9 +12,10 @@ add_scalar, listen, watch, - progress + progress, ) from .logging import logger +from ._protocol import VERSION as __version__ __all__ = [ "add_image", @@ -25,5 +26,5 @@ "logger", "watch", "listen", - "progress" + "progress", ] diff --git a/neetbox/_protocol.py b/neetbox/_protocol.py index fe9c01fb..68fc7e67 100644 --- a/neetbox/_protocol.py +++ b/neetbox/_protocol.py @@ -10,6 +10,9 @@ from enum import Enum from importlib.metadata import version from typing import Any, Union +from importlib.metadata import version + +VERSION = version("neetbox") # ===================== common things ===================== diff --git a/neetbox/config/_workspace.py b/neetbox/config/_workspace.py index 412b4b51..a79c6123 100644 --- a/neetbox/config/_workspace.py +++ b/neetbox/config/_workspace.py @@ -27,7 +27,7 @@ "name": os.path.basename(os.path.normpath(os.getcwd())), "version": NEETBOX_VERSION, "projectid": str(uuid4()), # later will be overwrite by workspace config file - "logging": {"level": "DEBUG", "logdir": "logs"}, + "logging": {"level": "INFO", "logdir": "logs"}, "extension": { "autoload": True, }, diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index 3455c1b5..d73f3e7b 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -87,5 +87,5 @@ def styled_text(text, style: LogStyle): def colored_text(text: str, color): - text = text.replace("[", "\[") + text = text.replace("[", r"\[") return f"[{color}]{text}[/{color}]" diff --git a/neetbox/server/_flask_server.py b/neetbox/server/_flask_server.py index 6aef5783..a25d52ff 100644 --- a/neetbox/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -13,6 +13,7 @@ import werkzeug from flask import Response, abort, json, redirect, request, send_from_directory +import neetbox from neetbox._protocol import * from neetbox.server._bridge import Bridge @@ -40,7 +41,7 @@ def get_flask_server(debug=False): app = Flask(__PROC_NAME, static_folder=None) - front_end_dist_path = os.path.join(os.path.dirname(__file__), "../frontend_dist") + front_end_dist_path = os.path.join(os.path.dirname(neetbox.__file__), "frontend_dist") @app.route("/") def static_serve_root(): From 309c774bd2ff7eaf54cf3b54aab7bf73dcdb03ec Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 18:44:01 +0800 Subject: [PATCH 19/20] fix r reference before assignment --- neetbox/client/_client.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/neetbox/client/_client.py b/neetbox/client/_client.py index acd59295..c0cdaa2e 100644 --- a/neetbox/client/_client.py +++ b/neetbox/client/_client.py @@ -101,11 +101,14 @@ def check_server_connectivity(cls, config=None): # check if daemon is alive def fetch_hello(root): + response = None try: - r = cls.get(api="hello", root=root) - assert r.json()["hello"] == "hello" + response = cls.get(api="hello", root=root) + assert response.json()["hello"] == "hello" except: - raise IOError(f"Daemon at {root} is not alive. ({r. status_code})") + raise IOError( + f"Daemon at {root} is not alive: {response.status_code if response else 'no response'}" + ) try: fetch_hello(http_root) @@ -143,7 +146,9 @@ def _connect(cls, config=None): server_host = config["host"] server_port = config["port"] if not cls.check_server_connectivity(): # if daemon not online - if not is_loopback(server_host): # daemon not running on localhost + if not ( + is_loopback(server_host) or server_host in ["127.0.0.1"] + ): # daemon not running on localhost logger.err( RuntimeError( f"No daemon running at {server_host}:{server_port}, daemon will not be attached, stopping..." From b50e83072b4a1b9e7288a730f5c42222888172bd Mon Sep 17 00:00:00 2001 From: VisualDust Date: Sun, 17 Dec 2023 18:50:21 +0800 Subject: [PATCH 20/20] fix Failed to connect to daemon after 10s --- neetbox/client/_client.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/neetbox/client/_client.py b/neetbox/client/_client.py index c0cdaa2e..27239c0e 100644 --- a/neetbox/client/_client.py +++ b/neetbox/client/_client.py @@ -170,11 +170,9 @@ def _connect(cls, config=None): _retry_timeout = 10 _time_begin = time.perf_counter() logger.log("Created daemon process, trying to connect to daemon...") - + online_flag = False while time.perf_counter() - _time_begin < 10: # try connect daemon - if cls.check_server_connectivity(): - break - else: + if not cls.check_server_connectivity(): exit_code = popen.poll() if exit_code is not None: logger.err( @@ -182,10 +180,16 @@ def _connect(cls, config=None): ) return False time.sleep(0.5) - logger.err( - RuntimeError(f"Failed to connect to daemon after {_retry_timeout}s, stopping..."), - reraise=True, - ) + else: + online_flag = True + break + if not online_flag: + logger.err( + RuntimeError( + f"Failed to connect to daemon after {_retry_timeout}s, stopping..." + ), + reraise=True, + ) cls.online_mode = True # enable online mode cls.ws_server_url = f"ws://{server_host}:{server_port + 1}" # ws server url