From 11e71273945631c17ae10a431ecdff9376abf0a7 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Wed, 27 Mar 2024 15:30:46 +1000 Subject: [PATCH 001/100] Added initial prototype for rathole --- packages/grid/rathole/client.toml | 6 +++ packages/grid/rathole/domain.dockerfile | 9 ++++ packages/grid/rathole/nginx.conf | 6 +++ packages/grid/rathole/rathole.dockerfile | 53 ++++++++++++++++++++++++ packages/grid/rathole/server.toml | 6 +++ packages/grid/rathole/start-client.sh | 4 ++ packages/grid/rathole/start-server.sh | 2 + 7 files changed, 86 insertions(+) create mode 100644 packages/grid/rathole/client.toml create mode 100644 packages/grid/rathole/domain.dockerfile create mode 100644 packages/grid/rathole/nginx.conf create mode 100644 packages/grid/rathole/rathole.dockerfile create mode 100644 packages/grid/rathole/server.toml create mode 100755 packages/grid/rathole/start-client.sh create mode 100755 packages/grid/rathole/start-server.sh diff --git a/packages/grid/rathole/client.toml b/packages/grid/rathole/client.toml new file mode 100644 index 00000000000..ba8b835a569 --- /dev/null +++ b/packages/grid/rathole/client.toml @@ -0,0 +1,6 @@ +[client] +remote_addr = "host.docker.internal:2333" # public IP and port of gateway + +[client.services.domain] +token = "domain-specific-rathole-secret" +local_addr = "localhost:8000" # nginx proxy diff --git a/packages/grid/rathole/domain.dockerfile b/packages/grid/rathole/domain.dockerfile new file mode 100644 index 00000000000..cdb657540e8 --- /dev/null +++ b/packages/grid/rathole/domain.dockerfile @@ -0,0 +1,9 @@ +ARG PYTHON_VERSION="3.12" +FROM python:${PYTHON_VERSION}-bookworm +RUN apt update && apt install -y netcat-openbsd vim +WORKDIR /app +CMD ["python3", "-m", "http.server", "8000"] +EXPOSE 8000 + +# docker build -f domain.dockerfile . -t domain +# docker run -it -p 8080:8000 domain diff --git a/packages/grid/rathole/nginx.conf b/packages/grid/rathole/nginx.conf new file mode 100644 index 00000000000..af1a3a752d7 --- /dev/null +++ b/packages/grid/rathole/nginx.conf @@ -0,0 +1,6 @@ +server { + listen 8000; + location / { + proxy_pass http://host.docker.internal:8080; + } +} \ No newline at end of file diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile new file mode 100644 index 00000000000..4ae1648e6d6 --- /dev/null +++ b/packages/grid/rathole/rathole.dockerfile @@ -0,0 +1,53 @@ +ARG RATHOLE_VERSION="0.5.0" +ARG PYTHON_VERSION="3.12" + +FROM rust as build +ARG RATHOLE_VERSION +ARG FEATURES +RUN apt update && apt install -y git +RUN git clone -b v${RATHOLE_VERSION} https://github.com/rapiz1/rathole + +WORKDIR /rathole +RUN cargo build --locked --release --features ${FEATURES:-default} + +FROM python:${PYTHON_VERSION}-bookworm +ARG RATHOLE_VERSION +ENV MODE="client" +COPY --from=build /rathole/target/release/rathole /app/rathole +RUN apt update && apt install -y netcat-openbsd vim +WORKDIR /app +COPY ./start-client.sh /app/start-client.sh +COPY ./start-server.sh /app/start-server.sh +COPY ./client.toml /app/client.toml +COPY ./server.toml /app/server.toml +COPY ./nginx.conf /etc/nginx/conf.d/default.conf + +CMD ["sh", "-c", "/app/start-$MODE.sh"] +EXPOSE 2333/udp +EXPOSE 2333 + +# build and run a fake domain to simulate a normal http container service +# docker build -f domain.dockerfile . -t domain +# docker run -it -d -p 8080:8000 domain + +# check the web server is running on 8080 +# curl localhost:8080 + +# build and run the rathole container +# docker build -f rathole.dockerfile . -t rathole + +# run the rathole server +# docker run -it -p 8001:8001 -p 8002:8002 -p 2333:2333 -e MODE=server rathole + +# check nothing is on port 8001 yet +# curl localhost:8001 + +# run the rathole client +# docker run -it -e MODE=client rathole + +# try port 8001 now +# curl localhost:8001 + +# add another client and edit the server.toml and client.toml for port 8002 + + diff --git a/packages/grid/rathole/server.toml b/packages/grid/rathole/server.toml new file mode 100644 index 00000000000..8145491e7cc --- /dev/null +++ b/packages/grid/rathole/server.toml @@ -0,0 +1,6 @@ +[server] +bind_addr = "0.0.0.0:2333" # public open port + +[server.services.domain] +token = "domain-specific-rathole-secret" +bind_addr = "0.0.0.0:8001" diff --git a/packages/grid/rathole/start-client.sh b/packages/grid/rathole/start-client.sh new file mode 100755 index 00000000000..60ace3fa31b --- /dev/null +++ b/packages/grid/rathole/start-client.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +apt update && apt install -y nginx +nginx & +/app/rathole client.toml diff --git a/packages/grid/rathole/start-server.sh b/packages/grid/rathole/start-server.sh new file mode 100755 index 00000000000..700b081a60d --- /dev/null +++ b/packages/grid/rathole/start-server.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +/app/rathole server.toml From 91a326a142524bd2140e4990d69517f4a58c95ac Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 15 Apr 2024 17:40:26 +0530 Subject: [PATCH 002/100] update dockerfile with additional hosts for docker internal --- packages/grid/rathole/rathole.dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 4ae1648e6d6..c4eb717696b 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -28,7 +28,9 @@ EXPOSE 2333 # build and run a fake domain to simulate a normal http container service # docker build -f domain.dockerfile . -t domain -# docker run -it -d -p 8080:8000 domain +# docker run --name domain1 -it -d -p 8080:8000 domain + + # check the web server is running on 8080 # curl localhost:8080 @@ -37,13 +39,13 @@ EXPOSE 2333 # docker build -f rathole.dockerfile . -t rathole # run the rathole server -# docker run -it -p 8001:8001 -p 8002:8002 -p 2333:2333 -e MODE=server rathole +# docker run --add-host host.docker.internal:host-gateway --name rathole-server -it -p 8001:8001 -p 8002:8002 -p 2333:2333 -e MODE=server rathole # check nothing is on port 8001 yet # curl localhost:8001 # run the rathole client -# docker run -it -e MODE=client rathole +# docker run --add-host host.docker.internal:host-gateway --name rathole-client -it -e MODE=client rathole # try port 8001 now # curl localhost:8001 From 191193d0e75ddb496f5a7c67764e343610e5f74f Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 18 Apr 2024 17:59:07 +0530 Subject: [PATCH 003/100] define intial frame for the rathole server app --- packages/grid/rathole/main.py | 72 +++++++++++++++++++++++++++++++++ packages/grid/rathole/models.py | 11 +++++ 2 files changed, 83 insertions(+) create mode 100644 packages/grid/rathole/main.py create mode 100644 packages/grid/rathole/models.py diff --git a/packages/grid/rathole/main.py b/packages/grid/rathole/main.py new file mode 100644 index 00000000000..55212b2fcbd --- /dev/null +++ b/packages/grid/rathole/main.py @@ -0,0 +1,72 @@ +# stdlib +import os +import sys + +# third party +from fastapi import FastAPI +from fastapi import status +from loguru import logger + +# relative +from .models import RatholeConfig +from .models import ResponseModel + +# Logging Configuration +log_level = os.getenv("APP_LOG_LEVEL", "INFO").upper() +logger.remove() +logger.add(sys.stderr, colorize=True, level=log_level) + +app = FastAPI(title="Rathole") + + +async def healthcheck() -> bool: + return True + + +@app.get( + "/healthcheck", + response_model=ResponseModel, + status_code=status.HTTP_200_OK, +) +async def healthcheck_endpoint() -> ResponseModel: + res = await healthcheck() + if res: + return ResponseModel(message="OK") + else: + return ResponseModel(message="FAIL") + + +@app.post( + "/config/", + response_model=ResponseModel, + status_code=status.HTTP_201_CREATED, +) +async def add_config(config: RatholeConfig) -> ResponseModel: + return ResponseModel(message="Config added successfully") + + +@app.delete( + "/config/{uuid}", + response_model=ResponseModel, + status_code=status.HTTP_200_OK, +) +async def remove_config(uuid: str) -> ResponseModel: + return ResponseModel(message="Config removed successfully") + + +@app.put( + "/config/{uuid}", + response_model=ResponseModel, + status_code=status.HTTP_200_OK, +) +async def update_config() -> ResponseModel: + return ResponseModel(message="Config updated successfully") + + +@app.get( + "/config/{uuid}", + response_model=RatholeConfig, + status_code=status.HTTP_201_CREATED, +) +async def get_config(uuid: str) -> RatholeConfig: + pass diff --git a/packages/grid/rathole/models.py b/packages/grid/rathole/models.py new file mode 100644 index 00000000000..5feb12cbdaa --- /dev/null +++ b/packages/grid/rathole/models.py @@ -0,0 +1,11 @@ +# third party +from pydantic import BaseModel + + +class ResponseModel(BaseModel): + message: str + + +class RatholeConfig(BaseModel): + uuid: str + secret_token: str From 02694ec8f658772a66c704eb1ecf73c2072fd45b Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 22 Apr 2024 15:22:06 +0530 Subject: [PATCH 004/100] define RatholeConfig model - added a tomlreaderwriter to read/write with locks - added a RatholeClientToml Reader/Writer - define a RatholeServerTole reader/writer --- packages/grid/rathole/models.py | 6 ++ packages/grid/rathole/nginx.conf | 17 +++- packages/grid/rathole/toml_writer.py | 26 +++++ packages/grid/rathole/utils.py | 137 +++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 packages/grid/rathole/toml_writer.py create mode 100644 packages/grid/rathole/utils.py diff --git a/packages/grid/rathole/models.py b/packages/grid/rathole/models.py index 5feb12cbdaa..27091510896 100644 --- a/packages/grid/rathole/models.py +++ b/packages/grid/rathole/models.py @@ -9,3 +9,9 @@ class ResponseModel(BaseModel): class RatholeConfig(BaseModel): uuid: str secret_token: str + local_addr_host: str + local_addr_port: int + + @property + def local_address(self) -> str: + return f"{self.local_addr_host}:{self.local_addr_port}" diff --git a/packages/grid/rathole/nginx.conf b/packages/grid/rathole/nginx.conf index af1a3a752d7..b660e980400 100644 --- a/packages/grid/rathole/nginx.conf +++ b/packages/grid/rathole/nginx.conf @@ -1,6 +1,15 @@ -server { - listen 8000; - location / { - proxy_pass http://host.docker.internal:8080; +http { + server { + listen 8000; + location / { + proxy_pass http://host.docker.internal:8080; + } + } + + server { + listen 8001; + location / { + proxy_pass http://host.docker.internal:8081; + } } } \ No newline at end of file diff --git a/packages/grid/rathole/toml_writer.py b/packages/grid/rathole/toml_writer.py new file mode 100644 index 00000000000..09c44801432 --- /dev/null +++ b/packages/grid/rathole/toml_writer.py @@ -0,0 +1,26 @@ +# stdlib +from pathlib import Path +import tomllib + +# third party +from filelock import FileLock + +FILE_LOCK_TIMEOUT = 30 + + +class TomlReaderWriter: + def __init__(self, lock: FileLock, filename: Path | str) -> None: + self.filename = Path(filename) + self.timeout = FILE_LOCK_TIMEOUT + self.lock = lock + + def write(self, toml_dict: dict) -> None: + with self.lock.acquire(timeout=self.timeout): + with open(str(self.filename), "wb") as fp: + tomllib.dump(toml_dict, fp) + + def read(self) -> dict: + with self.lock.acquire(timeout=self.timeout): + with open(str(self.filename), "rb") as fp: + toml = tomllib.load(fp) + return toml diff --git a/packages/grid/rathole/utils.py b/packages/grid/rathole/utils.py new file mode 100644 index 00000000000..166eee41c32 --- /dev/null +++ b/packages/grid/rathole/utils.py @@ -0,0 +1,137 @@ +# stdlib + +# third party +from filelock import FileLock + +# relative +from .models import RatholeConfig +from .toml_writer import TomlReaderWriter + +lock = FileLock("rathole.toml.lock") + + +class RatholeClientToml: + filename: str = "client.toml" + + def __init__(self) -> None: + self.client_toml = TomlReaderWriter(lock=lock, filename=self.filename) + + def set_remote_addr(self, remote_host: str) -> None: + """Add a new remote address to the client toml file.""" + + toml = self.client_toml.read() + + # Add the new remote address + if "client" not in toml: + toml["client"] = {} + + toml["client"]["remote_addr"] = remote_host + + if remote_host not in toml["client"]["remote"]: + toml["client"]["remote"].append(remote_host) + + self.client_toml.write(toml_dict=toml) + + def add_config(self, config: RatholeConfig) -> None: + """Add a new config to the toml file.""" + + toml = self.client_toml.read() + + # Add the new config + if "services" not in toml["client"]: + toml["client"]["services"] = {} + + if config.uuid not in toml["client"]["services"]: + toml["client"]["services"][config.uuid] = {} + + toml["client"]["services"][config.uuid] = { + "token": config.secret_token, + "local_addr": config.local_address, + } + + self.client_toml.write(toml) + + def remove_config(self, uuid: str) -> None: + """Remove a config from the toml file.""" + + toml = self.client_toml.read() + + # Remove the config + if "services" not in toml["client"]: + return + + if uuid not in toml["client"]["services"]: + return + + del toml["client"]["services"][uuid] + + self.client_toml.write(toml) + + def update_config(self, config: RatholeConfig) -> None: + """Update a config in the toml file.""" + + toml = self.client_toml.read() + + # Update the config + if "services" not in toml["client"]: + return + + if config.uuid not in toml["client"]["services"]: + return + + toml["client"]["services"][config.uuid] = { + "token": config.secret_token, + "local_addr": config.local_address, + } + + self.client_toml.write(toml) + + def get_config(self, uuid: str) -> RatholeConfig | None: + """Get a config from the toml file.""" + + toml = self.client_toml.read() + + # Get the config + if "services" not in toml["client"]: + return None + + if uuid not in toml["client"]["services"]: + return None + + service = toml["client"]["services"][uuid] + + return RatholeConfig( + uuid=uuid, + secret_token=service["token"], + local_addr_host=service["local_addr"].split(":")[0], + local_addr_port=service["local_addr"].split(":")[1], + ) + + def _validate(self) -> bool: + if not self.client_toml.filename.exists(): + return False + + toml = self.client_toml.read() + + if not toml["client"]["remote_addr"]: + return False + + for uuid, config in toml["client"]["services"].items(): + if not uuid: + return False + + if not config["token"] and not config["local_addr"]: + return False + + return True + + @property + def is_valid(self) -> bool: + return self._validate() + + +class ServerTomlReaderWriter: + filename: str = "server.toml" + + def __init__(self) -> None: + self.server_toml = TomlReaderWriter(lock=lock, filename=self.filename) From 4d26dab81669b5df8604ac22f278ef53ea365589 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 22 Apr 2024 16:11:04 +0530 Subject: [PATCH 005/100] implement server toml reader/writer class --- packages/grid/rathole/utils.py | 89 +++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/packages/grid/rathole/utils.py b/packages/grid/rathole/utils.py index 166eee41c32..e65896de439 100644 --- a/packages/grid/rathole/utils.py +++ b/packages/grid/rathole/utils.py @@ -120,7 +120,7 @@ def _validate(self) -> bool: if not uuid: return False - if not config["token"] and not config["local_addr"]: + if not config["token"] or not config["local_addr"]: return False return True @@ -130,8 +130,93 @@ def is_valid(self) -> bool: return self._validate() -class ServerTomlReaderWriter: +class RatholeServerToml: filename: str = "server.toml" def __init__(self) -> None: self.server_toml = TomlReaderWriter(lock=lock, filename=self.filename) + + def set_bind_address(self, bind_address: str) -> None: + """Set the bind address in the server toml file.""" + + toml = self.server_toml.read() + + # Set the bind address + toml["server"]["bind_addr"] = bind_address + + self.server_toml.write(toml) + + def add_config(self, config: RatholeConfig) -> None: + """Add a new config to the toml file.""" + + toml = self.server_toml.read() + + # Add the new config + if "services" not in toml["server"]: + toml["server"]["services"] = {} + + if config.uuid not in toml["server"]["services"]: + toml["server"]["services"][config.uuid] = {} + + toml["server"]["services"][config.uuid] = { + "token": config.secret_token, + "bind_addr": config.local_address, + } + + self.server_toml.write(toml) + + def remove_config(self, uuid: str) -> None: + """Remove a config from the toml file.""" + + toml = self.server_toml.read() + + # Remove the config + if "services" not in toml["server"]: + return + + if uuid not in toml["server"]["services"]: + return + + del toml["server"]["services"][uuid] + + self.server_toml.write(toml) + + def update_config(self, config: RatholeConfig) -> None: + """Update a config in the toml file.""" + + toml = self.server_toml.read() + + # Update the config + if "services" not in toml["server"]: + return + + if config.uuid not in toml["server"]["services"]: + return + + toml["server"]["services"][config.uuid] = { + "token": config.secret_token, + "bind_addr": config.local_address, + } + + self.server_toml.write(toml) + + def _validate(self) -> bool: + if not self.server_toml.filename.exists(): + return False + + toml = self.server_toml.read() + + if not toml["server"]["bind_addr"]: + return False + + for uuid, config in toml["server"]["services"].items(): + if not uuid: + return False + + if not config["token"] or not config["bind_addr"]: + return False + + return True + + def is_valid(self) -> bool: + return self._validate() From a875114506f8e1adf8f5a9ee3a6587acafcc0b94 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 22 Apr 2024 16:21:10 +0530 Subject: [PATCH 006/100] integrate rathole toml client/server manager with Rathole fastapi endpoints --- packages/grid/rathole/main.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/grid/rathole/main.py b/packages/grid/rathole/main.py index 55212b2fcbd..c23a664f98d 100644 --- a/packages/grid/rathole/main.py +++ b/packages/grid/rathole/main.py @@ -1,4 +1,5 @@ # stdlib +from enum import Enum import os import sys @@ -10,6 +11,8 @@ # relative from .models import RatholeConfig from .models import ResponseModel +from .utils import RatholeClientToml +from .utils import RatholeServerToml # Logging Configuration log_level = os.getenv("APP_LOG_LEVEL", "INFO").upper() @@ -19,6 +22,21 @@ app = FastAPI(title="Rathole") +class RatholeServiceType(Enum): + CLIENT = "client" + SERVER = "server" + + +ServiceType = os.getenv("RATHOLE_SERVICE_TYPE") + + +RatholeTomlManager = ( + RatholeServerToml() + if ServiceType == RatholeServiceType.SERVER.value + else RatholeClientToml() +) + + async def healthcheck() -> bool: return True @@ -42,6 +60,7 @@ async def healthcheck_endpoint() -> ResponseModel: status_code=status.HTTP_201_CREATED, ) async def add_config(config: RatholeConfig) -> ResponseModel: + RatholeTomlManager.add_config(config) return ResponseModel(message="Config added successfully") @@ -51,6 +70,7 @@ async def add_config(config: RatholeConfig) -> ResponseModel: status_code=status.HTTP_200_OK, ) async def remove_config(uuid: str) -> ResponseModel: + RatholeTomlManager.remove_config(uuid) return ResponseModel(message="Config removed successfully") @@ -59,7 +79,8 @@ async def remove_config(uuid: str) -> ResponseModel: response_model=ResponseModel, status_code=status.HTTP_200_OK, ) -async def update_config() -> ResponseModel: +async def update_config(config: RatholeConfig) -> ResponseModel: + RatholeTomlManager.update_config(config=config) return ResponseModel(message="Config updated successfully") @@ -69,4 +90,4 @@ async def update_config() -> ResponseModel: status_code=status.HTTP_201_CREATED, ) async def get_config(uuid: str) -> RatholeConfig: - pass + return RatholeTomlManager.get_config(uuid) From 59eff342f6251516b14fb00edb75f1437097fd01 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 22 Apr 2024 17:36:24 +0530 Subject: [PATCH 007/100] add a default value to Rathole service type --- packages/grid/rathole/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/grid/rathole/main.py b/packages/grid/rathole/main.py index c23a664f98d..b1b1fe59a09 100644 --- a/packages/grid/rathole/main.py +++ b/packages/grid/rathole/main.py @@ -22,17 +22,17 @@ app = FastAPI(title="Rathole") -class RatholeServiceType(Enum): +class RatholeMode(Enum): CLIENT = "client" SERVER = "server" -ServiceType = os.getenv("RATHOLE_SERVICE_TYPE") +ServiceType = os.getenv("RATHOLE_MODE", "client").lower() RatholeTomlManager = ( RatholeServerToml() - if ServiceType == RatholeServiceType.SERVER.value + if ServiceType == RatholeMode.SERVER.value else RatholeClientToml() ) From 6ed6b100d7ac9426093509d11a3379257e4aeacb Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 23 Apr 2024 14:35:26 +0530 Subject: [PATCH 008/100] added nginx conf builder --- packages/grid/rathole/nginx_builder.py | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 packages/grid/rathole/nginx_builder.py diff --git a/packages/grid/rathole/nginx_builder.py b/packages/grid/rathole/nginx_builder.py new file mode 100644 index 00000000000..53ef4ae64d7 --- /dev/null +++ b/packages/grid/rathole/nginx_builder.py @@ -0,0 +1,53 @@ +# stdlib +from pathlib import Path + +# third party +from filelock import FileLock +import nginx +from nginx import Conf + + +class NginxConfigBuilder: + def __init__(self, filename: str | Path) -> None: + self.filename = Path(filename) + self.lock = FileLock(f"{filename}.lock") + self.lock_timeout = 30 + + def read(self) -> Conf: + with self.lock.acquire(timeout=self.lock_timeout): + conf = nginx.loadf(self.filename) + + return conf + + def write(self, conf: Conf) -> None: + with self.lock.acquire(timeout=self.lock_timeout): + nginx.dumpf(conf, self.filename) + + def add_server(self, listen_port: int, location: str, proxy_pass: str) -> None: + conf = self.read() + server = conf.servers.add() + server.listen = listen_port + location = server.locations.add() + location.path = location + location.proxy_pass = proxy_pass + self.write(conf) + + def remove_server(self, listen_port: int) -> None: + conf = self.read() + for server in conf.servers: + if server.listen == listen_port: + conf.servers.remove(server) + break + self.write(conf) + + def modify_location_for_port( + self, listen_port: int, location: str, proxy_pass: str + ) -> None: + conf = self.read() + for server in conf.servers: + if server.listen == listen_port: + for loc in server.locations: + if loc.path == location: + loc.proxy_pass = proxy_pass + break + self.write(conf) From 0084befb2088d728856981f2d87d6ff690ecdfdc Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 23 Apr 2024 23:43:33 +0530 Subject: [PATCH 009/100] Fix RatholeConfigBuilder class --- packages/grid/rathole/nginx_builder.py | 65 ++++++++++++++++++-------- packages/grid/rathole/toml_writer.py | 2 +- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/packages/grid/rathole/nginx_builder.py b/packages/grid/rathole/nginx_builder.py index 53ef4ae64d7..3a25d34bf8a 100644 --- a/packages/grid/rathole/nginx_builder.py +++ b/packages/grid/rathole/nginx_builder.py @@ -7,9 +7,13 @@ from nginx import Conf -class NginxConfigBuilder: +class RatholeNginxConfigBuilder: def __init__(self, filename: str | Path) -> None: - self.filename = Path(filename) + self.filename = Path(filename).absolute() + + if not self.filename.exists(): + self.filename.touch() + self.lock = FileLock(f"{filename}.lock") self.lock_timeout = 30 @@ -24,30 +28,53 @@ def write(self, conf: Conf) -> None: nginx.dumpf(conf, self.filename) def add_server(self, listen_port: int, location: str, proxy_pass: str) -> None: - conf = self.read() - server = conf.servers.add() - server.listen = listen_port - location = server.locations.add() - location.path = location - location.proxy_pass = proxy_pass - self.write(conf) + n_config = self.read() + server_to_modify = self.find_server_with_listen_port(listen_port) + + if server_to_modify is not None: + server_to_modify.add( + nginx.Location(location, nginx.Key("proxy_pass", proxy_pass)) + ) + else: + server = nginx.Server( + nginx.Key("listen", listen_port), + nginx.Location(location, nginx.Key("proxy_pass", proxy_pass)), + ) + n_config.add(server) + self.write(n_config) def remove_server(self, listen_port: int) -> None: conf = self.read() for server in conf.servers: - if server.listen == listen_port: - conf.servers.remove(server) - break + for child in server.children: + if child.name == "listen" and int(child.value) == listen_port: + conf.remove(server) + break self.write(conf) - def modify_location_for_port( + def find_server_with_listen_port(self, listen_port: int) -> nginx.Server | None: + conf = self.read() + for server in conf.servers: + for child in server.children: + if child.name == "listen" and int(child.value) == listen_port: + return server + return None + + def modify_proxy_for_port( self, listen_port: int, location: str, proxy_pass: str ) -> None: conf = self.read() - for server in conf.servers: - if server.listen == listen_port: - for loc in server.locations: - if loc.path == location: - loc.proxy_pass = proxy_pass - break + server_to_modify = self.find_server_with_listen_port(listen_port) + + if server_to_modify is None: + raise ValueError(f"Server with listen port {listen_port} not found") + + for location in server_to_modify.locations: + if location.value != location: + continue + for key in location.keys: + if key.name == "proxy_pass": + key.value = proxy_pass + break + self.write(conf) diff --git a/packages/grid/rathole/toml_writer.py b/packages/grid/rathole/toml_writer.py index 09c44801432..a0e79aff627 100644 --- a/packages/grid/rathole/toml_writer.py +++ b/packages/grid/rathole/toml_writer.py @@ -10,7 +10,7 @@ class TomlReaderWriter: def __init__(self, lock: FileLock, filename: Path | str) -> None: - self.filename = Path(filename) + self.filename = Path(filename).absolute() self.timeout = FILE_LOCK_TIMEOUT self.lock = lock From bbb093f3b13886d03b87c57056707fec8f63df13 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 24 Apr 2024 14:29:44 +0530 Subject: [PATCH 010/100] - pass server name to nginx builder - integrate nginx builder with toml manager - add requirements.txt - update rathole dockerfile Co-authored-by: Khoa Nguyen --- packages/grid/rathole/__init__.py | 0 packages/grid/rathole/main.py | 2 +- packages/grid/rathole/models.py | 3 ++- packages/grid/rathole/nginx_builder.py | 14 +++++++++++++- packages/grid/rathole/rathole.dockerfile | 7 +++++++ packages/grid/rathole/requirements.txt | 5 +++++ packages/grid/rathole/start-client.sh | 11 +++++++++++ packages/grid/rathole/utils.py | 14 ++++++++++++++ 8 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 packages/grid/rathole/__init__.py create mode 100644 packages/grid/rathole/requirements.txt diff --git a/packages/grid/rathole/__init__.py b/packages/grid/rathole/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/grid/rathole/main.py b/packages/grid/rathole/main.py index b1b1fe59a09..fe4c8f8a081 100644 --- a/packages/grid/rathole/main.py +++ b/packages/grid/rathole/main.py @@ -27,7 +27,7 @@ class RatholeMode(Enum): SERVER = "server" -ServiceType = os.getenv("RATHOLE_MODE", "client").lower() +ServiceType = os.getenv("MODE", "client").lower() RatholeTomlManager = ( diff --git a/packages/grid/rathole/models.py b/packages/grid/rathole/models.py index 27091510896..c5921738885 100644 --- a/packages/grid/rathole/models.py +++ b/packages/grid/rathole/models.py @@ -11,7 +11,8 @@ class RatholeConfig(BaseModel): secret_token: str local_addr_host: str local_addr_port: int + server_name: str | None = None @property def local_address(self) -> str: - return f"{self.local_addr_host}:{self.local_addr_port}" + return f"http://{self.local_addr_host}:{self.local_addr_port}" diff --git a/packages/grid/rathole/nginx_builder.py b/packages/grid/rathole/nginx_builder.py index 3a25d34bf8a..3d1bd14ce1f 100644 --- a/packages/grid/rathole/nginx_builder.py +++ b/packages/grid/rathole/nginx_builder.py @@ -27,7 +27,13 @@ def write(self, conf: Conf) -> None: with self.lock.acquire(timeout=self.lock_timeout): nginx.dumpf(conf, self.filename) - def add_server(self, listen_port: int, location: str, proxy_pass: str) -> None: + def add_server( + self, + listen_port: int, + location: str, + proxy_pass: str, + server_name: str | None = None, + ) -> None: n_config = self.read() server_to_modify = self.find_server_with_listen_port(listen_port) @@ -35,12 +41,18 @@ def add_server(self, listen_port: int, location: str, proxy_pass: str) -> None: server_to_modify.add( nginx.Location(location, nginx.Key("proxy_pass", proxy_pass)) ) + if server_name is not None: + server_to_modify.add(nginx.Key("server_name", server_name)) else: server = nginx.Server( nginx.Key("listen", listen_port), nginx.Location(location, nginx.Key("proxy_pass", proxy_pass)), ) + if server_name is not None: + server.add(nginx.Key("server_name", server_name)) + n_config.add(server) + self.write(n_config) def remove_server(self, listen_port: int) -> None: diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index c4eb717696b..3916a0eb12f 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -13,6 +13,7 @@ RUN cargo build --locked --release --features ${FEATURES:-default} FROM python:${PYTHON_VERSION}-bookworm ARG RATHOLE_VERSION ENV MODE="client" +ENV APP_LOG_LEVEL="info" COPY --from=build /rathole/target/release/rathole /app/rathole RUN apt update && apt install -y netcat-openbsd vim WORKDIR /app @@ -21,7 +22,13 @@ COPY ./start-server.sh /app/start-server.sh COPY ./client.toml /app/client.toml COPY ./server.toml /app/server.toml COPY ./nginx.conf /etc/nginx/conf.d/default.conf +COPY ./main.py /app/main.py +COPY ./nginx_builder.py /app/nginx_builder.py +COPY ./utils.py /app/utils.py +COPY ./requirements.txt /app/requirements.txt + +RUN pip install --user -r requirements.txt CMD ["sh", "-c", "/app/start-$MODE.sh"] EXPOSE 2333/udp EXPOSE 2333 diff --git a/packages/grid/rathole/requirements.txt b/packages/grid/rathole/requirements.txt new file mode 100644 index 00000000000..4b379d83e8e --- /dev/null +++ b/packages/grid/rathole/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.110.0 +filelock==3.13.4 +loguru==0.7.2 +python-nginx +uvicorn[standard]==0.27.1 diff --git a/packages/grid/rathole/start-client.sh b/packages/grid/rathole/start-client.sh index 60ace3fa31b..850e71ab6e3 100755 --- a/packages/grid/rathole/start-client.sh +++ b/packages/grid/rathole/start-client.sh @@ -1,4 +1,15 @@ #!/usr/bin/env bash + +APP_MODULE=main:app +LOG_LEVEL=${LOG_LEVEL:-info} +HOST=${HOST:-0.0.0.0} +PORT=${PORT:-80} +RELOAD="" +DEBUG_CMD="" + + + apt update && apt install -y nginx nginx & +exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" & /app/rathole client.toml diff --git a/packages/grid/rathole/utils.py b/packages/grid/rathole/utils.py index e65896de439..485e4ae7e23 100644 --- a/packages/grid/rathole/utils.py +++ b/packages/grid/rathole/utils.py @@ -5,6 +5,7 @@ # relative from .models import RatholeConfig +from .nginx_builder import RatholeNginxConfigBuilder from .toml_writer import TomlReaderWriter lock = FileLock("rathole.toml.lock") @@ -15,6 +16,7 @@ class RatholeClientToml: def __init__(self) -> None: self.client_toml = TomlReaderWriter(lock=lock, filename=self.filename) + self.nginx_mananger = RatholeNginxConfigBuilder("nginx.conf") def set_remote_addr(self, remote_host: str) -> None: """Add a new remote address to the client toml file.""" @@ -51,6 +53,10 @@ def add_config(self, config: RatholeConfig) -> None: self.client_toml.write(toml) + self.nginx_mananger.add_server( + config.local_addr_port, location="/", proxy_pass="http://backend:80" + ) + def remove_config(self, uuid: str) -> None: """Remove a config from the toml file.""" @@ -135,6 +141,7 @@ class RatholeServerToml: def __init__(self) -> None: self.server_toml = TomlReaderWriter(lock=lock, filename=self.filename) + self.nginx_manager = RatholeNginxConfigBuilder("nginx.conf") def set_bind_address(self, bind_address: str) -> None: """Set the bind address in the server toml file.""" @@ -165,6 +172,13 @@ def add_config(self, config: RatholeConfig) -> None: self.server_toml.write(toml) + self.nginx_manager.add_server( + config.local_addr_port, + location="/", + proxy_pass=config.local_address, + server_name=f"{config.server_name}.local*", + ) + def remove_config(self, uuid: str) -> None: """Remove a config from the toml file.""" From 17e9a2f52410deba585c633734b26ddea2065599 Mon Sep 17 00:00:00 2001 From: khoaguin Date: Thu, 25 Apr 2024 09:57:40 +0700 Subject: [PATCH 011/100] fix relative import issues --- packages/grid/rathole/main.py | 10 ++++------ packages/grid/rathole/rathole.dockerfile | 4 +++- packages/grid/rathole/start-client.sh | 2 -- packages/grid/rathole/utils.py | 8 +++----- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/grid/rathole/main.py b/packages/grid/rathole/main.py index fe4c8f8a081..dfb5f6bd2a3 100644 --- a/packages/grid/rathole/main.py +++ b/packages/grid/rathole/main.py @@ -7,12 +7,10 @@ from fastapi import FastAPI from fastapi import status from loguru import logger - -# relative -from .models import RatholeConfig -from .models import ResponseModel -from .utils import RatholeClientToml -from .utils import RatholeServerToml +from models import RatholeConfig +from models import ResponseModel +from utils import RatholeClientToml +from utils import RatholeServerToml # Logging Configuration log_level = os.getenv("APP_LOG_LEVEL", "INFO").upper() diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 3916a0eb12f..5402b4291c1 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -22,12 +22,14 @@ COPY ./start-server.sh /app/start-server.sh COPY ./client.toml /app/client.toml COPY ./server.toml /app/server.toml COPY ./nginx.conf /etc/nginx/conf.d/default.conf +COPY ./__init__.py /app/__init__.py COPY ./main.py /app/main.py +COPY ./models.py /app/models.py COPY ./nginx_builder.py /app/nginx_builder.py COPY ./utils.py /app/utils.py +COPY ./toml_writer.py /app/toml_writer.py COPY ./requirements.txt /app/requirements.txt - RUN pip install --user -r requirements.txt CMD ["sh", "-c", "/app/start-$MODE.sh"] EXPOSE 2333/udp diff --git a/packages/grid/rathole/start-client.sh b/packages/grid/rathole/start-client.sh index 850e71ab6e3..035d370a1dc 100755 --- a/packages/grid/rathole/start-client.sh +++ b/packages/grid/rathole/start-client.sh @@ -7,8 +7,6 @@ PORT=${PORT:-80} RELOAD="" DEBUG_CMD="" - - apt update && apt install -y nginx nginx & exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" & diff --git a/packages/grid/rathole/utils.py b/packages/grid/rathole/utils.py index 485e4ae7e23..c6d54c8d7fe 100644 --- a/packages/grid/rathole/utils.py +++ b/packages/grid/rathole/utils.py @@ -2,11 +2,9 @@ # third party from filelock import FileLock - -# relative -from .models import RatholeConfig -from .nginx_builder import RatholeNginxConfigBuilder -from .toml_writer import TomlReaderWriter +from models import RatholeConfig +from nginx_builder import RatholeNginxConfigBuilder +from toml_writer import TomlReaderWriter lock = FileLock("rathole.toml.lock") From 3f1be01132c4c824cf93b175370b7dd687843d38 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 24 Apr 2024 22:45:39 +0530 Subject: [PATCH 012/100] fix rathole server --- packages/grid/rathole/nginx.conf | 7 ------- packages/grid/rathole/rathole.dockerfile | 7 +------ packages/grid/rathole/{ => server}/__init__.py | 0 packages/grid/rathole/{ => server}/main.py | 17 ++++++++++------- packages/grid/rathole/{ => server}/models.py | 0 .../grid/rathole/{ => server}/nginx_builder.py | 0 .../grid/rathole/{ => server}/toml_writer.py | 0 packages/grid/rathole/{ => server}/utils.py | 0 packages/grid/rathole/start-client.sh | 6 +++--- 9 files changed, 14 insertions(+), 23 deletions(-) rename packages/grid/rathole/{ => server}/__init__.py (100%) rename packages/grid/rathole/{ => server}/main.py (83%) rename packages/grid/rathole/{ => server}/models.py (100%) rename packages/grid/rathole/{ => server}/nginx_builder.py (100%) rename packages/grid/rathole/{ => server}/toml_writer.py (100%) rename packages/grid/rathole/{ => server}/utils.py (100%) diff --git a/packages/grid/rathole/nginx.conf b/packages/grid/rathole/nginx.conf index b660e980400..2be02976de4 100644 --- a/packages/grid/rathole/nginx.conf +++ b/packages/grid/rathole/nginx.conf @@ -5,11 +5,4 @@ http { proxy_pass http://host.docker.internal:8080; } } - - server { - listen 8001; - location / { - proxy_pass http://host.docker.internal:8081; - } - } } \ No newline at end of file diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 5402b4291c1..91f3a5d11e9 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -22,13 +22,8 @@ COPY ./start-server.sh /app/start-server.sh COPY ./client.toml /app/client.toml COPY ./server.toml /app/server.toml COPY ./nginx.conf /etc/nginx/conf.d/default.conf -COPY ./__init__.py /app/__init__.py -COPY ./main.py /app/main.py -COPY ./models.py /app/models.py -COPY ./nginx_builder.py /app/nginx_builder.py -COPY ./utils.py /app/utils.py -COPY ./toml_writer.py /app/toml_writer.py COPY ./requirements.txt /app/requirements.txt +COPY ./server/ /app/server/ RUN pip install --user -r requirements.txt CMD ["sh", "-c", "/app/start-$MODE.sh"] diff --git a/packages/grid/rathole/__init__.py b/packages/grid/rathole/server/__init__.py similarity index 100% rename from packages/grid/rathole/__init__.py rename to packages/grid/rathole/server/__init__.py diff --git a/packages/grid/rathole/main.py b/packages/grid/rathole/server/main.py similarity index 83% rename from packages/grid/rathole/main.py rename to packages/grid/rathole/server/main.py index dfb5f6bd2a3..201618c6432 100644 --- a/packages/grid/rathole/main.py +++ b/packages/grid/rathole/server/main.py @@ -7,10 +7,10 @@ from fastapi import FastAPI from fastapi import status from loguru import logger -from models import RatholeConfig -from models import ResponseModel -from utils import RatholeClientToml -from utils import RatholeServerToml +from server.models import RatholeConfig +from server.models import ResponseModel +from server.utils import RatholeClientToml +from server.utils import RatholeServerToml # Logging Configuration log_level = os.getenv("APP_LOG_LEVEL", "INFO").upper() @@ -40,7 +40,7 @@ async def healthcheck() -> bool: @app.get( - "/healthcheck", + "/", response_model=ResponseModel, status_code=status.HTTP_200_OK, ) @@ -84,8 +84,11 @@ async def update_config(config: RatholeConfig) -> ResponseModel: @app.get( "/config/{uuid}", - response_model=RatholeConfig, + response_model=RatholeConfig | ResponseModel, status_code=status.HTTP_201_CREATED, ) async def get_config(uuid: str) -> RatholeConfig: - return RatholeTomlManager.get_config(uuid) + config = RatholeTomlManager.get_config(uuid) + if config is None: + return ResponseModel(message="Config not found") + return config diff --git a/packages/grid/rathole/models.py b/packages/grid/rathole/server/models.py similarity index 100% rename from packages/grid/rathole/models.py rename to packages/grid/rathole/server/models.py diff --git a/packages/grid/rathole/nginx_builder.py b/packages/grid/rathole/server/nginx_builder.py similarity index 100% rename from packages/grid/rathole/nginx_builder.py rename to packages/grid/rathole/server/nginx_builder.py diff --git a/packages/grid/rathole/toml_writer.py b/packages/grid/rathole/server/toml_writer.py similarity index 100% rename from packages/grid/rathole/toml_writer.py rename to packages/grid/rathole/server/toml_writer.py diff --git a/packages/grid/rathole/utils.py b/packages/grid/rathole/server/utils.py similarity index 100% rename from packages/grid/rathole/utils.py rename to packages/grid/rathole/server/utils.py diff --git a/packages/grid/rathole/start-client.sh b/packages/grid/rathole/start-client.sh index 035d370a1dc..9b667f40e85 100755 --- a/packages/grid/rathole/start-client.sh +++ b/packages/grid/rathole/start-client.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -APP_MODULE=main:app +APP_MODULE=server.main:app LOG_LEVEL=${LOG_LEVEL:-info} HOST=${HOST:-0.0.0.0} -PORT=${PORT:-80} -RELOAD="" +PORT=${PORT:-5555} +RELOAD="--reload" DEBUG_CMD="" apt update && apt install -y nginx From f5bd6ff9a0d6adbed330d687d43dc6cfa0507fb0 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 26 Apr 2024 11:04:17 +0530 Subject: [PATCH 013/100] add rathole service to docker compose and traefik template --- packages/grid/default.env | 1 + packages/grid/docker-compose.build.yml | 6 ++++++ packages/grid/docker-compose.dev.yml | 6 ++++++ packages/grid/docker-compose.yml | 13 ++++++++++++- packages/grid/traefik/docker/dynamic.yml | 13 ++++++------- packages/hagrid/hagrid/cli.py | 14 ++++++++++++++ 6 files changed, 45 insertions(+), 8 deletions(-) diff --git a/packages/grid/default.env b/packages/grid/default.env index fb9cbd9a88d..bfb20ef1194 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -21,6 +21,7 @@ TRAEFIK_PUBLIC_TAG=traefik-public STACK_NAME=grid-openmined-org DOCKER_IMAGE_BACKEND=openmined/grid-backend DOCKER_IMAGE_FRONTEND=openmined/grid-frontend +DOCKER_IMAGE_RATHOLE=openmined/grid-rathole DOCKER_IMAGE_TRAEFIK=traefik TRAEFIK_VERSION=v2.11.0 REDIS_VERSION=6.2 diff --git a/packages/grid/docker-compose.build.yml b/packages/grid/docker-compose.build.yml index 7dc60d3fe41..cd43380ec18 100644 --- a/packages/grid/docker-compose.build.yml +++ b/packages/grid/docker-compose.build.yml @@ -22,3 +22,9 @@ services: context: ${RELATIVE_PATH}../ dockerfile: ./grid/backend/backend.dockerfile target: "backend" + + rathole: + build: + context: ${RELATIVE_PATH}../ + dockerfile: ./grid/rathole/rathole.dockerfile + target: "rathole" diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index d2b1f142053..c6a3c14d9e6 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -52,6 +52,12 @@ services: stdin_open: true tty: true + rathole: + volumes: + - ${RELATIVE_PATH}./rathole/server:/root/app/server + environment: + - DEV_MODE=True + # backend_stream: # volumes: # - ${RELATIVE_PATH}./backend/grid:/root/app/grid diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index c1b4599e300..425ed318565 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -50,13 +50,24 @@ services: - VERSION_HASH=${VERSION_HASH} - PORT=80 - HTTP_PORT=${HTTP_PORT} - - HTTPS_PORT=${HTTPS_PORT} + - HTTPS_PORT=${HTTPS_PORT}RELOAD - BACKEND_API_BASE_URL=${BACKEND_API_BASE_URL} extra_hosts: - "host.docker.internal:host-gateway" labels: - "orgs.openmined.syft=this is a syft frontend container" + rathole: + restart: always + image: "${DOCKER_IMAGE_RATHOLE?Variable not set}:${VERSION-latest}" + profiles: + - rathole + environment: + - SERVICE_NAME=rathole + - APP_LOG_LEVEL=${APP_LOG_LEVEL} + - MODE=${MODE} + - DEV_MODE=${DEV_MODE} + # redis: # restart: always # image: redis:${REDIS_VERSION?Variable not set} diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index cc6a7bb7ee4..3c6d0945077 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -48,14 +48,13 @@ http: middlewares: - "blob-storage-url" - "blob-storage-host" - vpn: - rule: "PathPrefix(`/vpn`)" + rathole: + rule: "PathPrefix(`/rathole`)" entryPoints: - web - - vpn - service: "headscale" + service: "rathole" middlewares: - - "vpn-url" + - "rathole-url" ping: rule: "PathPrefix(`/ping`)" entryPoints: @@ -73,7 +72,7 @@ http: stripprefix: prefixes: /blob forceslash: true - vpn-url: + rathole-url: stripprefix: - prefixes: /vpn + prefixes: /rathole forceslash: true diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 8fab98b4a83..cb0a3e9376c 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -496,6 +496,11 @@ def clean(location: str) -> None: is_flag=True, help="Enable auto approval of association requests", ) +@click.option( + "--rathole", + is_flag=True, + help="Enable rathole service", +) def launch(args: tuple[str], **kwargs: Any) -> None: verb = get_launch_verb() try: @@ -1314,6 +1319,7 @@ def create_launch_cmd( parsed_kwargs["headless"] = headless parsed_kwargs["tls"] = bool(kwargs["tls"]) + parsed_kwargs["enable_rathole"] = bool(kwargs["rathole"]) parsed_kwargs["test"] = bool(kwargs["test"]) parsed_kwargs["dev"] = bool(kwargs["dev"]) @@ -2241,6 +2247,11 @@ def create_launch_docker_cmd( else bool(kwargs["use_blob_storage"]) ) + enable_rathole = bool(kwargs.get("enable_rathole")) or str(node_type.input) in [ + "network", + "gateway", + ] + # use a docker volume host_path = "credentials-data" @@ -2411,6 +2422,9 @@ def create_launch_docker_cmd( if use_blob_storage: cmd += " --profile blob-storage" + if enable_rathole: + cmd += " --profile rathole" + # no frontend container so expect bad gateway on the / route if not bool(kwargs["headless"]): cmd += " --profile frontend" From 284ca0585a33260d738d0fdb2596580298564f38 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 26 Apr 2024 11:20:21 +0530 Subject: [PATCH 014/100] set rathole mode in hagrid cli --- packages/grid/docker-compose.dev.yml | 4 ++++ packages/grid/docker-compose.yml | 9 +++++++++ packages/grid/rathole/start-client.sh | 4 ++-- packages/hagrid/hagrid/cli.py | 7 +++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index c6a3c14d9e6..185cd461170 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -57,6 +57,10 @@ services: - ${RELATIVE_PATH}./rathole/server:/root/app/server environment: - DEV_MODE=True + stdin_open: true + tty: true + ports: + - "2333" # backend_stream: # volumes: diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 425ed318565..1039a80bc1a 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -67,6 +67,15 @@ services: - APP_LOG_LEVEL=${APP_LOG_LEVEL} - MODE=${MODE} - DEV_MODE=${DEV_MODE} + - APP_PORT=${APP_PORT} + - RATHOLE_PORT=${RATHOLE_PORT:-2333} + extra_hosts: + - "host.docker.internal:host-gateway" + labels: + - "orgs.openmined.syft=this is a syft rathole container" + ports: + - "${APP_PORT}:${APP_PORT}" + - "${RATHOLE_PORT}:${RATHOLE_PORT}" # redis: # restart: always diff --git a/packages/grid/rathole/start-client.sh b/packages/grid/rathole/start-client.sh index 9b667f40e85..9bbb7d9097d 100755 --- a/packages/grid/rathole/start-client.sh +++ b/packages/grid/rathole/start-client.sh @@ -3,11 +3,11 @@ APP_MODULE=server.main:app LOG_LEVEL=${LOG_LEVEL:-info} HOST=${HOST:-0.0.0.0} -PORT=${PORT:-5555} +PORT=${APP_PORT:-5555} RELOAD="--reload" DEBUG_CMD="" apt update && apt install -y nginx nginx & -exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" & +exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $APP_PORT --log-level $LOG_LEVEL "$APP_MODULE" & /app/rathole client.toml diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index cb0a3e9376c..1bd901d42ba 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -2263,6 +2263,10 @@ def create_launch_docker_cmd( # we might need to change this for the hagrid template mode host_path = f"{RELATIVE_PATH}./backend/grid/storage/{snake_name}" + rathole_mode = ( + "client" if enable_rathole and str(node_type.input) in ["domain"] else "server" + ) + envs = { "RELEASE": "production", "COMPOSE_DOCKER_CLI_BUILD": 1, @@ -2358,6 +2362,9 @@ def create_launch_docker_cmd( if "enable_signup" in kwargs: envs["ENABLE_SIGNUP"] = kwargs["enable_signup"] + if enable_rathole: + envs["MODE"] = rathole_mode + cmd = "" args = [] for k, v in envs.items(): From ce2d36eeec1450e9d259670f61f1be88f41b8bb1 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 26 Apr 2024 15:17:55 +0530 Subject: [PATCH 015/100] fix rathole config in docker compose files - fix imports in rathole utils - add rathole loadbalancer to traefik --- packages/grid/docker-compose.build.yml | 5 ++--- packages/grid/docker-compose.dev.yml | 6 ++++-- packages/grid/docker-compose.pull.yml | 3 +++ packages/grid/docker-compose.yml | 7 ++----- packages/grid/rathole/server/utils.py | 8 +++++--- packages/grid/rathole/start-server.sh | 11 +++++++++++ packages/grid/traefik/docker/dynamic.yml | 4 ++-- 7 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/grid/docker-compose.build.yml b/packages/grid/docker-compose.build.yml index cd43380ec18..a0175bc762a 100644 --- a/packages/grid/docker-compose.build.yml +++ b/packages/grid/docker-compose.build.yml @@ -25,6 +25,5 @@ services: rathole: build: - context: ${RELATIVE_PATH}../ - dockerfile: ./grid/rathole/rathole.dockerfile - target: "rathole" + context: ${RELATIVE_PATH}./rathole + dockerfile: rathole.dockerfile diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index 185cd461170..6ac138cba31 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -54,13 +54,15 @@ services: rathole: volumes: - - ${RELATIVE_PATH}./rathole/server:/root/app/server + - ${RELATIVE_PATH}./rathole/:/root/app/ environment: - DEV_MODE=True + - APP_PORT=5555 + - APP_LOG_LEVEL=debug stdin_open: true tty: true ports: - - "2333" + - "2333:2333" # backend_stream: # volumes: diff --git a/packages/grid/docker-compose.pull.yml b/packages/grid/docker-compose.pull.yml index db2329b04df..7fea3571f8a 100644 --- a/packages/grid/docker-compose.pull.yml +++ b/packages/grid/docker-compose.pull.yml @@ -24,3 +24,6 @@ services: # Temporary fix until we refactor pull, build, launch UI step during hagrid launch worker: image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" + + rathole: + image: "${DOCKER_IMAGE_RATHOLE?Variable not set}:${VERSION-latest}" diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 1039a80bc1a..9273a109a60 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -64,18 +64,15 @@ services: - rathole environment: - SERVICE_NAME=rathole - - APP_LOG_LEVEL=${APP_LOG_LEVEL} + - APP_LOG_LEVEL=${APP_LOG_LEVEL:-info} - MODE=${MODE} - DEV_MODE=${DEV_MODE} - - APP_PORT=${APP_PORT} + - APP_PORT=${APP_PORT:-5555} - RATHOLE_PORT=${RATHOLE_PORT:-2333} extra_hosts: - "host.docker.internal:host-gateway" labels: - "orgs.openmined.syft=this is a syft rathole container" - ports: - - "${APP_PORT}:${APP_PORT}" - - "${RATHOLE_PORT}:${RATHOLE_PORT}" # redis: # restart: always diff --git a/packages/grid/rathole/server/utils.py b/packages/grid/rathole/server/utils.py index c6d54c8d7fe..485e4ae7e23 100644 --- a/packages/grid/rathole/server/utils.py +++ b/packages/grid/rathole/server/utils.py @@ -2,9 +2,11 @@ # third party from filelock import FileLock -from models import RatholeConfig -from nginx_builder import RatholeNginxConfigBuilder -from toml_writer import TomlReaderWriter + +# relative +from .models import RatholeConfig +from .nginx_builder import RatholeNginxConfigBuilder +from .toml_writer import TomlReaderWriter lock = FileLock("rathole.toml.lock") diff --git a/packages/grid/rathole/start-server.sh b/packages/grid/rathole/start-server.sh index 700b081a60d..2ab8d52c7ed 100755 --- a/packages/grid/rathole/start-server.sh +++ b/packages/grid/rathole/start-server.sh @@ -1,2 +1,13 @@ #!/usr/bin/env bash + +APP_MODULE=server.main:app +LOG_LEVEL=${LOG_LEVEL:-info} +HOST=${HOST:-0.0.0.0} +PORT=${APP_PORT:-5555} +RELOAD="--reload" +DEBUG_CMD="" + +apt update && apt install -y nginx +nginx & +exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $APP_PORT --log-level $LOG_LEVEL "$APP_MODULE" & /app/rathole server.toml diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index 3c6d0945077..830ad50ad86 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -20,10 +20,10 @@ http: loadBalancer: servers: - url: "http://seaweedfs:4001" - headscale: + rathole: loadBalancer: servers: - - url: "http://headscale:8080" + - url: "http://rathole:5555" routers: frontend: rule: "PathPrefix(`/`)" From c179ddb579e387fd1448d1239afbdf4dac2e46bb Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Sun, 28 Apr 2024 18:29:46 +0530 Subject: [PATCH 016/100] fix client.toml --- packages/grid/docker-compose.dev.yml | 28 +------- packages/grid/docker-compose.pull.yml | 9 --- packages/grid/docker-compose.yml | 84 ------------------------ packages/grid/rathole/client.toml | 2 +- packages/grid/rathole/nginx.conf | 10 ++- packages/grid/traefik/docker/dynamic.yml | 18 ++++- 6 files changed, 23 insertions(+), 128 deletions(-) diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index 6ac138cba31..bcde98488e7 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -17,16 +17,6 @@ services: environment: - FRONTEND_TARGET=grid-ui-development - # redis: - # ports: - # - "6379" - - # queue: - # image: rabbitmq:3-management - # ports: - # - "15672" # admin web port - # # - "5672" # AMQP port - mongo: ports: - "27017" @@ -62,23 +52,7 @@ services: stdin_open: true tty: true ports: - - "2333:2333" - - # backend_stream: - # volumes: - # - ${RELATIVE_PATH}./backend/grid:/root/app/grid - # - ${RELATIVE_PATH}../syft:/root/app/syft - # - ${RELATIVE_PATH}./data/package-cache:/root/.cache - # environment: - # - DEV_MODE=True - - # celeryworker: - # volumes: - # - ${RELATIVE_PATH}./backend/grid:/root/app/grid - # - ${RELATIVE_PATH}../syft/:/root/app/syft - # - ${RELATIVE_PATH}./data/package-cache:/root/.cache - # environment: - # - DEV_MODE=True + - 2333:2333 seaweedfs: volumes: diff --git a/packages/grid/docker-compose.pull.yml b/packages/grid/docker-compose.pull.yml index 7fea3571f8a..e68ed03d968 100644 --- a/packages/grid/docker-compose.pull.yml +++ b/packages/grid/docker-compose.pull.yml @@ -1,17 +1,8 @@ version: "3.8" services: - # redis: - # image: redis:${REDIS_VERSION?Variable not set} - - # queue: - # image: rabbitmq:${RABBITMQ_VERSION?Variable not Set}${RABBITMQ_MANAGEMENT:-} - seaweedfs: image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${VERSION-latest}" - # docker-host: - # image: qoomon/docker-host - proxy: image: ${DOCKER_IMAGE_TRAEFIK?Variable not set}:${TRAEFIK_VERSION?Variable not set} diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 9273a109a60..8f7ec80765f 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -74,27 +74,6 @@ services: labels: - "orgs.openmined.syft=this is a syft rathole container" - # redis: - # restart: always - # image: redis:${REDIS_VERSION?Variable not set} - # volumes: - # - app-redis-data:/data - # - ./redis/redis.conf:/usr/local/etc/redis/redis.conf - # environment: - # - SERVICE_NAME=redis - # - RELEASE=${RELEASE:-production} - # env_file: - # - .env - - # queue: - # restart: always - # image: rabbitmq:3 - # environment: - # - SERVICE_NAME=queue - # - RELEASE=${RELEASE:-production} - # volumes: - # - ./rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf - worker: restart: always image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" @@ -183,69 +162,6 @@ services: labels: - "orgs.openmined.syft=this is a syft backend container" - # backend_stream: - # restart: always - # image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" - # depends_on: - # - proxy - # env_file: - # - .env - # environment: - # - SERVICE_NAME=backend_stream - # - RELEASE=${RELEASE:-production} - # - VERSION=${VERSION} - # - VERSION_HASH=${VERSION_HASH} - # - NODE_TYPE=${NODE_TYPE?Variable not set} - # - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} - # - STACK_API_KEY=${STACK_API_KEY} - # - PORT=8011 - # - STREAM_QUEUE=1 - # - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} - # - HTTP_PORT=${HTTP_PORT} - # - HTTPS_PORT=${HTTPS_PORT} - # - USE_BLOB_STORAGE=${USE_BLOB_STORAGE} - # - CONTAINER_HOST=${CONTAINER_HOST} - # - TRACE=${TRACE} - # - JAEGER_HOST=${JAEGER_HOST} - # - JAEGER_PORT=${JAEGER_PORT} - # - DEV_MODE=${DEV_MODE} - # network_mode: service:proxy - # volumes: - # - credentials-data:/root/data/creds/ - - # celeryworker: - # restart: always - # image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" - # depends_on: - # - proxy - # - queue - # env_file: - # - .env - # environment: - # - SERVICE_NAME=celeryworker - # - RELEASE=${RELEASE:-production} - # - VERSION=${VERSION} - # - VERSION_HASH=${VERSION_HASH} - # - NODE_TYPE=${NODE_TYPE?Variable not set} - # - DOMAIN_NAME=${DOMAIN_NAME?Variable not set} - # - C_FORCE_ROOT=1 - # - STACK_API_KEY=${STACK_API_KEY} - # - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} - # - HTTP_PORT=${HTTP_PORT} - # - HTTPS_PORT=${HTTPS_PORT} - # - USE_BLOB_STORAGE=${USE_BLOB_STORAGE} - # - CONTAINER_HOST=${CONTAINER_HOST} - # - NETWORK_CHECK_INTERVAL=${NETWORK_CHECK_INTERVAL} - # - DOMAIN_CHECK_INTERVAL=${DOMAIN_CHECK_INTERVAL} - # - TRACE=${TRACE} - # - JAEGER_HOST=${JAEGER_HOST} - # - JAEGER_PORT=${JAEGER_PORT} - # - DEV_MODE=${DEV_MODE} - # command: "/app/grid/worker-start.sh" - # network_mode: service:proxy - # volumes: - # - credentials-data:/storage - seaweedfs: profiles: - blob-storage diff --git a/packages/grid/rathole/client.toml b/packages/grid/rathole/client.toml index ba8b835a569..581f1af023c 100644 --- a/packages/grid/rathole/client.toml +++ b/packages/grid/rathole/client.toml @@ -1,5 +1,5 @@ [client] -remote_addr = "host.docker.internal:2333" # public IP and port of gateway +remote_addr = "localhost:2333" # public IP and port of gateway [client.services.domain] token = "domain-specific-rathole-secret" diff --git a/packages/grid/rathole/nginx.conf b/packages/grid/rathole/nginx.conf index 2be02976de4..447c482a9e4 100644 --- a/packages/grid/rathole/nginx.conf +++ b/packages/grid/rathole/nginx.conf @@ -1,8 +1,6 @@ -http { - server { - listen 8000; - location / { - proxy_pass http://host.docker.internal:8080; - } +server { + listen 8000; + location / { + proxy_pass http://test-domain-r:8001; } } \ No newline at end of file diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index 830ad50ad86..af5284cedcb 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -24,6 +24,10 @@ http: loadBalancer: servers: - url: "http://rathole:5555" + ratholeforward: + loadBalancer: + servers: + - url: "http://rathole:2333" routers: frontend: rule: "PathPrefix(`/`)" @@ -49,12 +53,19 @@ http: - "blob-storage-url" - "blob-storage-host" rathole: - rule: "PathPrefix(`/rathole`)" + rules: "PathPrefix(`/rathole`)" entryPoints: - web service: "rathole" middlewares: - "rathole-url" + ratholeforward: + rules: "HostRegexp(`{subdomain:[a-z]+}.local.rathole`)" + entryPoints: + - web + service: "ratholeforward" + middlewares: + - "rathole-redirect" ping: rule: "PathPrefix(`/ping`)" entryPoints: @@ -76,3 +87,8 @@ http: stripprefix: prefixes: /rathole forceslash: true + rathole-redirect: + redirectregex: + regex: "^.+" + replacement: "http://rathole:2333/" + permanent: true From 895448b19475b11516ca649977248aa1f7723a44 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Sun, 28 Apr 2024 18:31:35 +0530 Subject: [PATCH 017/100] fix client.toml and server.toml for testing --- packages/grid/rathole/client.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/rathole/client.toml b/packages/grid/rathole/client.toml index 581f1af023c..0d878798c95 100644 --- a/packages/grid/rathole/client.toml +++ b/packages/grid/rathole/client.toml @@ -1,5 +1,5 @@ [client] -remote_addr = "localhost:2333" # public IP and port of gateway +remote_addr = "20.197.23.137:2333" # public IP and port of gateway [client.services.domain] token = "domain-specific-rathole-secret" From e04fb02103eb2e207d178078bd3089a417286fa8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 29 Apr 2024 14:53:57 +0530 Subject: [PATCH 018/100] fix loadbalancer in dynamic yml for rathole forward --- packages/grid/docker-compose.yml | 2 ++ packages/grid/traefik/docker/dynamic.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 8f7ec80765f..bc8ae309055 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -62,6 +62,8 @@ services: image: "${DOCKER_IMAGE_RATHOLE?Variable not set}:${VERSION-latest}" profiles: - rathole + depends_on: + - proxy environment: - SERVICE_NAME=rathole - APP_LOG_LEVEL=${APP_LOG_LEVEL:-info} diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index af5284cedcb..6304a76be66 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -27,7 +27,7 @@ http: ratholeforward: loadBalancer: servers: - - url: "http://rathole:2333" + - url: "http://rathole:80" routers: frontend: rule: "PathPrefix(`/`)" @@ -90,5 +90,5 @@ http: rathole-redirect: redirectregex: regex: "^.+" - replacement: "http://rathole:2333/" + replacement: "http://rathole:80/" permanent: true From b22309eafddf36a38e338132f081be53bcc28345 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 29 Apr 2024 15:06:34 +0530 Subject: [PATCH 019/100] fix port number for rathole forwarding --- packages/grid/traefik/docker/dynamic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index 6304a76be66..e6d9b99c292 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -27,7 +27,7 @@ http: ratholeforward: loadBalancer: servers: - - url: "http://rathole:80" + - url: "http://rathole:8000" routers: frontend: rule: "PathPrefix(`/`)" @@ -90,5 +90,5 @@ http: rathole-redirect: redirectregex: regex: "^.+" - replacement: "http://rathole:80/" + replacement: "http://rathole:8000/" permanent: true From b61d70e478e93389e47ce4a92d9e6112d78b03a5 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 29 Apr 2024 17:35:40 +0530 Subject: [PATCH 020/100] retry fixing traefik --- packages/grid/traefik/docker/dynamic.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index e6d9b99c292..8ab8bbf8317 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -64,8 +64,8 @@ http: entryPoints: - web service: "ratholeforward" - middlewares: - - "rathole-redirect" + # middlewares: + # - "rathole-redirect" ping: rule: "PathPrefix(`/ping`)" entryPoints: @@ -90,5 +90,5 @@ http: rathole-redirect: redirectregex: regex: "^.+" - replacement: "http://rathole:8000/" + replacement: "http://rathole:8000$${1}" permanent: true From 6f0759ead00a0e7e93f40335269fcc7d7f4c0ce5 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 30 Apr 2024 11:00:19 +0530 Subject: [PATCH 021/100] add rathole service and statefilset yaml --- .../templates/rathole/rathole-service.yaml | 30 +++++++ .../rathole/rathole-statefulset.yaml | 89 +++++++++++++++++++ packages/grid/helm/syft/values.yaml | 24 +++++ 3 files changed, 143 insertions(+) create mode 100644 packages/grid/helm/syft/templates/rathole/rathole-service.yaml create mode 100644 packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml diff --git a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml new file mode 100644 index 00000000000..777e530262b --- /dev/null +++ b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: rathole + labels: + {{- include "common.labels" . | nindent 4 }} + app.kubernetes.io/component: rathole +spec: + type: ClusterIP + selector: + {{- include "common.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: rathole + ports: + - name: nginx + protocol: TCP + port: 80 + targetPort: 80 + - name: api + protocol: TCP + port: 5555 + targetPort: 5555 + type: NodePort + selector: + {{- include "common.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: rathole + ports: + - name: rathole + protocol: TCP + port: 2333 + targetPort: 2333 diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml new file mode 100644 index 00000000000..ff72a8cb593 --- /dev/null +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rathole + labels: + {{- include "common.labels" . | nindent 4 }} + app.kubernetes.io/component: rathole +spec: + replicas: 1 + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + {{- include "common.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: rathole + serviceName: rathole + podManagementPolicy: OrderedReady + template: + metadata: + labels: + {{- include "common.labels" . | nindent 8 }} + app.kubernetes.io/component: rathole + {{- if .Values.rathole.podLabels }} + {{- toYaml .Values.rathole.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.rathole.podAnnotations }} + annotations: {{- toYaml .Values.rathole.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.rathole.nodeSelector }} + nodeSelector: {{- .Values.rathole.nodeSelector | toYaml | nindent 8 }} + {{- end }} + containers: + - name: rathole + image: {{ .Values.global.registry }}/openmined/grid-rathole:{{ .Values.global.version }} + imagePullPolicy: Always + resources: {{ include "common.resources.set" (dict "resources" .Values.rathole.resources "preset" .Values.rathole.resourcesPreset) | nindent 12 }} + env: + - name: SERVICE_NAME + value: "rathole" + - name: APP_LOG_LEVEL + value: {{ .Values.rathole.appLogLevel | quote }} + - name: MODE + value: {{ .Values.rathole.mode | quote }} + - name: DEV_MODE + value: {{ .Values.rathole.devMode | quote }} + - name: APP_PORT + value: {{ .Values.rathole.appPort | quote }} + - name: RATHOLE_PORT + value: {{ .Values.rathole.ratholePort | quote }} + {{- if .Values.rathole.env }} + {{- toYaml .Values.rathole.env | nindent 12 }} + {{- end }} + ports: + - name: rathole-port + containerPort: 2333 + - name: api-port + containerPort: 5555 + - name: nginx-port + containerPort: 80 + startupProbe: + httpGet: + path: /?probe=startupProbe + port: api-port + failureThreshold: 30 + livenessProbe: + httpGet: + path: /ping?probe=livenessProbe + port: api-port + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + volumeMounts: + - name: rathole-data + mountPath: /data + readOnly: false + # TODO: Mount the .toml and nginx.conf files + + # Add any additional container configuration here + # such as environment variables, volumes, etc. + volumeClaimTemplates: + - metadata: + name: rathole-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.rathole.volumeSize | quote }} + storageClassName: {{ .Values.rathole.storageClassName | quote }} \ No newline at end of file diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index b390f67996e..1396616272e 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -222,3 +222,27 @@ ingress: # ================================================================================= extraResources: [] + + +# ================================================================================= + +rathole: + # Extra environment vars + env: null + + ratholePort: 2333 + appPort: 5555 + mode: "client" + devMode: "false" + appLogLevel: "info" + + # Pod labels & annotations + podLabels: null + podAnnotations: null + + # Node selector for pods + nodeSelector: null + + # Pod Resource Limits + resourcesPreset: small + resources: null From bffeefab9b22deeb37c9dff5e87a9be2b42c74b5 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 30 Apr 2024 21:55:16 +0530 Subject: [PATCH 022/100] add rathole to devspace and define a configmap to update server.toml --- packages/grid/devspace.yaml | 13 ++++++++++ .../templates/rathole/rathole-configmap.yaml | 11 ++++++++ .../rathole/rathole-statefulset.yaml | 25 +++++++++++++------ 3 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 75e2757e58a..ae0a17f7ec1 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -25,6 +25,7 @@ vars: DOCKER_IMAGE_BACKEND: openmined/grid-backend DOCKER_IMAGE_FRONTEND: openmined/grid-frontend DOCKER_IMAGE_SEAWEEDFS: openmined/grid-seaweedfs + DOCKER_IMAGE_RATHOLE: openmined/grid-rathole CONTAINER_REGISTRY: "docker.io" VERSION: "0.8.7-beta.2" PLATFORM: $(uname -m | grep -q 'arm64' && echo "arm64" || echo "amd64") @@ -58,6 +59,14 @@ images: context: ./seaweedfs tags: - dev-${DEVSPACE_TIMESTAMP} + rathole: + image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_RATHOLE}" + buildKit: + args: ["--platform", "linux/${PLATFORM}"] + dockerfile: ./rathole/rathole.dockerfile + context: ./rathole + tags: + - dev-${DEVSPACE_TIMESTAMP} # This is a list of `deployments` that DevSpace can create for this project deployments: @@ -109,6 +118,10 @@ dev: - path: ./backend/grid:/root/app/grid - path: ../syft:/root/app/syft ssh: {} + rathole: + labelSelector: + app.kubernetes.io/name: syft + app.kubernetes.io/component: rathole profiles: - name: gateway diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml new file mode 100644 index 00000000000..02aca907e6f --- /dev/null +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rathole-config + labels: + {{- include "common.labels" . | nindent 4 }} + app.kubernetes.io/component: rathole +data: + myserver.toml: | + [server] + bind_addr = "0.0.0.0:2333" diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index ff72a8cb593..992370bbbcb 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -34,7 +34,7 @@ spec: - name: rathole image: {{ .Values.global.registry }}/openmined/grid-rathole:{{ .Values.global.version }} imagePullPolicy: Always - resources: {{ include "common.resources.set" (dict "resources" .Values.rathole.resources "preset" .Values.rathole.resourcesPreset) | nindent 12 }} + resources: {{ include "common.resources.set" (dict "resources" .Values.rathole.resources "preset" .Values.rathole.resourcesPreset) | nindent 12 }} env: - name: SERVICE_NAME value: "rathole" @@ -65,25 +65,34 @@ spec: failureThreshold: 30 livenessProbe: httpGet: - path: /ping?probe=livenessProbe + path: /?probe=livenessProbe port: api-port periodSeconds: 15 timeoutSeconds: 5 failureThreshold: 3 volumeMounts: - - name: rathole-data - mountPath: /data + - name: rathole-config + mountPath: /app/data/myserver.toml + subPath: myserver.toml readOnly: false - # TODO: Mount the .toml and nginx.conf files + terminationGracePeriodSeconds: 5 + volumes: + - name: rathole-config + configMap: + name: rathole-config + # TODO: Mount the .toml and nginx.conf files # Add any additional container configuration here # such as environment variables, volumes, etc. volumeClaimTemplates: - metadata: name: rathole-data + labels: + {{- include "common.volumeLabels" . | nindent 8 }} + app.kubernetes.io/component: rathole spec: - accessModes: [ "ReadWriteOnce" ] + accessModes: + - ReadWriteOnce resources: requests: - storage: {{ .Values.rathole.volumeSize | quote }} - storageClassName: {{ .Values.rathole.storageClassName | quote }} \ No newline at end of file + storage: 10Mi \ No newline at end of file From eee0fc1502ed598b385af2389d8709983c133b58 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 1 May 2024 13:16:37 +0530 Subject: [PATCH 023/100] refactor and combine client and server.sh --- .../templates/rathole/rathole-configmap.yaml | 9 ++++++++- packages/grid/rathole/rathole.dockerfile | 3 +-- packages/grid/rathole/start-client.sh | 13 ------------ packages/grid/rathole/start-server.sh | 13 ------------ packages/grid/rathole/start.sh | 20 +++++++++++++++++++ 5 files changed, 29 insertions(+), 29 deletions(-) delete mode 100755 packages/grid/rathole/start-client.sh delete mode 100755 packages/grid/rathole/start-server.sh create mode 100755 packages/grid/rathole/start.sh diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index 02aca907e6f..7405235e0f0 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -6,6 +6,13 @@ metadata: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: rathole data: - myserver.toml: | + {{- if eq .Values.rathole.mode "server" }} + server.toml: | [server] bind_addr = "0.0.0.0:2333" + {{- end }} + + {{- if eq .Values.rathole.mode "client" }} + client.toml: | + [client] + remote_addr = "" diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 91f3a5d11e9..370b4233757 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -17,8 +17,7 @@ ENV APP_LOG_LEVEL="info" COPY --from=build /rathole/target/release/rathole /app/rathole RUN apt update && apt install -y netcat-openbsd vim WORKDIR /app -COPY ./start-client.sh /app/start-client.sh -COPY ./start-server.sh /app/start-server.sh +COPY ./start.sh /app/start.sh COPY ./client.toml /app/client.toml COPY ./server.toml /app/server.toml COPY ./nginx.conf /etc/nginx/conf.d/default.conf diff --git a/packages/grid/rathole/start-client.sh b/packages/grid/rathole/start-client.sh deleted file mode 100755 index 9bbb7d9097d..00000000000 --- a/packages/grid/rathole/start-client.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -APP_MODULE=server.main:app -LOG_LEVEL=${LOG_LEVEL:-info} -HOST=${HOST:-0.0.0.0} -PORT=${APP_PORT:-5555} -RELOAD="--reload" -DEBUG_CMD="" - -apt update && apt install -y nginx -nginx & -exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $APP_PORT --log-level $LOG_LEVEL "$APP_MODULE" & -/app/rathole client.toml diff --git a/packages/grid/rathole/start-server.sh b/packages/grid/rathole/start-server.sh deleted file mode 100755 index 2ab8d52c7ed..00000000000 --- a/packages/grid/rathole/start-server.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -APP_MODULE=server.main:app -LOG_LEVEL=${LOG_LEVEL:-info} -HOST=${HOST:-0.0.0.0} -PORT=${APP_PORT:-5555} -RELOAD="--reload" -DEBUG_CMD="" - -apt update && apt install -y nginx -nginx & -exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $APP_PORT --log-level $LOG_LEVEL "$APP_MODULE" & -/app/rathole server.toml diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh new file mode 100755 index 00000000000..45815250e93 --- /dev/null +++ b/packages/grid/rathole/start.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +APP_MODULE=server.main:app +LOG_LEVEL=${LOG_LEVEL:-info} +HOST=${HOST:-0.0.0.0} +PORT=${APP_PORT:-5555} +RELOAD="--reload" +DEBUG_CMD="" +RATHOLE_MODE=${RATHOLE_MODE:-server} + +apt update && apt install -y nginx +nginx & exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $APP_PORT --log-level $LOG_LEVEL "$APP_MODULE" & + +if [[ $RATHOLE_MODE == "server" ]]; then + /app/rathole server.toml +elif [[ $RATHOLE_MODE = "client" ]]; then + /app/rathole client.toml +else + echo "RATHOLE_MODE is set to an invalid value. Exiting." +fi From 0eb42a97bf0c4e16a161ef98a6f0a52193f256a0 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 1 May 2024 13:39:08 +0530 Subject: [PATCH 024/100] fix rathole configmap --- packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index 7405235e0f0..4b5fbea3633 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -16,3 +16,4 @@ data: client.toml: | [client] remote_addr = "" + {{- end }} From dc31f6be048aca2e5165c323cef85c2a26bab5d8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 1 May 2024 14:07:43 +0530 Subject: [PATCH 025/100] mount volume based on rathole mode if server or client --- .../syft/templates/rathole/rathole-statefulset.yaml | 13 +++++++++++-- packages/grid/helm/syft/values.yaml | 2 +- packages/grid/rathole/rathole.dockerfile | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index 992370bbbcb..860ea1b1881 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -71,10 +71,19 @@ spec: timeoutSeconds: 5 failureThreshold: 3 volumeMounts: + {{- if -eq .Values.rathole.mode "server" }} - name: rathole-config - mountPath: /app/data/myserver.toml - subPath: myserver.toml + mountPath: /app/data/server.toml + subPath: server.toml readOnly: false + {{- end }} + + {{- if -eq .Values.rathole.mode "client" }} + - name: rathole-config + mountPath: /app/data/client.toml + subPath: client.toml + readOnly: false + {{- end }} terminationGracePeriodSeconds: 5 volumes: - name: rathole-config diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 1396616272e..c59512e478b 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -232,7 +232,7 @@ rathole: ratholePort: 2333 appPort: 5555 - mode: "client" + mode: "server" devMode: "false" appLogLevel: "info" diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 370b4233757..0d8bebc3590 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -25,7 +25,7 @@ COPY ./requirements.txt /app/requirements.txt COPY ./server/ /app/server/ RUN pip install --user -r requirements.txt -CMD ["sh", "-c", "/app/start-$MODE.sh"] +CMD ["sh", "-c", "/app/start.sh"] EXPOSE 2333/udp EXPOSE 2333 From 00cc225d6808e4ea93369051c8ccea54be1b11a3 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 1 May 2024 14:53:48 +0530 Subject: [PATCH 026/100] mount server.toml and client.toml based on rathole mode --- .../helm/syft/templates/rathole/rathole-configmap.yaml | 4 ++++ .../helm/syft/templates/rathole/rathole-statefulset.yaml | 8 ++++---- packages/grid/rathole/client.toml | 6 ------ packages/grid/rathole/rathole.dockerfile | 2 -- packages/grid/rathole/server.toml | 6 ------ 5 files changed, 8 insertions(+), 18 deletions(-) delete mode 100644 packages/grid/rathole/client.toml delete mode 100644 packages/grid/rathole/server.toml diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index 4b5fbea3633..3e843fbc1cc 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -10,6 +10,10 @@ data: server.toml: | [server] bind_addr = "0.0.0.0:2333" + + [server.services.domain] + token = "domain-specific-rathole-secret" + bind_addr = "0.0.0.0:8001" {{- end }} {{- if eq .Values.rathole.mode "client" }} diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index 860ea1b1881..e44b5c4a442 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -71,16 +71,16 @@ spec: timeoutSeconds: 5 failureThreshold: 3 volumeMounts: - {{- if -eq .Values.rathole.mode "server" }} + {{- if eq .Values.rathole.mode "server" }} - name: rathole-config - mountPath: /app/data/server.toml + mountPath: /app/server.toml subPath: server.toml readOnly: false {{- end }} - {{- if -eq .Values.rathole.mode "client" }} + {{- if eq .Values.rathole.mode "client" }} - name: rathole-config - mountPath: /app/data/client.toml + mountPath: /app/client.toml subPath: client.toml readOnly: false {{- end }} diff --git a/packages/grid/rathole/client.toml b/packages/grid/rathole/client.toml deleted file mode 100644 index 0d878798c95..00000000000 --- a/packages/grid/rathole/client.toml +++ /dev/null @@ -1,6 +0,0 @@ -[client] -remote_addr = "20.197.23.137:2333" # public IP and port of gateway - -[client.services.domain] -token = "domain-specific-rathole-secret" -local_addr = "localhost:8000" # nginx proxy diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 0d8bebc3590..1d1b82785af 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -18,8 +18,6 @@ COPY --from=build /rathole/target/release/rathole /app/rathole RUN apt update && apt install -y netcat-openbsd vim WORKDIR /app COPY ./start.sh /app/start.sh -COPY ./client.toml /app/client.toml -COPY ./server.toml /app/server.toml COPY ./nginx.conf /etc/nginx/conf.d/default.conf COPY ./requirements.txt /app/requirements.txt COPY ./server/ /app/server/ diff --git a/packages/grid/rathole/server.toml b/packages/grid/rathole/server.toml deleted file mode 100644 index 8145491e7cc..00000000000 --- a/packages/grid/rathole/server.toml +++ /dev/null @@ -1,6 +0,0 @@ -[server] -bind_addr = "0.0.0.0:2333" # public open port - -[server.services.domain] -token = "domain-specific-rathole-secret" -bind_addr = "0.0.0.0:8001" From 79bcc23c1f5cedff639c7712b63a1cc54ce77628 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 2 May 2024 20:15:21 +0530 Subject: [PATCH 027/100] update configmap to seperate path of conf loaded and used in rathole add a script to copy toml file from loaded path to used path cleanup start.sh --- packages/grid/devspace.yaml | 3 ++ .../syft/templates/proxy/proxy-configmap.yaml | 9 +++++ .../templates/proxy/proxy-deployment.yaml | 7 ++++ .../templates/rathole/rathole-service.yaml | 14 +------ .../rathole/rathole-statefulset.yaml | 37 ++++--------------- packages/grid/rathole/start.sh | 23 ++++++------ 6 files changed, 39 insertions(+), 54 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index ae0a17f7ec1..3c22db1fd91 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -133,6 +133,9 @@ profiles: path: images.seaweedfs - op: remove path: dev.seaweedfs + - op: replace + path: deployments.syft.helm.values.rathole.mode + value: "server" - name: gcp patches: diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index 1989f399161..2654d49c8a1 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -72,4 +72,13 @@ data: providers: file: filename: /etc/traefik/dynamic.yml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: proxy-config-dynamic + labels: + {{- include "common.labels" . | nindent 4 }} + app.kubernetes.io/component: proxy +data: {} \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml index 6adb42f6c9c..f06051b9e0b 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml @@ -45,6 +45,10 @@ spec: - mountPath: /etc/traefik name: traefik-conf readOnly: false + volumeMounts: + - mountPath: /etc/traefik/dynamic + name: traefik-conf-dynamic + readOnly: false startupProbe: null livenessProbe: httpGet: @@ -59,3 +63,6 @@ spec: - configMap: name: proxy-config name: traefik-conf + - configMap: + name: proxy-config-dynamic + name: traefik-conf-dynamic \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml index 777e530262b..5d6184e4795 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml @@ -7,19 +7,6 @@ metadata: app.kubernetes.io/component: rathole spec: type: ClusterIP - selector: - {{- include "common.selectorLabels" . | nindent 4 }} - app.kubernetes.io/component: rathole - ports: - - name: nginx - protocol: TCP - port: 80 - targetPort: 80 - - name: api - protocol: TCP - port: 5555 - targetPort: 5555 - type: NodePort selector: {{- include "common.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: rathole @@ -28,3 +15,4 @@ spec: protocol: TCP port: 2333 targetPort: 2333 + diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index e44b5c4a442..0f07516e352 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -54,45 +54,22 @@ spec: ports: - name: rathole-port containerPort: 2333 - - name: api-port - containerPort: 5555 - - name: nginx-port - containerPort: 80 - startupProbe: - httpGet: - path: /?probe=startupProbe - port: api-port - failureThreshold: 30 - livenessProbe: - httpGet: - path: /?probe=livenessProbe - port: api-port - periodSeconds: 15 - timeoutSeconds: 5 - failureThreshold: 3 + startupProbe: null + livenessProbe: null volumeMounts: - {{- if eq .Values.rathole.mode "server" }} - - name: rathole-config - mountPath: /app/server.toml - subPath: server.toml + - name: mount-config + mountPath: /conf/ readOnly: false - {{- end }} - - {{- if eq .Values.rathole.mode "client" }} - name: rathole-config - mountPath: /app/client.toml - subPath: client.toml + mountPath: /app/conf/ readOnly: false - {{- end }} terminationGracePeriodSeconds: 5 volumes: - name: rathole-config + emptyDir: {} + - name: mount-config configMap: name: rathole-config - # TODO: Mount the .toml and nginx.conf files - - # Add any additional container configuration here - # such as environment variables, volumes, etc. volumeClaimTemplates: - metadata: name: rathole-data diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh index 45815250e93..907972bb50a 100755 --- a/packages/grid/rathole/start.sh +++ b/packages/grid/rathole/start.sh @@ -1,20 +1,21 @@ #!/usr/bin/env bash - -APP_MODULE=server.main:app -LOG_LEVEL=${LOG_LEVEL:-info} -HOST=${HOST:-0.0.0.0} -PORT=${APP_PORT:-5555} -RELOAD="--reload" -DEBUG_CMD="" RATHOLE_MODE=${RATHOLE_MODE:-server} -apt update && apt install -y nginx -nginx & exec python -m $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $APP_PORT --log-level $LOG_LEVEL "$APP_MODULE" & +cp -L -r -f /conf/* conf/ +#!/bin/bash if [[ $RATHOLE_MODE == "server" ]]; then - /app/rathole server.toml + /app/rathole conf/server.toml & elif [[ $RATHOLE_MODE = "client" ]]; then - /app/rathole client.toml + /app/rathole conf/client.toml & else echo "RATHOLE_MODE is set to an invalid value. Exiting." fi + +while true +do + # Execute your script here + cp -L -r -f /conf/* conf/ + # Sleep for 10 seconds + sleep 10 +done \ No newline at end of file From c693d3ff2d7ad91eeeb02bfd350e2d0a8348ca3f Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 3 May 2024 11:29:00 +0530 Subject: [PATCH 028/100] add some comment to start script --- packages/grid/rathole/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh index 907972bb50a..e948973decf 100755 --- a/packages/grid/rathole/start.sh +++ b/packages/grid/rathole/start.sh @@ -3,7 +3,6 @@ RATHOLE_MODE=${RATHOLE_MODE:-server} cp -L -r -f /conf/* conf/ -#!/bin/bash if [[ $RATHOLE_MODE == "server" ]]; then /app/rathole conf/server.toml & elif [[ $RATHOLE_MODE = "client" ]]; then @@ -12,6 +11,7 @@ else echo "RATHOLE_MODE is set to an invalid value. Exiting." fi +# reload config every 10 seconds while true do # Execute your script here From 3d7759e43b9d1bf6027560204e06386e2b31fe84 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Sun, 5 May 2024 14:50:41 +0530 Subject: [PATCH 029/100] rathole: remove fastapi and nginx ports mapping from traefik --- .../syft/templates/proxy/proxy-configmap.yaml | 3 +- .../templates/proxy/proxy-deployment.yaml | 17 +++++----- .../templates/rathole/rathole-service.yaml | 1 - packages/grid/traefik/docker/dynamic.yml | 31 ------------------- 4 files changed, 9 insertions(+), 43 deletions(-) diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index 2654d49c8a1..6e7dd9c0fe0 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -81,4 +81,5 @@ metadata: labels: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: proxy -data: {} \ No newline at end of file +data: + rathole-dynamic.yml: | diff --git a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml index f06051b9e0b..db5bef8a813 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml @@ -45,10 +45,6 @@ spec: - mountPath: /etc/traefik name: traefik-conf readOnly: false - volumeMounts: - - mountPath: /etc/traefik/dynamic - name: traefik-conf-dynamic - readOnly: false startupProbe: null livenessProbe: httpGet: @@ -60,9 +56,10 @@ spec: readinessProbe: null terminationGracePeriodSeconds: 5 volumes: - - configMap: - name: proxy-config - name: traefik-conf - - configMap: - name: proxy-config-dynamic - name: traefik-conf-dynamic \ No newline at end of file + - name: traefik-conf + projected: + sources: + - configMap: + name: proxy-config + - configMap: + name: proxy-config-dynamic \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml index 5d6184e4795..d9050f0d693 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml @@ -15,4 +15,3 @@ spec: protocol: TCP port: 2333 targetPort: 2333 - diff --git a/packages/grid/traefik/docker/dynamic.yml b/packages/grid/traefik/docker/dynamic.yml index 8ab8bbf8317..61e68e7ad03 100644 --- a/packages/grid/traefik/docker/dynamic.yml +++ b/packages/grid/traefik/docker/dynamic.yml @@ -20,14 +20,6 @@ http: loadBalancer: servers: - url: "http://seaweedfs:4001" - rathole: - loadBalancer: - servers: - - url: "http://rathole:5555" - ratholeforward: - loadBalancer: - servers: - - url: "http://rathole:8000" routers: frontend: rule: "PathPrefix(`/`)" @@ -52,20 +44,6 @@ http: middlewares: - "blob-storage-url" - "blob-storage-host" - rathole: - rules: "PathPrefix(`/rathole`)" - entryPoints: - - web - service: "rathole" - middlewares: - - "rathole-url" - ratholeforward: - rules: "HostRegexp(`{subdomain:[a-z]+}.local.rathole`)" - entryPoints: - - web - service: "ratholeforward" - # middlewares: - # - "rathole-redirect" ping: rule: "PathPrefix(`/ping`)" entryPoints: @@ -83,12 +61,3 @@ http: stripprefix: prefixes: /blob forceslash: true - rathole-url: - stripprefix: - prefixes: /rathole - forceslash: true - rathole-redirect: - redirectregex: - regex: "^.+" - replacement: "http://rathole:8000$${1}" - permanent: true From a1a379b954646e8247b0691d2e688510d9be3b01 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 8 May 2024 15:19:28 +0530 Subject: [PATCH 030/100] add toml_w and tomli packages - add methods to extract configmap by name --- packages/grid/devspace.yaml | 2 ++ .../syft/templates/proxy/proxy-configmap.yaml | 12 ++++++++++++ .../syft/templates/rathole/rathole-service.yaml | 4 ++-- packages/grid/helm/syft/values.yaml | 1 + packages/grid/helm/values.dev.yaml | 3 +++ packages/syft/setup.cfg | 2 ++ packages/syft/src/syft/custom_worker/k8s.py | 15 +++++++++++++++ 7 files changed, 37 insertions(+), 2 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 3c22db1fd91..214ea0a23b4 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -122,6 +122,8 @@ dev: labelSelector: app.kubernetes.io/name: syft app.kubernetes.io/component: rathole + ports: + - port: "2333" # rathole profiles: - name: gateway diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index 6e7dd9c0fe0..8853fae80b2 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -83,3 +83,15 @@ metadata: app.kubernetes.io/component: proxy data: rathole-dynamic.yml: | + # http: + # services: + # rathole_domain_1: + # loadBalancer: + # servers: + # - url: "http://rathole-0.rathole.syft.svc.cluster.local:2333" + # routers: + # rathole_domain_1: + # rule: "Host('domain1.domain.syft.local')" + # entryPoints: + # - "web" + # service: "rathole_domain_1" diff --git a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml index d9050f0d693..01fa305ac77 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml @@ -6,12 +6,12 @@ metadata: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: rathole spec: - type: ClusterIP + clusterIP: None selector: {{- include "common.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: rathole ports: - name: rathole - protocol: TCP port: 2333 targetPort: 2333 + protocol: TCP diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index c59512e478b..6e0bdfbd32d 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -229,6 +229,7 @@ extraResources: [] rathole: # Extra environment vars env: null + enabled: true ratholePort: 2333 appPort: 5555 diff --git a/packages/grid/helm/values.dev.yaml b/packages/grid/helm/values.dev.yaml index 0951f7e906c..6d0bd129028 100644 --- a/packages/grid/helm/values.dev.yaml +++ b/packages/grid/helm/values.dev.yaml @@ -42,3 +42,6 @@ frontend: proxy: resourcesPreset: null resources: null + +rathole: + enabled: "true" diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 54e059b73ac..a6ceb8b4bc1 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -68,6 +68,8 @@ syft = PyYAML==6.0.1 azure-storage-blob==12.19.1 ipywidgets==8.1.2 + tomli==2.0.1 # Later for python 3.11 > we can just use tomlib that comes with python + tomli_w==1.0.0 install_requires = %(syft)s diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index cb4b5765e62..60557c86afb 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -9,6 +9,7 @@ # third party import kr8s from kr8s.objects import APIObject +from kr8s.objects import ConfigMap from kr8s.objects import Pod from kr8s.objects import Secret from pydantic import BaseModel @@ -171,6 +172,20 @@ def b64encode_secret(data: str) -> str: """Convert the data to base64 encoded string for Secret.""" return base64.b64encode(data.encode()).decode() + @staticmethod + def get_configmap(client: kr8s.Api, name: str) -> ConfigMap | None: + config_map = client.get("configmaps", name) + return config_map[0] if config_map else None + + @staticmethod + def update_configmap( + config_map: ConfigMap, + patch: dict, + ) -> None: + existing_data = config_map.raw + existing_data.update(patch) + config_map.patch(patch=existing_data) + @staticmethod def create_dockerconfig_secret( secret_name: str, From 619e119f7b3390d0a4dd951bee3a309831374aa0 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 9 May 2024 12:50:40 +0530 Subject: [PATCH 031/100] add classes to handle CRUD ops for toml server and client files in rathole - add RatholeServer class to add client info to rathole server - add method to alter endpoints in traefik - add method to add config to rathole dynamic traefik config via configmaps --- .../backend/backend-service-account.yaml | 2 +- .../src/syft/service/network/node_peer.py | 1 + .../syft/src/syft/service/network/rathole.py | 173 +++++++++++++ .../src/syft/service/network/rathole_toml.py | 237 ++++++++++++++++++ 4 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 packages/syft/src/syft/service/network/rathole.py create mode 100644 packages/syft/src/syft/service/network/rathole_toml.py diff --git a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml index a466d0c3fe4..7b542adfc0b 100644 --- a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml @@ -26,7 +26,7 @@ metadata: app.kubernetes.io/component: backend rules: - apiGroups: [""] - resources: ["pods", "configmaps", "secrets"] + resources: ["pods", "configmaps", "secrets", "service"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] - apiGroups: [""] resources: ["pods/log"] diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 70e6f9bfb40..453b7f3563d 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -66,6 +66,7 @@ class NodePeer(SyftObject): node_routes: list[NodeRouteType] = [] node_type: NodeType admin_email: str + rathole_token: str | None = None def existed_route( self, route: NodeRouteType | None = None, route_id: UID | None = None diff --git a/packages/syft/src/syft/service/network/rathole.py b/packages/syft/src/syft/service/network/rathole.py new file mode 100644 index 00000000000..964c1dde27a --- /dev/null +++ b/packages/syft/src/syft/service/network/rathole.py @@ -0,0 +1,173 @@ +# stdlib +from typing import Self +from typing import cast + +# third party +import yaml + +# relative +from ...custom_worker.k8s import KubeUtils +from ...custom_worker.k8s import get_kr8s_client +from ...serde import serializable +from ...types.base import SyftBaseModel +from .node_peer import NodePeer +from .rathole_toml import RatholeServerToml +from .routes import HTTPNodeRoute + +RATHOLE_TOML_CONFIG_MAP = "rathole-config" +RATHOLE_PROXY_CONFIG_MAP = "rathole-proxy-config" +RATHOLE_DEFAULT_BIND_ADDRESS = "http://0.0.0.0:2333" +PROXY_CONFIG_MAP = "proxy-config" + + +@serializable() +class RatholeConfig(SyftBaseModel): + uuid: str + secret_token: str + local_addr_host: str + local_addr_port: int + server_name: str | None = None + + @property + def local_address(self) -> str: + return f"http://{self.local_addr_host}:{self.local_addr_port}" + + @classmethod + def from_peer(cls, peer: NodePeer) -> Self: + high_priority_route = peer.pick_highest_priority_route() + + if not isinstance(high_priority_route, HTTPNodeRoute): + raise ValueError("Rathole only supports HTTPNodeRoute") + + return cls( + uuid=peer.id, + secret_token=peer.rathole_token, + local_addr_host=high_priority_route.host_or_ip, + local_addr_port=high_priority_route.port, + server_name=peer.name, + ) + + +# class RatholeProxyConfigWriter: +# def get_config(self, *args, **kwargs): +# pass + +# def save_config(self, *args, **kwargs): +# pass + +# def add_service(url: str, service_name: str, port: int, hostname: str): +# pass + +# def delete_service(self, *args, **kwargs): +# pass + + +class RatholeService: + def __init__(self) -> None: + self.k8rs_client = get_kr8s_client() + + def add_client_to_server(self, peer: NodePeer) -> None: + """Add a client to the rathole server toml file.""" + + route = cast(HTTPNodeRoute, peer.pick_highest_priority_route()) + + config = RatholeConfig( + uuid=peer.id, + secret_token=peer.rathole_token, + local_addr_host="localhost", + local_addr_port=route.port, + server_name=peer.name, + ) + + # Get rathole toml config map + rathole_config_map = KubeUtils.get_configmap( + client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP + ) + + client_filename = RatholeServerToml.filename + + toml_str = rathole_config_map.data[client_filename] + + # Add the peer info to the toml file + rathole_toml = RatholeServerToml(toml_str) + rathole_toml.add_config(config=config) + + if not rathole_toml.get_bind_address(): + # First time adding a peer + rathole_toml.set_bind_address(RATHOLE_DEFAULT_BIND_ADDRESS) + + rathole_config_map.data[client_filename] = rathole_toml.toml_str + + # Update the rathole config map + KubeUtils.update_configmap( + client=self.k8rs_client, + name=RATHOLE_TOML_CONFIG_MAP, + data=rathole_config_map.data, + ) + + # Add the peer info to the proxy config map + self.add_port_to_proxy(config) + + def add_port_to_proxy(self, config: RatholeConfig, entrypoint: str = "web") -> None: + """Add a port to the rathole proxy config map.""" + + rathole_proxy_config_map = KubeUtils.get_configmap( + self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP + ) + + rathole_proxy = rathole_proxy_config_map.data["rathole-proxy.yml"] + + if not rathole_proxy: + rathole_proxy = {"http": {"routers": {}, "services": {}}} + + # TODO: config.port, this should be a random port + + rathole_proxy["http"]["services"][config.server_name] = { + "loadBalancer": { + "servers": [{"url": f"http://rathole:{config.local_addr_port}"}] + } + } + + rathole_proxy["http"]["routers"][config.server_name] = { + "rule": f"Host(`{config.server_name}.syft.local`)", + "service": config.server_name, + "entryPoints": [entrypoint], + } + + KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, rathole_proxy) + + def add_entrypoint(self, port: int, peer_name: str) -> None: + """Add an entrypoint to the traefik config map.""" + + proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) + + data = proxy_config_map.data + + traefik_config_str = data["traefik.yml"] + + traefik_config = yaml.safe_load(traefik_config_str) + + traefik_config["entryPoints"][f"{peer_name}-entrypoint"] = { + "address": f":{port}" + } + + data["traefik.yml"] = yaml.safe_dump(traefik_config) + + KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, data) + + def remove_endpoint(self, peer_name: str) -> None: + """Remove an entrypoint from the traefik config map.""" + + proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) + + data = proxy_config_map.data + + traefik_config_str = data["traefik.yml"] + + traefik_config = yaml.safe_load(traefik_config_str) + + del traefik_config["entryPoints"][f"{peer_name}-entrypoint"] + + data["traefik.yml"] = yaml.safe_dump(traefik_config) + + KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, data) diff --git a/packages/syft/src/syft/service/network/rathole_toml.py b/packages/syft/src/syft/service/network/rathole_toml.py new file mode 100644 index 00000000000..e50c830c5d8 --- /dev/null +++ b/packages/syft/src/syft/service/network/rathole_toml.py @@ -0,0 +1,237 @@ +# third party +import tomli +import tomli_w + +# relative +from .rathole import RatholeConfig + + +class TomlReaderWriter: + @staticmethod + def load(toml_str: str) -> dict: + return tomli.loads(toml_str) + + @staticmethod + def dump(toml_dict: str) -> str: + return tomli_w.dumps(toml_dict) + + +class RatholeBaseToml: + filename: str + + def __init__(self, toml_str: str) -> None: + self.toml_writer = TomlReaderWriter + self.toml_str = toml_str + + def read(self) -> dict: + return self.toml_writer.load(self.toml_str) + + def save(self, toml_dict: dict) -> None: + self.toml_str = self.toml_writer.dump(self.toml_str) + + def _validate(self) -> bool: + raise NotImplementedError + + @property + def is_valid(self) -> bool: + return self._validate() + + +class RatholeClientToml(RatholeBaseToml): + filename: str = "client.toml" + + def set_remote_addr(self, remote_host: str) -> None: + """Add a new remote address to the client toml file.""" + + toml = self.read() + + # Add the new remote address + if "client" not in toml: + toml["client"] = {} + + toml["client"]["remote_addr"] = remote_host + + if remote_host not in toml["client"]["remote"]: + toml["client"]["remote"].append(remote_host) + + self.save(toml) + + def add_config(self, config: RatholeConfig) -> None: + """Add a new config to the toml file.""" + + toml = self.read() + + # Add the new config + if "services" not in toml["client"]: + toml["client"]["services"] = {} + + if config.uuid not in toml["client"]["services"]: + toml["client"]["services"][config.uuid] = {} + + toml["client"]["services"][config.uuid] = { + "token": config.secret_token, + "local_addr": config.local_address, + } + + self.save(toml) + + def remove_config(self, uuid: str) -> None: + """Remove a config from the toml file.""" + + toml = self.read() + + # Remove the config + if "services" not in toml["client"]: + return + + if uuid not in toml["client"]["services"]: + return + + del toml["client"]["services"][uuid] + + self.save(toml) + + def update_config(self, config: RatholeConfig) -> None: + """Update a config in the toml file.""" + + toml = self.read() + + # Update the config + if "services" not in toml["client"]: + return + + if config.uuid not in toml["client"]["services"]: + return + + toml["client"]["services"][config.uuid] = { + "token": config.secret_token, + "local_addr": config.local_address, + } + + self.save(toml) + + def get_config(self, uuid: str) -> RatholeConfig | None: + """Get a config from the toml file.""" + + toml = self.read() + + # Get the config + if "services" not in toml["client"]: + return None + + if uuid not in toml["client"]["services"]: + return None + + service = toml["client"]["services"][uuid] + + return RatholeConfig( + uuid=uuid, + secret_token=service["token"], + local_addr_host=service["local_addr"].split(":")[0], + local_addr_port=service["local_addr"].split(":")[1], + ) + + def _validate(self) -> bool: + toml = self.read() + + if not toml["client"]["remote_addr"]: + return False + + for uuid, config in toml["client"]["services"].items(): + if not uuid: + return False + + if not config["token"] or not config["local_addr"]: + return False + + return True + + +class RatholeServerToml(RatholeBaseToml): + filename: str = "server.toml" + + def set_bind_address(self, bind_address: str) -> None: + """Set the bind address in the server toml file.""" + + toml = self.read() + + # Set the bind address + toml["server"]["bind_addr"] = bind_address + + self.save(toml) + + def get_bind_address(self) -> str: + """Get the bind address from the server toml file.""" + + toml = self.read() + + return toml["server"]["bind_addr"] + + def add_config(self, config: RatholeConfig) -> None: + """Add a new config to the toml file.""" + + toml = self.read() + + # Add the new config + if "services" not in toml["server"]: + toml["server"]["services"] = {} + + if config.uuid not in toml["server"]["services"]: + toml["server"]["services"][config.uuid] = {} + + toml["server"]["services"][config.uuid] = { + "token": config.secret_token, + "bind_addr": config.local_address, + } + + self.save(toml) + + def remove_config(self, uuid: str) -> None: + """Remove a config from the toml file.""" + + toml = self.read() + + # Remove the config + if "services" not in toml["server"]: + return + + if uuid not in toml["server"]["services"]: + return + + del toml["server"]["services"][uuid] + + self.save(toml) + + def update_config(self, config: RatholeConfig) -> None: + """Update a config in the toml file.""" + + toml = self.read() + + # Update the config + if "services" not in toml["server"]: + return + + if config.uuid not in toml["server"]["services"]: + return + + toml["server"]["services"][config.uuid] = { + "token": config.secret_token, + "bind_addr": config.local_address, + } + + self.save(toml) + + def _validate(self) -> bool: + toml = self.read() + + if not toml["server"]["bind_addr"]: + return False + + for uuid, config in toml["server"]["services"].items(): + if not uuid: + return False + + if not config["token"] or not config["bind_addr"]: + return False + + return True From 55272f8448cffff72b87ee7b5cac88d6ce2c728d Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 9 May 2024 15:13:36 +0530 Subject: [PATCH 032/100] add method to add host to client.toml in configmap - add methods to forward rathole port to proxy --- .../syft/src/syft/service/network/rathole.py | 82 ++++++++++++++++--- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/packages/syft/src/syft/service/network/rathole.py b/packages/syft/src/syft/service/network/rathole.py index 964c1dde27a..56d25af36a9 100644 --- a/packages/syft/src/syft/service/network/rathole.py +++ b/packages/syft/src/syft/service/network/rathole.py @@ -1,4 +1,5 @@ # stdlib +import secrets from typing import Self from typing import cast @@ -11,6 +12,7 @@ from ...serde import serializable from ...types.base import SyftBaseModel from .node_peer import NodePeer +from .rathole_toml import RatholeClientToml from .rathole_toml import RatholeServerToml from .routes import HTTPNodeRoute @@ -66,13 +68,13 @@ class RatholeService: def __init__(self) -> None: self.k8rs_client = get_kr8s_client() - def add_client_to_server(self, peer: NodePeer) -> None: - """Add a client to the rathole server toml file.""" + def add_host_to_server(self, peer: NodePeer) -> None: + """Add a host to the rathole server toml file.""" route = cast(HTTPNodeRoute, peer.pick_highest_priority_route()) config = RatholeConfig( - uuid=peer.id, + uuid=peer.id.to_string(), secret_token=peer.rathole_token, local_addr_host="localhost", local_addr_port=route.port, @@ -106,21 +108,81 @@ def add_client_to_server(self, peer: NodePeer) -> None: ) # Add the peer info to the proxy config map - self.add_port_to_proxy(config) + self.add_dynamic_addr_to_rathole(config) + + def get_random_port(self) -> int: + """Get a random port number.""" + return secrets.randbits(15) + + def add_host_to_client(self, peer: NodePeer) -> None: + """Add a host to the rathole client toml file.""" + + random_port = self.get_random_port() + + config = RatholeConfig( + uuid=peer.id.to_string(), + secret_token=peer.rathole_token, + local_addr_host="localhost", + local_addr_port=random_port, + server_name=peer.name, + ) + + # Get rathole toml config map + rathole_config_map = KubeUtils.get_configmap( + client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP + ) + + client_filename = RatholeClientToml.filename + + toml_str = rathole_config_map.data[client_filename] + + rathole_toml = RatholeClientToml(toml_str=toml_str) + + rathole_toml.add_config(config=config) + + self.add_entrypoint(port=random_port, peer_name=peer.name) - def add_port_to_proxy(self, config: RatholeConfig, entrypoint: str = "web") -> None: + self.forward_port_to_proxy(config=config, entrypoint=peer.name) + + def forward_port_to_proxy( + self, config: RatholeConfig, entrypoint: str = "web" + ) -> None: """Add a port to the rathole proxy config map.""" rathole_proxy_config_map = KubeUtils.get_configmap( self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP ) - rathole_proxy = rathole_proxy_config_map.data["rathole-proxy.yml"] + rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] if not rathole_proxy: rathole_proxy = {"http": {"routers": {}, "services": {}}} - # TODO: config.port, this should be a random port + rathole_proxy["http"]["services"][config.server_name] = { + "loadBalancer": {"servers": [{"url": "http://proxy:8001"}]} + } + + rathole_proxy["http"]["routers"][config.server_name] = { + "rule": "PathPrefix(`/`)", + "service": config.server_name, + "entryPoints": [entrypoint], + } + + KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, rathole_proxy) + + def add_dynamic_addr_to_rathole( + self, config: RatholeConfig, entrypoint: str = "web" + ) -> None: + """Add a port to the rathole proxy config map.""" + + rathole_proxy_config_map = KubeUtils.get_configmap( + self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP + ) + + rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] + + if not rathole_proxy: + rathole_proxy = {"http": {"routers": {}, "services": {}}} rathole_proxy["http"]["services"][config.server_name] = { "loadBalancer": { @@ -147,9 +209,7 @@ def add_entrypoint(self, port: int, peer_name: str) -> None: traefik_config = yaml.safe_load(traefik_config_str) - traefik_config["entryPoints"][f"{peer_name}-entrypoint"] = { - "address": f":{port}" - } + traefik_config["entryPoints"][f"{peer_name}"] = {"address": f":{port}"} data["traefik.yml"] = yaml.safe_dump(traefik_config) @@ -166,7 +226,7 @@ def remove_endpoint(self, peer_name: str) -> None: traefik_config = yaml.safe_load(traefik_config_str) - del traefik_config["entryPoints"][f"{peer_name}-entrypoint"] + del traefik_config["entryPoints"][f"{peer_name}"] data["traefik.yml"] = yaml.safe_dump(traefik_config) From 7b8e461cd66de72132ba416938733087d6196593 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 13 May 2024 13:12:42 +0530 Subject: [PATCH 033/100] move RatholeService to seperate file - fix bugs with saving toml - add resourceVersion --- packages/grid/default.env | 3 + .../backend/backend-statefulset.yaml | 8 + .../syft/templates/proxy/proxy-configmap.yaml | 13 +- .../templates/rathole/rathole-configmap.yaml | 1 + packages/grid/helm/syft/values.yaml | 4 +- .../syft/src/syft/service/network/rathole.py | 206 +----------------- .../syft/service/network/rathole_service.py | 206 ++++++++++++++++++ .../src/syft/service/network/rathole_toml.py | 8 +- 8 files changed, 233 insertions(+), 216 deletions(-) create mode 100644 packages/syft/src/syft/service/network/rathole_service.py diff --git a/packages/grid/default.env b/packages/grid/default.env index f6efe8aa463..906251ee865 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -110,3 +110,6 @@ ENABLE_SIGNUP=False # Enclave Attestation DOCKER_IMAGE_ENCLAVE_ATTESTATION=openmined/grid-enclave-attestation + +# Rathole Config +RATHOLE_PORT=2333 \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index 8048f262e5e..f9b2dd42353 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -82,9 +82,17 @@ spec: {{- if .Values.node.debuggerEnabled }} - name: DEBUGGER_ENABLED value: "true" + {{- end }} + {{- if eq .Values.node.type "gateway" }} - name: ASSOCIATION_REQUEST_AUTO_APPROVAL value: {{ .Values.node.associationRequestAutoApproval | quote }} {{- end }} + {{- if .Values.rathole.enabled }} + - name: RATHOLE_PORT + value: {{ .Values.rathole.port | quote }} + - name: RATHOLE_ENABLED + value: "true" + {{- end }} # MongoDB - name: MONGO_PORT value: {{ .Values.mongo.port | quote }} diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index 8853fae80b2..ee5d2316e99 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: proxy-config + resourceVersion: "" labels: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: proxy @@ -83,15 +84,3 @@ metadata: app.kubernetes.io/component: proxy data: rathole-dynamic.yml: | - # http: - # services: - # rathole_domain_1: - # loadBalancer: - # servers: - # - url: "http://rathole-0.rathole.syft.svc.cluster.local:2333" - # routers: - # rathole_domain_1: - # rule: "Host('domain1.domain.syft.local')" - # entryPoints: - # - "web" - # service: "rathole_domain_1" diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index 3e843fbc1cc..46b141d68d9 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: rathole-config + resourceVersion: "" labels: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: rathole diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 6d2bf1761ac..ee6db586bb6 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -227,9 +227,9 @@ rathole: env: null enabled: true - ratholePort: 2333 + port: 2333 appPort: 5555 - mode: "server" + mode: "client" devMode: "false" appLogLevel: "info" diff --git a/packages/syft/src/syft/service/network/rathole.py b/packages/syft/src/syft/service/network/rathole.py index 56d25af36a9..a90e6f34030 100644 --- a/packages/syft/src/syft/service/network/rathole.py +++ b/packages/syft/src/syft/service/network/rathole.py @@ -1,25 +1,15 @@ # stdlib -import secrets from typing import Self -from typing import cast - -# third party -import yaml # relative -from ...custom_worker.k8s import KubeUtils -from ...custom_worker.k8s import get_kr8s_client -from ...serde import serializable +from ...serde.serializable import serializable from ...types.base import SyftBaseModel +from ...util.util import get_env from .node_peer import NodePeer -from .rathole_toml import RatholeClientToml -from .rathole_toml import RatholeServerToml -from .routes import HTTPNodeRoute -RATHOLE_TOML_CONFIG_MAP = "rathole-config" -RATHOLE_PROXY_CONFIG_MAP = "rathole-proxy-config" -RATHOLE_DEFAULT_BIND_ADDRESS = "http://0.0.0.0:2333" -PROXY_CONFIG_MAP = "proxy-config" + +def get_rathole_port() -> int: + return int(get_env("RATHOLE_PORT", "2333")) @serializable() @@ -36,6 +26,9 @@ def local_address(self) -> str: @classmethod def from_peer(cls, peer: NodePeer) -> Self: + # relative + from .routes import HTTPNodeRoute + high_priority_route = peer.pick_highest_priority_route() if not isinstance(high_priority_route, HTTPNodeRoute): @@ -48,186 +41,3 @@ def from_peer(cls, peer: NodePeer) -> Self: local_addr_port=high_priority_route.port, server_name=peer.name, ) - - -# class RatholeProxyConfigWriter: -# def get_config(self, *args, **kwargs): -# pass - -# def save_config(self, *args, **kwargs): -# pass - -# def add_service(url: str, service_name: str, port: int, hostname: str): -# pass - -# def delete_service(self, *args, **kwargs): -# pass - - -class RatholeService: - def __init__(self) -> None: - self.k8rs_client = get_kr8s_client() - - def add_host_to_server(self, peer: NodePeer) -> None: - """Add a host to the rathole server toml file.""" - - route = cast(HTTPNodeRoute, peer.pick_highest_priority_route()) - - config = RatholeConfig( - uuid=peer.id.to_string(), - secret_token=peer.rathole_token, - local_addr_host="localhost", - local_addr_port=route.port, - server_name=peer.name, - ) - - # Get rathole toml config map - rathole_config_map = KubeUtils.get_configmap( - client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP - ) - - client_filename = RatholeServerToml.filename - - toml_str = rathole_config_map.data[client_filename] - - # Add the peer info to the toml file - rathole_toml = RatholeServerToml(toml_str) - rathole_toml.add_config(config=config) - - if not rathole_toml.get_bind_address(): - # First time adding a peer - rathole_toml.set_bind_address(RATHOLE_DEFAULT_BIND_ADDRESS) - - rathole_config_map.data[client_filename] = rathole_toml.toml_str - - # Update the rathole config map - KubeUtils.update_configmap( - client=self.k8rs_client, - name=RATHOLE_TOML_CONFIG_MAP, - data=rathole_config_map.data, - ) - - # Add the peer info to the proxy config map - self.add_dynamic_addr_to_rathole(config) - - def get_random_port(self) -> int: - """Get a random port number.""" - return secrets.randbits(15) - - def add_host_to_client(self, peer: NodePeer) -> None: - """Add a host to the rathole client toml file.""" - - random_port = self.get_random_port() - - config = RatholeConfig( - uuid=peer.id.to_string(), - secret_token=peer.rathole_token, - local_addr_host="localhost", - local_addr_port=random_port, - server_name=peer.name, - ) - - # Get rathole toml config map - rathole_config_map = KubeUtils.get_configmap( - client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP - ) - - client_filename = RatholeClientToml.filename - - toml_str = rathole_config_map.data[client_filename] - - rathole_toml = RatholeClientToml(toml_str=toml_str) - - rathole_toml.add_config(config=config) - - self.add_entrypoint(port=random_port, peer_name=peer.name) - - self.forward_port_to_proxy(config=config, entrypoint=peer.name) - - def forward_port_to_proxy( - self, config: RatholeConfig, entrypoint: str = "web" - ) -> None: - """Add a port to the rathole proxy config map.""" - - rathole_proxy_config_map = KubeUtils.get_configmap( - self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP - ) - - rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] - - if not rathole_proxy: - rathole_proxy = {"http": {"routers": {}, "services": {}}} - - rathole_proxy["http"]["services"][config.server_name] = { - "loadBalancer": {"servers": [{"url": "http://proxy:8001"}]} - } - - rathole_proxy["http"]["routers"][config.server_name] = { - "rule": "PathPrefix(`/`)", - "service": config.server_name, - "entryPoints": [entrypoint], - } - - KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, rathole_proxy) - - def add_dynamic_addr_to_rathole( - self, config: RatholeConfig, entrypoint: str = "web" - ) -> None: - """Add a port to the rathole proxy config map.""" - - rathole_proxy_config_map = KubeUtils.get_configmap( - self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP - ) - - rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] - - if not rathole_proxy: - rathole_proxy = {"http": {"routers": {}, "services": {}}} - - rathole_proxy["http"]["services"][config.server_name] = { - "loadBalancer": { - "servers": [{"url": f"http://rathole:{config.local_addr_port}"}] - } - } - - rathole_proxy["http"]["routers"][config.server_name] = { - "rule": f"Host(`{config.server_name}.syft.local`)", - "service": config.server_name, - "entryPoints": [entrypoint], - } - - KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, rathole_proxy) - - def add_entrypoint(self, port: int, peer_name: str) -> None: - """Add an entrypoint to the traefik config map.""" - - proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) - - data = proxy_config_map.data - - traefik_config_str = data["traefik.yml"] - - traefik_config = yaml.safe_load(traefik_config_str) - - traefik_config["entryPoints"][f"{peer_name}"] = {"address": f":{port}"} - - data["traefik.yml"] = yaml.safe_dump(traefik_config) - - KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, data) - - def remove_endpoint(self, peer_name: str) -> None: - """Remove an entrypoint from the traefik config map.""" - - proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) - - data = proxy_config_map.data - - traefik_config_str = data["traefik.yml"] - - traefik_config = yaml.safe_load(traefik_config_str) - - del traefik_config["entryPoints"][f"{peer_name}"] - - data["traefik.yml"] = yaml.safe_dump(traefik_config) - - KubeUtils.update_configmap(self.k8rs_client, PROXY_CONFIG_MAP, data) diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py new file mode 100644 index 00000000000..53ea27f5c38 --- /dev/null +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -0,0 +1,206 @@ +# stdlib +import secrets + +# third party +import yaml + +# relative +from ...custom_worker.k8s import KubeUtils +from ...custom_worker.k8s import get_kr8s_client +from .node_peer import NodePeer +from .rathole import RatholeConfig +from .rathole import get_rathole_port +from .rathole_toml import RatholeClientToml +from .rathole_toml import RatholeServerToml + +RATHOLE_TOML_CONFIG_MAP = "rathole-config" +RATHOLE_PROXY_CONFIG_MAP = "proxy-config-dynamic" +PROXY_CONFIG_MAP = "proxy-config" + + +class RatholeService: + def __init__(self) -> None: + self.k8rs_client = get_kr8s_client() + + def add_host_to_server(self, peer: NodePeer) -> None: + """Add a host to the rathole server toml file. + + Args: + peer (NodePeer): The peer to be added to the rathole server. + + Returns: + None + """ + + random_port = self.get_random_port() + + config = RatholeConfig( + uuid=peer.id.to_string(), + secret_token=peer.rathole_token, + local_addr_host="localhost", + local_addr_port=random_port, + server_name=peer.name, + ) + + # Get rathole toml config map + rathole_config_map = KubeUtils.get_configmap( + client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP + ) + + client_filename = RatholeServerToml.filename + + toml_str = rathole_config_map.data[client_filename] + + # Add the peer info to the toml file + rathole_toml = RatholeServerToml(toml_str) + rathole_toml.add_config(config=config) + + # First time adding a peer + if not rathole_toml.get_rathole_listener_addr(): + bind_addr = f"http://localhost:{get_rathole_port()}" + rathole_toml.set_rathole_listener_addr(bind_addr) + + data = {client_filename: rathole_toml.toml_str} + + # Update the rathole config map + KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) + + # Add the peer info to the proxy config map + self.add_dynamic_addr_to_rathole(config) + + def get_random_port(self) -> int: + """Get a random port number.""" + return secrets.randbits(15) + + def add_host_to_client(self, peer: NodePeer) -> None: + """Add a host to the rathole client toml file.""" + + random_port = self.get_random_port() + + config = RatholeConfig( + uuid=peer.id.to_string(), + secret_token=peer.rathole_token, + local_addr_host="localhost", + local_addr_port=random_port, + server_name=peer.name, + ) + + # Get rathole toml config map + rathole_config_map = KubeUtils.get_configmap( + client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP + ) + + client_filename = RatholeClientToml.filename + + toml_str = rathole_config_map.data[client_filename] + + rathole_toml = RatholeClientToml(toml_str=toml_str) + + rathole_toml.add_config(config=config) + + data = {client_filename: rathole_toml.toml_str} + + # Update the rathole config map + KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) + + self.add_entrypoint(port=random_port, peer_name=peer.name) + + self.forward_port_to_proxy(config=config, entrypoint=peer.name) + + def forward_port_to_proxy( + self, config: RatholeConfig, entrypoint: str = "web" + ) -> None: + """Add a port to the rathole proxy config map.""" + + rathole_proxy_config_map = KubeUtils.get_configmap( + self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP + ) + + rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] + + if not rathole_proxy: + rathole_proxy = {"http": {"routers": {}, "services": {}}} + else: + rathole_proxy = yaml.safe_load(rathole_proxy) + + rathole_proxy["http"]["services"][config.server_name] = { + "loadBalancer": {"servers": [{"url": "http://proxy:8001"}]} + } + + rathole_proxy["http"]["routers"][config.server_name] = { + "rule": "PathPrefix(`/`)", + "service": config.server_name, + "entryPoints": [entrypoint], + } + + KubeUtils.update_configmap( + config_map=rathole_proxy_config_map, + patch={"data": {"rathole-dynamic.yml": yaml.safe_dump(rathole_proxy)}}, + ) + + def add_dynamic_addr_to_rathole( + self, config: RatholeConfig, entrypoint: str = "web" + ) -> None: + """Add a port to the rathole proxy config map.""" + + rathole_proxy_config_map = KubeUtils.get_configmap( + self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP + ) + + rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] + + if not rathole_proxy: + rathole_proxy = {"http": {"routers": {}, "services": {}}} + else: + rathole_proxy = yaml.safe_load(rathole_proxy) + + rathole_proxy["http"]["services"][config.server_name] = { + "loadBalancer": { + "servers": [{"url": f"http://rathole:{config.local_addr_port}"}] + } + } + + rathole_proxy["http"]["routers"][config.server_name] = { + "rule": f"Host(`{config.server_name}.syft.local`)", + "service": config.server_name, + "entryPoints": [entrypoint], + } + + KubeUtils.update_configmap( + config_map=rathole_proxy_config_map, + patch={"data": {"rathole-dynamic.yml": yaml.safe_dump(rathole_proxy)}}, + ) + + def add_entrypoint(self, port: int, peer_name: str) -> None: + """Add an entrypoint to the traefik config map.""" + + proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) + + data = proxy_config_map.data + + traefik_config_str = data["traefik.yml"] + + traefik_config = yaml.safe_load(traefik_config_str) + + traefik_config["entryPoints"][f"{peer_name}"] = {"address": f":{port}"} + + data["traefik.yml"] = yaml.safe_dump(traefik_config) + + KubeUtils.update_configmap(config_map=proxy_config_map, patch={"data": data}) + + def remove_endpoint(self, peer_name: str) -> None: + """Remove an entrypoint from the traefik config map.""" + + proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) + + data = proxy_config_map.data + + traefik_config_str = data["traefik.yml"] + + traefik_config = yaml.safe_load(traefik_config_str) + + del traefik_config["entryPoints"][f"{peer_name}"] + + data["traefik.yml"] = yaml.safe_dump(traefik_config) + + KubeUtils.update_configmap(config_map=proxy_config_map, patch={"data": data}) diff --git a/packages/syft/src/syft/service/network/rathole_toml.py b/packages/syft/src/syft/service/network/rathole_toml.py index e50c830c5d8..7ca69be6d14 100644 --- a/packages/syft/src/syft/service/network/rathole_toml.py +++ b/packages/syft/src/syft/service/network/rathole_toml.py @@ -27,7 +27,7 @@ def read(self) -> dict: return self.toml_writer.load(self.toml_str) def save(self, toml_dict: dict) -> None: - self.toml_str = self.toml_writer.dump(self.toml_str) + self.toml_str = self.toml_writer.dump(toml_dict) def _validate(self) -> bool: raise NotImplementedError @@ -150,17 +150,17 @@ def _validate(self) -> bool: class RatholeServerToml(RatholeBaseToml): filename: str = "server.toml" - def set_bind_address(self, bind_address: str) -> None: + def set_rathole_listener_addr(self, bind_addr: str) -> None: """Set the bind address in the server toml file.""" toml = self.read() # Set the bind address - toml["server"]["bind_addr"] = bind_address + toml["server"]["bind_addr"] = bind_addr self.save(toml) - def get_bind_address(self) -> str: + def get_rathole_listener_addr(self) -> str: """Get the bind address from the server toml file.""" toml = self.read() From 13e2dd015792fb282da522ef885e264050a020a4 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 13 May 2024 16:03:29 +0530 Subject: [PATCH 034/100] fix add host to client method to pass remote addr - integrate Rathole service to Network service --- .../service/network/association_request.py | 10 +++++++++- .../syft/service/network/network_service.py | 19 +++++++++++++++++++ .../syft/service/network/rathole_service.py | 16 ++++++++++------ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index 70c08a52e56..0cdb78375fb 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -1,5 +1,6 @@ # stdlib import secrets +from typing import cast # third party from result import Err @@ -72,14 +73,21 @@ def _run( except Exception as e: return Err(SyftError(message=str(e))) - network_stash = service_ctx.node.get_service(NetworkService).stash + network_service = cast( + NetworkService, service_ctx.node.get_service(NetworkService) + ) + + network_stash = network_service.stash result = network_stash.create_or_update_peer( service_ctx.node.verify_key, self.remote_peer ) + if result.is_err(): return Err(SyftError(message=str(result.err()))) + network_service.rathole_service.add_host_to_server(self.remote_peer) + # this way they can match up who we are with who they think we are # Sending a signed messages for the peer to verify self_node_peer = self.self_node_route.validate_with_context(context=service_ctx) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index fd937f8491f..dfb7c8cdce2 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -1,5 +1,6 @@ # stdlib from collections.abc import Callable +from hashlib import sha256 import secrets from typing import Any @@ -48,6 +49,8 @@ from ..warnings import CRUDWarning from .association_request import AssociationRequestChange from .node_peer import NodePeer +from .rathole import get_rathole_port +from .rathole_service import RatholeService from .routes import HTTPNodeRoute from .routes import NodeRoute from .routes import NodeRouteType @@ -140,6 +143,7 @@ class NetworkService(AbstractService): def __init__(self, store: DocumentStore) -> None: self.store = store self.stash = NetworkStash(store=store) + self.rathole_service = RatholeService() # TODO: Check with MADHAVA, can we even allow guest user to introduce routes to # domain nodes? @@ -172,6 +176,9 @@ def exchange_credentials_with( ) random_challenge = secrets.token_bytes(16) + rathole_token = self._generate_token() + self_node_peer.rathole_token = rathole_token + # ask the remote client to add this node (represented by `self_node_peer`) as a peer remote_res = remote_client.api.services.network.add_peer( peer=self_node_peer, @@ -195,12 +202,24 @@ def exchange_credentials_with( if result.is_err(): return SyftError(message=str(result.err())) + remote_addr = f"{remote_node_route.protocol}://{remote_node_route.host_or_ip}:{get_rathole_port()()}" + + self.rathole_service.add_host_to_client( + peer_name=self_node_peer.name, + peer_id=self_node_peer.id.to_string(), + rathole_token=self_node_peer.rathole_token, + remote_addr=remote_addr, + ) + return ( SyftSuccess(message="Routes Exchanged") if association_request_approved else remote_res ) + def _generate_token(self) -> str: + return sha256(secrets.token_bytes(16)).hexdigest() + @service_method(path="network.add_peer", name="add_peer", roles=GUEST_ROLE_LEVEL) def add_peer( self, diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 53ea27f5c38..c9556f69f4e 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -72,17 +72,19 @@ def get_random_port(self) -> int: """Get a random port number.""" return secrets.randbits(15) - def add_host_to_client(self, peer: NodePeer) -> None: + def add_host_to_client( + self, peer_name: str, peer_id: str, rathole_token: str, remote_addr: str + ) -> None: """Add a host to the rathole client toml file.""" random_port = self.get_random_port() config = RatholeConfig( - uuid=peer.id.to_string(), - secret_token=peer.rathole_token, + uuid=peer_id, + secret_token=rathole_token, local_addr_host="localhost", local_addr_port=random_port, - server_name=peer.name, + server_name=peer_name, ) # Get rathole toml config map @@ -98,14 +100,16 @@ def add_host_to_client(self, peer: NodePeer) -> None: rathole_toml.add_config(config=config) + rathole_toml.set_remote_addr(remote_addr) + data = {client_filename: rathole_toml.toml_str} # Update the rathole config map KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) - self.add_entrypoint(port=random_port, peer_name=peer.name) + self.add_entrypoint(port=random_port, peer_name=peer_name) - self.forward_port_to_proxy(config=config, entrypoint=peer.name) + self.forward_port_to_proxy(config=config, entrypoint=peer_name) def forward_port_to_proxy( self, config: RatholeConfig, entrypoint: str = "web" From ad838ac7e5ba4c21f612788e053801d8cf0600c0 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 15 May 2024 14:50:10 +0530 Subject: [PATCH 035/100] fix values.yml not being correctly propogated --- packages/grid/devspace.yaml | 11 +++++++---- packages/grid/helm/syft/values.yaml | 4 ---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index dc999c80063..cd777d5ef44 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -76,15 +76,16 @@ deployments: releaseName: syft-dev chart: name: ./helm/syft + # anything that does not need devspace $env vars should go in values.dev.yaml + valuesFiles: + - ./helm/syft/values.yaml + - ./helm/values.dev.yaml values: global: registry: ${CONTAINER_REGISTRY} version: dev-${DEVSPACE_TIMESTAMP} node: type: domain # required for the gateway profile - # anything that does not need devspace $env vars should go in values.dev.yaml - valuesFiles: - - ./helm/values.dev.yaml dev: mongo: @@ -139,9 +140,11 @@ profiles: path: images.seaweedfs - op: remove path: dev.seaweedfs + + # Patch mode to server - op: replace path: deployments.syft.helm.values.rathole.mode - value: "server" + value: server # Port Re-Mapping # Mongo diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index ee6db586bb6..f03f81d619c 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -228,10 +228,6 @@ rathole: enabled: true port: 2333 - appPort: 5555 - mode: "client" - devMode: "false" - appLogLevel: "info" # Pod labels & annotations podLabels: null From 01fe5e82268a8fffff2b545854e56b727e01f9bb Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 15 May 2024 19:50:27 +0530 Subject: [PATCH 036/100] add mode to rathole in values.yml --- packages/grid/devspace.yaml | 2 ++ packages/grid/helm/syft/values.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index cd777d5ef44..b52069a4d5a 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -86,6 +86,8 @@ deployments: version: dev-${DEVSPACE_TIMESTAMP} node: type: domain # required for the gateway profile + rathole: + mode: client dev: mongo: diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index f03f81d619c..7df55c978da 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -228,6 +228,7 @@ rathole: enabled: true port: 2333 + mode: client # Pod labels & annotations podLabels: null From f7be2806354217562db335e3650fbb4fae0613e3 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 15 May 2024 22:20:35 +0530 Subject: [PATCH 037/100] fix set remote addr method in Rathole service - skip remote ping check if rathole --- .../service/network/association_request.py | 55 ++++++++++--------- .../syft/service/network/network_service.py | 8 ++- .../src/syft/service/network/rathole_toml.py | 5 +- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index 0cdb78375fb..ac64b0c57f2 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -43,35 +43,36 @@ def _run( service_ctx = context.to_service_ctx() - try: - remote_client: SyftClient = self.remote_peer.client_with_context( - context=service_ctx - ) - if remote_client.is_err(): - return SyftError( - message=f"Failed to create remote client for peer: " - f"{self.remote_peer.id}. Error: {remote_client.err()}" + if self.remote_peer.rathole_token is None: + try: + remote_client: SyftClient = self.remote_peer.client_with_context( + context=service_ctx ) - remote_client = remote_client.ok() - random_challenge = secrets.token_bytes(16) - remote_res = remote_client.api.services.network.ping( - challenge=random_challenge - ) - except Exception as e: - return SyftError(message="Remote Peer cannot ping peer:" + str(e)) + if remote_client.is_err(): + return SyftError( + message=f"Failed to create remote client for peer: " + f"{self.remote_peer.id}. Error: {remote_client.err()}" + ) + remote_client = remote_client.ok() + random_challenge = secrets.token_bytes(16) + remote_res = remote_client.api.services.network.ping( + challenge=random_challenge + ) + except Exception as e: + return SyftError(message="Remote Peer cannot ping peer:" + str(e)) - if isinstance(remote_res, SyftError): - return Err(remote_res) + if isinstance(remote_res, SyftError): + return Err(remote_res) - challenge_signature = remote_res + challenge_signature = remote_res - # Verifying if the challenge is valid - try: - self.remote_peer.verify_key.verify_key.verify( - random_challenge, challenge_signature - ) - except Exception as e: - return Err(SyftError(message=str(e))) + # Verifying if the challenge is valid + try: + self.remote_peer.verify_key.verify_key.verify( + random_challenge, challenge_signature + ) + except Exception as e: + return Err(SyftError(message=str(e))) network_service = cast( NetworkService, service_ctx.node.get_service(NetworkService) @@ -79,6 +80,8 @@ def _run( network_stash = network_service.stash + network_service.rathole_service.add_host_to_server(self.remote_peer) + result = network_stash.create_or_update_peer( service_ctx.node.verify_key, self.remote_peer ) @@ -86,8 +89,6 @@ def _run( if result.is_err(): return Err(SyftError(message=str(result.err()))) - network_service.rathole_service.add_host_to_server(self.remote_peer) - # this way they can match up who we are with who they think we are # Sending a signed messages for the peer to verify self_node_peer = self.self_node_route.validate_with_context(context=service_ctx) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index dfb7c8cdce2..4976119f5a9 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -188,7 +188,9 @@ def exchange_credentials_with( ) if isinstance(remote_res, SyftError): - return remote_res + return SyftError( + message=f"returned error from add peer: {remote_res.message}" + ) association_request_approved = not isinstance(remote_res, Request) @@ -200,7 +202,9 @@ def exchange_credentials_with( remote_node_peer, ) if result.is_err(): - return SyftError(message=str(result.err())) + return SyftError( + message=f"Failed to update remote node peer: {str(result.err())}" + ) remote_addr = f"{remote_node_route.protocol}://{remote_node_route.host_or_ip}:{get_rathole_port()()}" diff --git a/packages/syft/src/syft/service/network/rathole_toml.py b/packages/syft/src/syft/service/network/rathole_toml.py index 7ca69be6d14..e5fe17b59e9 100644 --- a/packages/syft/src/syft/service/network/rathole_toml.py +++ b/packages/syft/src/syft/service/network/rathole_toml.py @@ -49,10 +49,7 @@ def set_remote_addr(self, remote_host: str) -> None: if "client" not in toml: toml["client"] = {} - toml["client"]["remote_addr"] = remote_host - - if remote_host not in toml["client"]["remote"]: - toml["client"]["remote"].append(remote_host) + toml["client"]["remote_addr"] = remote_host self.save(toml) From c5c3fa6243bf0caddf59c5c92ef85eba6f81b443 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 15 May 2024 22:45:09 +0530 Subject: [PATCH 038/100] rename RATHOLE_MODE to MODE in rathole/start.sh - removed extra braces from get_rathole_port --- packages/grid/rathole/start.sh | 8 ++++---- packages/syft/src/syft/service/network/network_service.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh index e948973decf..d9527196ce9 100755 --- a/packages/grid/rathole/start.sh +++ b/packages/grid/rathole/start.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash -RATHOLE_MODE=${RATHOLE_MODE:-server} +MODE=${MODE:-server} cp -L -r -f /conf/* conf/ -if [[ $RATHOLE_MODE == "server" ]]; then +if [[ $MODE == "server" ]]; then /app/rathole conf/server.toml & -elif [[ $RATHOLE_MODE = "client" ]]; then +elif [[ $MODE = "client" ]]; then /app/rathole conf/client.toml & else - echo "RATHOLE_MODE is set to an invalid value. Exiting." + echo "RATHOLE MODE is set to an invalid value. Exiting." fi # reload config every 10 seconds diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 4976119f5a9..2a8aa986846 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -206,7 +206,7 @@ def exchange_credentials_with( message=f"Failed to update remote node peer: {str(result.err())}" ) - remote_addr = f"{remote_node_route.protocol}://{remote_node_route.host_or_ip}:{get_rathole_port()()}" + remote_addr = f"{remote_node_route.protocol}://{remote_node_route.host_or_ip}:{get_rathole_port()}" self.rathole_service.add_host_to_client( peer_name=self_node_peer.name, From 401d97417266f6eccdb29feea4424d94c2d94c77 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 16 May 2024 00:04:09 +0530 Subject: [PATCH 039/100] add a retry if client.toml is invalid when no connections are setup --- .../syft/templates/rathole/rathole-configmap.yaml | 2 +- packages/grid/rathole/start.sh | 11 ++++++++++- packages/syft/src/syft/service/network/rathole.py | 2 +- .../syft/src/syft/service/network/rathole_service.py | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index 46b141d68d9..d9e004d3c5e 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -20,5 +20,5 @@ data: {{- if eq .Values.rathole.mode "client" }} client.toml: | [client] - remote_addr = "" + remote_addr = "0.0.0.0:2333" {{- end }} diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh index d9527196ce9..4095f30f3aa 100755 --- a/packages/grid/rathole/start.sh +++ b/packages/grid/rathole/start.sh @@ -6,7 +6,16 @@ cp -L -r -f /conf/* conf/ if [[ $MODE == "server" ]]; then /app/rathole conf/server.toml & elif [[ $MODE = "client" ]]; then - /app/rathole conf/client.toml & + while true; do + /app/rathole conf/client.toml + status=$? + if [ $status -eq 0 ]; then + break + else + echo "Failed to load client.toml, retrying in 5 seconds..." + sleep 10 + fi + done & else echo "RATHOLE MODE is set to an invalid value. Exiting." fi diff --git a/packages/syft/src/syft/service/network/rathole.py b/packages/syft/src/syft/service/network/rathole.py index a90e6f34030..e102134ada6 100644 --- a/packages/syft/src/syft/service/network/rathole.py +++ b/packages/syft/src/syft/service/network/rathole.py @@ -22,7 +22,7 @@ class RatholeConfig(SyftBaseModel): @property def local_address(self) -> str: - return f"http://{self.local_addr_host}:{self.local_addr_port}" + return f"{self.local_addr_host}:{self.local_addr_port}" @classmethod def from_peer(cls, peer: NodePeer) -> Self: diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index c9556f69f4e..250c56c21cb 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -57,7 +57,7 @@ def add_host_to_server(self, peer: NodePeer) -> None: # First time adding a peer if not rathole_toml.get_rathole_listener_addr(): - bind_addr = f"http://localhost:{get_rathole_port()}" + bind_addr = f"localhost:{get_rathole_port()}" rathole_toml.set_rathole_listener_addr(bind_addr) data = {client_filename: rathole_toml.toml_str} From e4e55f38947d42829b5a02bce44c0afc3346503f Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 16 May 2024 11:19:30 +0530 Subject: [PATCH 040/100] remove fastapi from grid/rathole --- packages/grid/rathole/domain.dockerfile | 9 - packages/grid/rathole/nginx.conf | 6 - packages/grid/rathole/requirements.txt | 5 - packages/grid/rathole/server/__init__.py | 0 packages/grid/rathole/server/main.py | 94 ------- packages/grid/rathole/server/models.py | 18 -- packages/grid/rathole/server/nginx_builder.py | 92 ------- packages/grid/rathole/server/toml_writer.py | 26 -- packages/grid/rathole/server/utils.py | 236 ------------------ 9 files changed, 486 deletions(-) delete mode 100644 packages/grid/rathole/domain.dockerfile delete mode 100644 packages/grid/rathole/nginx.conf delete mode 100644 packages/grid/rathole/requirements.txt delete mode 100644 packages/grid/rathole/server/__init__.py delete mode 100644 packages/grid/rathole/server/main.py delete mode 100644 packages/grid/rathole/server/models.py delete mode 100644 packages/grid/rathole/server/nginx_builder.py delete mode 100644 packages/grid/rathole/server/toml_writer.py delete mode 100644 packages/grid/rathole/server/utils.py diff --git a/packages/grid/rathole/domain.dockerfile b/packages/grid/rathole/domain.dockerfile deleted file mode 100644 index cdb657540e8..00000000000 --- a/packages/grid/rathole/domain.dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG PYTHON_VERSION="3.12" -FROM python:${PYTHON_VERSION}-bookworm -RUN apt update && apt install -y netcat-openbsd vim -WORKDIR /app -CMD ["python3", "-m", "http.server", "8000"] -EXPOSE 8000 - -# docker build -f domain.dockerfile . -t domain -# docker run -it -p 8080:8000 domain diff --git a/packages/grid/rathole/nginx.conf b/packages/grid/rathole/nginx.conf deleted file mode 100644 index 447c482a9e4..00000000000 --- a/packages/grid/rathole/nginx.conf +++ /dev/null @@ -1,6 +0,0 @@ -server { - listen 8000; - location / { - proxy_pass http://test-domain-r:8001; - } -} \ No newline at end of file diff --git a/packages/grid/rathole/requirements.txt b/packages/grid/rathole/requirements.txt deleted file mode 100644 index 4b379d83e8e..00000000000 --- a/packages/grid/rathole/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -fastapi==0.110.0 -filelock==3.13.4 -loguru==0.7.2 -python-nginx -uvicorn[standard]==0.27.1 diff --git a/packages/grid/rathole/server/__init__.py b/packages/grid/rathole/server/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/grid/rathole/server/main.py b/packages/grid/rathole/server/main.py deleted file mode 100644 index 201618c6432..00000000000 --- a/packages/grid/rathole/server/main.py +++ /dev/null @@ -1,94 +0,0 @@ -# stdlib -from enum import Enum -import os -import sys - -# third party -from fastapi import FastAPI -from fastapi import status -from loguru import logger -from server.models import RatholeConfig -from server.models import ResponseModel -from server.utils import RatholeClientToml -from server.utils import RatholeServerToml - -# Logging Configuration -log_level = os.getenv("APP_LOG_LEVEL", "INFO").upper() -logger.remove() -logger.add(sys.stderr, colorize=True, level=log_level) - -app = FastAPI(title="Rathole") - - -class RatholeMode(Enum): - CLIENT = "client" - SERVER = "server" - - -ServiceType = os.getenv("MODE", "client").lower() - - -RatholeTomlManager = ( - RatholeServerToml() - if ServiceType == RatholeMode.SERVER.value - else RatholeClientToml() -) - - -async def healthcheck() -> bool: - return True - - -@app.get( - "/", - response_model=ResponseModel, - status_code=status.HTTP_200_OK, -) -async def healthcheck_endpoint() -> ResponseModel: - res = await healthcheck() - if res: - return ResponseModel(message="OK") - else: - return ResponseModel(message="FAIL") - - -@app.post( - "/config/", - response_model=ResponseModel, - status_code=status.HTTP_201_CREATED, -) -async def add_config(config: RatholeConfig) -> ResponseModel: - RatholeTomlManager.add_config(config) - return ResponseModel(message="Config added successfully") - - -@app.delete( - "/config/{uuid}", - response_model=ResponseModel, - status_code=status.HTTP_200_OK, -) -async def remove_config(uuid: str) -> ResponseModel: - RatholeTomlManager.remove_config(uuid) - return ResponseModel(message="Config removed successfully") - - -@app.put( - "/config/{uuid}", - response_model=ResponseModel, - status_code=status.HTTP_200_OK, -) -async def update_config(config: RatholeConfig) -> ResponseModel: - RatholeTomlManager.update_config(config=config) - return ResponseModel(message="Config updated successfully") - - -@app.get( - "/config/{uuid}", - response_model=RatholeConfig | ResponseModel, - status_code=status.HTTP_201_CREATED, -) -async def get_config(uuid: str) -> RatholeConfig: - config = RatholeTomlManager.get_config(uuid) - if config is None: - return ResponseModel(message="Config not found") - return config diff --git a/packages/grid/rathole/server/models.py b/packages/grid/rathole/server/models.py deleted file mode 100644 index c5921738885..00000000000 --- a/packages/grid/rathole/server/models.py +++ /dev/null @@ -1,18 +0,0 @@ -# third party -from pydantic import BaseModel - - -class ResponseModel(BaseModel): - message: str - - -class RatholeConfig(BaseModel): - uuid: str - secret_token: str - local_addr_host: str - local_addr_port: int - server_name: str | None = None - - @property - def local_address(self) -> str: - return f"http://{self.local_addr_host}:{self.local_addr_port}" diff --git a/packages/grid/rathole/server/nginx_builder.py b/packages/grid/rathole/server/nginx_builder.py deleted file mode 100644 index 3d1bd14ce1f..00000000000 --- a/packages/grid/rathole/server/nginx_builder.py +++ /dev/null @@ -1,92 +0,0 @@ -# stdlib -from pathlib import Path - -# third party -from filelock import FileLock -import nginx -from nginx import Conf - - -class RatholeNginxConfigBuilder: - def __init__(self, filename: str | Path) -> None: - self.filename = Path(filename).absolute() - - if not self.filename.exists(): - self.filename.touch() - - self.lock = FileLock(f"{filename}.lock") - self.lock_timeout = 30 - - def read(self) -> Conf: - with self.lock.acquire(timeout=self.lock_timeout): - conf = nginx.loadf(self.filename) - - return conf - - def write(self, conf: Conf) -> None: - with self.lock.acquire(timeout=self.lock_timeout): - nginx.dumpf(conf, self.filename) - - def add_server( - self, - listen_port: int, - location: str, - proxy_pass: str, - server_name: str | None = None, - ) -> None: - n_config = self.read() - server_to_modify = self.find_server_with_listen_port(listen_port) - - if server_to_modify is not None: - server_to_modify.add( - nginx.Location(location, nginx.Key("proxy_pass", proxy_pass)) - ) - if server_name is not None: - server_to_modify.add(nginx.Key("server_name", server_name)) - else: - server = nginx.Server( - nginx.Key("listen", listen_port), - nginx.Location(location, nginx.Key("proxy_pass", proxy_pass)), - ) - if server_name is not None: - server.add(nginx.Key("server_name", server_name)) - - n_config.add(server) - - self.write(n_config) - - def remove_server(self, listen_port: int) -> None: - conf = self.read() - for server in conf.servers: - for child in server.children: - if child.name == "listen" and int(child.value) == listen_port: - conf.remove(server) - break - self.write(conf) - - def find_server_with_listen_port(self, listen_port: int) -> nginx.Server | None: - conf = self.read() - for server in conf.servers: - for child in server.children: - if child.name == "listen" and int(child.value) == listen_port: - return server - return None - - def modify_proxy_for_port( - self, listen_port: int, location: str, proxy_pass: str - ) -> None: - conf = self.read() - server_to_modify = self.find_server_with_listen_port(listen_port) - - if server_to_modify is None: - raise ValueError(f"Server with listen port {listen_port} not found") - - for location in server_to_modify.locations: - if location.value != location: - continue - for key in location.keys: - if key.name == "proxy_pass": - key.value = proxy_pass - break - - self.write(conf) diff --git a/packages/grid/rathole/server/toml_writer.py b/packages/grid/rathole/server/toml_writer.py deleted file mode 100644 index a0e79aff627..00000000000 --- a/packages/grid/rathole/server/toml_writer.py +++ /dev/null @@ -1,26 +0,0 @@ -# stdlib -from pathlib import Path -import tomllib - -# third party -from filelock import FileLock - -FILE_LOCK_TIMEOUT = 30 - - -class TomlReaderWriter: - def __init__(self, lock: FileLock, filename: Path | str) -> None: - self.filename = Path(filename).absolute() - self.timeout = FILE_LOCK_TIMEOUT - self.lock = lock - - def write(self, toml_dict: dict) -> None: - with self.lock.acquire(timeout=self.timeout): - with open(str(self.filename), "wb") as fp: - tomllib.dump(toml_dict, fp) - - def read(self) -> dict: - with self.lock.acquire(timeout=self.timeout): - with open(str(self.filename), "rb") as fp: - toml = tomllib.load(fp) - return toml diff --git a/packages/grid/rathole/server/utils.py b/packages/grid/rathole/server/utils.py deleted file mode 100644 index 485e4ae7e23..00000000000 --- a/packages/grid/rathole/server/utils.py +++ /dev/null @@ -1,236 +0,0 @@ -# stdlib - -# third party -from filelock import FileLock - -# relative -from .models import RatholeConfig -from .nginx_builder import RatholeNginxConfigBuilder -from .toml_writer import TomlReaderWriter - -lock = FileLock("rathole.toml.lock") - - -class RatholeClientToml: - filename: str = "client.toml" - - def __init__(self) -> None: - self.client_toml = TomlReaderWriter(lock=lock, filename=self.filename) - self.nginx_mananger = RatholeNginxConfigBuilder("nginx.conf") - - def set_remote_addr(self, remote_host: str) -> None: - """Add a new remote address to the client toml file.""" - - toml = self.client_toml.read() - - # Add the new remote address - if "client" not in toml: - toml["client"] = {} - - toml["client"]["remote_addr"] = remote_host - - if remote_host not in toml["client"]["remote"]: - toml["client"]["remote"].append(remote_host) - - self.client_toml.write(toml_dict=toml) - - def add_config(self, config: RatholeConfig) -> None: - """Add a new config to the toml file.""" - - toml = self.client_toml.read() - - # Add the new config - if "services" not in toml["client"]: - toml["client"]["services"] = {} - - if config.uuid not in toml["client"]["services"]: - toml["client"]["services"][config.uuid] = {} - - toml["client"]["services"][config.uuid] = { - "token": config.secret_token, - "local_addr": config.local_address, - } - - self.client_toml.write(toml) - - self.nginx_mananger.add_server( - config.local_addr_port, location="/", proxy_pass="http://backend:80" - ) - - def remove_config(self, uuid: str) -> None: - """Remove a config from the toml file.""" - - toml = self.client_toml.read() - - # Remove the config - if "services" not in toml["client"]: - return - - if uuid not in toml["client"]["services"]: - return - - del toml["client"]["services"][uuid] - - self.client_toml.write(toml) - - def update_config(self, config: RatholeConfig) -> None: - """Update a config in the toml file.""" - - toml = self.client_toml.read() - - # Update the config - if "services" not in toml["client"]: - return - - if config.uuid not in toml["client"]["services"]: - return - - toml["client"]["services"][config.uuid] = { - "token": config.secret_token, - "local_addr": config.local_address, - } - - self.client_toml.write(toml) - - def get_config(self, uuid: str) -> RatholeConfig | None: - """Get a config from the toml file.""" - - toml = self.client_toml.read() - - # Get the config - if "services" not in toml["client"]: - return None - - if uuid not in toml["client"]["services"]: - return None - - service = toml["client"]["services"][uuid] - - return RatholeConfig( - uuid=uuid, - secret_token=service["token"], - local_addr_host=service["local_addr"].split(":")[0], - local_addr_port=service["local_addr"].split(":")[1], - ) - - def _validate(self) -> bool: - if not self.client_toml.filename.exists(): - return False - - toml = self.client_toml.read() - - if not toml["client"]["remote_addr"]: - return False - - for uuid, config in toml["client"]["services"].items(): - if not uuid: - return False - - if not config["token"] or not config["local_addr"]: - return False - - return True - - @property - def is_valid(self) -> bool: - return self._validate() - - -class RatholeServerToml: - filename: str = "server.toml" - - def __init__(self) -> None: - self.server_toml = TomlReaderWriter(lock=lock, filename=self.filename) - self.nginx_manager = RatholeNginxConfigBuilder("nginx.conf") - - def set_bind_address(self, bind_address: str) -> None: - """Set the bind address in the server toml file.""" - - toml = self.server_toml.read() - - # Set the bind address - toml["server"]["bind_addr"] = bind_address - - self.server_toml.write(toml) - - def add_config(self, config: RatholeConfig) -> None: - """Add a new config to the toml file.""" - - toml = self.server_toml.read() - - # Add the new config - if "services" not in toml["server"]: - toml["server"]["services"] = {} - - if config.uuid not in toml["server"]["services"]: - toml["server"]["services"][config.uuid] = {} - - toml["server"]["services"][config.uuid] = { - "token": config.secret_token, - "bind_addr": config.local_address, - } - - self.server_toml.write(toml) - - self.nginx_manager.add_server( - config.local_addr_port, - location="/", - proxy_pass=config.local_address, - server_name=f"{config.server_name}.local*", - ) - - def remove_config(self, uuid: str) -> None: - """Remove a config from the toml file.""" - - toml = self.server_toml.read() - - # Remove the config - if "services" not in toml["server"]: - return - - if uuid not in toml["server"]["services"]: - return - - del toml["server"]["services"][uuid] - - self.server_toml.write(toml) - - def update_config(self, config: RatholeConfig) -> None: - """Update a config in the toml file.""" - - toml = self.server_toml.read() - - # Update the config - if "services" not in toml["server"]: - return - - if config.uuid not in toml["server"]["services"]: - return - - toml["server"]["services"][config.uuid] = { - "token": config.secret_token, - "bind_addr": config.local_address, - } - - self.server_toml.write(toml) - - def _validate(self) -> bool: - if not self.server_toml.filename.exists(): - return False - - toml = self.server_toml.read() - - if not toml["server"]["bind_addr"]: - return False - - for uuid, config in toml["server"]["services"].items(): - if not uuid: - return False - - if not config["token"] or not config["bind_addr"]: - return False - - return True - - def is_valid(self) -> bool: - return self._validate() From dd9a037203b0118ed2ba51763780787aae89c2b2 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 16 May 2024 12:35:59 +0530 Subject: [PATCH 041/100] use rathole image to build rathole --- packages/grid/rathole/rathole.dockerfile | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 1d1b82785af..5f3a8762677 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -1,32 +1,23 @@ ARG RATHOLE_VERSION="0.5.0" ARG PYTHON_VERSION="3.12" -FROM rust as build -ARG RATHOLE_VERSION -ARG FEATURES -RUN apt update && apt install -y git -RUN git clone -b v${RATHOLE_VERSION} https://github.com/rapiz1/rathole - -WORKDIR /rathole -RUN cargo build --locked --release --features ${FEATURES:-default} +FROM rapiz1/rathole:v${RATHOLE_VERSION} as build FROM python:${PYTHON_VERSION}-bookworm ARG RATHOLE_VERSION ENV MODE="client" -ENV APP_LOG_LEVEL="info" -COPY --from=build /rathole/target/release/rathole /app/rathole RUN apt update && apt install -y netcat-openbsd vim +COPY --from=build /app/rathole /app/rathole + WORKDIR /app COPY ./start.sh /app/start.sh -COPY ./nginx.conf /etc/nginx/conf.d/default.conf -COPY ./requirements.txt /app/requirements.txt -COPY ./server/ /app/server/ -RUN pip install --user -r requirements.txt -CMD ["sh", "-c", "/app/start.sh"] EXPOSE 2333/udp EXPOSE 2333 +CMD ["sh", "-c", "/app/start.sh"] + + # build and run a fake domain to simulate a normal http container service # docker build -f domain.dockerfile . -t domain # docker run --name domain1 -it -d -p 8080:8000 domain From f9a3fcf7d143608476c7318b95923385e69ddc88 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 17 May 2024 10:28:06 +0530 Subject: [PATCH 042/100] configure rathole toml to use websockets - add a path in ingress to resolve to rathole service - fix remote addr url in client.toml --- packages/grid/helm/syft/templates/global/ingress.yaml | 11 +++++++++-- .../syft/templates/rathole/rathole-configmap.yaml | 10 ++++++++++ .../syft/src/syft/service/network/network_service.py | 8 ++++++-- packages/syft/src/syft/types/grid_url.py | 4 ++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/grid/helm/syft/templates/global/ingress.yaml b/packages/grid/helm/syft/templates/global/ingress.yaml index 677a66313a6..71bac8d6964 100644 --- a/packages/grid/helm/syft/templates/global/ingress.yaml +++ b/packages/grid/helm/syft/templates/global/ingress.yaml @@ -27,13 +27,20 @@ spec: - host: {{ .Values.ingress.hostname | quote }} http: paths: - - backend: + - path: / + pathType: Prefix + backend: service: name: proxy port: number: 80 - path: / + - path: /rathole pathType: Prefix + backend: + service: + name: rathole + port: + number: 2333 {{- if .Values.ingress.tls.enabled }} tls: - hosts: diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index d9e004d3c5e..cd5453f1ea2 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -12,6 +12,11 @@ data: [server] bind_addr = "0.0.0.0:2333" + [server.transport] + type = "websocket" + [server.transport.websocket] + tls = false + [server.services.domain] token = "domain-specific-rathole-secret" bind_addr = "0.0.0.0:8001" @@ -21,4 +26,9 @@ data: client.toml: | [client] remote_addr = "0.0.0.0:2333" + + [client.transport] + type = "websocket" + [client.transport.websocket] + tls = false {{- end }} diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 2a8aa986846..e0143541728 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -49,7 +49,6 @@ from ..warnings import CRUDWarning from .association_request import AssociationRequestChange from .node_peer import NodePeer -from .rathole import get_rathole_port from .rathole_service import RatholeService from .routes import HTTPNodeRoute from .routes import NodeRoute @@ -206,7 +205,12 @@ def exchange_credentials_with( message=f"Failed to update remote node peer: {str(result.err())}" ) - remote_addr = f"{remote_node_route.protocol}://{remote_node_route.host_or_ip}:{get_rathole_port()}" + remote_url = GridURL( + host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port + ) + rathole_remote_addr = remote_url.with_path("/rathole").as_container_host() + + remote_addr = rathole_remote_addr.url_no_protocol self.rathole_service.add_host_to_client( peer_name=self_node_peer.name, diff --git a/packages/syft/src/syft/types/grid_url.py b/packages/syft/src/syft/types/grid_url.py index 91cf53e46d7..9db8de440a8 100644 --- a/packages/syft/src/syft/types/grid_url.py +++ b/packages/syft/src/syft/types/grid_url.py @@ -135,6 +135,10 @@ def base_url(self) -> str: def base_url_no_port(self) -> str: return f"{self.protocol}://{self.host_or_ip}" + @property + def url_no_protocol(self) -> str: + return f"{self.host_or_ip}:{self.port}{self.path}" + @property def url_path(self) -> str: return f"{self.path}{self.query_string}" From 299dbac78e589cecb99ff923cafbe06f5bd3bda8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 21 May 2024 23:14:13 +0530 Subject: [PATCH 043/100] configure to use same port and path for both http and websocket - add a traefik rule to route requests with Header with websocket to rathole --- packages/grid/helm/syft/templates/global/ingress.yaml | 11 ++--------- .../helm/syft/templates/proxy/proxy-configmap.yaml | 11 ++++++++++- .../syft/src/syft/service/network/network_service.py | 2 +- .../syft/src/syft/service/network/rathole_service.py | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/grid/helm/syft/templates/global/ingress.yaml b/packages/grid/helm/syft/templates/global/ingress.yaml index 71bac8d6964..677a66313a6 100644 --- a/packages/grid/helm/syft/templates/global/ingress.yaml +++ b/packages/grid/helm/syft/templates/global/ingress.yaml @@ -27,20 +27,13 @@ spec: - host: {{ .Values.ingress.hostname | quote }} http: paths: - - path: / - pathType: Prefix - backend: + - backend: service: name: proxy port: number: 80 - - path: /rathole + path: / pathType: Prefix - backend: - service: - name: rathole - port: - number: 2333 {{- if .Values.ingress.tls.enabled }} tls: - hosts: diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index ee5d2316e99..9a3cacf23b2 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: proxy-config - resourceVersion: "" labels: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: proxy @@ -22,7 +21,16 @@ data: loadBalancer: servers: - url: "http://seaweedfs:8333" + rathole: + loadBalancer: + servers: + - url: "http://rathole:2333" routers: + rathole: + rule: "PathPrefix(`/`) && Headers(`Upgrade`, `websocket`)" + entryPoints: + - "web" + service: "rathole" frontend: rule: "PathPrefix(`/`)" entryPoints: @@ -84,3 +92,4 @@ metadata: app.kubernetes.io/component: proxy data: rathole-dynamic.yml: | + diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index e0143541728..daff8c411f3 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -208,7 +208,7 @@ def exchange_credentials_with( remote_url = GridURL( host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port ) - rathole_remote_addr = remote_url.with_path("/rathole").as_container_host() + rathole_remote_addr = remote_url.as_container_host() remote_addr = rathole_remote_addr.url_no_protocol diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 250c56c21cb..9be55c25fcf 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -82,8 +82,8 @@ def add_host_to_client( config = RatholeConfig( uuid=peer_id, secret_token=rathole_token, - local_addr_host="localhost", - local_addr_port=random_port, + local_addr_host="proxy", + local_addr_port=8001, server_name=peer_name, ) From f67774c31315b5a9e6d1222d33d688601fd29d0b Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 22 May 2024 10:42:48 +0530 Subject: [PATCH 044/100] fix proxy port set to the client toml --- packages/syft/src/syft/service/network/rathole_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 9be55c25fcf..c9287535a0a 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -83,7 +83,7 @@ def add_host_to_client( uuid=peer_id, secret_token=rathole_token, local_addr_host="proxy", - local_addr_port=8001, + local_addr_port=80, server_name=peer_name, ) From fd60a9b889cbf17ae9a68dcd8263f89fdf573ac1 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Wed, 22 May 2024 18:33:19 +1000 Subject: [PATCH 045/100] Added build step for rathole for arm64 platforms --- packages/grid/rathole/rathole.dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 5f3a8762677..7166afd63dd 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -1,13 +1,20 @@ ARG RATHOLE_VERSION="0.5.0" ARG PYTHON_VERSION="3.12" -FROM rapiz1/rathole:v${RATHOLE_VERSION} as build +FROM rust as build +ARG RATHOLE_VERSION +ARG FEATURES +RUN apt update && apt install -y git +RUN git clone -b v${RATHOLE_VERSION} https://github.com/rapiz1/rathole + +WORKDIR /rathole +RUN cargo build --locked --release --features ${FEATURES:-default} FROM python:${PYTHON_VERSION}-bookworm ARG RATHOLE_VERSION ENV MODE="client" RUN apt update && apt install -y netcat-openbsd vim -COPY --from=build /app/rathole /app/rathole +COPY --from=build /rathole/target/release/rathole /app/rathole WORKDIR /app COPY ./start.sh /app/start.sh From 72e7daa10a3a187ccf75b4ccf8b381906b5911b4 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 22 May 2024 21:42:08 +0530 Subject: [PATCH 046/100] start traefik in watch mode with watch on directory instead of a single file - replace localhost with 0.0.0.0 during local addr bind in rathole client.toml - fix allow backend to access and perform crud on resource Services --- .../syft/templates/backend/backend-service-account.yaml | 2 +- .../grid/helm/syft/templates/proxy/proxy-configmap.yaml | 4 +++- packages/syft/src/syft/service/network/rathole_service.py | 8 +------- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml index 7b542adfc0b..76d70afee70 100644 --- a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml @@ -26,7 +26,7 @@ metadata: app.kubernetes.io/component: backend rules: - apiGroups: [""] - resources: ["pods", "configmaps", "secrets", "service"] + resources: ["pods", "configmaps", "secrets", "services"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] - apiGroups: [""] resources: ["pods/log"] diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index 9a3cacf23b2..c7628bc2ec4 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -80,7 +80,9 @@ data: providers: file: - filename: /etc/traefik/dynamic.yml + directory: /etc/traefik/ + watch: true + --- apiVersion: v1 diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index c9287535a0a..a514d499d98 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -37,7 +37,7 @@ def add_host_to_server(self, peer: NodePeer) -> None: config = RatholeConfig( uuid=peer.id.to_string(), secret_token=peer.rathole_token, - local_addr_host="localhost", + local_addr_host="0.0.0.0", local_addr_port=random_port, server_name=peer.name, ) @@ -77,8 +77,6 @@ def add_host_to_client( ) -> None: """Add a host to the rathole client toml file.""" - random_port = self.get_random_port() - config = RatholeConfig( uuid=peer_id, secret_token=rathole_token, @@ -107,10 +105,6 @@ def add_host_to_client( # Update the rathole config map KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) - self.add_entrypoint(port=random_port, peer_name=peer_name) - - self.forward_port_to_proxy(config=config, entrypoint=peer_name) - def forward_port_to_proxy( self, config: RatholeConfig, entrypoint: str = "web" ) -> None: From 59ecc5d108d9ffef06ae8db42e557badf3ddd8c7 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 23 May 2024 17:25:38 +0530 Subject: [PATCH 047/100] update method to expose port on rathole service --- packages/syft/setup.cfg | 1 - packages/syft/src/syft/custom_worker/k8s.py | 6 ++ .../syft/service/network/rathole_service.py | 65 +++++++++++++------ 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 6f69310c34e..a64a91fc049 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -91,7 +91,6 @@ data_science = opendp==0.9.2 evaluate==0.4.1 recordlinkage==0.16 - dm-haiku==0.0.10 torch[cpu]==2.2.1 dev = diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index 60557c86afb..d9702e72f3f 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -12,6 +12,7 @@ from kr8s.objects import ConfigMap from kr8s.objects import Pod from kr8s.objects import Secret +from kr8s.objects import Service from pydantic import BaseModel from typing_extensions import Self @@ -177,6 +178,11 @@ def get_configmap(client: kr8s.Api, name: str) -> ConfigMap | None: config_map = client.get("configmaps", name) return config_map[0] if config_map else None + @staticmethod + def get_service(client: kr8s.Api, name: str) -> Service | None: + service = client.get("services", name) + return service[0] if service else None + @staticmethod def update_configmap( config_map: ConfigMap, diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index a514d499d98..0894544de44 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -1,12 +1,15 @@ # stdlib import secrets +from typing import cast # third party +from kr8s.objects import Service import yaml # relative from ...custom_worker.k8s import KubeUtils from ...custom_worker.k8s import get_kr8s_client +from ...types.uid import UID from .node_peer import NodePeer from .rathole import RatholeConfig from .rathole import get_rathole_port @@ -34,8 +37,10 @@ def add_host_to_server(self, peer: NodePeer) -> None: random_port = self.get_random_port() + peer_id = cast(UID, peer.id) + config = RatholeConfig( - uuid=peer.id.to_string(), + uuid=peer_id.to_string(), secret_token=peer.rathole_token, local_addr_host="0.0.0.0", local_addr_port=random_port, @@ -47,6 +52,9 @@ def add_host_to_server(self, peer: NodePeer) -> None: client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP ) + if rathole_config_map is None: + raise Exception("Rathole config map not found.") + client_filename = RatholeServerToml.filename toml_str = rathole_config_map.data[client_filename] @@ -90,6 +98,9 @@ def add_host_to_client( client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP ) + if rathole_config_map is None: + raise Exception("Rathole config map not found.") + client_filename = RatholeClientToml.filename toml_str = rathole_config_map.data[client_filename] @@ -114,6 +125,9 @@ def forward_port_to_proxy( self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP ) + if rathole_proxy_config_map is None: + raise Exception("Rathole proxy config map not found.") + rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] if not rathole_proxy: @@ -145,6 +159,9 @@ def add_dynamic_addr_to_rathole( self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP ) + if rathole_proxy_config_map is None: + raise Exception("Rathole proxy config map not found.") + rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] if not rathole_proxy: @@ -169,36 +186,42 @@ def add_dynamic_addr_to_rathole( patch={"data": {"rathole-dynamic.yml": yaml.safe_dump(rathole_proxy)}}, ) - def add_entrypoint(self, port: int, peer_name: str) -> None: - """Add an entrypoint to the traefik config map.""" + self.expose_port_on_rathole_service(config.server_name, config.local_addr_port) - proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) + def expose_port_on_rathole_service(self, port_name: str, port: int) -> None: + """Expose a port on the rathole service.""" - data = proxy_config_map.data + rathole_service = KubeUtils.get_service(self.k8rs_client, "rathole") - traefik_config_str = data["traefik.yml"] + rathole_service = cast(Service, rathole_service) - traefik_config = yaml.safe_load(traefik_config_str) + config = rathole_service.raw - traefik_config["entryPoints"][f"{peer_name}"] = {"address": f":{port}"} - - data["traefik.yml"] = yaml.safe_dump(traefik_config) - - KubeUtils.update_configmap(config_map=proxy_config_map, patch={"data": data}) + config["spec"]["ports"].append( + { + "name": port_name, + "port": port, + "targetPort": port, + "protocol": "TCP", + } + ) - def remove_endpoint(self, peer_name: str) -> None: - """Remove an entrypoint from the traefik config map.""" + rathole_service.patch(config) - proxy_config_map = KubeUtils.get_configmap(self.k8rs_client, PROXY_CONFIG_MAP) + def remove_port_on_rathole_service(self, port_name: str) -> None: + """Remove a port from the rathole service.""" - data = proxy_config_map.data + rathole_service = KubeUtils.get_service(self.k8rs_client, "rathole") - traefik_config_str = data["traefik.yml"] + rathole_service = cast(Service, rathole_service) - traefik_config = yaml.safe_load(traefik_config_str) + config = rathole_service.raw - del traefik_config["entryPoints"][f"{peer_name}"] + ports = config["spec"]["ports"] - data["traefik.yml"] = yaml.safe_dump(traefik_config) + for port in ports: + if port["name"] == port_name: + ports.remove(port) + break - KubeUtils.update_configmap(config_map=proxy_config_map, patch={"data": data}) + rathole_service.patch(config) From 7c540961a09ed5dbae5573270d3f735c6434b6fa Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 23 May 2024 22:25:43 +0530 Subject: [PATCH 048/100] fix proxy rule for dynamically added router rules for rathole - set RUST_LOG level to trace for debugging --- packages/grid/rathole/rathole.dockerfile | 2 +- packages/grid/rathole/start.sh | 4 ++-- packages/syft/src/syft/service/network/rathole_service.py | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 7166afd63dd..42d147527c7 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -13,7 +13,7 @@ RUN cargo build --locked --release --features ${FEATURES:-default} FROM python:${PYTHON_VERSION}-bookworm ARG RATHOLE_VERSION ENV MODE="client" -RUN apt update && apt install -y netcat-openbsd vim +RUN apt update && apt install -y netcat-openbsd vim rsync COPY --from=build /rathole/target/release/rathole /app/rathole WORKDIR /app diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh index 4095f30f3aa..0e708908836 100755 --- a/packages/grid/rathole/start.sh +++ b/packages/grid/rathole/start.sh @@ -4,10 +4,10 @@ MODE=${MODE:-server} cp -L -r -f /conf/* conf/ if [[ $MODE == "server" ]]; then - /app/rathole conf/server.toml & + RUST_LOG=trace /app/rathole conf/server.toml & elif [[ $MODE = "client" ]]; then while true; do - /app/rathole conf/client.toml + RUST_LOG=trace /app/rathole conf/client.toml status=$? if [ $status -eq 0 ]; then break diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 0894544de44..2051035dd6a 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -175,8 +175,13 @@ def add_dynamic_addr_to_rathole( } } + proxy_rule = ( + f"Host(`{config.server_name}.syft.local`) || " + f"HostHeader(`{config.server_name}.syft.local`) && PathPrefix(`/`)" + ) + rathole_proxy["http"]["routers"][config.server_name] = { - "rule": f"Host(`{config.server_name}.syft.local`)", + "rule": proxy_rule, "service": config.server_name, "entryPoints": [entrypoint], } From 57b35b0b0f3d7741c32684d402330ada9218f63d Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 27 May 2024 16:09:30 +0530 Subject: [PATCH 049/100] fix lint --- packages/syft/src/syft/service/network/network_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 447713a6cff..e3a1c827a67 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -287,7 +287,7 @@ def exchange_credentials_with( self.rathole_service.add_host_to_client( peer_name=self_node_peer.name, - peer_id=self_node_peer.id.to_string(), + peer_id=str(self_node_peer.id), rathole_token=self_node_peer.rathole_token, remote_addr=remote_addr, ) From 1fbd8beb9f0c64adefc4a16f123ee395c47f780f Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 28 May 2024 18:30:06 +0530 Subject: [PATCH 050/100] move rathole token to http routes - update get and post to patch host headers if rathole token is present - fix stream endpoint to work with rathole --- packages/syft/src/syft/client/client.py | 66 +++++++++++++++---- packages/syft/src/syft/node/routes.py | 18 ++--- .../service/network/association_request.py | 4 +- .../syft/service/network/network_service.py | 37 +++++------ .../src/syft/service/network/node_peer.py | 1 - .../syft/service/network/rathole_service.py | 4 +- .../syft/src/syft/service/network/routes.py | 2 + packages/syft/src/syft/util/util.py | 5 ++ 8 files changed, 90 insertions(+), 47 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index ba4dfc38c80..858822771dc 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -4,6 +4,7 @@ # stdlib import base64 from collections.abc import Callable +from collections.abc import Iterator from copy import deepcopy from enum import Enum from getpass import getpass @@ -48,9 +49,11 @@ from ..service.user.user_service import UserService from ..types.grid_url import GridURL from ..types.syft_object import SYFT_OBJECT_VERSION_2 +from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.uid import UID from ..util.logger import debug from ..util.telemetry import instrument +from ..util.util import generate_token from ..util.util import prompt_warning_message from ..util.util import thread_ident from ..util.util import verify_tls @@ -68,11 +71,6 @@ from ..service.network.node_peer import NodePeer -# use to enable mitm proxy -# from syft.grid.connections.http_connection import HTTPConnection -# HTTPConnection.proxies = {"http": "http://127.0.0.1:8080"} - - def upgrade_tls(url: GridURL, response: Response) -> GridURL: try: if response.url.startswith("https://") and url.protocol == "http": @@ -117,6 +115,7 @@ def forward_message_to_proxy( API_PATH = "/api/v2" DEFAULT_PYGRID_PORT = 80 DEFAULT_PYGRID_ADDRESS = f"http://localhost:{DEFAULT_PYGRID_PORT}" +INTERNAL_PROXY_URL = "http://proxy:80" class Routes(Enum): @@ -129,15 +128,16 @@ class Routes(Enum): STREAM = f"{API_PATH}/stream" -@serializable(attrs=["proxy_target_uid", "url"]) +@serializable(attrs=["proxy_target_uid", "url", "rathole_token"]) class HTTPConnection(NodeConnection): __canonical_name__ = "HTTPConnection" - __version__ = SYFT_OBJECT_VERSION_2 + __version__ = SYFT_OBJECT_VERSION_3 url: GridURL proxy_target_uid: UID | None = None routes: type[Routes] = Routes session_cache: Session | None = None + rathole_token: str | None = None @field_validator("url", mode="before") @classmethod @@ -149,7 +149,11 @@ def make_url(cls, v: Any) -> Any: ) def with_proxy(self, proxy_target_uid: UID) -> Self: - return HTTPConnection(url=self.url, proxy_target_uid=proxy_target_uid) + return HTTPConnection( + url=self.url, + proxy_target_uid=proxy_target_uid, + rathole_token=self.rathole_token, + ) def stream_via(self, proxy_uid: UID, url_path: str) -> GridURL: # Update the presigned url path to @@ -182,10 +186,24 @@ def session(self) -> Session: self.session_cache = session return self.session_cache - def _make_get(self, path: str, params: dict | None = None) -> bytes: - url = self.url.with_path(path) + def _make_get( + self, path: str, params: dict | None = None, stream: bool = False + ) -> bytes | Iterator[Any]: + headers = {} + url = self.url + + if self.rathole_token: + url = GridURL.from_url(INTERNAL_PROXY_URL) + headers = {"Host": self.host_or_ip} + + url = url.with_path(path) response = self.session.get( - str(url), verify=verify_tls(), proxies={}, params=params + str(url), + verify=verify_tls(), + proxies={}, + params=params, + headers=headers, + stream=stream, ) if response.status_code != 200: raise requests.ConnectionError( @@ -195,6 +213,9 @@ def _make_get(self, path: str, params: dict | None = None) -> bytes: # upgrade to tls if available self.url = upgrade_tls(self.url, response) + if stream: + return response.iter_content(chunk_size=None) + return response.content def _make_post( @@ -203,9 +224,21 @@ def _make_post( json: dict[str, Any] | None = None, data: bytes | None = None, ) -> bytes: - url = self.url.with_path(path) + headers = {} + url = self.url + + if self.rathole_token: + url = GridURL.from_url(INTERNAL_PROXY_URL) + headers = {"Host": self.host_or_ip} + + url = url.with_path(path) response = self.session.post( - str(url), verify=verify_tls(), json=json, proxies={}, data=data + str(url), + verify=verify_tls(), + json=json, + proxies={}, + data=data, + headers=headers, ) if response.status_code != 200: raise requests.ConnectionError( @@ -683,7 +716,10 @@ def guest(self) -> Self: ) def exchange_route( - self, client: Self, protocol: SyftProtocol = SyftProtocol.HTTP + self, + client: Self, + protocol: SyftProtocol = SyftProtocol.HTTP, + reverse_tunnel: bool = False, ) -> SyftSuccess | SyftError: # relative from ..service.network.routes import connection_to_route @@ -694,6 +730,8 @@ def exchange_route( if client.metadata is None: return SyftError(f"client {client}'s metadata is None!") + self_node_route.rathole_token = generate_token() if reverse_tunnel else None + return self.api.services.network.exchange_credentials_with( self_node_route=self_node_route, remote_node_route=remote_node_route, diff --git a/packages/syft/src/syft/node/routes.py b/packages/syft/src/syft/node/routes.py index 5b25774ff18..f32aa5c3bd9 100644 --- a/packages/syft/src/syft/node/routes.py +++ b/packages/syft/src/syft/node/routes.py @@ -18,6 +18,7 @@ # relative from ..abstract_node import AbstractNode +from ..client.connection import NodeConnection from ..protocol.data_protocol import PROTOCOL_TYPE from ..serde.deserialize import _deserialize as deserialize from ..serde.serialize import _serialize as serialize @@ -50,7 +51,7 @@ def make_routes(worker: Worker) -> APIRouter: async def get_body(request: Request) -> bytes: return await request.body() - def _blob_url(peer_uid: UID, presigned_url: str) -> str: + def _get_node_connection(peer_uid: UID) -> NodeConnection: # relative from ..service.network.node_peer import route_to_connection @@ -58,9 +59,7 @@ def _blob_url(peer_uid: UID, presigned_url: str) -> str: peer = network_service.stash.get_by_uid(worker.verify_key, peer_uid).ok() peer_node_route = peer.pick_highest_priority_route() connection = route_to_connection(route=peer_node_route) - url = connection.to_blob_route(presigned_url) - - return str(url) + return connection @router.get("/stream/{peer_uid}/{url_path}/", name="stream") async def stream(peer_uid: str, url_path: str) -> StreamingResponse: @@ -71,17 +70,14 @@ async def stream(peer_uid: str, url_path: str) -> StreamingResponse: peer_uid_parsed = UID.from_string(peer_uid) - url = _blob_url(peer_uid=peer_uid_parsed, presigned_url=url_path_parsed) - try: - resp = requests.get(url=url, stream=True) # nosec - resp.raise_for_status() + peer_connection = _get_node_connection(peer_uid_parsed) + url = peer_connection.to_blob_route(url_path_parsed) + stream_response = peer_connection._make_get(url.path, stream=True) except requests.RequestException: raise HTTPException(404, "Failed to retrieve data from domain.") - return StreamingResponse( - resp.iter_content(chunk_size=None), media_type="text/event-stream" - ) + return StreamingResponse(stream_response, media_type="text/event-stream") @router.get( "/", diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index e963aec638e..ede16865685 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -44,7 +44,9 @@ def _run( service_ctx = context.to_service_ctx() - if self.remote_peer.rathole_token is None: + highest_route = self.remote_peer.pick_highest_priority_route() + + if highest_route.rathole_token is None: try: remote_client: SyftClient = self.remote_peer.client_with_context( context=service_ctx diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index e3a1c827a67..ec5d1c361b5 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -1,7 +1,6 @@ # stdlib from collections.abc import Callable from enum import Enum -from hashlib import sha256 import secrets from typing import Any @@ -186,9 +185,6 @@ def exchange_credentials_with( ) remote_node_peer = NodePeer.from_client(remote_client) - rathole_token = self._generate_token() - self_node_peer.rathole_token = rathole_token - # ask the remote client to add this node (represented by `self_node_peer`) as a peer # check locally if the remote node already exists as a peer existing_peer_result = self.stash.get_by_uid( @@ -278,19 +274,20 @@ def exchange_credentials_with( if result.is_err(): return SyftError(message="Failed to update route information.") - remote_url = GridURL( - host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port - ) - rathole_remote_addr = remote_url.as_container_host() + if self_node_peer.rathole_token: + remote_url = GridURL( + host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port + ) + rathole_remote_addr = remote_url.as_container_host() - remote_addr = rathole_remote_addr.url_no_protocol + remote_addr = rathole_remote_addr.url_no_protocol - self.rathole_service.add_host_to_client( - peer_name=self_node_peer.name, - peer_id=str(self_node_peer.id), - rathole_token=self_node_peer.rathole_token, - remote_addr=remote_addr, - ) + self.rathole_service.add_host_to_client( + peer_name=self_node_peer.name, + peer_id=str(self_node_peer.id), + rathole_token=self_node_peer.rathole_token, + remote_addr=remote_addr, + ) return ( SyftSuccess(message="Routes Exchanged") @@ -298,9 +295,6 @@ def exchange_credentials_with( else remote_res ) - def _generate_token(self) -> str: - return sha256(secrets.token_bytes(16)).hexdigest() - @service_method(path="network.add_peer", name="add_peer", roles=GUEST_ROLE_LEVEL) def add_peer( self, @@ -940,6 +934,7 @@ def from_grid_url(context: TransformContext) -> TransformContext: context.output["private"] = False context.output["proxy_target_uid"] = context.obj.proxy_target_uid context.output["priority"] = 1 + context.output["rathole_token"] = context.obj.rathole_token return context @@ -976,7 +971,11 @@ def node_route_to_http_connection( url = GridURL( protocol=obj.protocol, host_or_ip=obj.host_or_ip, port=obj.port ).as_container_host() - return HTTPConnection(url=url, proxy_target_uid=obj.proxy_target_uid) + return HTTPConnection( + url=url, + proxy_target_uid=obj.proxy_target_uid, + rathole_token=obj.rathole_token, + ) @transform(NodeMetadataV3, NodePeer) diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index a6cb6d3eea3..35292dd89dd 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -85,7 +85,6 @@ class NodePeer(SyftObject): ping_status: NodePeerConnectionStatus | None = None ping_status_message: str | None = None pinged_timestamp: DateTime | None = None - rathole_token: str | None = None def existed_route( self, route: NodeRouteType | None = None, route_id: UID | None = None diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 2051035dd6a..67d4f78eddd 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -35,13 +35,15 @@ def add_host_to_server(self, peer: NodePeer) -> None: None """ + rathole_route = peer.pick_highest_priority_route() + random_port = self.get_random_port() peer_id = cast(UID, peer.id) config = RatholeConfig( uuid=peer_id.to_string(), - secret_token=peer.rathole_token, + secret_token=rathole_route.rathole_token, local_addr_host="0.0.0.0", local_addr_port=random_port, server_name=peer.name, diff --git a/packages/syft/src/syft/service/network/routes.py b/packages/syft/src/syft/service/network/routes.py index f3fa9b1ad1a..9e49f82c689 100644 --- a/packages/syft/src/syft/service/network/routes.py +++ b/packages/syft/src/syft/service/network/routes.py @@ -95,6 +95,7 @@ class HTTPNodeRoute(SyftObject, NodeRoute): port: int = 80 proxy_target_uid: UID | None = None priority: int = 1 + rathole_token: str | None = None def __eq__(self, other: Any) -> bool: if not isinstance(other, HTTPNodeRoute): @@ -107,6 +108,7 @@ def __hash__(self) -> int: + hash(self.port) + hash(self.protocol) + hash(self.proxy_target_uid) + + hash(self.rathole_token) ) def __str__(self) -> str: diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index b0affa2b1a0..a93c2ba8fe1 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -21,6 +21,7 @@ import platform import random import re +import secrets from secrets import randbelow import socket import sys @@ -919,3 +920,7 @@ def get_queue_address(port: int) -> str: def get_dev_mode() -> bool: return str_to_bool(os.getenv("DEV_MODE", "False")) + + +def generate_token() -> str: + return hashlib.sha256(secrets.token_bytes(16)).hexdigest() From 9c7983cbcacfb4352d0fe0f479825079c2d00b74 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 29 May 2024 00:49:25 +0530 Subject: [PATCH 051/100] fix passing host name in case of rathole connection - pass reverse tunnel via connect_to_gateway - add method to get rathole route --- packages/syft/src/syft/client/client.py | 21 +++++++++++----- .../syft/src/syft/client/domain_client.py | 7 +++++- .../src/syft/protocol/protocol_version.json | 24 +++++++++++++++++++ .../service/network/association_request.py | 4 ++-- .../syft/service/network/network_service.py | 12 ++++++++-- .../src/syft/service/network/node_peer.py | 6 +++++ .../syft/service/network/rathole_service.py | 2 +- .../syft/src/syft/service/network/routes.py | 3 ++- 8 files changed, 66 insertions(+), 13 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 858822771dc..28e975efa03 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -53,7 +53,6 @@ from ..types.uid import UID from ..util.logger import debug from ..util.telemetry import instrument -from ..util.util import generate_token from ..util.util import prompt_warning_message from ..util.util import thread_ident from ..util.util import verify_tls @@ -194,7 +193,7 @@ def _make_get( if self.rathole_token: url = GridURL.from_url(INTERNAL_PROXY_URL) - headers = {"Host": self.host_or_ip} + headers = {"Host": self.url.host_or_ip} url = url.with_path(path) response = self.session.get( @@ -229,7 +228,7 @@ def _make_post( if self.rathole_token: url = GridURL.from_url(INTERNAL_PROXY_URL) - headers = {"Host": self.host_or_ip} + headers = {"Host": self.url.host_or_ip} url = url.with_path(path) response = self.session.post( @@ -336,9 +335,20 @@ def register(self, new_user: UserCreate) -> SyftSigningKey: def make_call(self, signed_call: SignedSyftAPICall) -> Any | SyftError: msg_bytes: bytes = _serialize(obj=signed_call, to_bytes=True) + + headers = {} + + if self.rathole_token: + api_url = GridURL.from_url(INTERNAL_PROXY_URL) + api_url = api_url.with_path(self.routes.ROUTE_API_CALL.value) + headers = {"Host": self.url.host_or_ip} + else: + api_url = self.api_url + response = requests.post( # nosec - url=str(self.api_url), + url=api_url, data=msg_bytes, + headers=headers, ) if response.status_code != 200: @@ -730,12 +740,11 @@ def exchange_route( if client.metadata is None: return SyftError(f"client {client}'s metadata is None!") - self_node_route.rathole_token = generate_token() if reverse_tunnel else None - return self.api.services.network.exchange_credentials_with( self_node_route=self_node_route, remote_node_route=remote_node_route, remote_node_verify_key=client.metadata.to(NodeMetadataV3).verify_key, + reverse_tunnel=reverse_tunnel, ) else: raise ValueError( diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index 8f1e7cb9dc8..6428fd7b851 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -288,6 +288,7 @@ def connect_to_gateway( email: str | None = None, password: str | None = None, protocol: str | SyftProtocol = SyftProtocol.HTTP, + reverse_tunnel: bool = False, ) -> SyftSuccess | SyftError | None: if isinstance(protocol, str): protocol = SyftProtocol(protocol) @@ -305,7 +306,11 @@ def connect_to_gateway( if isinstance(client, SyftError): return client - res = self.exchange_route(client, protocol=protocol) + res = self.exchange_route( + client, + protocol=protocol, + reverse_tunnel=reverse_tunnel, + ) if isinstance(res, SyftSuccess): if self.metadata: return SyftSuccess( diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index b4050ab030f..04e7b652bcb 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -198,6 +198,30 @@ "hash": "e5f099940a7623f145f51f3e15b97a910a1d7fda1f67739420fed3035d1f2995", "action": "add" } + }, + "HTTPConnection": { + "2": { + "version": 2, + "hash": "68409295f8916ceb22a8cf4abf89f5e4bcff0d75dc37e16ede37250ada28df59", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "5e363abe2875beec89a3f4f4f5c53e15f9893fb98e5da71e2fa6c0f619883b1f", + "action": "add" + } + }, + "HTTPNodeRoute": { + "2": { + "version": 2, + "hash": "2134ea812f7c6ea41522727ae087245c4b1195ffbad554db638070861cd9eb1c", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "89ace8067c392b802fe23a99446a8ae464a9dad0b49d8b2c3871b631451acec4", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index ede16865685..2590b3b42fe 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -44,9 +44,9 @@ def _run( service_ctx = context.to_service_ctx() - highest_route = self.remote_peer.pick_highest_priority_route() + rathole_route = self.remote_peer.get_rathole_route() - if highest_route.rathole_token is None: + if rathole_route.rathole_token is None: try: remote_client: SyftClient = self.remote_peer.client_with_context( context=service_ctx diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index ec5d1c361b5..de560747ac6 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -30,6 +30,7 @@ from ...types.transforms import transform_method from ...types.uid import UID from ...util.telemetry import instrument +from ...util.util import generate_token from ...util.util import prompt_warning_message from ..context import AuthedServiceContext from ..data_subject.data_subject import NamePartitionKey @@ -166,6 +167,7 @@ def exchange_credentials_with( self_node_route: NodeRoute, remote_node_route: NodeRoute, remote_node_verify_key: SyftVerifyKey, + reverse_tunnel: bool = False, ) -> Request | SyftSuccess | SyftError: """ Exchange Route With Another Node. If there is a pending association request, return it @@ -174,6 +176,11 @@ def exchange_credentials_with( # Step 1: Validate the Route self_node_peer = self_node_route.validate_with_context(context=context) + if reverse_tunnel: + _rathole_route = self_node_peer.node_routes[-1] + _rathole_route.rathole_token = generate_token() + _rathole_route.host_or_ip = f"{self_node_peer.name}.syft.local" + if isinstance(self_node_peer, SyftError): return self_node_peer @@ -274,7 +281,8 @@ def exchange_credentials_with( if result.is_err(): return SyftError(message="Failed to update route information.") - if self_node_peer.rathole_token: + if reverse_tunnel: + rathole_route = self_node_peer.get_rathole_route() remote_url = GridURL( host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port ) @@ -285,7 +293,7 @@ def exchange_credentials_with( self.rathole_service.add_host_to_client( peer_name=self_node_peer.name, peer_id=str(self_node_peer.id), - rathole_token=self_node_peer.rathole_token, + rathole_token=rathole_route.rathole_token, remote_addr=remote_addr, ) diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 35292dd89dd..e6ac045e9f1 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -269,6 +269,12 @@ def pick_highest_priority_route(self) -> NodeRoute: highest_priority_route = route return highest_priority_route + def get_rathole_route(self) -> NodeRoute | None: + for route in self.node_routes: + if hasattr(route, "rathole_token") and route.rathole_token: + return route + return None + def delete_route( self, route: NodeRouteType | None = None, route_id: UID | None = None ) -> SyftError | None: diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 67d4f78eddd..ad5e783c7dd 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -35,7 +35,7 @@ def add_host_to_server(self, peer: NodePeer) -> None: None """ - rathole_route = peer.pick_highest_priority_route() + rathole_route = peer.get_rathole_route() random_port = self.get_random_port() diff --git a/packages/syft/src/syft/service/network/routes.py b/packages/syft/src/syft/service/network/routes.py index 9e49f82c689..1d9ec116467 100644 --- a/packages/syft/src/syft/service/network/routes.py +++ b/packages/syft/src/syft/service/network/routes.py @@ -19,6 +19,7 @@ from ...serde.serializable import serializable from ...types.syft_object import SYFT_OBJECT_VERSION_1 from ...types.syft_object import SYFT_OBJECT_VERSION_2 +from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SyftObject from ...types.transforms import TransformContext from ...types.uid import UID @@ -87,7 +88,7 @@ def validate_with_context( @serializable() class HTTPNodeRoute(SyftObject, NodeRoute): __canonical_name__ = "HTTPNodeRoute" - __version__ = SYFT_OBJECT_VERSION_2 + __version__ = SYFT_OBJECT_VERSION_3 host_or_ip: str private: bool = False From 2b7172e20a9025bdf240feb02bb0a54e03d1698d Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Sun, 26 May 2024 22:35:23 +0200 Subject: [PATCH 052/100] add widget --- .../syft/assets/css/tabulator_pysyft.min.css | 1655 ++++++++++++++++- .../syft/src/syft/assets/jinja/table.jinja2 | 6 +- packages/syft/src/syft/assets/js/table.js | 76 +- .../syft/src/syft/service/sync/diff_state.py | 7 + .../src/syft/service/sync/resolve_widget.py | 160 ++ .../components/tabulator_template.py | 21 +- 6 files changed, 1913 insertions(+), 12 deletions(-) diff --git a/packages/syft/src/syft/assets/css/tabulator_pysyft.min.css b/packages/syft/src/syft/assets/css/tabulator_pysyft.min.css index f474df40562..fde1ee7edc8 100644 --- a/packages/syft/src/syft/assets/css/tabulator_pysyft.min.css +++ b/packages/syft/src/syft/assets/css/tabulator_pysyft.min.css @@ -1,6 +1,1651 @@ -:root{--tabulator-background-color:#fff;--tabulator-border-color:rgba(0,0,0,.12);--tabulator-text-size:16px;--tabulator-header-background-color:#f5f5f5;--tabulator-header-text-color:#555;--tabulator-header-border-color:rgba(0,0,0,.12);--tabulator-header-separator-color:rgba(0,0,0,.12);--tabulator-header-margin:4px;--tabulator-sort-arrow-hover:#555;--tabulator-sort-arrow-active:#666;--tabulator-sort-arrow-inactive:#bbb;--tabulator-column-resize-guide-color:#999;--tabulator-row-background-color:#fff;--tabulator-row-alt-background-color:#f8f8f8;--tabulator-row-border-color:rgba(0,0,0,.12);--tabulator-row-text-color:#333;--tabulator-row-hover-background:#e1f5fe;--tabulator-row-selected-background:#17161d;--tabulator-row-selected-background-hover:#17161d;--tabulator-edit-box-color:#17161d;--tabulator-error-color:#d00;--tabulator-footer-background-color:transparent;--tabulator-footer-text-color:#555;--tabulator-footer-border-color:rgba(0,0,0,.12);--tabulator-footer-separator-color:rgba(0,0,0,.12);--tabulator-footer-active-color:#17161d;--tabulator-spreadsheet-active-tab-color:#fff;--tabulator-range-border-color:#17161d;--tabulator-range-handle-color:#17161d;--tabulator-range-header-selected-background:var( - --tabulator-range-border-color - );--tabulator-range-header-selected-text-color:#fff;--tabulator-range-header-highlight-background:colors-gray-timberwolf;--tabulator-range-header-text-highlight-background:#fff;--tabulator-pagination-button-background:#fff;--tabulator-pagination-button-background-hover:#06c;--tabulator-pagination-button-color:#999;--tabulator-pagination-button-color-hover:#fff;--tabulator-pagination-button-color-active:#000;--tabulator-cell-padding:15px}body.vscode-dark,body[data-jp-theme-light=false]{--tabulator-background-color:#080808;--tabulator-border-color:#666;--tabulator-text-size:16px;--tabulator-header-background-color:#212121;--tabulator-header-text-color:#555;--tabulator-header-border-color:#666;--tabulator-header-separator-color:#666;--tabulator-header-margin:4px;--tabulator-sort-arrow-hover:#fff;--tabulator-sort-arrow-active:#e6e6e6;--tabulator-sort-arrow-inactive:#666;--tabulator-column-resize-guide-color:#999;--tabulator-row-background-color:#080808;--tabulator-row-alt-background-color:#212121;--tabulator-row-border-color:#666;--tabulator-row-text-color:#f8f8f8;--tabulator-row-hover-background:#333;--tabulator-row-selected-background:#241e1e;--tabulator-row-selected-background-hover:#333;--tabulator-edit-box-color:#333;--tabulator-error-color:#d00;--tabulator-footer-background-color:transparent;--tabulator-footer-text-color:#555;--tabulator-footer-border-color:rgba(0,0,0,.12);--tabulator-footer-separator-color:rgba(0,0,0,.12);--tabulator-footer-active-color:#17161d;--tabulator-spreadsheet-active-tab-color:#fff;--tabulator-range-border-color:#17161d;--tabulator-range-handle-color:var(--tabulator-range-border-color);--tabulator-range-header-selected-background:var( - --tabulator-range-border-color - );--tabulator-range-header-selected-text-color:#fff;--tabulator-range-header-highlight-background:#d6d6d6;--tabulator-range-header-text-highlight-background:#fff;--tabulator-pagination-button-background:#212121;--tabulator-pagination-button-background-hover:#555;--tabulator-pagination-button-color:#999;--tabulator-pagination-button-color-hover:#fff;--tabulator-pagination-button-color-active:#fff;--tabulator-cell-padding:15px}.tabulator{border:1px solid var(--tabulator-border-color);font-size:var(--tabulator-text-size);overflow:hidden;position:relative;text-align:left;-webkit-transform:translateZ(0);-moz-transform:translateZ(0);-ms-transform:translateZ(0);-o-transform:translateZ(0);transform:translateZ(0)}.tabulator[tabulator-layout=fitDataFill] .tabulator-tableholder .tabulator-table{min-width:100%}.tabulator[tabulator-layout=fitDataTable]{display:inline-block}.tabulator.tabulator-block-select,.tabulator.tabulator-ranges .tabulator-cell:not(.tabulator-editing){user-select:none}.tabulator .tabulator-header{background-color:var(--tabulator-header-background-color);border-bottom:1px solid var(--tabulator-header-separator-color);box-sizing:border-box;color:var(--tabulator-header-text-color);font-weight:700;outline:none;overflow:hidden;position:relative;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none;white-space:nowrap;width:100%}.tabulator .tabulator-header.tabulator-header-hidden{display:none}.tabulator .tabulator-header .tabulator-header-contents{overflow:hidden;position:relative}.tabulator .tabulator-header .tabulator-header-contents .tabulator-headers{display:inline-block}.tabulator .tabulator-header .tabulator-col{background:var(--tabulator-header-background-color);border-right:1px solid var(--tabulator-header-border-color);box-sizing:border-box;display:inline-flex;flex-direction:column;justify-content:flex-start;overflow:hidden;position:relative;text-align:left;vertical-align:bottom}.tabulator .tabulator-header .tabulator-col.tabulator-moving{background:hsl(var(--tabulator-header-background-color),calc(var(--tabulator-header-background-color) - 5%))!important;border:1px solid var(--tabulator-header-separator-color);pointer-events:none;position:absolute}.tabulator .tabulator-header .tabulator-col.tabulator-range-highlight{background-color:var(--tabulator-range-header-highlight-background);color:var(--tabulator-range-header-text-highlight-background)}.tabulator .tabulator-header .tabulator-col.tabulator-range-selected{background-color:var(--tabulator-range-header-selected-background);color:var(--tabulator-range-header-selected-text-color)}.tabulator .tabulator-header .tabulator-col .tabulator-col-content{box-sizing:border-box;padding:4px;position:relative}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-header-popup-button{padding:0 8px}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-header-popup-button:hover{cursor:pointer;opacity:.6}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title-holder{position:relative}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title{box-sizing:border-box;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom;white-space:nowrap;width:100%}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title.tabulator-col-title-wrap{text-overflow:clip;white-space:normal}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title .tabulator-title-editor{background:#fff;border:1px solid #999;box-sizing:border-box;padding:1px;width:100%}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title .tabulator-header-popup-button+.tabulator-title-editor{width:calc(100% - 22px)}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter{align-items:center;bottom:0;display:flex;position:absolute;right:4px;top:0}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter .tabulator-arrow{border-bottom:6px solid var(--tabulator-sort-arrow-inactive);border-left:6px solid transparent;border-right:6px solid transparent;height:0;width:0}.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols{border-top:1px solid var(--tabulator-header-border-color);display:flex;margin-right:-1px;overflow:hidden;position:relative}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter{box-sizing:border-box;margin-top:2px;position:relative;text-align:center;width:100%}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter textarea{height:auto!important}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter svg{margin-top:3px}.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input::-ms-clear{height:0;width:0}.tabulator .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title{padding-right:25px}@media (hover:hover) and (pointer:fine){.tabulator .tabulator-header .tabulator-col.tabulator-sortable.tabulator-col-sorter-element:hover{background-color:hsl(var(--tabulator-header-background-color),calc(var(--tabulator-header-background-color) - 10%))!important;cursor:pointer}}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=none] .tabulator-col-content .tabulator-col-sorter{color:var(--tabulator-sort-arrow-inactive)}@media (hover:hover) and (pointer:fine){.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=none] .tabulator-col-content .tabulator-col-sorter.tabulator-col-sorter-element .tabulator-arrow:hover{border-bottom:6px solid var(--tabulator-sort-arrow-hover);cursor:pointer}}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=none] .tabulator-col-content .tabulator-col-sorter .tabulator-arrow{border-bottom:6px solid var(--tabulator-sort-arrow-inactive);border-top:none}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=ascending] .tabulator-col-content .tabulator-col-sorter{color:var(--tabulator-sort-arrow-active)}@media (hover:hover) and (pointer:fine){.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=ascending] .tabulator-col-content .tabulator-col-sorter.tabulator-col-sorter-element .tabulator-arrow:hover{border-bottom:6px solid var(--tabulator-sort-arrow-hover);cursor:pointer}}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=ascending] .tabulator-col-content .tabulator-col-sorter .tabulator-arrow{border-bottom:6px solid var(--tabulator-sort-arrow-active);border-top:none}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=descending] .tabulator-col-content .tabulator-col-sorter{color:var(--tabulator-sort-arrow-active)}@media (hover:hover) and (pointer:fine){.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=descending] .tabulator-col-content .tabulator-col-sorter.tabulator-col-sorter-element .tabulator-arrow:hover{border-top:6px solid var(--tabulator-sort-arrow-hover);cursor:pointer}}.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=descending] .tabulator-col-content .tabulator-col-sorter .tabulator-arrow{border-bottom:none;border-top:6px solid var(--tabulator-sort-arrow-active);color:var(--tabulator-sort-arrow-active)}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical .tabulator-col-content .tabulator-col-title{align-items:center;display:flex;justify-content:center;text-orientation:mixed;writing-mode:vertical-rl}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-col-vertical-flip .tabulator-col-title{transform:rotate(180deg)}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-col-title{padding-right:0;padding-top:20px}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable.tabulator-col-vertical-flip .tabulator-col-title{padding-bottom:20px;padding-right:0}.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-col-sorter{bottom:auto;justify-content:center;left:0;right:0;top:4px}.tabulator .tabulator-header .tabulator-frozen{left:0;position:sticky;z-index:11}.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-left{border-right:2px solid var(--tabulator-row-border-color)}.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-right{border-left:2px solid var(--tabulator-row-border-color)}.tabulator .tabulator-header .tabulator-calcs-holder{border-bottom:1px solid var(--tabulator-header-border-color);border-top:1px solid var(--tabulator-row-border-color);box-sizing:border-box;display:inline-block}.tabulator .tabulator-header .tabulator-calcs-holder,.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row{background:hsl(var(--tabulator-header-background-color),calc(var(--tabulator-header-background-color) + 5%))!important}.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle{display:none}.tabulator .tabulator-header .tabulator-frozen-rows-holder{display:inline-block}.tabulator .tabulator-tableholder{-webkit-overflow-scrolling:touch;overflow:auto;position:relative;white-space:nowrap;width:100%}.tabulator .tabulator-tableholder:focus{outline:none}.tabulator .tabulator-tableholder .tabulator-placeholder{align-items:center;box-sizing:border-box;display:flex;justify-content:center;min-width:100%;width:100%}.tabulator .tabulator-tableholder .tabulator-placeholder[tabulator-render-mode=virtual]{min-height:100%}.tabulator .tabulator-tableholder .tabulator-placeholder .tabulator-placeholder-contents{color:#ccc;display:inline-block;font-size:20px;font-weight:700;padding:10px;text-align:center;white-space:normal}.tabulator .tabulator-tableholder .tabulator-table{background-color:var(--tabulator-row-background-color);color:var(--tabulator-row-text-color);display:inline-block;overflow:visible;position:relative;white-space:nowrap}.tabulator .tabulator-tableholder .tabulator-table .tabulator-row.tabulator-calcs{background:hsl(var(--tabulator-row-atl-background-color),calc(var(--tabulator-row-alt-background-color) - 5%))!important;font-weight:700}.tabulator .tabulator-tableholder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-top{border-bottom:2px solid var(--tabulator-row-border-color)}.tabulator .tabulator-tableholder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-bottom{border-top:2px solid var(--tabulator-row-border-color)}.tabulator .tabulator-tableholder .tabulator-range-overlay{inset:0;pointer-events:none;position:absolute;z-index:10}.tabulator .tabulator-tableholder .tabulator-range-overlay .tabulator-range{border:1px solid var(--tabulator-range-border-color);box-sizing:border-box;position:absolute}.tabulator .tabulator-tableholder .tabulator-range-overlay .tabulator-range.tabulator-range-active:after{background-color:var(--tabulator-range-handle-color);border-radius:999px;bottom:-3px;content:"";height:6px;position:absolute;right:-3px;width:6px}.tabulator .tabulator-tableholder .tabulator-range-overlay .tabulator-range-cell-active{border:2px solid var(--tabulator-range-border-color);box-sizing:border-box;position:absolute}.tabulator .tabulator-footer{border-top:1px solid var(--tabulator-footer-separator-color);color:var(--tabulator-footer-text-color);font-weight:700;user-select:none;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none;white-space:nowrap}.tabulator .tabulator-footer .tabulator-footer-contents{align-items:center;display:flex;flex-direction:row;justify-content:space-between;padding:5px 10px}.tabulator .tabulator-footer .tabulator-footer-contents:empty{display:none}.tabulator .tabulator-footer .tabulator-spreadsheet-tabs{margin-top:-5px;overflow-x:auto}.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab{border:1px solid var(--tabulator-border-color);border-bottom-left-radius:5px;border-bottom-right-radius:5px;border-top:none;display:inline-block;font-size:.9em;padding:5px}.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab:hover{cursor:pointer;opacity:.7}.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab.tabulator-spreadsheet-tab-active{background:var(--tabulator-spreadsheet-active-tab-color)}.tabulator .tabulator-footer .tabulator-calcs-holder{border-bottom:1px solid var(--tabulator-row-border-color);border-top:1px solid var(--tabulator-row-border-color);box-sizing:border-box;overflow:hidden;text-align:left;width:100%}.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row{display:inline-block}.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle{display:none}.tabulator .tabulator-footer .tabulator-calcs-holder:only-child{border-bottom:none;margin-bottom:-5px}.tabulator .tabulator-footer>*+.tabulator-page-counter{margin-left:10px}.tabulator .tabulator-footer .tabulator-page-counter{font-weight:400}.tabulator .tabulator-footer .tabulator-paginator{color:var(--tabulator-footer-text-color);flex:1;font-family:inherit;font-size:inherit;font-weight:inherit;text-align:right}.tabulator .tabulator-footer .tabulator-page-size{border:1px solid var(--tabulator-footer-border-color);border-radius:3px;display:inline-block;margin:0 5px;padding:2px 5px}.tabulator .tabulator-footer .tabulator-pages{margin:0 7px}.tabulator .tabulator-footer .tabulator-page{background:hsla(0,0%,100%,.2);border:1px solid var(--tabulator-footer-border-color);border-radius:3px;display:inline-block;margin:0 2px;padding:2px 5px}.tabulator .tabulator-footer .tabulator-page.active{color:var(--tabulator-footer-active-color)}.tabulator .tabulator-footer .tabulator-page:disabled{opacity:.5}@media (hover:hover) and (pointer:fine){.tabulator .tabulator-footer .tabulator-page:not(disabled):hover{background:rgba(0,0,0,.2);color:#fff;cursor:pointer}}.tabulator .tabulator-col-resize-handle{display:inline-block;margin-left:-3px;margin-right:-3px;position:relative;vertical-align:middle;width:6px;z-index:11}@media (hover:hover) and (pointer:fine){.tabulator .tabulator-col-resize-handle:hover{cursor:ew-resize}}.tabulator .tabulator-col-resize-handle:last-of-type{margin-right:0;width:3px}.tabulator .tabulator-col-resize-guide{height:100%;margin-left:-.5px;top:0;width:4px}.tabulator .tabulator-col-resize-guide,.tabulator .tabulator-row-resize-guide{background-color:var(--tabulator-column-resize-guide-color);opacity:.5;position:absolute}.tabulator .tabulator-row-resize-guide{height:4px;left:0;margin-top:-.5px;width:100%}.tabulator .tabulator-alert{align-items:center;background:rgba(0,0,0,.4);display:flex;height:100%;left:0;position:absolute;text-align:center;top:0;width:100%;z-index:100}.tabulator .tabulator-alert .tabulator-alert-msg{background:#fff;border-radius:10px;display:inline-block;font-size:16px;font-weight:700;margin:0 auto;padding:10px 20px}.tabulator .tabulator-alert .tabulator-alert-msg.tabulator-alert-state-msg{border:4px solid #333;color:#000}.tabulator .tabulator-alert .tabulator-alert-msg.tabulator-alert-state-error{border:4px solid #d00;color:#590000}.tabulator-row{background-color:var(--tabulator-row-background-color);box-sizing:border-box;min-height:calc(var(--tabulator-text-size) + var(--tabulator-header-margin)*2);position:relative}.tabulator-row.tabulator-row-even{background-color:var(--tabulator-row-alt-background-color)}@media (hover:hover) and (pointer:fine){.tabulator-row.tabulator-selectable:hover{background-color:var(--tabulator-row-hover-background);cursor:pointer}}.tabulator-row.tabulator-selected{background-color:var(--tabulator-row-selected-background)}@media (hover:hover) and (pointer:fine){.tabulator-row.tabulator-selected:hover{background-color:var(--tabulator-row-selected-background-hover);cursor:pointer}}.tabulator-row.tabulator-row-moving{background:#fff;border:1px solid #000}.tabulator-row.tabulator-moving{border-bottom:1px solid var(--tabulator-row-border-color);border-top:1px solid var(--tabulator-row-border-color);pointer-events:none;position:absolute;z-index:15}.tabulator-row.tabulator-range-highlight .tabulator-cell.tabulator-range-row-header{background-color:var(--tabulator-range-header-highlight-background);color:var(--tabulator-range-header-text-highlight-background)}.tabulator-row.tabulator-range-highlight.tabulator-range-selected .tabulator-cell.tabulator-range-row-header,.tabulator-row.tabulator-range-selected .tabulator-cell.tabulator-range-row-header{background-color:var(--tabulator-range-header-selected-background);color:var(--tabulator-range-header-selected-text-color)}.tabulator-row .tabulator-row-resize-handle{bottom:0;height:5px;left:0;position:absolute;right:0}.tabulator-row .tabulator-row-resize-handle.prev{bottom:auto;top:0}@media (hover:hover) and (pointer:fine){.tabulator-row .tabulator-row-resize-handle:hover{cursor:ns-resize}}.tabulator-row .tabulator-responsive-collapse{border-bottom:1px solid var(--tabulator-row-border-color);border-top:1px solid var(--tabulator-row-border-color);box-sizing:border-box;padding:5px}.tabulator-row .tabulator-responsive-collapse:empty{display:none}.tabulator-row .tabulator-responsive-collapse table{font-size:var(--tabulator-text-size)}.tabulator-row .tabulator-responsive-collapse table tr td{position:relative}.tabulator-row .tabulator-responsive-collapse table tr td:first-of-type{padding-right:10px}.tabulator-row .tabulator-cell{border-right:1px solid var(--tabulator-row-border-color);box-sizing:border-box;display:inline-block;outline:none;overflow:hidden;padding:4px;position:relative;text-overflow:ellipsis;vertical-align:middle;white-space:nowrap}.tabulator-row .tabulator-cell.tabulator-row-header{border-bottom:1px solid var(--tabulator-row-border-color)}.tabulator-row .tabulator-cell.tabulator-frozen{background-color:inherit;display:inline-block;left:0;position:sticky;z-index:11}.tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-left{border-right:2px solid var(--tabulator-row-border-color)}.tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-right{border-left:2px solid var(--tabulator-row-border-color)}.tabulator-row .tabulator-cell.tabulator-editing{border:1px solid var(--tabulator-edit-box-color);outline:none;padding:0}.tabulator-row .tabulator-cell.tabulator-editing input,.tabulator-row .tabulator-cell.tabulator-editing select{background:transparent;border:1px;outline:none}.tabulator-row .tabulator-cell.tabulator-validation-fail{border:1px solid var(--tabulator-error-color)}.tabulator-row .tabulator-cell.tabulator-validation-fail input,.tabulator-row .tabulator-cell.tabulator-validation-fail select{background:transparent;border:1px;color:var(--tabulator-error-color)}.tabulator-row .tabulator-cell.tabulator-row-handle{align-items:center;display:inline-flex;justify-content:center;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none}.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box{width:80%}.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box .tabulator-row-handle-bar{background:#666;height:3px;margin-top:2px;width:100%}.tabulator-row .tabulator-cell.tabulator-range-selected:not(.tabulator-range-only-cell-selected):not(.tabulator-range-row-header){background-color:var(--tabulator-row-selected-background)}.tabulator-row .tabulator-cell .tabulator-data-tree-branch-empty{display:inline-block;width:7px}.tabulator-row .tabulator-cell .tabulator-data-tree-branch{border-bottom:2px solid var(--tabulator-row-border-color);border-bottom-left-radius:1px;border-left:2px solid var(--tabulator-row-border-color);display:inline-block;height:9px;margin-right:5px;margin-top:-9px;vertical-align:middle;width:7px}.tabulator-row .tabulator-cell .tabulator-data-tree-control{align-items:center;background:rgba(0,0,0,.1);border:1px solid var(--tabulator-row-text-color);border-radius:2px;display:inline-flex;height:11px;justify-content:center;margin-right:5px;overflow:hidden;vertical-align:middle;width:11px}@media (hover:hover) and (pointer:fine){.tabulator-row .tabulator-cell .tabulator-data-tree-control:hover{background:rgba(0,0,0,.2);cursor:pointer}}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse{background:transparent;display:inline-block;height:7px;position:relative;width:1px}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after{background:var(--tabulator-row-text-color);content:"";height:1px;left:-3px;position:absolute;top:3px;width:7px}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand{background:var(--tabulator-row-text-color);display:inline-block;height:7px;position:relative;width:1px}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand:after{background:var(--tabulator-row-text-color);content:"";height:1px;left:-3px;position:absolute;top:3px;width:7px}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle{align-items:center;background:#666;border-radius:20px;color:var(--tabulator-row-background-color);display:inline-flex;font-size:1.1em;font-weight:700;height:15px;justify-content:center;-moz-user-select:none;-khtml-user-select:none;-webkit-user-select:none;-o-user-select:none;width:15px}@media (hover:hover) and (pointer:fine){.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle:hover{cursor:pointer;opacity:.7}}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-close{display:initial}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-open{display:none}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle svg{stroke:var(--tabulator-row-background-color)}.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle .tabulator-responsive-collapse-toggle-close{display:none}.tabulator-row .tabulator-cell .tabulator-traffic-light{border-radius:14px;display:inline-block;height:14px;width:14px}.tabulator-row.tabulator-group{background:#ccc;border-bottom:1px solid #999;border-right:1px solid var(--tabulator-row-border-color);border-top:1px solid #999;box-sizing:border-box;font-weight:700;min-width:100%;padding:5px 5px 5px 10px}@media (hover:hover) and (pointer:fine){.tabulator-row.tabulator-group:hover{background-color:rgba(0,0,0,.1);cursor:pointer}}.tabulator-row.tabulator-group.tabulator-group-visible .tabulator-arrow{border-bottom:0;border-left:6px solid transparent;border-right:6px solid transparent;border-top:6px solid var(--tabulator-sort-arrow-active);margin-right:10px}.tabulator-row.tabulator-group.tabulator-group-level-1{padding-left:30px}.tabulator-row.tabulator-group.tabulator-group-level-2{padding-left:50px}.tabulator-row.tabulator-group.tabulator-group-level-3{padding-left:70px}.tabulator-row.tabulator-group.tabulator-group-level-4{padding-left:90px}.tabulator-row.tabulator-group.tabulator-group-level-5{padding-left:110px}.tabulator-row.tabulator-group .tabulator-group-toggle{display:inline-block}.tabulator-row.tabulator-group .tabulator-arrow{border-bottom:6px solid transparent;border-left:6px solid var(--tabulator-sort-arrow-active);border-right:0;border-top:6px solid transparent;display:inline-block;height:0;margin-right:16px;vertical-align:middle;width:0}.tabulator-row.tabulator-group span{color:#d00}.tabulator-toggle{background:#dcdcdc;border:1px solid #ccc;box-sizing:border-box;display:flex;flex-direction:row}.tabulator-toggle.tabulator-toggle-on{background:#1c6cc2}.tabulator-toggle .tabulator-toggle-switch{background:#fff;border:1px solid #ccc;box-sizing:border-box}.tabulator-popup-container{-webkit-overflow-scrolling:touch;background:var(--tabulator-row-background-color);border:1px solid var(--tabulator-row-border-color);box-shadow:0 0 5px 0 rgba(0,0,0,.2);box-sizing:border-box;display:inline-block;font-size:var(--tabulator-text-size);overflow-y:auto;position:absolute;z-index:10000}.tabulator-popup{border-radius:3px;padding:5px}.tabulator-tooltip{border-radius:2px;box-shadow:none;font-size:12px;max-width:min(500px,100%);padding:3px 5px;pointer-events:none}.tabulator-menu .tabulator-menu-item{box-sizing:border-box;padding:5px 10px;position:relative;user-select:none}.tabulator-menu .tabulator-menu-item.tabulator-menu-item-disabled{opacity:.5}@media (hover:hover) and (pointer:fine){.tabulator-menu .tabulator-menu-item:not(.tabulator-menu-item-disabled):hover{background:var(--tabulator-row-alt-background-color);cursor:pointer}}.tabulator-menu .tabulator-menu-item.tabulator-menu-item-submenu{padding-right:25px}.tabulator-menu .tabulator-menu-item.tabulator-menu-item-submenu:after{border-color:var(--tabulator-row-border-color);border-style:solid;border-width:1px 1px 0 0;content:"";display:inline-block;height:7px;position:absolute;right:10px;top:calc(5px + .4em);transform:rotate(45deg);vertical-align:top;width:7px}.tabulator-menu .tabulator-menu-separator{border-top:1px solid var(--tabulator-row-border-color)}.tabulator-edit-list{-webkit-overflow-scrolling:touch;font-size:var(--tabulator-text-size);max-height:200px;overflow-y:auto}.tabulator-edit-list .tabulator-edit-list-item{color:var(--tabulator-row-text-color);outline:none;padding:4px}.tabulator-edit-list .tabulator-edit-list-item.active{background:var(--tabulator-edit-box-color);color:var(--tabulator-row-background-color)}.tabulator-edit-list .tabulator-edit-list-item.active.focused{outline:1px solid rgba(var(--tabulator-row-background-color),.5)}.tabulator-edit-list .tabulator-edit-list-item.focused{outline:1px solid var(--tabulator-edit-box-color)}@media (hover:hover) and (pointer:fine){.tabulator-edit-list .tabulator-edit-list-item:hover{background:var(--tabulator-edit-box-color);color:var(--tabulator-row-background-color);cursor:pointer}}.tabulator-edit-list .tabulator-edit-list-placeholder{color:var(--tabulator-row-text-color);padding:4px;text-align:center}.tabulator-edit-list .tabulator-edit-list-group{border-bottom:1px solid var(--tabulator-row-border-color);color:var(--tabulator-row-text-color);font-weight:700;padding:6px 4px 4px}.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-2,.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-2{padding-left:12px}.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-3,.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-3{padding-left:20px}.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-4,.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-4{padding-left:28px}.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-5,.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-5{padding-left:36px}.tabulator.tabulator-ltr{direction:ltr}.tabulator.tabulator-rtl{direction:rtl;text-align:initial}.tabulator.tabulator-rtl .tabulator-header .tabulator-col{border-left:1px solid var(--tabulator-header-border-color);border-right:initial;text-align:initial}.tabulator.tabulator-rtl .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols{margin-left:-1px;margin-right:0}.tabulator.tabulator-rtl .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title{padding-left:25px;padding-right:0}.tabulator.tabulator-rtl .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter{left:8px;right:auto}.tabulator.tabulator-rtl .tabulator-tableholder .tabulator-range-overlay .tabulator-range.tabulator-range-active:after{background-color:var(--tabulator-range-handle-color);border-radius:999px;bottom:-3px;content:"";height:6px;left:-3px;position:absolute;right:auto;width:6px}.tabulator.tabulator-rtl .tabulator-row .tabulator-cell{border-left:1px solid var(--tabulator-row-border-color);border-right:initial}.tabulator.tabulator-rtl .tabulator-row .tabulator-cell .tabulator-data-tree-branch{border-bottom-left-radius:0;border-bottom-right-radius:1px;border-left:initial;border-right:2px solid var(--tabulator-row-border-color);margin-left:5px;margin-right:0}.tabulator.tabulator-rtl .tabulator-row .tabulator-cell .tabulator-data-tree-control{margin-left:5px;margin-right:0}.tabulator.tabulator-rtl .tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-left{border-left:2px solid var(--tabulator-row-border-color)}.tabulator.tabulator-rtl .tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-right{border-right:2px solid var(--tabulator-row-border-color)}.tabulator.tabulator-rtl .tabulator-row .tabulator-col-resize-handle:last-of-type{margin-left:0;margin-right:-3px;width:3px}.tabulator.tabulator-rtl .tabulator-footer .tabulator-calcs-holder{text-align:initial}.tabulator-print-fullscreen{bottom:0;left:0;position:absolute;right:0;top:0;z-index:10000}body.tabulator-print-fullscreen-hide>:not(.tabulator-print-fullscreen){display:none!important}.tabulator-print-table{border-collapse:collapse}.tabulator-print-table .tabulator-data-tree-branch{border-bottom:2px solid var(--tabulator-row-border-color);border-bottom-left-radius:1px;border-left:2px solid var(--tabulator-row-border-color);display:inline-block;height:9px;margin-right:5px;margin-top:-9px;vertical-align:middle;width:7px}.tabulator-print-table .tabulator-print-table-group{background:#ccc;border-bottom:1px solid #999;border-right:1px solid var(--tabulator-row-border-color);border-top:1px solid #999;box-sizing:border-box;font-weight:700;min-width:100%;padding:5px 5px 5px 10px}@media (hover:hover) and (pointer:fine){.tabulator-print-table .tabulator-print-table-group:hover{background-color:rgba(0,0,0,.1);cursor:pointer}}.tabulator-print-table .tabulator-print-table-group.tabulator-group-visible .tabulator-arrow{border-bottom:0;border-left:6px solid transparent;border-right:6px solid transparent;border-top:6px solid var(--tabulator-sort-arrow-active);margin-right:10px}.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-1 td{padding-left:30px!important}.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-2 td{padding-left:50px!important}.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-3 td{padding-left:70px!important}.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-4 td{padding-left:90px!important}.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-5 td{padding-left:110px!important}.tabulator-print-table .tabulator-print-table-group .tabulator-group-toggle{display:inline-block}.tabulator-print-table .tabulator-print-table-group .tabulator-arrow{border-bottom:6px solid transparent;border-left:6px solid var(--tabulator-sort-arrow-active);border-right:0;border-top:6px solid transparent;display:inline-block;height:0;margin-right:16px;vertical-align:middle;width:0}.tabulator-print-table .tabulator-print-table-group span{color:#d00}.tabulator-print-table .tabulator-data-tree-control{align-items:center;background:rgba(0,0,0,.1);border:1px solid var(--tabulator-row-text-color);border-radius:2px;display:inline-flex;height:11px;justify-content:center;margin-right:5px;overflow:hidden;vertical-align:middle;width:11px}@media (hover:hover) and (pointer:fine){.tabulator-print-table .tabulator-data-tree-control:hover{background:rgba(0,0,0,.2);cursor:pointer}}.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-collapse{background:transparent;display:inline-block;height:7px;position:relative;width:1px}.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after{background:var(--tabulator-row-text-color);content:"";height:1px;left:-3px;position:absolute;top:3px;width:7px}.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand{background:var(--tabulator-row-text-color);display:inline-block;height:7px;position:relative;width:1px}.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand:after{background:var(--tabulator-row-text-color);content:"";height:1px;left:-3px;position:absolute;top:3px;width:7px}.tabulator{background-color:var(--tabulator-background-color);max-width:100%;width:100%}.tabulator .tabulator-header{color:inherit}.tabulator .tabulator-header .tabulator-col{border-top:none}.tabulator .tabulator-header .tabulator-col:first-of-type{border-left:none}.tabulator .tabulator-header .tabulator-col:last-of-type{border-right:none}.tabulator .tabulator-header .tabulator-col:not(first-of-type),.tabulator .tabulator-header .tabulator-col:not(last-of-type){border-right:1px solid var(--tabulator-header-border-color)}.tabulator .tabulator-header .tabulator-col .tabulator-col-content{padding:var(--tabulator-cell-padding)}.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter{right:-10px}.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols{border-top:1px solid var(--tabulator-border-color)}.tabulator .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title{padding-right:10px}.tabulator .tabulator-header .tabulator-calcs-holder{border-bottom:1px solid var(--tabulator-header-separator-color);width:100%}.tabulator .tabulator-header .tabulator-frozen-rows-holder{min-width:600%}.tabulator .tabulator-header .tabulator-frozen-rows-holder:empty{display:none}.tabulator .tabulator-header .tabulator-frozen .tabulator-frozen-left,.tabulator .tabulator-header .tabulator-frozen .tabulator-frozen-right{background:inherit}.tabulator .tabulator-tableholder .tabulator-table{color:inherit}.tabulator .tabulator-footer{background-color:var(--tabulator-footer-background-color);color:inherit}.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab{font-weight:400;padding:8px 12px}.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab.tabulator-spreadsheet-tab-active{color:var(--tabulator-footer-active-color)}.tabulator .tabulator-footer .tabulator-paginator{color:inherit}.tabulator .tabulator-footer .tabulator-page{background:var(--tabulator-pagination-button-background);border-radius:0;border-right:none;color:var(--tabulator-pagination-button-color);margin:5px 0 0;padding:8px 12px}.tabulator .tabulator-footer .tabulator-page:first-of-type,.tabulator .tabulator-footer .tabulator-page[data-page=next]{border-bottom-left-radius:4px;border-top-left-radius:4px}.tabulator .tabulator-footer .tabulator-page:last-of-type,.tabulator .tabulator-footer .tabulator-page[data-page=prev]{border:1px solid var(--tabulator-footer-border-color);border-bottom-right-radius:4px;border-top-right-radius:4px}.tabulator .tabulator-footer .tabulator-page:not(disabled):hover{background:var(--tabulator-pagination-button-background-hover);color:var(--tabulator-pagination-button-color-hover)}.tabulator .tabulator-footer .tabulator-page.active,.tabulator .tabulator-footer .tabulator-page[data-page=first] :not(disabled):not(:hover),.tabulator .tabulator-footer .tabulator-page[data-page=last] :not(disabled):not(:hover),.tabulator .tabulator-footer .tabulator-page[data-page=next] :not(disabled):not(:hover),.tabulator .tabulator-footer .tabulator-page[data-page=prev] :not(disabled):not(:hover){color:var(--tabulator-pagination-button-color-active)}.tabulator.striped .tabulator-row:nth-child(2n){background-color:var(--tabulator-row-alt-background-color)}.tabulator.striped .tabulator-row:nth-child(2n).tabulator-selected{background-color:var(--tabulator-row-selected-background)!important}@media (hover:hover) and (pointer:fine){.tabulator.striped .tabulator-row:nth-child(2n).tabulator-selectable:hover{background-color:var(--tabulator-row-hover-background);cursor:pointer}.tabulator.striped .tabulator-row:nth-child(2n).tabulator-selected:hover{background-color:var(--tabulator-row-selected-background-hover)!important;cursor:pointer}}.tabulator-row{border-bottom:1px solid var(--tabulator-row-border-color);min-height:calc(var(--tabulator-text-size) + var(--tabulator-cell-padding)*2)}.tabulator-row.tabulator-row-even{background-color:var(--tabulator-row-background-color)}.tabulator-row .tabulator-cell{padding:var(--tabulator-cell-padding)}.tabulator-row .tabulator-cell:last-of-type{border-right:none}.tabulator-row .tabulator-cell.tabulator-row-header{background:var(--tabulator-header-background-color);border-bottom:none;border-right:1px solid var(--tabulator-border-color)}.tabulator-row .tabulator-cell .tabulator-data-tree-control{border:1px solid #ccc}.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after,.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand,.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand:after{background:#ccc}.tabulator-row.tabulator-group{background:#fafafa}.tabulator-row.tabulator-group span{color:#666;margin-left:10px}.tabulator-edit-select-list{background:var(--tabulator-header-background-color)}.tabulator-edit-select-list .tabulator-edit-select-list-item{color:inherit}.tabulator-edit-select-list .tabulator-edit-select-list-item.active{color:var(--tabulator-header-background-color)}.tabulator-edit-select-list .tabulator-edit-select-list-item.active.focused{outline:1px solid rgba(var(--tabulator-header-background-color),.5)}@media (hover:hover) and (pointer:fine){.tabulator-edit-select-list .tabulator-edit-select-list-item:hover{color:var(--tabulator-header-background-color)}}.tabulator-edit-select-list .tabulator-edit-select-list-group,.tabulator-edit-select-list .tabulator-edit-select-list-notice{color:inherit}.tabulator.tabulator-rtl .tabulator-header .tabulator-col{border-left:none;border-right:none}.tabulator-print-table .tabulator-print-table-group{background:#fafafa}.tabulator-print-table .tabulator-print-table-group span{color:#666;margin-left:10px}.tabulator-print-table .tabulator-data-tree-control{border:1px solid #ccc}.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after,.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand,.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand:after{background:#ccc} +:root { + --tabulator-background-color: #fff; + --tabulator-border-color: rgba(0, 0, 0, .12); + --tabulator-text-size: 16px; + --tabulator-header-background-color: #f5f5f5; + --tabulator-header-text-color: #555; + --tabulator-header-border-color: rgba(0, 0, 0, .12); + --tabulator-header-separator-color: rgba(0, 0, 0, .12); + --tabulator-header-margin: 4px; + --tabulator-sort-arrow-hover: #555; + --tabulator-sort-arrow-active: #666; + --tabulator-sort-arrow-inactive: #bbb; + --tabulator-column-resize-guide-color: #999; + --tabulator-row-background-color: #fff; + --tabulator-row-alt-background-color: #f8f8f8; + --tabulator-row-border-color: rgba(0, 0, 0, .12); + --tabulator-row-text-color: #333; + --tabulator-row-hover-background: #e1f5fe; + --tabulator-row-selected-background: #ace5ff; + --tabulator-row-selected-background-hover: #9bcfe8; + --tabulator-edit-box-color: #17161d; + --tabulator-error-color: #d00; + --tabulator-footer-background-color: transparent; + --tabulator-footer-text-color: #555; + --tabulator-footer-border-color: rgba(0, 0, 0, .12); + --tabulator-footer-separator-color: rgba(0, 0, 0, .12); + --tabulator-footer-active-color: #17161d; + --tabulator-spreadsheet-active-tab-color: #fff; + --tabulator-range-border-color: #17161d; + --tabulator-range-handle-color: #17161d; + --tabulator-range-header-selected-background: var(--tabulator-range-border-color); + --tabulator-range-header-selected-text-color: #fff; + --tabulator-range-header-highlight-background: colors-gray-timberwolf; + --tabulator-range-header-text-highlight-background: #fff; + --tabulator-pagination-button-background: #fff; + --tabulator-pagination-button-background-hover: #06c; + --tabulator-pagination-button-color: #999; + --tabulator-pagination-button-color-hover: #fff; + --tabulator-pagination-button-color-active: #000; + --tabulator-cell-padding: 15px +} + +body.vscode-dark, +body[data-jp-theme-light=false] { + --tabulator-background-color: #080808; + --tabulator-border-color: #666; + --tabulator-text-size: 16px; + --tabulator-header-background-color: #212121; + --tabulator-header-text-color: #555; + --tabulator-header-border-color: #666; + --tabulator-header-separator-color: #666; + --tabulator-header-margin: 4px; + --tabulator-sort-arrow-hover: #fff; + --tabulator-sort-arrow-active: #e6e6e6; + --tabulator-sort-arrow-inactive: #666; + --tabulator-column-resize-guide-color: #999; + --tabulator-row-background-color: #080808; + --tabulator-row-alt-background-color: #212121; + --tabulator-row-border-color: #666; + --tabulator-row-text-color: #f8f8f8; + --tabulator-row-hover-background: #333; + --tabulator-row-selected-background: #3d355d; + --tabulator-row-selected-background-hover: #483f69; + --tabulator-edit-box-color: #333; + --tabulator-error-color: #d00; + --tabulator-footer-background-color: transparent; + --tabulator-footer-text-color: #555; + --tabulator-footer-border-color: rgba(0, 0, 0, .12); + --tabulator-footer-separator-color: rgba(0, 0, 0, .12); + --tabulator-footer-active-color: #17161d; + --tabulator-spreadsheet-active-tab-color: #fff; + --tabulator-range-border-color: #17161d; + --tabulator-range-handle-color: var(--tabulator-range-border-color); + --tabulator-range-header-selected-background: var(--tabulator-range-border-color); + --tabulator-range-header-selected-text-color: #fff; + --tabulator-range-header-highlight-background: #d6d6d6; + --tabulator-range-header-text-highlight-background: #fff; + --tabulator-pagination-button-background: #212121; + --tabulator-pagination-button-background-hover: #555; + --tabulator-pagination-button-color: #999; + --tabulator-pagination-button-color-hover: #fff; + --tabulator-pagination-button-color-active: #fff; + --tabulator-cell-padding: 15px +} + +.tabulator { + border: 1px solid var(--tabulator-border-color); + font-size: var(--tabulator-text-size); + overflow: hidden; + position: relative; + text-align: left; + -webkit-transform: translateZ(0); + -moz-transform: translateZ(0); + -ms-transform: translateZ(0); + -o-transform: translateZ(0); + transform: translateZ(0) +} + +.tabulator[tabulator-layout=fitDataFill] .tabulator-tableholder .tabulator-table { + min-width: 100% +} + +.tabulator[tabulator-layout=fitDataTable] { + display: inline-block +} + +.tabulator.tabulator-block-select, +.tabulator.tabulator-ranges .tabulator-cell:not(.tabulator-editing) { + user-select: none +} + +.tabulator .tabulator-header { + background-color: var(--tabulator-header-background-color); + border-bottom: 1px solid var(--tabulator-header-separator-color); + box-sizing: border-box; + color: var(--tabulator-header-text-color); + font-weight: 700; + outline: none; + overflow: hidden; + position: relative; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none; + white-space: nowrap; + width: 100% +} + +.tabulator .tabulator-header.tabulator-header-hidden { + display: none +} + +.tabulator .tabulator-header .tabulator-header-contents { + overflow: hidden; + position: relative +} + +.tabulator .tabulator-header .tabulator-header-contents .tabulator-headers { + display: inline-block +} + +.tabulator .tabulator-header .tabulator-col { + background: var(--tabulator-header-background-color); + border-right: 1px solid var(--tabulator-header-border-color); + box-sizing: border-box; + display: inline-flex; + flex-direction: column; + justify-content: flex-start; + overflow: hidden; + position: relative; + text-align: left; + vertical-align: bottom +} + +.tabulator .tabulator-header .tabulator-col.tabulator-moving { + background: hsl(var(--tabulator-header-background-color), calc(var(--tabulator-header-background-color) - 5%)) !important; + border: 1px solid var(--tabulator-header-separator-color); + pointer-events: none; + position: absolute +} + +.tabulator .tabulator-header .tabulator-col.tabulator-range-highlight { + background-color: var(--tabulator-range-header-highlight-background); + color: var(--tabulator-range-header-text-highlight-background) +} + +.tabulator .tabulator-header .tabulator-col.tabulator-range-selected { + background-color: var(--tabulator-range-header-selected-background); + color: var(--tabulator-range-header-selected-text-color) +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content { + box-sizing: border-box; + padding: 4px; + position: relative +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-header-popup-button { + padding: 0 8px +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-header-popup-button:hover { + cursor: pointer; + opacity: .6 +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title-holder { + position: relative +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title { + box-sizing: border-box; + overflow: hidden; + text-overflow: ellipsis; + vertical-align: bottom; + white-space: nowrap; + width: 100% +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title.tabulator-col-title-wrap { + text-overflow: clip; + white-space: normal +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title .tabulator-title-editor { + background: #fff; + border: 1px solid #999; + box-sizing: border-box; + padding: 1px; + width: 100% +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-title .tabulator-header-popup-button+.tabulator-title-editor { + width: calc(100% - 22px) +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter { + align-items: center; + bottom: 0; + display: flex; + position: absolute; + right: 4px; + top: 0 +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter .tabulator-arrow { + border-bottom: 6px solid var(--tabulator-sort-arrow-inactive); + border-left: 6px solid transparent; + border-right: 6px solid transparent; + height: 0; + width: 0 +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols { + border-top: 1px solid var(--tabulator-header-border-color); + display: flex; + margin-right: -1px; + overflow: hidden; + position: relative +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter { + box-sizing: border-box; + margin-top: 2px; + position: relative; + text-align: center; + width: 100% +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter textarea { + height: auto !important +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter svg { + margin-top: 3px +} + +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input::-ms-clear { + height: 0; + width: 0 +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title { + padding-right: 25px +} + +@media (hover:hover) and (pointer:fine) { + .tabulator .tabulator-header .tabulator-col.tabulator-sortable.tabulator-col-sorter-element:hover { + background-color: hsl(var(--tabulator-header-background-color), calc(var(--tabulator-header-background-color) - 10%)) !important; + cursor: pointer + } +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=none] .tabulator-col-content .tabulator-col-sorter { + color: var(--tabulator-sort-arrow-inactive) +} + +@media (hover:hover) and (pointer:fine) { + .tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=none] .tabulator-col-content .tabulator-col-sorter.tabulator-col-sorter-element .tabulator-arrow:hover { + border-bottom: 6px solid var(--tabulator-sort-arrow-hover); + cursor: pointer + } +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=none] .tabulator-col-content .tabulator-col-sorter .tabulator-arrow { + border-bottom: 6px solid var(--tabulator-sort-arrow-inactive); + border-top: none +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=ascending] .tabulator-col-content .tabulator-col-sorter { + color: var(--tabulator-sort-arrow-active) +} + +@media (hover:hover) and (pointer:fine) { + .tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=ascending] .tabulator-col-content .tabulator-col-sorter.tabulator-col-sorter-element .tabulator-arrow:hover { + border-bottom: 6px solid var(--tabulator-sort-arrow-hover); + cursor: pointer + } +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=ascending] .tabulator-col-content .tabulator-col-sorter .tabulator-arrow { + border-bottom: 6px solid var(--tabulator-sort-arrow-active); + border-top: none +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=descending] .tabulator-col-content .tabulator-col-sorter { + color: var(--tabulator-sort-arrow-active) +} + +@media (hover:hover) and (pointer:fine) { + .tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=descending] .tabulator-col-content .tabulator-col-sorter.tabulator-col-sorter-element .tabulator-arrow:hover { + border-top: 6px solid var(--tabulator-sort-arrow-hover); + cursor: pointer + } +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable[aria-sort=descending] .tabulator-col-content .tabulator-col-sorter .tabulator-arrow { + border-bottom: none; + border-top: 6px solid var(--tabulator-sort-arrow-active); + color: var(--tabulator-sort-arrow-active) +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical .tabulator-col-content .tabulator-col-title { + align-items: center; + display: flex; + justify-content: center; + text-orientation: mixed; + writing-mode: vertical-rl +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-col-vertical-flip .tabulator-col-title { + transform: rotate(180deg) +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-col-title { + padding-right: 0; + padding-top: 20px +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable.tabulator-col-vertical-flip .tabulator-col-title { + padding-bottom: 20px; + padding-right: 0 +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-vertical.tabulator-sortable .tabulator-col-sorter { + bottom: auto; + justify-content: center; + left: 0; + right: 0; + top: 4px +} + +.tabulator .tabulator-header .tabulator-frozen { + left: 0; + position: sticky; + z-index: 11 +} + +.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-left { + border-right: 2px solid var(--tabulator-row-border-color) +} + +.tabulator .tabulator-header .tabulator-frozen.tabulator-frozen-right { + border-left: 2px solid var(--tabulator-row-border-color) +} + +.tabulator .tabulator-header .tabulator-calcs-holder { + border-bottom: 1px solid var(--tabulator-header-border-color); + border-top: 1px solid var(--tabulator-row-border-color); + box-sizing: border-box; + display: inline-block +} + +.tabulator .tabulator-header .tabulator-calcs-holder, +.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row { + background: hsl(var(--tabulator-header-background-color), calc(var(--tabulator-header-background-color) + 5%)) !important +} + +.tabulator .tabulator-header .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle { + display: none +} + +.tabulator .tabulator-header .tabulator-frozen-rows-holder { + display: inline-block +} + +.tabulator .tabulator-tableholder { + -webkit-overflow-scrolling: touch; + overflow: auto; + position: relative; + white-space: nowrap; + width: 100% +} + +.tabulator .tabulator-tableholder:focus { + outline: none +} + +.tabulator .tabulator-tableholder .tabulator-placeholder { + align-items: center; + box-sizing: border-box; + display: flex; + justify-content: center; + min-width: 100%; + width: 100% +} + +.tabulator .tabulator-tableholder .tabulator-placeholder[tabulator-render-mode=virtual] { + min-height: 100% +} + +.tabulator .tabulator-tableholder .tabulator-placeholder .tabulator-placeholder-contents { + color: #ccc; + display: inline-block; + font-size: 20px; + font-weight: 700; + padding: 10px; + text-align: center; + white-space: normal +} + +.tabulator .tabulator-tableholder .tabulator-table { + background-color: var(--tabulator-row-background-color); + color: var(--tabulator-row-text-color); + display: inline-block; + overflow: visible; + position: relative; + white-space: nowrap +} + +.tabulator .tabulator-tableholder .tabulator-table .tabulator-row.tabulator-calcs { + background: hsl(var(--tabulator-row-atl-background-color), calc(var(--tabulator-row-alt-background-color) - 5%)) !important; + font-weight: 700 +} + +.tabulator .tabulator-tableholder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-top { + border-bottom: 2px solid var(--tabulator-row-border-color) +} + +.tabulator .tabulator-tableholder .tabulator-table .tabulator-row.tabulator-calcs.tabulator-calcs-bottom { + border-top: 2px solid var(--tabulator-row-border-color) +} + +.tabulator .tabulator-tableholder .tabulator-range-overlay { + inset: 0; + pointer-events: none; + position: absolute; + z-index: 10 +} + +.tabulator .tabulator-tableholder .tabulator-range-overlay .tabulator-range { + border: 1px solid var(--tabulator-range-border-color); + box-sizing: border-box; + position: absolute +} + +.tabulator .tabulator-tableholder .tabulator-range-overlay .tabulator-range.tabulator-range-active:after { + background-color: var(--tabulator-range-handle-color); + border-radius: 999px; + bottom: -3px; + content: ""; + height: 6px; + position: absolute; + right: -3px; + width: 6px +} + +.tabulator .tabulator-tableholder .tabulator-range-overlay .tabulator-range-cell-active { + border: 2px solid var(--tabulator-range-border-color); + box-sizing: border-box; + position: absolute +} + +.tabulator .tabulator-footer { + color: var(--tabulator-footer-text-color); + font-weight: 700; + user-select: none; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none; + white-space: nowrap +} + +.tabulator .tabulator-footer .tabulator-footer-contents { + align-items: center; + display: flex; + flex-direction: row; + justify-content: space-between; + padding: 5px 10px +} + +.tabulator .tabulator-footer .tabulator-footer-contents:empty { + display: none +} + +.tabulator .tabulator-footer .tabulator-spreadsheet-tabs { + margin-top: -5px; + overflow-x: auto +} + +.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab { + border: 1px solid var(--tabulator-border-color); + border-bottom-left-radius: 5px; + border-bottom-right-radius: 5px; + border-top: none; + display: inline-block; + font-size: .9em; + padding: 5px +} + +.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab:hover { + cursor: pointer; + opacity: .7 +} + +.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab.tabulator-spreadsheet-tab-active { + background: var(--tabulator-spreadsheet-active-tab-color) +} + +.tabulator .tabulator-footer .tabulator-calcs-holder { + border-bottom: 1px solid var(--tabulator-row-border-color); + border-top: 1px solid var(--tabulator-row-border-color); + box-sizing: border-box; + overflow: hidden; + text-align: left; + width: 100% +} + +.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row { + display: inline-block +} + +.tabulator .tabulator-footer .tabulator-calcs-holder .tabulator-row .tabulator-col-resize-handle { + display: none +} + +.tabulator .tabulator-footer .tabulator-calcs-holder:only-child { + border-bottom: none; + margin-bottom: -5px +} + +.tabulator .tabulator-footer>*+.tabulator-page-counter { + margin-left: 10px +} + +.tabulator .tabulator-footer .tabulator-page-counter { + font-weight: 400 +} + +.tabulator .tabulator-footer .tabulator-paginator { + color: var(--tabulator-footer-text-color); + flex: 1; + font-family: inherit; + font-size: inherit; + font-weight: inherit; + text-align: right +} + +.tabulator .tabulator-footer .tabulator-page-size { + border: 1px solid var(--tabulator-footer-border-color); + border-radius: 3px; + display: inline-block; + margin: 0 5px; + padding: 2px 5px +} + +.tabulator .tabulator-footer .tabulator-pages { + margin: 0 7px +} + +.tabulator .tabulator-footer .tabulator-page { + background: hsla(0, 0%, 100%, .2); + border: 1px solid var(--tabulator-footer-border-color); + border-radius: 3px; + display: inline-block; + margin: 0 2px; + padding: 2px 5px +} + +.tabulator .tabulator-footer .tabulator-page.active { + color: var(--tabulator-footer-active-color) +} + +.tabulator .tabulator-footer .tabulator-page:disabled { + opacity: .5 +} + +@media (hover:hover) and (pointer:fine) { + .tabulator .tabulator-footer .tabulator-page:not(disabled):hover { + background: rgba(0, 0, 0, .2); + color: #fff; + cursor: pointer + } +} + +.tabulator .tabulator-col-resize-handle { + display: inline-block; + margin-left: -3px; + margin-right: -3px; + position: relative; + vertical-align: middle; + width: 6px; + z-index: 11 +} + +@media (hover:hover) and (pointer:fine) { + .tabulator .tabulator-col-resize-handle:hover { + cursor: ew-resize + } +} + +.tabulator .tabulator-col-resize-handle:last-of-type { + margin-right: 0; + width: 3px +} + +.tabulator .tabulator-col-resize-guide { + height: 100%; + margin-left: -.5px; + top: 0; + width: 4px +} + +.tabulator .tabulator-col-resize-guide, +.tabulator .tabulator-row-resize-guide { + background-color: var(--tabulator-column-resize-guide-color); + opacity: .5; + position: absolute +} + +.tabulator .tabulator-row-resize-guide { + height: 4px; + left: 0; + margin-top: -.5px; + width: 100% +} + +.tabulator .tabulator-alert { + align-items: center; + background: rgba(0, 0, 0, .4); + display: flex; + height: 100%; + left: 0; + position: absolute; + text-align: center; + top: 0; + width: 100%; + z-index: 100 +} + +.tabulator .tabulator-alert .tabulator-alert-msg { + background: #fff; + border-radius: 10px; + display: inline-block; + font-size: 16px; + font-weight: 700; + margin: 0 auto; + padding: 10px 20px +} + +.tabulator .tabulator-alert .tabulator-alert-msg.tabulator-alert-state-msg { + border: 4px solid #333; + color: #000 +} + +.tabulator .tabulator-alert .tabulator-alert-msg.tabulator-alert-state-error { + border: 4px solid #d00; + color: #590000 +} + +.tabulator-row { + background-color: var(--tabulator-row-background-color); + box-sizing: border-box; + min-height: calc(var(--tabulator-text-size) + var(--tabulator-header-margin)*2); + position: relative +} + +.tabulator-row.tabulator-row-even { + background-color: var(--tabulator-row-alt-background-color) +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-row.tabulator-selectable:hover { + background-color: var(--tabulator-row-hover-background); + cursor: pointer + } +} + +.tabulator-row.tabulator-selected { + background-color: var(--tabulator-row-selected-background) !important; +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-row.tabulator-selected:hover { + background-color: var(--tabulator-row-selected-background-hover) !important; + cursor: pointer + } +} + +.tabulator-row.tabulator-row-moving { + background: #fff; + border: 1px solid #000 +} + +.tabulator-row.tabulator-moving { + border-bottom: 1px solid var(--tabulator-row-border-color); + border-top: 1px solid var(--tabulator-row-border-color); + pointer-events: none; + position: absolute; + z-index: 15 +} + +.tabulator-row.tabulator-range-highlight .tabulator-cell.tabulator-range-row-header { + background-color: var(--tabulator-range-header-highlight-background); + color: var(--tabulator-range-header-text-highlight-background) +} + +.tabulator-row.tabulator-range-highlight.tabulator-range-selected .tabulator-cell.tabulator-range-row-header, +.tabulator-row.tabulator-range-selected .tabulator-cell.tabulator-range-row-header { + background-color: var(--tabulator-range-header-selected-background); + color: var(--tabulator-range-header-selected-text-color) +} + +.tabulator-row .tabulator-row-resize-handle { + bottom: 0; + height: 5px; + left: 0; + position: absolute; + right: 0 +} + +.tabulator-row .tabulator-row-resize-handle.prev { + bottom: auto; + top: 0 +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-row .tabulator-row-resize-handle:hover { + cursor: ns-resize + } +} + +.tabulator-row .tabulator-responsive-collapse { + border-bottom: 1px solid var(--tabulator-row-border-color); + border-top: 1px solid var(--tabulator-row-border-color); + box-sizing: border-box; + padding: 5px +} + +.tabulator-row .tabulator-responsive-collapse:empty { + display: none +} + +.tabulator-row .tabulator-responsive-collapse table { + font-size: var(--tabulator-text-size) +} + +.tabulator-row .tabulator-responsive-collapse table tr td { + position: relative +} + +.tabulator-row .tabulator-responsive-collapse table tr td:first-of-type { + padding-right: 10px +} + +.tabulator-row .tabulator-cell { + border-right: 1px solid var(--tabulator-row-border-color); + box-sizing: border-box; + display: inline-block; + outline: none; + overflow: hidden; + padding: 4px; + position: relative; + text-overflow: ellipsis; + vertical-align: middle; + white-space: nowrap +} + +.tabulator-row .tabulator-cell.tabulator-row-header { + border-bottom: 1px solid var(--tabulator-row-border-color) +} + +.tabulator-row .tabulator-cell.tabulator-frozen { + background-color: inherit; + display: inline-block; + left: 0; + position: sticky; + z-index: 11 +} + +.tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-left { + border-right: 2px solid var(--tabulator-row-border-color) +} + +.tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-right { + border-left: 2px solid var(--tabulator-row-border-color) +} + +.tabulator-row .tabulator-cell.tabulator-editing { + border: 1px solid var(--tabulator-edit-box-color); + outline: none; + padding: 0 +} + +.tabulator-row .tabulator-cell.tabulator-editing input, +.tabulator-row .tabulator-cell.tabulator-editing select { + background: transparent; + border: 1px; + outline: none +} + +.tabulator-row .tabulator-cell.tabulator-validation-fail { + border: 1px solid var(--tabulator-error-color) +} + +.tabulator-row .tabulator-cell.tabulator-validation-fail input, +.tabulator-row .tabulator-cell.tabulator-validation-fail select { + background: transparent; + border: 1px; + color: var(--tabulator-error-color) +} + +.tabulator-row .tabulator-cell.tabulator-row-handle { + align-items: center; + display: inline-flex; + justify-content: center; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none +} + +.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box { + width: 80% +} + +.tabulator-row .tabulator-cell.tabulator-row-handle .tabulator-row-handle-box .tabulator-row-handle-bar { + background: #666; + height: 3px; + margin-top: 2px; + width: 100% +} + +.tabulator-row .tabulator-cell.tabulator-range-selected:not(.tabulator-range-only-cell-selected):not(.tabulator-range-row-header) { + background-color: var(--tabulator-row-selected-background) +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-branch-empty { + display: inline-block; + width: 7px +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-branch { + border-bottom: 2px solid var(--tabulator-row-border-color); + border-bottom-left-radius: 1px; + border-left: 2px solid var(--tabulator-row-border-color); + display: inline-block; + height: 9px; + margin-right: 5px; + margin-top: -9px; + vertical-align: middle; + width: 7px +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control { + align-items: center; + background: rgba(0, 0, 0, .1); + border: 1px solid var(--tabulator-row-text-color); + border-radius: 2px; + display: inline-flex; + height: 11px; + justify-content: center; + margin-right: 5px; + overflow: hidden; + vertical-align: middle; + width: 11px +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-row .tabulator-cell .tabulator-data-tree-control:hover { + background: rgba(0, 0, 0, .2); + cursor: pointer + } +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse { + background: transparent; + display: inline-block; + height: 7px; + position: relative; + width: 1px +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after { + background: var(--tabulator-row-text-color); + content: ""; + height: 1px; + left: -3px; + position: absolute; + top: 3px; + width: 7px +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand { + background: var(--tabulator-row-text-color); + display: inline-block; + height: 7px; + position: relative; + width: 1px +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand:after { + background: var(--tabulator-row-text-color); + content: ""; + height: 1px; + left: -3px; + position: absolute; + top: 3px; + width: 7px +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle { + align-items: center; + background: #666; + border-radius: 20px; + color: var(--tabulator-row-background-color); + display: inline-flex; + font-size: 1.1em; + font-weight: 700; + height: 15px; + justify-content: center; + -moz-user-select: none; + -khtml-user-select: none; + -webkit-user-select: none; + -o-user-select: none; + width: 15px +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle:hover { + cursor: pointer; + opacity: .7 + } +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-close { + display: initial +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle.open .tabulator-responsive-collapse-toggle-open { + display: none +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle svg { + stroke: var(--tabulator-row-background-color) +} + +.tabulator-row .tabulator-cell .tabulator-responsive-collapse-toggle .tabulator-responsive-collapse-toggle-close { + display: none +} + +.tabulator-row .tabulator-cell .tabulator-traffic-light { + border-radius: 14px; + display: inline-block; + height: 14px; + width: 14px +} + +.tabulator-row.tabulator-group { + background: #ccc; + border-bottom: 1px solid #999; + border-right: 1px solid var(--tabulator-row-border-color); + border-top: 1px solid #999; + box-sizing: border-box; + font-weight: 700; + min-width: 100%; + padding: 5px 5px 5px 10px +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-row.tabulator-group:hover { + background-color: rgba(0, 0, 0, .1); + cursor: pointer + } +} + +.tabulator-row.tabulator-group.tabulator-group-visible .tabulator-arrow { + border-bottom: 0; + border-left: 6px solid transparent; + border-right: 6px solid transparent; + border-top: 6px solid var(--tabulator-sort-arrow-active); + margin-right: 10px +} + +.tabulator-row.tabulator-group.tabulator-group-level-1 { + padding-left: 30px +} + +.tabulator-row.tabulator-group.tabulator-group-level-2 { + padding-left: 50px +} + +.tabulator-row.tabulator-group.tabulator-group-level-3 { + padding-left: 70px +} + +.tabulator-row.tabulator-group.tabulator-group-level-4 { + padding-left: 90px +} + +.tabulator-row.tabulator-group.tabulator-group-level-5 { + padding-left: 110px +} + +.tabulator-row.tabulator-group .tabulator-group-toggle { + display: inline-block +} + +.tabulator-row.tabulator-group .tabulator-arrow { + border-bottom: 6px solid transparent; + border-left: 6px solid var(--tabulator-sort-arrow-active); + border-right: 0; + border-top: 6px solid transparent; + display: inline-block; + height: 0; + margin-right: 16px; + vertical-align: middle; + width: 0 +} + +.tabulator-row.tabulator-group span { + color: #d00 +} + +.tabulator-toggle { + background: #dcdcdc; + border: 1px solid #ccc; + box-sizing: border-box; + display: flex; + flex-direction: row +} + +.tabulator-toggle.tabulator-toggle-on { + background: #1c6cc2 +} + +.tabulator-toggle .tabulator-toggle-switch { + background: #fff; + border: 1px solid #ccc; + box-sizing: border-box +} + +.tabulator-popup-container { + -webkit-overflow-scrolling: touch; + background: var(--tabulator-row-background-color); + border: 1px solid var(--tabulator-row-border-color); + box-shadow: 0 0 5px 0 rgba(0, 0, 0, .2); + box-sizing: border-box; + display: inline-block; + font-size: var(--tabulator-text-size); + overflow-y: auto; + position: absolute; + z-index: 10000 +} + +.tabulator-popup { + border-radius: 3px; + padding: 5px +} + +.tabulator-tooltip { + border-radius: 2px; + box-shadow: none; + font-size: 12px; + max-width: min(500px, 100%); + padding: 3px 5px; + pointer-events: none +} + +.tabulator-menu .tabulator-menu-item { + box-sizing: border-box; + padding: 5px 10px; + position: relative; + user-select: none +} + +.tabulator-menu .tabulator-menu-item.tabulator-menu-item-disabled { + opacity: .5 +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-menu .tabulator-menu-item:not(.tabulator-menu-item-disabled):hover { + background: var(--tabulator-row-alt-background-color); + cursor: pointer + } +} + +.tabulator-menu .tabulator-menu-item.tabulator-menu-item-submenu { + padding-right: 25px +} + +.tabulator-menu .tabulator-menu-item.tabulator-menu-item-submenu:after { + border-color: var(--tabulator-row-border-color); + border-style: solid; + border-width: 1px 1px 0 0; + content: ""; + display: inline-block; + height: 7px; + position: absolute; + right: 10px; + top: calc(5px + .4em); + transform: rotate(45deg); + vertical-align: top; + width: 7px +} + +.tabulator-menu .tabulator-menu-separator { + border-top: 1px solid var(--tabulator-row-border-color) +} + +.tabulator-edit-list { + -webkit-overflow-scrolling: touch; + font-size: var(--tabulator-text-size); + max-height: 200px; + overflow-y: auto +} + +.tabulator-edit-list .tabulator-edit-list-item { + color: var(--tabulator-row-text-color); + outline: none; + padding: 4px +} + +.tabulator-edit-list .tabulator-edit-list-item.active { + background: var(--tabulator-edit-box-color); + color: var(--tabulator-row-background-color) +} + +.tabulator-edit-list .tabulator-edit-list-item.active.focused { + outline: 1px solid rgba(var(--tabulator-row-background-color), .5) +} + +.tabulator-edit-list .tabulator-edit-list-item.focused { + outline: 1px solid var(--tabulator-edit-box-color) +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-edit-list .tabulator-edit-list-item:hover { + background: var(--tabulator-edit-box-color); + color: var(--tabulator-row-background-color); + cursor: pointer + } +} + +.tabulator-edit-list .tabulator-edit-list-placeholder { + color: var(--tabulator-row-text-color); + padding: 4px; + text-align: center +} + +.tabulator-edit-list .tabulator-edit-list-group { + border-bottom: 1px solid var(--tabulator-row-border-color); + color: var(--tabulator-row-text-color); + font-weight: 700; + padding: 6px 4px 4px +} + +.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-2, +.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-2 { + padding-left: 12px +} + +.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-3, +.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-3 { + padding-left: 20px +} + +.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-4, +.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-4 { + padding-left: 28px +} + +.tabulator-edit-list .tabulator-edit-list-group.tabulator-edit-list-group-level-5, +.tabulator-edit-list .tabulator-edit-list-item.tabulator-edit-list-group-level-5 { + padding-left: 36px +} + +.tabulator.tabulator-ltr { + direction: ltr +} + +.tabulator.tabulator-rtl { + direction: rtl; + text-align: initial +} + +.tabulator.tabulator-rtl .tabulator-header .tabulator-col { + border-left: 1px solid var(--tabulator-header-border-color); + border-right: initial; + text-align: initial +} + +.tabulator.tabulator-rtl .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols { + margin-left: -1px; + margin-right: 0 +} + +.tabulator.tabulator-rtl .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title { + padding-left: 25px; + padding-right: 0 +} + +.tabulator.tabulator-rtl .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter { + left: 8px; + right: auto +} + +.tabulator.tabulator-rtl .tabulator-tableholder .tabulator-range-overlay .tabulator-range.tabulator-range-active:after { + background-color: var(--tabulator-range-handle-color); + border-radius: 999px; + bottom: -3px; + content: ""; + height: 6px; + left: -3px; + position: absolute; + right: auto; + width: 6px +} + +.tabulator.tabulator-rtl .tabulator-row .tabulator-cell { + border-left: 1px solid var(--tabulator-row-border-color); + border-right: initial +} + +.tabulator.tabulator-rtl .tabulator-row .tabulator-cell .tabulator-data-tree-branch { + border-bottom-left-radius: 0; + border-bottom-right-radius: 1px; + border-left: initial; + border-right: 2px solid var(--tabulator-row-border-color); + margin-left: 5px; + margin-right: 0 +} + +.tabulator.tabulator-rtl .tabulator-row .tabulator-cell .tabulator-data-tree-control { + margin-left: 5px; + margin-right: 0 +} + +.tabulator.tabulator-rtl .tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-left { + border-left: 2px solid var(--tabulator-row-border-color) +} + +.tabulator.tabulator-rtl .tabulator-row .tabulator-cell.tabulator-frozen.tabulator-frozen-right { + border-right: 2px solid var(--tabulator-row-border-color) +} + +.tabulator.tabulator-rtl .tabulator-row .tabulator-col-resize-handle:last-of-type { + margin-left: 0; + margin-right: -3px; + width: 3px +} + +.tabulator.tabulator-rtl .tabulator-footer .tabulator-calcs-holder { + text-align: initial +} + +.tabulator-print-fullscreen { + bottom: 0; + left: 0; + position: absolute; + right: 0; + top: 0; + z-index: 10000 +} + +body.tabulator-print-fullscreen-hide>:not(.tabulator-print-fullscreen) { + display: none !important +} + +.tabulator-print-table { + border-collapse: collapse +} + +.tabulator-print-table .tabulator-data-tree-branch { + border-bottom: 2px solid var(--tabulator-row-border-color); + border-bottom-left-radius: 1px; + border-left: 2px solid var(--tabulator-row-border-color); + display: inline-block; + height: 9px; + margin-right: 5px; + margin-top: -9px; + vertical-align: middle; + width: 7px +} + +.tabulator-print-table .tabulator-print-table-group { + background: #ccc; + border-bottom: 1px solid #999; + border-right: 1px solid var(--tabulator-row-border-color); + border-top: 1px solid #999; + box-sizing: border-box; + font-weight: 700; + min-width: 100%; + padding: 5px 5px 5px 10px +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-print-table .tabulator-print-table-group:hover { + background-color: rgba(0, 0, 0, .1); + cursor: pointer + } +} + +.tabulator-print-table .tabulator-print-table-group.tabulator-group-visible .tabulator-arrow { + border-bottom: 0; + border-left: 6px solid transparent; + border-right: 6px solid transparent; + border-top: 6px solid var(--tabulator-sort-arrow-active); + margin-right: 10px +} + +.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-1 td { + padding-left: 30px !important +} + +.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-2 td { + padding-left: 50px !important +} + +.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-3 td { + padding-left: 70px !important +} + +.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-4 td { + padding-left: 90px !important +} + +.tabulator-print-table .tabulator-print-table-group.tabulator-group-level-5 td { + padding-left: 110px !important +} + +.tabulator-print-table .tabulator-print-table-group .tabulator-group-toggle { + display: inline-block +} + +.tabulator-print-table .tabulator-print-table-group .tabulator-arrow { + border-bottom: 6px solid transparent; + border-left: 6px solid var(--tabulator-sort-arrow-active); + border-right: 0; + border-top: 6px solid transparent; + display: inline-block; + height: 0; + margin-right: 16px; + vertical-align: middle; + width: 0 +} + +.tabulator-print-table .tabulator-print-table-group span { + color: #d00 +} + +.tabulator-print-table .tabulator-data-tree-control { + align-items: center; + background: rgba(0, 0, 0, .1); + border: 1px solid var(--tabulator-row-text-color); + border-radius: 2px; + display: inline-flex; + height: 11px; + justify-content: center; + margin-right: 5px; + overflow: hidden; + vertical-align: middle; + width: 11px +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-print-table .tabulator-data-tree-control:hover { + background: rgba(0, 0, 0, .2); + cursor: pointer + } +} + +.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-collapse { + background: transparent; + display: inline-block; + height: 7px; + position: relative; + width: 1px +} + +.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after { + background: var(--tabulator-row-text-color); + content: ""; + height: 1px; + left: -3px; + position: absolute; + top: 3px; + width: 7px +} + +.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand { + background: var(--tabulator-row-text-color); + display: inline-block; + height: 7px; + position: relative; + width: 1px +} + +.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand:after { + background: var(--tabulator-row-text-color); + content: ""; + height: 1px; + left: -3px; + position: absolute; + top: 3px; + width: 7px +} + +.tabulator { + background-color: var(--tabulator-background-color); + max-width: 100%; + width: 100% +} + +.tabulator .tabulator-header { + color: inherit +} + +.tabulator .tabulator-header .tabulator-col { + border-top: none +} + +.tabulator .tabulator-header .tabulator-col:first-of-type { + border-left: none +} + +.tabulator .tabulator-header .tabulator-col:last-of-type { + border-right: none +} + +.tabulator .tabulator-header .tabulator-col:not(first-of-type), +.tabulator .tabulator-header .tabulator-col:not(last-of-type) { + border-right: 1px solid var(--tabulator-header-border-color) +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content { + padding: var(--tabulator-cell-padding) +} + +.tabulator .tabulator-header .tabulator-col .tabulator-col-content .tabulator-col-sorter { + right: -10px +} + +.tabulator .tabulator-header .tabulator-col.tabulator-col-group .tabulator-col-group-cols { + border-top: 1px solid var(--tabulator-border-color) +} + +.tabulator .tabulator-header .tabulator-col.tabulator-sortable .tabulator-col-title { + padding-right: 10px +} + +.tabulator .tabulator-header .tabulator-calcs-holder { + border-bottom: 1px solid var(--tabulator-header-separator-color); + width: 100% +} + +.tabulator .tabulator-header .tabulator-frozen-rows-holder { + min-width: 600% +} + +.tabulator .tabulator-header .tabulator-frozen-rows-holder:empty { + display: none +} + +.tabulator .tabulator-header .tabulator-frozen .tabulator-frozen-left, +.tabulator .tabulator-header .tabulator-frozen .tabulator-frozen-right { + background: inherit +} + +.tabulator .tabulator-tableholder .tabulator-table { + color: inherit +} + +.tabulator .tabulator-footer { + background-color: var(--tabulator-footer-background-color); + color: inherit +} + +.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab { + font-weight: 400; + padding: 8px 12px +} + +.tabulator .tabulator-footer .tabulator-spreadsheet-tabs .tabulator-spreadsheet-tab.tabulator-spreadsheet-tab-active { + color: var(--tabulator-footer-active-color) +} + +.tabulator .tabulator-footer .tabulator-paginator { + color: inherit +} + +.tabulator .tabulator-footer .tabulator-page { + background: var(--tabulator-pagination-button-background); + border-radius: 0; + border-right: none; + color: var(--tabulator-pagination-button-color); + margin: 5px 0 0; + padding: 8px 12px +} + +.tabulator .tabulator-footer .tabulator-page:first-of-type, +.tabulator .tabulator-footer .tabulator-page[data-page=next] { + border-bottom-left-radius: 4px; + border-top-left-radius: 4px +} + +.tabulator .tabulator-footer .tabulator-page:last-of-type, +.tabulator .tabulator-footer .tabulator-page[data-page=prev] { + border: 1px solid var(--tabulator-footer-border-color); + border-bottom-right-radius: 4px; + border-top-right-radius: 4px +} + +.tabulator .tabulator-footer .tabulator-page:not(disabled):hover { + background: var(--tabulator-pagination-button-background-hover); + color: var(--tabulator-pagination-button-color-hover) +} + +.tabulator .tabulator-footer .tabulator-page.active, +.tabulator .tabulator-footer .tabulator-page[data-page=first] :not(disabled):not(:hover), +.tabulator .tabulator-footer .tabulator-page[data-page=last] :not(disabled):not(:hover), +.tabulator .tabulator-footer .tabulator-page[data-page=next] :not(disabled):not(:hover), +.tabulator .tabulator-footer .tabulator-page[data-page=prev] :not(disabled):not(:hover) { + color: var(--tabulator-pagination-button-color-active) +} + +.tabulator.striped .tabulator-row:nth-child(2n) { + background-color: var(--tabulator-row-alt-background-color) +} + +.tabulator.striped .tabulator-row:nth-child(2n).tabulator-selected { + background-color: var(--tabulator-row-selected-background) !important +} + +@media (hover:hover) and (pointer:fine) { + .tabulator.striped .tabulator-row:nth-child(2n).tabulator-selectable:hover { + background-color: var(--tabulator-row-hover-background); + cursor: pointer + } + + .tabulator.striped .tabulator-row:nth-child(2n).tabulator-selected:hover { + background-color: var(--tabulator-row-selected-background-hover) !important; + cursor: pointer + } +} + +.tabulator-row { + border-bottom: 1px solid var(--tabulator-row-border-color); + min-height: calc(var(--tabulator-text-size) + var(--tabulator-cell-padding)*2) +} + +.tabulator-row.tabulator-row-even { + background-color: var(--tabulator-row-background-color) +} + +.tabulator-row .tabulator-cell { + padding: var(--tabulator-cell-padding) +} + +.tabulator-row .tabulator-cell:last-of-type { + border-right: none +} + +.tabulator-row .tabulator-cell.tabulator-row-header { + background: var(--tabulator-header-background-color); + border-bottom: none; + border-right: 1px solid var(--tabulator-border-color) +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control { + border: 1px solid #ccc +} + +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after, +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand, +.tabulator-row .tabulator-cell .tabulator-data-tree-control .tabulator-data-tree-control-expand:after { + background: #ccc +} + +.tabulator-row.tabulator-group { + background: #fafafa +} + +.tabulator-row.tabulator-group span { + color: #666; + margin-left: 10px +} + +.tabulator-edit-select-list { + background: var(--tabulator-header-background-color) +} + +.tabulator-edit-select-list .tabulator-edit-select-list-item { + color: inherit +} + +.tabulator-edit-select-list .tabulator-edit-select-list-item.active { + color: var(--tabulator-header-background-color) +} + +.tabulator-edit-select-list .tabulator-edit-select-list-item.active.focused { + outline: 1px solid rgba(var(--tabulator-header-background-color), .5) +} + +@media (hover:hover) and (pointer:fine) { + .tabulator-edit-select-list .tabulator-edit-select-list-item:hover { + color: var(--tabulator-header-background-color) + } +} + +.tabulator-edit-select-list .tabulator-edit-select-list-group, +.tabulator-edit-select-list .tabulator-edit-select-list-notice { + color: inherit +} + +.tabulator.tabulator-rtl .tabulator-header .tabulator-col { + border-left: none; + border-right: none +} + +.tabulator-print-table .tabulator-print-table-group { + background: #fafafa +} + +.tabulator-print-table .tabulator-print-table-group span { + color: #666; + margin-left: 10px +} + +.tabulator-print-table .tabulator-data-tree-control { + border: 1px solid #ccc +} + +.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-collapse:after, +.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand, +.tabulator-print-table .tabulator-data-tree-control .tabulator-data-tree-control-expand:after { + background: #ccc +} + /*# sourceMappingURL=tabulator_pysyft.min.css.map */ \ No newline at end of file diff --git a/packages/syft/src/syft/assets/jinja/table.jinja2 b/packages/syft/src/syft/assets/jinja/table.jinja2 index 1eb580ef01a..f750a80d0ab 100644 --- a/packages/syft/src/syft/assets/jinja/table.jinja2 +++ b/packages/syft/src/syft/assets/jinja/table.jinja2 @@ -38,11 +38,13 @@ diff --git a/packages/syft/src/syft/assets/js/table.js b/packages/syft/src/syft/assets/js/table.js index 1a257627d6a..ff6c5e84d12 100644 --- a/packages/syft/src/syft/assets/js/table.js +++ b/packages/syft/src/syft/assets/js/table.js @@ -6,7 +6,6 @@ TABULATOR_CSS = document.querySelectorAll(".escape-unfocus").forEach((input) => { input.addEventListener("keydown", (event) => { if (event.key === "Escape") { - console.log("Escape key pressed"); event.stopPropagation(); input.blur(); } @@ -58,7 +57,14 @@ function load_tabulator(elementId) { }); } -function buildTable(columns, rowHeader, data, uid) { +function buildTable( + columns, + rowHeader, + data, + uid, + pagination = true, + maxHeight = null, +) { const tableId = `table-${uid}`; const searchBarId = `search-${uid}`; const numrowsId = `numrows-${uid}`; @@ -73,11 +79,13 @@ function buildTable(columns, rowHeader, data, uid) { data: data, columns: columns, rowHeader: rowHeader, + index: "_table_repr_index", layout: "fitDataStretch", resizableColumnFit: true, resizableColumnGuide: true, - pagination: "local", + pagination: pagination, paginationSize: 5, + maxHeight: maxHeight, }); // Events needed for cell overflow: @@ -100,6 +108,7 @@ function buildTable(columns, rowHeader, data, uid) { numrowsElement.innerHTML = data.length; } + configureHighlightSingleRow(table, uid); configureSearch(table, searchBarId, columns); return table; @@ -129,3 +138,64 @@ function configureSearch(table, searchBarId, columns) { table.setFilter([filterArray]); }); } + +function configureHighlightSingleRow(table, uid) { + // Listener for rowHighlight events, with fields: + // uid: string, table uid + // index: number | string, row index to highlight + // jumpToRow: bool, if true, jumps to page where the row is located + document.addEventListener("rowHighlight", function (e) { + if (e.detail.uid === uid) { + let row_idx = e.detail.index; + let rows = table.getRows(); + for (let row of rows) { + if (row.getIndex() == row_idx) { + row.select(); + if (e.detail.jumpToRow) { + table.setPageToRow(row_idx); + table.scrollToRow(row_idx, "top", false); + } + } else { + row.deselect(); + } + } + } + }); +} + +function waitForTable(uid, timeout = 1000) { + return new Promise((resolve, reject) => { + // Check if the table is ready immediately + if (window["table_" + uid]) { + resolve(); + } else { + // Otherwise, check every 100ms until the table is ready or the timeout is reached + var startTime = Date.now(); + var checkTableInterval = setInterval(function () { + if (window["table_" + uid]) { + clearInterval(checkTableInterval); + resolve(); + } else if (Date.now() - startTime > timeout) { + clearInterval(checkTableInterval); + reject(`Timeout: table_"${uid}" not found.`); + } + }, 100); + } + }); +} + +function highlightSingleRow(uid, index = null, jumpToRow = false) { + // Highlight a single row in the table with the given uid + // If index is not provided or doesn't exist, all rows are deselected + waitForTable(uid) + .then(() => { + document.dispatchEvent( + new CustomEvent("rowHighlight", { + detail: { uid, index, jumpToRow }, + }), + ); + }) + .catch((error) => { + console.log(error); + }); +} diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index 3cb360eafa4..f176f852afa 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -11,6 +11,7 @@ from typing import Literal # third party +import ipywidgets from loguru import logger import pandas as pd from pydantic import model_validator @@ -1118,6 +1119,12 @@ class NodeDiff(SyftObject): include_ignored: bool = False + def resolve(self) -> ipywidgets.Widget: + # relative + from .resolve_widget import PaginatedResolveWidget + + return PaginatedResolveWidget(batches=self.batches).build() + def __getitem__(self, idx: Any) -> ObjectDiffBatch: return self.batches[idx] diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index dd9dadc505e..9d0f21cdcda 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -1,11 +1,14 @@ # stdlib +from collections.abc import Callable from enum import Enum from enum import auto import html +import secrets from typing import Any from uuid import uuid4 # third party +from IPython import display import ipywidgets as widgets from ipywidgets import Button from ipywidgets import Checkbox @@ -22,6 +25,8 @@ from ...util.notebook_ui.components.sync import MainDescription from ...util.notebook_ui.components.sync import SyncWidgetHeader from ...util.notebook_ui.components.sync import TypeLabel +from ...util.notebook_ui.components.tabulator_template import build_tabulator_table +from ...util.notebook_ui.components.tabulator_template import highlight_single_row from ...util.notebook_ui.styles import CSS_CODE from ..action.action_object import ActionObject from ..api.api import TwinAPIEndpoint @@ -590,3 +595,158 @@ def separator(self) -> widgets.HTML: def build_header(self) -> HTML: header_html = SyncWidgetHeader(diff_batch=self.obj_diff_batch).to_html() return HTML(value=header_html) + + +class PaginationControl: + def __init__(self, data: list, callback: Callable[[int], None]): + self.data = data + self.callback = callback + self.current_index = 0 + self.index_label = widgets.Label(value=f"Index: {self.current_index}") + + self.first_button = widgets.Button(description="First") + self.previous_button = widgets.Button(description="Previous") + self.next_button = widgets.Button(description="Next") + self.last_button = widgets.Button(description="Last") + + self.first_button.on_click(self.go_to_first) + self.previous_button.on_click(self.go_to_previous) + self.next_button.on_click(self.go_to_next) + self.last_button.on_click(self.go_to_last) + self.output = widgets.Output() + + self.buttons = widgets.HBox( + [ + self.first_button, + self.previous_button, + self.next_button, + self.last_button, + ] + ) + self.update_buttons() + self.update_index_callback() + + def update_index_label(self) -> None: + self.index_label.value = f"Current: {self.current_index}" + + def update_buttons(self) -> None: + self.first_button.disabled = self.current_index == 0 + self.previous_button.disabled = self.current_index == 0 + self.next_button.disabled = self.current_index == len(self.data) - 1 + self.last_button.disabled = self.current_index == len(self.data) - 1 + + def go_to_first(self, b: Button) -> None: + self.current_index = 0 + self.update_index_callback() + + def go_to_previous(self, b: Button) -> None: + if self.current_index > 0: + self.current_index -= 1 + self.update_index_callback() + + def go_to_next(self, b: Button) -> None: + if self.current_index < len(self.data) - 1: + self.current_index += 1 + self.update_index_callback() + + def go_to_last(self, b: Button) -> None: + self.current_index = len(self.data) - 1 + self.update_index_callback() + + def update_index_callback(self) -> None: + self.update_index_label() + self.update_buttons() + + # NOTE self.output is required to display IPython.display.HTML + # IPython.display.HTML is used to execute JS code + with self.output: + self.callback(self.current_index) + + def build(self) -> widgets.VBox: + return widgets.VBox( + [widgets.HBox([self.buttons, self.index_label]), self.output] + ) + + +class PaginatedWidget: + def __init__( + self, children: list, on_paginate_callback: Callable[[int], None] | None = None + ): + # on_paginate_callback is an optional secondary callback, + # called after updating the page index and displaying the new widget + self.children = children + self.on_paginate_callback = on_paginate_callback + self.current_index = 0 + self.container = widgets.VBox() + + self.pagination_control = PaginationControl(children, self.on_paginate) + + # Initial display + self.on_paginate(self.pagination_control.current_index) + + def __getitem__(self, index: int) -> widgets.Widget: + return self.children[index] + + def on_paginate(self, index: int) -> None: + self.container.children = [self.children[index]] + if self.on_paginate_callback: + self.on_paginate_callback(index) + + def build(self) -> widgets.VBox: + return widgets.VBox([self.pagination_control.build(), self.container]) + + +class PaginatedResolveWidget: + """ + PaginatedResolveWidget is a widget that displays + a ResolveWidget for each ObjectDiffBatch, + paginated by a PaginationControl widget. + """ + + def __init__(self, batches: list[ObjectDiffBatch]): + self.batches = batches + self.resolve_widgets = [ + ResolveWidget(obj_diff_batch=batch) for batch in self.batches + ] + + self.table_uid = secrets.token_hex(4) + + # Disable the table pagination to avoid the double pagination buttons + self.batch_table = build_tabulator_table( + obj=batches, + uid=self.table_uid, + max_height=500, + pagination=False, + ) + + self.paginated_widget = PaginatedWidget( + children=[widget.widget for widget in self.resolve_widgets], + on_paginate_callback=self.on_paginate, + ) + + self.table_output = widgets.Output() + with self.table_output: + display.display(display.HTML(self.batch_table)) + highlight_single_row( + self.table_uid, self.paginated_widget.current_index, jump_to_row=True + ) + + def on_paginate(self, index: int) -> None: + return highlight_single_row(self.table_uid, index, jump_to_row=True) + + def build(self) -> widgets.VBox: + return widgets.VBox([self.table_output, self.paginated_widget.build()]) + + def click_sync(self, index: int) -> SyftSuccess | SyftError: + return self.resolve_widgets[index].click_sync() + + def click_share_all_private_data(self, index: int) -> None: + self.resolve_widgets[index].click_share_all_private_data() + + def _share_all(self) -> None: + for widget in self.resolve_widgets: + widget.click_share_all_private_data() + + def _sync_all(self) -> None: + for widget in self.resolve_widgets: + widget.click_sync() diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index 60ba8da4915..e79016ab7a9 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -92,7 +92,12 @@ def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: return formatted -def build_tabulator_table(obj: Any) -> str | None: +def build_tabulator_table( + obj: Any, + uid: str | None = None, + max_height: int | None = None, + pagination: bool = True, +) -> str | None: try: table_data, table_metadata = prepare_table_data(obj) if len(table_data) == 0: @@ -113,10 +118,11 @@ def build_tabulator_table(obj: Any) -> str | None: if icon is None: icon = Icon.TABLE.svg + uid = uid if uid is not None else secrets.token_hex(4) column_data, row_header = create_tabulator_columns(table_metadata["columns"]) table_data = format_table_data(table_data) table_html = table_template.render( - uid=secrets.token_hex(4), + uid=uid, columns=json.dumps(column_data), row_header=json.dumps(row_header), data=json.dumps(table_data), @@ -127,6 +133,8 @@ def build_tabulator_table(obj: Any) -> str | None: name=table_metadata["name"], tabulator_js=tabulator_js, tabulator_css=tabulator_css, + max_height=json.dumps(max_height), + pagination=json.dumps(pagination), ) return table_html @@ -140,3 +148,12 @@ def show_table(obj: Any) -> None: table = build_tabulator_table(obj) if table is not None: display(HTML(table)) + + +def highlight_single_row( + table_uid: str, + index: int | str | None = None, + jump_to_row: bool = True, +) -> None: + js_code = f"" + display(HTML(js_code)) From 667c4d4d0c940cc4036bc38b003cf0f4816029a8 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Mon, 27 May 2024 10:34:34 +0200 Subject: [PATCH 053/100] minor fixes --- packages/syft/src/syft/service/sync/diff_state.py | 2 +- packages/syft/src/syft/service/sync/resolve_widget.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index f176f852afa..1b2b36bf2f6 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1123,7 +1123,7 @@ def resolve(self) -> ipywidgets.Widget: # relative from .resolve_widget import PaginatedResolveWidget - return PaginatedResolveWidget(batches=self.batches).build() + return PaginatedResolveWidget(batches=self.batches) def __getitem__(self, idx: Any) -> ObjectDiffBatch: return self.batches[idx] diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 9d0f21cdcda..41aa071be18 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -731,6 +731,11 @@ def __init__(self, batches: list[ObjectDiffBatch]): self.table_uid, self.paginated_widget.current_index, jump_to_row=True ) + self.widget = self.build() + + def __getitem__(self, index: int) -> ResolveWidget: + return self.resolve_widgets[index] + def on_paginate(self, index: int) -> None: return highlight_single_row(self.table_uid, index, jump_to_row=True) @@ -750,3 +755,6 @@ def _share_all(self) -> None: def _sync_all(self) -> None: for widget in self.resolve_widgets: widget.click_sync() + + def _repr_mimebundle_(self, **kwargs: dict) -> dict[str, str] | None: + return self.widget._repr_mimebundle_(**kwargs) From 6262647b50adebc129475264211011226d5c51bd Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Mon, 27 May 2024 14:51:55 +0200 Subject: [PATCH 054/100] add decision column, disable sort --- .../syft/src/syft/assets/jinja/table.jinja2 | 1 + packages/syft/src/syft/assets/js/table.js | 29 ++++++++++++- .../syft/src/syft/service/sync/diff_state.py | 19 ++++++++- .../src/syft/service/sync/resolve_widget.py | 41 +++++++++++++++---- .../components/tabulator_template.py | 19 ++++++++- 5 files changed, 97 insertions(+), 12 deletions(-) diff --git a/packages/syft/src/syft/assets/jinja/table.jinja2 b/packages/syft/src/syft/assets/jinja/table.jinja2 index f750a80d0ab..7a44d798540 100644 --- a/packages/syft/src/syft/assets/jinja/table.jinja2 +++ b/packages/syft/src/syft/assets/jinja/table.jinja2 @@ -45,6 +45,7 @@ "{{ uid }}", pagination={{ pagination }}, maxHeight={{ max_height }}, + headerSort={{ header_sort }}, ) diff --git a/packages/syft/src/syft/assets/js/table.js b/packages/syft/src/syft/assets/js/table.js index ff6c5e84d12..35fee482bd9 100644 --- a/packages/syft/src/syft/assets/js/table.js +++ b/packages/syft/src/syft/assets/js/table.js @@ -64,6 +64,7 @@ function buildTable( uid, pagination = true, maxHeight = null, + headerSort = true, ) { const tableId = `table-${uid}`; const searchBarId = `search-${uid}`; @@ -86,6 +87,7 @@ function buildTable( pagination: pagination, paginationSize: 5, maxHeight: maxHeight, + headerSort: headerSort, }); // Events needed for cell overflow: @@ -152,7 +154,8 @@ function configureHighlightSingleRow(table, uid) { if (row.getIndex() == row_idx) { row.select(); if (e.detail.jumpToRow) { - table.setPageToRow(row_idx); + // catch promise in case the table does not have pagination + table.setPageToRow(row_idx).catch((_) => {}); table.scrollToRow(row_idx, "top", false); } } else { @@ -169,7 +172,7 @@ function waitForTable(uid, timeout = 1000) { if (window["table_" + uid]) { resolve(); } else { - // Otherwise, check every 100ms until the table is ready or the timeout is reached + // Otherwise, poll until the table is ready or timeout var startTime = Date.now(); var checkTableInterval = setInterval(function () { if (window["table_" + uid]) { @@ -199,3 +202,25 @@ function highlightSingleRow(uid, index = null, jumpToRow = false) { console.log(error); }); } + +function updateTableCell(uid, index, field, value) { + // Update the value of a cell in the table with the given uid + waitForTable(uid) + .then(() => { + const table = window["table_" + uid]; + if (!table) { + throw new Error(`Table with uid ${uid} not found.`); + } + + const row = table.getRow(index); + if (!row) { + throw new Error(`Row with index ${index} not found.`); + } + + // Update the cell value + row.update({ [field]: value }); + }) + .catch((error) => { + console.error(error); + }); +} diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index 1b2b36bf2f6..efef97e6335 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -39,6 +39,7 @@ from ...types.uid import UID from ...util import options from ...util.colors import SURFACE +from ...util.notebook_ui.components.sync import Label from ...util.notebook_ui.components.sync import SyncTableObject from ...util.notebook_ui.icons import Icon from ...util.notebook_ui.styles import FONT_CSS @@ -705,6 +706,21 @@ def root_id(self) -> UID: def root_type(self) -> type: return self.root_diff.obj_type + def decision_badge(self) -> str: + if self.decision is None: + return "" + if self.decision == SyncDecision.IGNORE: + decision_str = "IGNORED" + badge_color = "label-red" + if self.decision == SyncDecision.SKIP: + decision_str = "SKIPPED" + badge_color = "label-gray" + else: + decision_str = "SYNCED" + badge_color = "label-green" + + return Label(value=decision_str, label_class=badge_color).to_html() + @property def is_ignored(self) -> bool: return self.decision == SyncDecision.IGNORE @@ -847,9 +863,10 @@ def _coll_repr_(self) -> dict[str, Any]: high_html = SyncTableObject(object=self.root_diff.high_obj).to_html() return { - "Merge status": self.status_badge(), + "Diff status": self.status_badge(), "Public Sync State": low_html, "Private sync state": high_html, + "Decision": self.decision_badge(), } @property diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 41aa071be18..60856d752f2 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -2,6 +2,7 @@ from collections.abc import Callable from enum import Enum from enum import auto +from functools import partial import html import secrets from typing import Any @@ -27,6 +28,7 @@ from ...util.notebook_ui.components.sync import TypeLabel from ...util.notebook_ui.components.tabulator_template import build_tabulator_table from ...util.notebook_ui.components.tabulator_template import highlight_single_row +from ...util.notebook_ui.components.tabulator_template import update_table_cell from ...util.notebook_ui.styles import CSS_CODE from ..action.action_object import ActionObject from ..api.api import TwinAPIEndpoint @@ -411,11 +413,14 @@ def _on_share_private_data_change(self, change: Any) -> None: class ResolveWidget: - def __init__(self, obj_diff_batch: ObjectDiffBatch): + def __init__( + self, obj_diff_batch: ObjectDiffBatch, on_sync_callback: Callable | None = None + ): self.obj_diff_batch: ObjectDiffBatch = obj_diff_batch self.id2widget: dict[ UID, CollapsableObjectDiffWidget | MainObjectDiffWidget ] = {} + self.on_sync_callback = on_sync_callback self.main_widget = self.build() self.result_widget = VBox() # Placeholder for SyftSuccess / SyftError self.widget = VBox( @@ -468,6 +473,8 @@ def click_sync(self, *args: list, **kwargs: dict) -> SyftSuccess | SyftError: ) self.set_widget_result_state(res) + if self.on_sync_callback: + self.on_sync_callback() return res @property @@ -635,21 +642,21 @@ def update_buttons(self) -> None: self.next_button.disabled = self.current_index == len(self.data) - 1 self.last_button.disabled = self.current_index == len(self.data) - 1 - def go_to_first(self, b: Button) -> None: + def go_to_first(self, b: Button | None) -> None: self.current_index = 0 self.update_index_callback() - def go_to_previous(self, b: Button) -> None: + def go_to_previous(self, b: Button | None) -> None: if self.current_index > 0: self.current_index -= 1 self.update_index_callback() - def go_to_next(self, b: Button) -> None: + def go_to_next(self, b: Button | None) -> None: if self.current_index < len(self.data) - 1: self.current_index += 1 self.update_index_callback() - def go_to_last(self, b: Button) -> None: + def go_to_last(self, b: Button | None) -> None: self.current_index = len(self.data) - 1 self.update_index_callback() @@ -705,8 +712,12 @@ class PaginatedResolveWidget: def __init__(self, batches: list[ObjectDiffBatch]): self.batches = batches - self.resolve_widgets = [ - ResolveWidget(obj_diff_batch=batch) for batch in self.batches + self.resolve_widgets: list[ResolveWidget] = [ + ResolveWidget( + obj_diff_batch=batch, + on_sync_callback=partial(self.on_click_sync, i), + ) + for i, batch in enumerate(self.batches) ] self.table_uid = secrets.token_hex(4) @@ -717,6 +728,7 @@ def __init__(self, batches: list[ObjectDiffBatch]): uid=self.table_uid, max_height=500, pagination=False, + header_sort=False, ) self.paginated_widget = PaginatedWidget( @@ -733,6 +745,21 @@ def __init__(self, batches: list[ObjectDiffBatch]): self.widget = self.build() + def on_click_sync(self, index: int) -> None: + self.update_table_sync_decision(index) + if self.batches[index].decision is not None: + self.paginated_widget.pagination_control.go_to_next(None) + + def update_table_sync_decision(self, index: int) -> None: + new_decision = self.batches[index].decision_badge() + with self.table_output: + update_table_cell( + uid=self.table_uid, + index=index, + field="Decision", + value=new_decision, + ) + def __getitem__(self, index: int) -> ResolveWidget: return self.resolve_widgets[index] diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index e79016ab7a9..ee0576cc206 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -23,6 +23,7 @@ def create_tabulator_columns( column_names: list[str], column_widths: dict | None = None, + header_sort: bool = True, ) -> tuple[list[dict], dict | None]: """Returns tuple of (columns, row_header) for tabulator table""" if column_widths is None: @@ -33,10 +34,10 @@ def create_tabulator_columns( if TABLE_INDEX_KEY in column_names: row_header = { "field": TABLE_INDEX_KEY, - "headerSort": True, "frozen": True, "widthGrow": 0.3, "minWidth": 60, + "headerSort": header_sort, } for colname in column_names: @@ -48,6 +49,7 @@ def create_tabulator_columns( "resizable": True, "minWidth": 60, "maxInitialWidth": 500, + "headerSort": header_sort, } if colname in column_widths: column["widthGrow"] = column_widths[colname] @@ -97,6 +99,7 @@ def build_tabulator_table( uid: str | None = None, max_height: int | None = None, pagination: bool = True, + header_sort: bool = True, ) -> str | None: try: table_data, table_metadata = prepare_table_data(obj) @@ -119,7 +122,9 @@ def build_tabulator_table( icon = Icon.TABLE.svg uid = uid if uid is not None else secrets.token_hex(4) - column_data, row_header = create_tabulator_columns(table_metadata["columns"]) + column_data, row_header = create_tabulator_columns( + table_metadata["columns"], header_sort=header_sort + ) table_data = format_table_data(table_data) table_html = table_template.render( uid=uid, @@ -135,6 +140,7 @@ def build_tabulator_table( tabulator_css=tabulator_css, max_height=json.dumps(max_height), pagination=json.dumps(pagination), + header_sort=json.dumps(header_sort), ) return table_html @@ -157,3 +163,12 @@ def highlight_single_row( ) -> None: js_code = f"" display(HTML(js_code)) + + +def update_table_cell(uid: str, index: int, field: str, value: str) -> None: + js_code = f""" + + """ + display(HTML(js_code)) From 17cc6d384af599394bc2bbda0aacacb0e79b7f66 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Mon, 27 May 2024 15:08:08 +0200 Subject: [PATCH 055/100] fix old resolve method + typing --- packages/syft/src/syft/client/syncing.py | 12 +++++++++--- packages/syft/src/syft/service/sync/diff_state.py | 8 ++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/syft/src/syft/client/syncing.py b/packages/syft/src/syft/client/syncing.py index 7185cb5316e..52bf80a5486 100644 --- a/packages/syft/src/syft/client/syncing.py +++ b/packages/syft/src/syft/client/syncing.py @@ -86,9 +86,15 @@ def get_user_input_for_resolve() -> SyncDecision: print(f"Please choose between {options_str}") -def resolve(obj_diff_batch: ObjectDiffBatch) -> ResolveWidget: - widget = ResolveWidget(obj_diff_batch) - return widget +def resolve(obj: ObjectDiffBatch | NodeDiff) -> ResolveWidget: + if isinstance(obj, NodeDiff): + return obj.resolve() + elif isinstance(obj, ObjectDiffBatch): + return ResolveWidget(obj) + else: + raise ValueError( + f"Invalid type: could not resolve object with type {type(obj).__qualname__}" + ) @deprecated(reason="resolve_single has been renamed to resolve", return_syfterror=True) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index efef97e6335..e91b405136a 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -9,9 +9,9 @@ from typing import Any from typing import ClassVar from typing import Literal +from typing import TYPE_CHECKING # third party -import ipywidgets from loguru import logger import pandas as pd from pydantic import model_validator @@ -61,6 +61,10 @@ from ..user.user import UserView from .sync_state import SyncState +if TYPE_CHECKING: + # relative + from .resolve_widget import PaginatedResolveWidget + sketchy_tab = "‎ " * 4 @@ -1136,7 +1140,7 @@ class NodeDiff(SyftObject): include_ignored: bool = False - def resolve(self) -> ipywidgets.Widget: + def resolve(self) -> "PaginatedResolveWidget": # relative from .resolve_widget import PaginatedResolveWidget From 810bd655210d73af61f3abd6d319b7c3540e3e71 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Mon, 27 May 2024 16:36:16 +0200 Subject: [PATCH 056/100] add comments --- packages/syft/src/syft/client/syncing.py | 29 +++++-------------- .../syft/src/syft/service/sync/diff_state.py | 7 +++++ .../src/syft/service/sync/resolve_widget.py | 7 ++++- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/packages/syft/src/syft/client/syncing.py b/packages/syft/src/syft/client/syncing.py index 52bf80a5486..428117634ef 100644 --- a/packages/syft/src/syft/client/syncing.py +++ b/packages/syft/src/syft/client/syncing.py @@ -8,6 +8,7 @@ from ..service.sync.diff_state import NodeDiff from ..service.sync.diff_state import ObjectDiffBatch from ..service.sync.diff_state import SyncInstruction +from ..service.sync.resolve_widget import PaginatedResolveWidget from ..service.sync.resolve_widget import ResolveWidget from ..service.sync.sync_state import SyncState from ..types.uid import UID @@ -71,34 +72,18 @@ def compare_clients( ) -def get_user_input_for_resolve() -> SyncDecision: - options = [x.value for x in SyncDecision] - options_str = ", ".join(options[:-1]) + f" or {options[-1]}" - print(f"How do you want to sync these objects? choose between {options_str}") - - while True: - decision = input() - decision = decision.lower() - - try: - return SyncDecision(decision) - except ValueError: - print(f"Please choose between {options_str}") - - -def resolve(obj: ObjectDiffBatch | NodeDiff) -> ResolveWidget: - if isinstance(obj, NodeDiff): - return obj.resolve() - elif isinstance(obj, ObjectDiffBatch): - return ResolveWidget(obj) - else: +def resolve(obj: ObjectDiffBatch | NodeDiff) -> ResolveWidget | PaginatedResolveWidget: + if not isinstance(obj, ObjectDiffBatch | NodeDiff): raise ValueError( f"Invalid type: could not resolve object with type {type(obj).__qualname__}" ) + return obj.resolve() @deprecated(reason="resolve_single has been renamed to resolve", return_syfterror=True) -def resolve_single(obj_diff_batch: ObjectDiffBatch) -> ResolveWidget: +def resolve_single( + obj_diff_batch: ObjectDiffBatch, +) -> ResolveWidget | PaginatedResolveWidget: return resolve(obj_diff_batch) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index e91b405136a..f943174d75a 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -64,6 +64,7 @@ if TYPE_CHECKING: # relative from .resolve_widget import PaginatedResolveWidget + from .resolve_widget import ResolveWidget sketchy_tab = "‎ " * 4 @@ -560,6 +561,12 @@ class ObjectDiffBatch(SyftObject): root_diff: ObjectDiff sync_direction: SyncDirection | None + def resolve(self) -> "ResolveWidget": + # relative + from .resolve_widget import ResolveWidget + + return ResolveWidget(self) + def walk_graph( self, deps: dict[UID, list[UID]], diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 60856d752f2..7d683139aae 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -699,8 +699,13 @@ def on_paginate(self, index: int) -> None: if self.on_paginate_callback: self.on_paginate_callback(index) + def spacer(self, height: int) -> widgets.HTML: + return widgets.HTML(f"
") + def build(self) -> widgets.VBox: - return widgets.VBox([self.pagination_control.build(), self.container]) + return widgets.VBox( + [self.pagination_control.build(), self.spacer(8), self.container] + ) class PaginatedResolveWidget: From 307c4b8f6a5a3b0d60d75d82f00c2947594a2994 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Mon, 27 May 2024 16:50:06 +0200 Subject: [PATCH 057/100] fix tests --- packages/syft/src/syft/service/sync/resolve_widget.py | 2 +- .../syft/tests/syft/service/sync/sync_resolve_single_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 7d683139aae..261ed28e075 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -719,7 +719,7 @@ def __init__(self, batches: list[ObjectDiffBatch]): self.batches = batches self.resolve_widgets: list[ResolveWidget] = [ ResolveWidget( - obj_diff_batch=batch, + batch, on_sync_callback=partial(self.on_click_sync, i), ) for i, batch in enumerate(self.batches) diff --git a/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py b/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py index b3972532521..07585c6de87 100644 --- a/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py +++ b/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py @@ -36,7 +36,7 @@ def compare_and_resolve( diff_state_before = compare_clients(from_client, to_client) for obj_diff_batch in diff_state_before.active_batches: widget = resolve( - obj_diff_batch=obj_diff_batch, + obj_diff_batch, ) if decision_callback: decision = decision_callback(obj_diff_batch) From 5a931f675da29a04c2b85ed0301bcb838da65939 Mon Sep 17 00:00:00 2001 From: Julian Cardonnet Date: Thu, 16 May 2024 17:23:29 -0300 Subject: [PATCH 058/100] Use autosplat for settings update. Add docstring to Update service --- .../syft/service/settings/settings_service.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index 30e8242ceb9..f93a019e26a 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -61,10 +61,36 @@ def set( else: return SyftError(message=result.err()) - @service_method(path="settings.update", name="update") + @service_method(path="settings.update", name="update", autosplat=["settings"]) def update( self, context: AuthedServiceContext, settings: NodeSettingsUpdate ) -> Result[SyftSuccess, SyftError]: + """ + Update the Node Settings using the provided values. + + Args: + name: Optional[str] + Node name + organization: Optional[str] + Organization name + description: Optional[str] + Node description + on_board: Optional[bool] + Show onboarding panel when a user logs in for the first time + signup_enabled: Optional[bool] + Enable/Disable registration + admin_email: Optional[str] + Administrator email + association_request_auto_approval: Optional[bool] + + Returns: + Result[SyftSuccess, SyftError]: A result indicating the success or failure of the update operation. + + Example: + >>> node_client.update(name='foo', organization='bar', description='baz', signup_enabled=True) + SyftSuccess: Settings updated successfully. + """ + result = self.stash.get_all(context.credentials) if result.is_ok(): current_settings = result.ok() From b1d96e43ab6d38f8920a6e426b9099a48676de3a Mon Sep 17 00:00:00 2001 From: Ionesio Junior Date: Mon, 20 May 2024 12:27:10 -0300 Subject: [PATCH 059/100] Fix autosplat when it's called from inner services --- packages/syft/src/syft/service/service.py | 3 ++- packages/syft/src/syft/service/settings/settings_service.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index c28fc1157d1..e7ccd5c25ad 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -340,6 +340,7 @@ def wrapper(func: Any) -> Callable: _path = class_name + "." + func_name signature = inspect.signature(func) signature = signature_remove_self(signature) + signature_with_context = deepcopy(signature) signature = signature_remove_context(signature) input_signature = deepcopy(signature) @@ -353,7 +354,7 @@ def _decorator(self: Any, *args: Any, **kwargs: Any) -> Callable: ) if autosplat is not None and len(autosplat) > 0: args, kwargs = reconstruct_args_kwargs( - signature=input_signature, + signature=signature_with_context, autosplat=autosplat, args=args, kwargs=kwargs, diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index f93a019e26a..80ac0cdcc55 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -165,7 +165,7 @@ def allow_guest_signup( result = method(context=context, settings=settings) - if result.is_err(): + if isinstance(result, SyftError): return SyftError(message=f"Failed to update settings: {result.err()}") message = "enabled" if enable else "disabled" From 0245797694e60a3d8ccdc59bfc4bd1d9a53ee808 Mon Sep 17 00:00:00 2001 From: Ionesio Junior Date: Mon, 20 May 2024 14:32:22 -0300 Subject: [PATCH 060/100] Fix unit/notebook tests --- packages/syft/src/syft/service/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index e7ccd5c25ad..9576ac2b92e 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -288,6 +288,7 @@ def reconstruct_args_kwargs( final_kwargs[param_key] = param.default else: raise Exception(f"Missing {param_key} not in kwargs.") + final_kwargs['context'] = kwargs['context'] if 'context' in kwargs else None return (args, final_kwargs) @@ -340,7 +341,6 @@ def wrapper(func: Any) -> Callable: _path = class_name + "." + func_name signature = inspect.signature(func) signature = signature_remove_self(signature) - signature_with_context = deepcopy(signature) signature = signature_remove_context(signature) input_signature = deepcopy(signature) @@ -354,7 +354,7 @@ def _decorator(self: Any, *args: Any, **kwargs: Any) -> Callable: ) if autosplat is not None and len(autosplat) > 0: args, kwargs = reconstruct_args_kwargs( - signature=signature_with_context, + signature=input_signature, autosplat=autosplat, args=args, kwargs=kwargs, From ba11b4dc769f880bd2d456ae70c0f7ebda0b8c6d Mon Sep 17 00:00:00 2001 From: Ionesio Junior Date: Mon, 20 May 2024 15:27:42 -0300 Subject: [PATCH 061/100] Update settings test --- .../syft/tests/syft/settings/settings_service_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/syft/tests/syft/settings/settings_service_test.py b/packages/syft/tests/syft/settings/settings_service_test.py index 13592539a7f..56bf414f373 100644 --- a/packages/syft/tests/syft/settings/settings_service_test.py +++ b/packages/syft/tests/syft/settings/settings_service_test.py @@ -151,7 +151,7 @@ def mock_stash_get_all(root_verify_key) -> Ok: monkeypatch.setattr(settings_service.stash, "get_all", mock_stash_get_all) # update the settings in the settings stash using settings_service - response = settings_service.update(authed_context, update_settings) + response = settings_service.update(context=authed_context, settings=update_settings) # not_updated_settings = response.ok()[1] @@ -174,7 +174,7 @@ def mock_stash_get_all_error(credentials) -> Err: return Err(mock_error_message) monkeypatch.setattr(settings_service.stash, "get_all", mock_stash_get_all_error) - response = settings_service.update(authed_context, update_settings) + response = settings_service.update(context=authed_context, settings=update_settings) assert isinstance(response, SyftError) assert response.message == mock_error_message @@ -185,7 +185,7 @@ def test_settingsservice_update_stash_empty( update_settings: NodeSettingsUpdate, authed_context: AuthedServiceContext, ) -> None: - response = settings_service.update(authed_context, update_settings) + response = settings_service.update(context=authed_context, settings=update_settings) assert isinstance(response, SyftError) assert response.message == "No settings found" @@ -214,7 +214,7 @@ def mock_stash_update_error(credentials, update_settings: NodeSettings) -> Err: monkeypatch.setattr(settings_service.stash, "update", mock_stash_update_error) - response = settings_service.update(authed_context, update_settings) + response = settings_service.update(context=authed_context, settings=update_settings) assert isinstance(response, SyftError) assert response.message == mock_update_error_message From 552a8ed2e85f7e37f5efca1b738f92005b593256 Mon Sep 17 00:00:00 2001 From: alexnicita Date: Fri, 24 May 2024 15:53:30 -0400 Subject: [PATCH 062/100] rename new_project.send() to new_project.start() --- notebooks/api/0.8/01-submit-code.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/api/0.8/01-submit-code.ipynb b/notebooks/api/0.8/01-submit-code.ipynb index 761d1a96e7a..8448680171b 100644 --- a/notebooks/api/0.8/01-submit-code.ipynb +++ b/notebooks/api/0.8/01-submit-code.ipynb @@ -482,7 +482,7 @@ "outputs": [], "source": [ "# Once we start the project, it will submit the project along with the code request to the Domain Server\n", - "project = new_project.send()\n", + "project = new_project.start()\n", "project" ] }, From 46a0a765b4817d229c7242d33062a2cce0341626 Mon Sep 17 00:00:00 2001 From: S Rasswanth <43314053+rasswanth-s@users.noreply.github.com> Date: Mon, 27 May 2024 16:36:07 +0530 Subject: [PATCH 063/100] Revert "rename new_project.send() to new_project.start()" --- notebooks/api/0.8/01-submit-code.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/api/0.8/01-submit-code.ipynb b/notebooks/api/0.8/01-submit-code.ipynb index 8448680171b..761d1a96e7a 100644 --- a/notebooks/api/0.8/01-submit-code.ipynb +++ b/notebooks/api/0.8/01-submit-code.ipynb @@ -482,7 +482,7 @@ "outputs": [], "source": [ "# Once we start the project, it will submit the project along with the code request to the Domain Server\n", - "project = new_project.start()\n", + "project = new_project.send()\n", "project" ] }, From 7a8368a64928ce6c7913cd903d81e834935e8eb5 Mon Sep 17 00:00:00 2001 From: Ionesio Junior Date: Mon, 27 May 2024 09:51:40 -0300 Subject: [PATCH 064/100] Fix lint --- packages/syft/src/syft/service/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index 9576ac2b92e..77a4679e1a3 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -288,7 +288,7 @@ def reconstruct_args_kwargs( final_kwargs[param_key] = param.default else: raise Exception(f"Missing {param_key} not in kwargs.") - final_kwargs['context'] = kwargs['context'] if 'context' in kwargs else None + final_kwargs["context"] = kwargs["context"] if "context" in kwargs else None return (args, final_kwargs) From 30100454ad1a91c77a99830adbc535ea8f612e35 Mon Sep 17 00:00:00 2001 From: Ionesio Junior Date: Mon, 27 May 2024 09:58:57 -0300 Subject: [PATCH 065/100] Fix unit tests --- packages/syft/tests/syft/users/user_service_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/syft/tests/syft/users/user_service_test.py b/packages/syft/tests/syft/users/user_service_test.py index 54a32bef836..7c0cc32562a 100644 --- a/packages/syft/tests/syft/users/user_service_test.py +++ b/packages/syft/tests/syft/users/user_service_test.py @@ -229,7 +229,7 @@ def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Ok | Err: expected_output = [guest_user.to(UserView)] # Search via id - response = user_service.search(authed_context, id=guest_user.id) + response = user_service.search(context=authed_context, id=guest_user.id) assert isinstance(response, list) assert all( r.to_dict() == expected.to_dict() @@ -238,7 +238,7 @@ def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Ok | Err: # assert response.to_dict() == expected_output.to_dict() # Search via email - response = user_service.search(authed_context, email=guest_user.email) + response = user_service.search(context=authed_context, email=guest_user.email) assert isinstance(response, list) assert all( r.to_dict() == expected.to_dict() @@ -246,7 +246,7 @@ def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Ok | Err: ) # Search via name - response = user_service.search(authed_context, name=guest_user.name) + response = user_service.search(context=authed_context, name=guest_user.name) assert isinstance(response, list) assert all( r.to_dict() == expected.to_dict() @@ -255,7 +255,7 @@ def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Ok | Err: # Search via verify_key response = user_service.search( - authed_context, + context=authed_context, verify_key=guest_user.verify_key, ) assert isinstance(response, list) @@ -266,7 +266,7 @@ def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Ok | Err: # Search via multiple kwargs response = user_service.search( - authed_context, name=guest_user.name, email=guest_user.email + context=authed_context, name=guest_user.name, email=guest_user.email ) assert isinstance(response, list) assert all( @@ -279,7 +279,7 @@ def test_userservice_search_with_invalid_kwargs( user_service: UserService, authed_context: AuthedServiceContext ) -> None: # Search with invalid kwargs - response = user_service.search(authed_context, role=ServiceRole.GUEST) + response = user_service.search(context=authed_context, role=ServiceRole.GUEST) assert isinstance(response, SyftError) assert "Invalid Search parameters" in response.message From fc2b703327a2793e3c5babcb411100678beff1ee Mon Sep 17 00:00:00 2001 From: Ionesio Junior Date: Mon, 27 May 2024 10:25:50 -0300 Subject: [PATCH 066/100] Fix lint --- packages/syft/src/syft/service/service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index 77a4679e1a3..55f2c1f4b5d 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -288,7 +288,10 @@ def reconstruct_args_kwargs( final_kwargs[param_key] = param.default else: raise Exception(f"Missing {param_key} not in kwargs.") - final_kwargs["context"] = kwargs["context"] if "context" in kwargs else None + + if "context": + final_kwargs["context"] = kwargs["context"] + return (args, final_kwargs) From 3e9286311d246fb8c53d5b08e5696f0b0c5a9e8c Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Wed, 29 May 2024 11:01:28 +1000 Subject: [PATCH 067/100] Fixed linting / tests adding error handling for None types --- packages/grid/backend/grid/core/config.py | 3 +++ .../backend/backend-statefulset.yaml | 2 +- .../service/network/association_request.py | 2 +- .../syft/service/network/network_service.py | 19 +++++++++++++++++-- .../src/syft/service/network/node_peer.py | 2 +- .../syft/src/syft/service/network/rathole.py | 4 ++-- .../syft/service/network/rathole_service.py | 2 ++ tox.ini | 2 ++ 8 files changed, 29 insertions(+), 7 deletions(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 8c55b8cd3f7..33d65719fe8 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -155,6 +155,9 @@ def get_emails_enabled(self) -> Self: ASSOCIATION_REQUEST_AUTO_APPROVAL: bool = str_to_bool( os.getenv("ASSOCIATION_REQUEST_AUTO_APPROVAL", "False") ) + REVERSE_TUNNEL_RATHOLE_ENABLED: bool = str_to_bool( + os.getenv("REVERSE_TUNNEL_RATHOLE_ENABLED", "false") + ) model_config = SettingsConfigDict(case_sensitive=True) diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index f9b2dd42353..1fed68448dc 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -90,7 +90,7 @@ spec: {{- if .Values.rathole.enabled }} - name: RATHOLE_PORT value: {{ .Values.rathole.port | quote }} - - name: RATHOLE_ENABLED + - name: REVERSE_TUNNEL_RATHOLE_ENABLED value: "true" {{- end }} # MongoDB diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index 2590b3b42fe..0d1363b54e6 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -46,7 +46,7 @@ def _run( rathole_route = self.remote_peer.get_rathole_route() - if rathole_route.rathole_token is None: + if rathole_route and rathole_route.rathole_token is None: try: remote_client: SyftClient = self.remote_peer.client_with_context( context=service_ctx diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index de560747ac6..f8e9b29658d 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -31,7 +31,9 @@ from ...types.uid import UID from ...util.telemetry import instrument from ...util.util import generate_token +from ...util.util import get_env from ...util.util import prompt_warning_message +from ...util.util import str_to_bool from ..context import AuthedServiceContext from ..data_subject.data_subject import NamePartitionKey from ..metadata.node_metadata import NodeMetadataV3 @@ -61,6 +63,12 @@ NodeTypePartitionKey = PartitionKey(key="node_type", type_=NodeType) OrderByNamePartitionKey = PartitionKey(key="name", type_=str) +REVERSE_TUNNEL_RATHOLE_ENABLED = "REVERSE_TUNNEL_RATHOLE_ENABLED" + + +def get_rathole_enabled() -> bool: + return str_to_bool(get_env(REVERSE_TUNNEL_RATHOLE_ENABLED, "false")) + @serializable() class NodePeerAssociationStatus(Enum): @@ -151,7 +159,8 @@ class NetworkService(AbstractService): def __init__(self, store: DocumentStore) -> None: self.store = store self.stash = NetworkStash(store=store) - self.rathole_service = RatholeService() + if get_rathole_enabled(): + self.rathole_service = RatholeService() # TODO: Check with MADHAVA, can we even allow guest user to introduce routes to # domain nodes? @@ -281,8 +290,14 @@ def exchange_credentials_with( if result.is_err(): return SyftError(message="Failed to update route information.") - if reverse_tunnel: + if reverse_tunnel and get_rathole_enabled(): rathole_route = self_node_peer.get_rathole_route() + if not rathole_route: + raise Exception( + "Failed to exchange credentials. " + + f"Peer: {self_node_peer} has no rathole route: {rathole_route}" + ) + remote_url = GridURL( host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port ) diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index e6ac045e9f1..4fe2b5f38b2 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -269,7 +269,7 @@ def pick_highest_priority_route(self) -> NodeRoute: highest_priority_route = route return highest_priority_route - def get_rathole_route(self) -> NodeRoute | None: + def get_rathole_route(self) -> HTTPNodeRoute | None: for route in self.node_routes: if hasattr(route, "rathole_token") and route.rathole_token: return route diff --git a/packages/syft/src/syft/service/network/rathole.py b/packages/syft/src/syft/service/network/rathole.py index e102134ada6..d311bfa9c2b 100644 --- a/packages/syft/src/syft/service/network/rathole.py +++ b/packages/syft/src/syft/service/network/rathole.py @@ -1,5 +1,5 @@ -# stdlib -from typing import Self +# third party +from typing_extensions import Self # relative from ...serde.serializable import serializable diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index ad5e783c7dd..2bcde4fd2f4 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -36,6 +36,8 @@ def add_host_to_server(self, peer: NodePeer) -> None: """ rathole_route = peer.get_rathole_route() + if not rathole_route: + raise Exception(f"Peer: {peer} has no rathole route: {rathole_route}") random_port = self.get_random_port() diff --git a/tox.ini b/tox.ini index 35cb08c9fb7..c282c971edc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,7 @@ [tox] envlist = + dev.k8s.launch.domain + dev.k8s.launch.gateway dev.k8s.registry dev.k8s.start dev.k8s.deploy From 74ad27711ca2a4904feceff77fdcce63fa323f2b Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Wed, 29 May 2024 16:52:07 +1000 Subject: [PATCH 068/100] Re-add changed node route --- packages/syft/src/syft/service/network/network_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index f8e9b29658d..4532201aaae 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -189,6 +189,7 @@ def exchange_credentials_with( _rathole_route = self_node_peer.node_routes[-1] _rathole_route.rathole_token = generate_token() _rathole_route.host_or_ip = f"{self_node_peer.name}.syft.local" + self_node_peer.node_routes[-1] = _rathole_route if isinstance(self_node_peer, SyftError): return self_node_peer From 47d5852c03b36436d05fabe3cda2d9d590bfd0e9 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Thu, 30 May 2024 16:30:45 +1000 Subject: [PATCH 069/100] Change rathole port in gateway dev mode --- packages/grid/devspace.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 616eaa3ea4c..ed6dd9bbf11 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -164,6 +164,11 @@ profiles: path: dev.backend.containers.backend-container.ssh.localPort value: 3481 + # Mongo + - op: replace + path: dev.rathole.ports[0].port + value: 2334:2333 + - name: gcp patches: - op: replace From 6ac32533e23400c722830df08e8c603c85a09d66 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 31 May 2024 00:24:41 +0530 Subject: [PATCH 070/100] update network service to add override rathole config is already exists --- .../service/network/association_request.py | 35 +++- .../syft/service/network/network_service.py | 170 ++++++++---------- .../src/syft/service/network/node_peer.py | 10 ++ .../syft/service/network/rathole_service.py | 61 ++----- 4 files changed, 131 insertions(+), 145 deletions(-) diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index 0d1363b54e6..043bdbc101e 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -33,6 +33,16 @@ class AssociationRequestChange(Change): def _run( self, context: ChangeContext, apply: bool ) -> Result[tuple[bytes, NodePeer], SyftError]: + """ + Executes the association request. + + Args: + context (ChangeContext): The change context. + apply (bool): A flag indicating whether to apply the association request. + + Returns: + Result[tuple[bytes, NodePeer], SyftError]: The result of the association request. + """ # relative from .network_service import NetworkService @@ -42,11 +52,25 @@ def _run( SyftError(message="Undo not supported for AssociationRequestChange") ) + # Get the network service service_ctx = context.to_service_ctx() + network_service = cast( + NetworkService, service_ctx.node.get_service(NetworkService) + ) + network_stash = network_service.stash + # Check if remote peer to be added is via rathole rathole_route = self.remote_peer.get_rathole_route() + add_rathole_route = ( + rathole_route is not None + and self.remote_peer.latest_added_route == rathole_route + ) - if rathole_route and rathole_route.rathole_token is None: + # If the remote peer is added via rathole, we don't need to ping the peer + if add_rathole_route: + network_service.rathole_service.add_host_to_server(self.remote_peer) + else: + # Pinging the remote peer to verify the connection try: remote_client: SyftClient = self.remote_peer.client_with_context( context=service_ctx @@ -77,14 +101,7 @@ def _run( except Exception as e: return Err(SyftError(message=str(e))) - network_service = cast( - NetworkService, service_ctx.node.get_service(NetworkService) - ) - - network_stash = network_service.stash - - network_service.rathole_service.add_host_to_server(self.remote_peer) - + # Adding the remote peer to the network stash result = network_stash.create_or_update_peer( service_ctx.node.verify_key, self.remote_peer ) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 4532201aaae..a1607694ef4 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -1,10 +1,12 @@ # stdlib from collections.abc import Callable from enum import Enum +import logging import secrets from typing import Any # third party +from loguru import logger from result import Err from result import Result @@ -130,10 +132,9 @@ def create_or_update_peer( existing = existing.ok() existing.update_routes(peer.node_routes) result = self.update(credentials, existing) - return result else: result = self.set(credentials, peer) - return result + return result def get_by_verify_key( self, credentials: SyftVerifyKey, verify_key: SyftVerifyKey @@ -202,115 +203,53 @@ def exchange_credentials_with( ) remote_node_peer = NodePeer.from_client(remote_client) - # ask the remote client to add this node (represented by `self_node_peer`) as a peer - # check locally if the remote node already exists as a peer - existing_peer_result = self.stash.get_by_uid( - context.node.verify_key, remote_node_peer.id + # Step 3: Check remotely if the self node already exists as a peer + # Update the peer if it exists, otherwise add it + remote_self_node_peer = remote_client.api.services.network.get_peer_by_name( + name=self_node_peer.name ) - if ( - existing_peer_result.is_ok() - and (existing_peer := existing_peer_result.ok()) is not None - ): - msg = [ - ( - f"{existing_peer.node_type} peer '{existing_peer.name}' already exist for " - f"{self_node_peer.node_type} '{self_node_peer.name}'." - ) - ] - if existing_peer != remote_node_peer: - result = self.stash.create_or_update_peer( - context.node.verify_key, - remote_node_peer, - ) - msg.append( - f"{existing_peer.node_type} peer '{existing_peer.name}' information change detected." - ) - if result.is_err(): - msg.append( - f"Attempt to update peer '{existing_peer.name}' information failed." - ) - return SyftError(message="\n".join(msg)) - msg.append( - f"{existing_peer.node_type} peer '{existing_peer.name}' information successfully updated." - ) - # Also check remotely if the self node already exists as a peer - remote_self_node_peer = remote_client.api.services.network.get_peer_by_name( - name=self_node_peer.name - ) - if isinstance(remote_self_node_peer, NodePeer): - msg.append( - f"{self_node_peer.node_type} '{self_node_peer.name}' already exist " - f"as a peer for {remote_node_peer.node_type} '{remote_node_peer.name}'." - ) - if remote_self_node_peer != self_node_peer: - result = remote_client.api.services.network.update_peer( - peer=self_node_peer, - ) - msg.append( - f"{self_node_peer.node_type} peer '{self_node_peer.name}' information change detected." - ) - if isinstance(result, SyftError): - msg.apnpend( - f"Attempt to remotely update {self_node_peer.node_type} peer " - f"'{self_node_peer.name}' information remotely failed." - ) - return SyftError(message="\n".join(msg)) - msg.append( - f"{self_node_peer.node_type} peer '{self_node_peer.name}' " - f"information successfully updated." - ) - msg.append( - f"Routes between {remote_node_peer.node_type} '{remote_node_peer.name}' and " - f"{self_node_peer.node_type} '{self_node_peer.name}' already exchanged." + association_request_approved = True + if isinstance(remote_self_node_peer, NodePeer): + result = remote_client.api.services.network.update_peer(peer=self_node_peer) + if isinstance(result, SyftError): + return SyftError( + message=f"Failed to add peer information on remote client : {remote_client.id}" ) - return SyftSuccess(message="\n".join(msg)) # If peer does not exist, ask the remote client to add this node # (represented by `self_node_peer`) as a peer - random_challenge = secrets.token_bytes(16) - remote_res = remote_client.api.services.network.add_peer( - peer=self_node_peer, - challenge=random_challenge, - self_node_route=remote_node_route, - verify_key=remote_node_verify_key, - ) - - if isinstance(remote_res, SyftError): - return SyftError( - message=f"returned error from add peer: {remote_res.message}" + if remote_self_node_peer is None: + random_challenge = secrets.token_bytes(16) + remote_res = remote_client.api.services.network.add_peer( + peer=self_node_peer, + challenge=random_challenge, + self_node_route=remote_node_route, + verify_key=remote_node_verify_key, ) - association_request_approved = not isinstance(remote_res, Request) + if isinstance(remote_res, SyftError): + return SyftError( + message=f"Failed to add peer to remote client: {remote_client.id}. Error: {remote_res.message}" + ) + + association_request_approved = not isinstance(remote_res, Request) - # save the remote peer for later + # Step 4: Save the remote peer for later result = self.stash.create_or_update_peer( context.node.verify_key, remote_node_peer, ) if result.is_err(): + logging.error( + f"Failed to save peer: {remote_node_peer}. Error: {result.err()}" + ) return SyftError(message="Failed to update route information.") + # Step 5: Save rathole config to enable reverse tunneling if reverse_tunnel and get_rathole_enabled(): - rathole_route = self_node_peer.get_rathole_route() - if not rathole_route: - raise Exception( - "Failed to exchange credentials. " - + f"Peer: {self_node_peer} has no rathole route: {rathole_route}" - ) - - remote_url = GridURL( - host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port - ) - rathole_remote_addr = remote_url.as_container_host() - - remote_addr = rathole_remote_addr.url_no_protocol - - self.rathole_service.add_host_to_client( - peer_name=self_node_peer.name, - peer_id=str(self_node_peer.id), - rathole_token=rathole_route.rathole_token, - remote_addr=remote_addr, + self._add_reverse_tunneling_config_for_peer( + self_node_peer=self_node_peer, remote_node_route=remote_node_route ) return ( @@ -319,6 +258,33 @@ def exchange_credentials_with( else remote_res ) + def _add_reverse_tunneling_config_for_peer( + self, + self_node_peer: NodePeer, + remote_node_route: NodeRoute, + ) -> None: + + rathole_route = self_node_peer.get_rathole_route() + if not rathole_route: + return SyftError( + "Failed to exchange routes via . " + + f"Peer: {self_node_peer} has no rathole route: {rathole_route}" + ) + + remote_url = GridURL( + host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port + ) + rathole_remote_addr = remote_url.as_container_host() + + remote_addr = rathole_remote_addr.url_no_protocol + + self.rathole_service.add_host_to_client( + peer_name=self_node_peer.name, + peer_id=str(self_node_peer.id), + rathole_token=rathole_route.rathole_token, + remote_addr=remote_addr, + ) + @service_method(path="network.add_peer", name="add_peer", roles=GUEST_ROLE_LEVEL) def add_peer( self, @@ -524,6 +490,22 @@ def update_peer( return SyftError( message=f"Failed to update peer '{peer.name}'. Error: {result.err()}" ) + + if context.node.node_side_type == NodeType.GATEWAY: + rathole_route = peer.get_rathole_route() + self.rathole_service.add_host_to_server(peer) if rathole_route else None + else: + self_node_peer: NodePeer = context.node.settings.to(NodePeer) + rathole_route = self_node_peer.get_rathole_route() + ( + self._add_reverse_tunneling_config_for_peer( + self_node_peer=self_node_peer, + remote_node_route=peer.pick_highest_priority_route(), + ) + if rathole_route + else None + ) + return SyftSuccess( message=f"Peer '{result.ok().name}' information successfully updated." ) diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 4fe2b5f38b2..59015e21dfb 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -215,6 +215,16 @@ def from_client(client: SyftClient) -> "NodePeer": peer.node_routes.append(route) return peer + @property + def latest_added_route(self) -> NodeRoute | None: + """ + Returns the latest added route from the list of node routes. + + Returns: + NodeRoute | None: The latest added route, or None if there are no routes. + """ + return self.node_routes[-1] if self.node_routes else None + def client_with_context( self, context: NodeServiceContext ) -> Result[type[SyftClient], str]: diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 2bcde4fd2f4..e2d729de069 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -120,40 +120,6 @@ def add_host_to_client( # Update the rathole config map KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) - def forward_port_to_proxy( - self, config: RatholeConfig, entrypoint: str = "web" - ) -> None: - """Add a port to the rathole proxy config map.""" - - rathole_proxy_config_map = KubeUtils.get_configmap( - self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP - ) - - if rathole_proxy_config_map is None: - raise Exception("Rathole proxy config map not found.") - - rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] - - if not rathole_proxy: - rathole_proxy = {"http": {"routers": {}, "services": {}}} - else: - rathole_proxy = yaml.safe_load(rathole_proxy) - - rathole_proxy["http"]["services"][config.server_name] = { - "loadBalancer": {"servers": [{"url": "http://proxy:8001"}]} - } - - rathole_proxy["http"]["routers"][config.server_name] = { - "rule": "PathPrefix(`/`)", - "service": config.server_name, - "entryPoints": [entrypoint], - } - - KubeUtils.update_configmap( - config_map=rathole_proxy_config_map, - patch={"data": {"rathole-dynamic.yml": yaml.safe_dump(rathole_proxy)}}, - ) - def add_dynamic_addr_to_rathole( self, config: RatholeConfig, entrypoint: str = "web" ) -> None: @@ -206,14 +172,25 @@ def expose_port_on_rathole_service(self, port_name: str, port: int) -> None: config = rathole_service.raw - config["spec"]["ports"].append( - { - "name": port_name, - "port": port, - "targetPort": port, - "protocol": "TCP", - } - ) + existing_port_idx = None + for idx, port in enumerate(config["spec"]["ports"]): + if port["name"] == port_name: + print("Port already exists.", existing_port_idx, port_name) + existing_port_idx = idx + break + + if existing_port_idx is not None: + config["spec"]["ports"][existing_port_idx]["port"] = port + config["spec"]["ports"][existing_port_idx]["targetPort"] = port + else: + config["spec"]["ports"].append( + { + "name": port_name, + "port": port, + "targetPort": port, + "protocol": "TCP", + } + ) rathole_service.patch(config) From e27c5cd1a05d33aff6d20319da5b5bafb0f4fdd6 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Sat, 1 Jun 2024 14:44:32 +0530 Subject: [PATCH 071/100] make id optional in NodeConnection and its subclasses - remove using route id to check for existing routes --- packages/syft/src/syft/client/client.py | 3 +-- packages/syft/src/syft/client/connection.py | 7 +++-- .../src/syft/protocol/protocol_version.json | 16 ++++++++++-- .../syft/service/network/network_service.py | 10 +++---- .../src/syft/service/network/node_peer.py | 26 +++++-------------- .../syft/service/network/rathole_service.py | 4 +-- .../syft/src/syft/service/network/routes.py | 5 ++-- 7 files changed, 36 insertions(+), 35 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 28e975efa03..33a509dfc18 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -48,7 +48,6 @@ from ..service.user.user_roles import ServiceRole from ..service.user.user_service import UserService from ..types.grid_url import GridURL -from ..types.syft_object import SYFT_OBJECT_VERSION_2 from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.uid import UID from ..util.logger import debug @@ -391,7 +390,7 @@ def get_client_type(self) -> type[SyftClient] | SyftError: @serializable() class PythonConnection(NodeConnection): __canonical_name__ = "PythonConnection" - __version__ = SYFT_OBJECT_VERSION_2 + __version__ = SYFT_OBJECT_VERSION_3 node: AbstractNode proxy_target_uid: UID | None = None diff --git a/packages/syft/src/syft/client/connection.py b/packages/syft/src/syft/client/connection.py index e82db863e8a..0899532818a 100644 --- a/packages/syft/src/syft/client/connection.py +++ b/packages/syft/src/syft/client/connection.py @@ -2,13 +2,16 @@ from typing import Any # relative -from ..types.syft_object import SYFT_OBJECT_VERSION_2 +from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.syft_object import SyftObject +from ..types.uid import UID class NodeConnection(SyftObject): __canonical_name__ = "NodeConnection" - __version__ = SYFT_OBJECT_VERSION_2 + __version__ = SYFT_OBJECT_VERSION_3 + + id: UID | None = None # type: ignore def get_cache_key(self) -> str: raise NotImplementedError diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 395f1263ef1..a32451b058b 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -217,7 +217,7 @@ }, "3": { "version": 3, - "hash": "5e363abe2875beec89a3f4f4f5c53e15f9893fb98e5da71e2fa6c0f619883b1f", + "hash": "9162f038f0f6401c4cf4d1b517c40805d1f291bd69a6e76f3c1ee9e5095de2e5", "action": "add" } }, @@ -229,7 +229,7 @@ }, "3": { "version": 3, - "hash": "89ace8067c392b802fe23a99446a8ae464a9dad0b49d8b2c3871b631451acec4", + "hash": "9e7e3700a2f7b1a67f054efbcb31edc71bbf358c469c85ed7760b81233803bac", "action": "add" } }, @@ -239,6 +239,18 @@ "hash": "010d9aaca95f3fdfc8d1f97d01c1bd66483da774a59275b310c08d6912f7f863", "action": "add" } + }, + "PythonConnection": { + "2": { + "version": 2, + "hash": "eb479c671fc112b2acbedb88bc5624dfdc9592856c04c22c66410f6c863e1708", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "1084c85a59c0436592530b5fe9afc2394088c8d16faef2b19fdb9fb83ff0f0e2", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index a1607694ef4..fdfe6c184ec 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -4,9 +4,9 @@ import logging import secrets from typing import Any +from typing import cast # third party -from loguru import logger from result import Err from result import Result @@ -263,10 +263,9 @@ def _add_reverse_tunneling_config_for_peer( self_node_peer: NodePeer, remote_node_route: NodeRoute, ) -> None: - rathole_route = self_node_peer.get_rathole_route() if not rathole_route: - return SyftError( + raise Exception( "Failed to exchange routes via . " + f"Peer: {self_node_peer} has no rathole route: {rathole_route}" ) @@ -491,7 +490,8 @@ def update_peer( message=f"Failed to update peer '{peer.name}'. Error: {result.err()}" ) - if context.node.node_side_type == NodeType.GATEWAY: + node_side_type = cast(NodeType, context.node.node_side_type) + if node_side_type == NodeType.GATEWAY: rathole_route = peer.get_rathole_route() self.rathole_service.add_host_to_server(peer) if rathole_route else None else: @@ -899,7 +899,7 @@ def _get_remote_node_peer_by_verify_key( remote_node_peer = remote_node_peer.ok() if remote_node_peer is None: return SyftError( - message=f"Can't retrive {remote_node_peer.name} from the store of peers (None)." + message=f"Can't retrieve {remote_node_peer.name} from the store of peers (None)." ) return remote_node_peer diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 59015e21dfb..b08a793e96f 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -86,22 +86,17 @@ class NodePeer(SyftObject): ping_status_message: str | None = None pinged_timestamp: DateTime | None = None - def existed_route( - self, route: NodeRouteType | None = None, route_id: UID | None = None - ) -> tuple[bool, int | None]: + def existed_route(self, route: NodeRouteType) -> tuple[bool, int | None]: """Check if a route exists in self.node_routes Args: route: the route to be checked. For now it can be either - HTTPNodeRoute or PythonNodeRoute or VeilidNodeRoute - route_id: the id of the route to be checked + HTTPNodeRoute or PythonNodeRoute Returns: if the route exists, returns (True, index of the existed route in self.node_routes) if the route does not exist returns (False, None) """ - if route_id is None and route is None: - raise ValueError("Either route or route_id should be provided in args") if route: if not isinstance(route, HTTPNodeRoute | PythonNodeRoute | VeilidNodeRoute): @@ -110,11 +105,6 @@ def existed_route( if route == r: return (True, i) - elif route_id: - for i, r in enumerate(self.node_routes): - if r.id == route_id: - return (True, i) - return (False, None) def assign_highest_priority(self, route: NodeRoute) -> NodeRoute: @@ -131,7 +121,7 @@ def assign_highest_priority(self, route: NodeRoute) -> NodeRoute: route.priority = current_max_priority + 1 return route - def update_route(self, route: NodeRoute) -> NodeRoute | None: + def update_route(self, route: NodeRoute) -> None: """ Update the route for the node. If the route already exists, return it. @@ -140,17 +130,13 @@ def update_route(self, route: NodeRoute) -> NodeRoute | None: Args: route (NodeRoute): The new route to be added to the peer. - - Returns: - NodeRoute | None: if the route already exists, return it, else returns None """ - existed, _ = self.existed_route(route) + existed, idx = self.existed_route(route) if existed: - return route + self.node_routes[idx] = route # type: ignore else: new_route = self.assign_highest_priority(route) self.node_routes.append(new_route) - return None def update_routes(self, new_routes: list[NodeRoute]) -> None: """ @@ -191,7 +177,7 @@ def update_existed_route_priority( message="Priority must be greater than 0. Now it is {priority}." ) - existed, index = self.existed_route(route_id=route.id) + existed, index = self.existed_route(route=route) if not existed or index is None: return SyftError(message=f"Route with id {route.id} does not exist.") diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index e2d729de069..2b879f2df27 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -173,8 +173,8 @@ def expose_port_on_rathole_service(self, port_name: str, port: int) -> None: config = rathole_service.raw existing_port_idx = None - for idx, port in enumerate(config["spec"]["ports"]): - if port["name"] == port_name: + for idx, existing_port in enumerate(config["spec"]["ports"]): + if existing_port["name"] == port_name: print("Port already exists.", existing_port_idx, port_name) existing_port_idx = idx break diff --git a/packages/syft/src/syft/service/network/routes.py b/packages/syft/src/syft/service/network/routes.py index 1d9ec116467..b1ff68f6b72 100644 --- a/packages/syft/src/syft/service/network/routes.py +++ b/packages/syft/src/syft/service/network/routes.py @@ -18,7 +18,6 @@ from ...node.worker_settings import WorkerSettings from ...serde.serializable import serializable from ...types.syft_object import SYFT_OBJECT_VERSION_1 -from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SyftObject from ...types.transforms import TransformContext @@ -90,6 +89,7 @@ class HTTPNodeRoute(SyftObject, NodeRoute): __canonical_name__ = "HTTPNodeRoute" __version__ = SYFT_OBJECT_VERSION_3 + id: UID | None = None # type: ignore host_or_ip: str private: bool = False protocol: str = "http" @@ -119,8 +119,9 @@ def __str__(self) -> str: @serializable() class PythonNodeRoute(SyftObject, NodeRoute): __canonical_name__ = "PythonNodeRoute" - __version__ = SYFT_OBJECT_VERSION_2 + __version__ = SYFT_OBJECT_VERSION_3 + id: UID | None = None # type: ignore worker_settings: WorkerSettings proxy_target_uid: UID | None = None priority: int = 1 From b9a0e6c72b529a9316167b91bec7c8bc0026dab5 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 3 Jun 2024 11:17:08 +0530 Subject: [PATCH 072/100] add /rathole prefix to dynamic proxy config --- .../syft/src/syft/protocol/protocol_version.json | 12 ++++++++++++ .../syft/src/syft/service/network/rathole_service.py | 9 +++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index a32451b058b..9aa5efd25fb 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -251,6 +251,18 @@ "hash": "1084c85a59c0436592530b5fe9afc2394088c8d16faef2b19fdb9fb83ff0f0e2", "action": "add" } + }, + "PythonNodeRoute": { + "2": { + "version": 2, + "hash": "3eca5767ae4a8fbe67744509e58c6d9fb78f38fa0a0f7fcf5960ab4250acc1f0", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "1bc413ec7c1d498ec945878e21e00affd9bd6d53b564b1e10e52feb09f177d04", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index 2b879f2df27..abc3bc878a5 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -135,7 +135,7 @@ def add_dynamic_addr_to_rathole( rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] if not rathole_proxy: - rathole_proxy = {"http": {"routers": {}, "services": {}}} + rathole_proxy = {"http": {"routers": {}, "services": {}, "middlewares": {}}} else: rathole_proxy = yaml.safe_load(rathole_proxy) @@ -145,15 +145,20 @@ def add_dynamic_addr_to_rathole( } } + rathole_proxy["http"]["middlewares"]["strip-rathole-prefix"] = { + "replacePathRegex:": {"regex": "^/rathole/(.*)", "replacement": "/$1"} + } + proxy_rule = ( f"Host(`{config.server_name}.syft.local`) || " - f"HostHeader(`{config.server_name}.syft.local`) && PathPrefix(`/`)" + f"HostHeader(`{config.server_name}.syft.local`) && PathPrefix(`/rathole`)" ) rathole_proxy["http"]["routers"][config.server_name] = { "rule": proxy_rule, "service": config.server_name, "entryPoints": [entrypoint], + "middlewares": ["strip-rathole-prefix"], } KubeUtils.update_configmap( From a7de40706d813ddd41a0cc50e9d491e8bc70c1cc Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 3 Jun 2024 11:22:29 +0530 Subject: [PATCH 073/100] add rathole prefix in http connection when rathole token present --- packages/syft/src/syft/client/client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 33a509dfc18..7e74a19261d 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -124,6 +124,7 @@ class Routes(Enum): ROUTE_API_CALL = f"{API_PATH}/api_call" ROUTE_BLOB_STORE = "/blob" STREAM = f"{API_PATH}/stream" + RATHOLE = "/rathole" @serializable(attrs=["proxy_target_uid", "url", "rathole_token"]) @@ -191,7 +192,7 @@ def _make_get( url = self.url if self.rathole_token: - url = GridURL.from_url(INTERNAL_PROXY_URL) + url = GridURL.from_url(INTERNAL_PROXY_URL).with_path(Routes.RATHOLE.value) headers = {"Host": self.url.host_or_ip} url = url.with_path(path) @@ -226,7 +227,7 @@ def _make_post( url = self.url if self.rathole_token: - url = GridURL.from_url(INTERNAL_PROXY_URL) + url = GridURL.from_url(INTERNAL_PROXY_URL).with_path(Routes.RATHOLE.value) headers = {"Host": self.url.host_or_ip} url = url.with_path(path) @@ -338,7 +339,9 @@ def make_call(self, signed_call: SignedSyftAPICall) -> Any | SyftError: headers = {} if self.rathole_token: - api_url = GridURL.from_url(INTERNAL_PROXY_URL) + api_url = GridURL.from_url(INTERNAL_PROXY_URL).with_path( + Routes.RATHOLE.value + ) api_url = api_url.with_path(self.routes.ROUTE_API_CALL.value) headers = {"Host": self.url.host_or_ip} else: From d52643bbb3c3e25ff499b56be248300e72ef4dbd Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Sun, 9 Jun 2024 23:56:40 +0530 Subject: [PATCH 074/100] update internal proxy url to include /rathole - fix syntax in middleware to strip rathole prefix - fix proxy config map to ignore if PathPrefix doesn't match --- .../helm/syft/templates/proxy/proxy-configmap.yaml | 6 +++--- packages/syft/src/syft/client/client.py | 11 ++++------- .../syft/src/syft/service/network/network_service.py | 5 ++--- .../syft/src/syft/service/network/rathole_service.py | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index e5055f03f95..748dfee78c4 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -28,17 +28,17 @@ data: - url: "http://rathole:2333" routers: rathole: - rule: "PathPrefix(`/`) && Headers(`Upgrade`, `websocket`)" + rule: "PathPrefix(`/`) && Headers(`Upgrade`, `websocket`) && !PathPrefix(`/rathole`)" entryPoints: - "web" service: "rathole" frontend: - rule: "PathPrefix(`/`)" + rule: "PathPrefix(`/`) && !PathPrefix(`/rathole`)" entryPoints: - "web" service: "frontend" backend: - rule: "PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)" + rule: "(PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)) && !PathPrefix(`/rathole`)" entryPoints: - "web" service: "backend" diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 7e74a19261d..3a1c047ac1f 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -113,7 +113,7 @@ def forward_message_to_proxy( API_PATH = "/api/v2" DEFAULT_PYGRID_PORT = 80 DEFAULT_PYGRID_ADDRESS = f"http://localhost:{DEFAULT_PYGRID_PORT}" -INTERNAL_PROXY_URL = "http://proxy:80" +INTERNAL_PROXY_TO_RATHOLE = "http://proxy:80/rathole/" class Routes(Enum): @@ -124,7 +124,6 @@ class Routes(Enum): ROUTE_API_CALL = f"{API_PATH}/api_call" ROUTE_BLOB_STORE = "/blob" STREAM = f"{API_PATH}/stream" - RATHOLE = "/rathole" @serializable(attrs=["proxy_target_uid", "url", "rathole_token"]) @@ -192,7 +191,7 @@ def _make_get( url = self.url if self.rathole_token: - url = GridURL.from_url(INTERNAL_PROXY_URL).with_path(Routes.RATHOLE.value) + url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) headers = {"Host": self.url.host_or_ip} url = url.with_path(path) @@ -227,7 +226,7 @@ def _make_post( url = self.url if self.rathole_token: - url = GridURL.from_url(INTERNAL_PROXY_URL).with_path(Routes.RATHOLE.value) + url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) headers = {"Host": self.url.host_or_ip} url = url.with_path(path) @@ -339,9 +338,7 @@ def make_call(self, signed_call: SignedSyftAPICall) -> Any | SyftError: headers = {} if self.rathole_token: - api_url = GridURL.from_url(INTERNAL_PROXY_URL).with_path( - Routes.RATHOLE.value - ) + api_url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) api_url = api_url.with_path(self.routes.ROUTE_API_CALL.value) headers = {"Host": self.url.host_or_ip} else: diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index fdfe6c184ec..adb117be61a 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -608,11 +608,10 @@ def add_route( if isinstance(remote_node_peer, SyftError): return remote_node_peer # add and update the priority for the peer - existed_route: NodeRoute | None = remote_node_peer.update_route(route) - if existed_route: + if route in remote_node_peer.node_routes: return SyftSuccess( message=f"The route already exists between '{context.node.name}' and " - f"peer '{remote_node_peer.name}' with id '{existed_route.id}'." + f"peer '{remote_node_peer.name}'." ) # update the peer in the store with the updated routes result = self.stash.update( diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_service.py index abc3bc878a5..afdf48503d7 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_service.py @@ -146,7 +146,7 @@ def add_dynamic_addr_to_rathole( } rathole_proxy["http"]["middlewares"]["strip-rathole-prefix"] = { - "replacePathRegex:": {"regex": "^/rathole/(.*)", "replacement": "/$1"} + "replacePathRegex": {"regex": "^/rathole/(.*)", "replacement": "/$1"} } proxy_rule = ( From ed6ce110c19cb62e419dfa25d2f46cac5b6f4906 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 10 Jun 2024 00:43:22 +0530 Subject: [PATCH 075/100] update protocol version --- .../src/syft/protocol/protocol_version.json | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 900bf516616..98c6b4576ba 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -284,6 +284,30 @@ "hash": "1bc413ec7c1d498ec945878e21e00affd9bd6d53b564b1e10e52feb09f177d04", "action": "add" } + }, + "HTTPConnection": { + "2": { + "version": 2, + "hash": "68409295f8916ceb22a8cf4abf89f5e4bcff0d75dc37e16ede37250ada28df59", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "9162f038f0f6401c4cf4d1b517c40805d1f291bd69a6e76f3c1ee9e5095de2e5", + "action": "add" + } + }, + "HTTPNodeRoute": { + "2": { + "version": 2, + "hash": "2134ea812f7c6ea41522727ae087245c4b1195ffbad554db638070861cd9eb1c", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "9e7e3700a2f7b1a67f054efbcb31edc71bbf358c469c85ed7760b81233803bac", + "action": "add" + } } } } From 6ea2cff3e59bc7a0b6ec97615041529eb8d2aff8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 10 Jun 2024 15:53:56 +0530 Subject: [PATCH 076/100] fix parameter name reference in update peer api - fix reference of node side instead of node side type from context - return the updated object in mongo --- .../syft/src/syft/service/network/network_service.py | 10 +++++++--- packages/syft/src/syft/store/mongo_document_store.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 86b744cc36b..eba9a7a6b3c 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -7,6 +7,7 @@ from typing import cast # third party +from loguru import logger from result import Result # relative @@ -216,9 +217,12 @@ def exchange_credentials_with( id=self_node_peer.id, node_routes=self_node_peer.node_routes ) result = remote_client.api.services.network.update_peer( - update_peer=updated_peer + peer_update=updated_peer ) if isinstance(result, SyftError): + logger.error( + f"Failed to update peer information on remote client. {result.message}" + ) return SyftError( message=f"Failed to add peer information on remote client : {remote_client.id}" ) @@ -502,8 +506,8 @@ def update_peer( peer = result.ok() - node_side_type = cast(NodeType, context.node.node_side_type) - if node_side_type == NodeType.GATEWAY: + node_side_type = cast(NodeType, context.node.node_type) + if node_side_type.value == NodeType.GATEWAY.value: rathole_route = peer.get_rathole_route() self.rathole_service.add_host_to_server(peer) if rathole_route else None else: diff --git a/packages/syft/src/syft/store/mongo_document_store.py b/packages/syft/src/syft/store/mongo_document_store.py index 59d6799c2bb..60040ce4a9d 100644 --- a/packages/syft/src/syft/store/mongo_document_store.py +++ b/packages/syft/src/syft/store/mongo_document_store.py @@ -376,7 +376,7 @@ def _update( except Exception as e: return Err(f"Failed to update obj: {obj} with qk: {qk}. Error: {e}") - return Ok(obj) + return Ok(prev_obj) else: return Err(f"Failed to update obj {obj}, you have no permission") From 4dd0b7fb878dd20d859d98219f513cbc86dfe996 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 10 Jun 2024 17:16:36 +0530 Subject: [PATCH 077/100] added notebook --- notebooks/Experimental/Network.ipynb | 8036 ++++++++++++++++++++++++++ 1 file changed, 8036 insertions(+) create mode 100644 notebooks/Experimental/Network.ipynb diff --git a/notebooks/Experimental/Network.ipynb b/notebooks/Experimental/Network.ipynb new file mode 100644 index 00000000000..88240421fb3 --- /dev/null +++ b/notebooks/Experimental/Network.ipynb @@ -0,0 +1,8036 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "bd9a2226-3e53-4f27-9213-75a8c3ff9176", + "metadata": {}, + "outputs": [], + "source": [ + "import syft as sy" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fddf8d07-d154-4284-a27b-d74e35d3f851", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "gateway_client = sy.login(url=\"http://localhost\", port=9081, email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "8f7b106d-b784-45d8-b54d-4ce2de2da453", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "domain_client = sy.login(url=\"http://localhost\", port=9082, email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ff504949-620d-4e26-beee-0d39e0e502eb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: Connected domain 'syft-dev-node' to gateway 'syft-dev-node'. Routes Exchanged

" + ], + "text/plain": [ + "SyftSuccess: Connected domain 'syft-dev-node' to gateway 'syft-dev-node'. Routes Exchanged" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "domain_client.connect_to_gateway(gateway_client, reverse_tunnel=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ba7bc71a-4e6a-4429-9588-7b3d0ed19e27", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

Request List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gateway_client.api.services.request" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "5b4984e1-331e-4fd8-b012-768fc613f48a", + "metadata": {}, + "outputs": [], + "source": [ + "# gateway_client.api.services.request[0].approve()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "90dc44bd", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

NodePeer List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "[syft.service.network.node_peer.NodePeer]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "node_peers = gateway_client.api.network.get_all_peers()\n", + "node_peers" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "8c06aaa6-4157-42d1-959f-9d47722a3420", + "metadata": {}, + "outputs": [], + "source": [ + "node_peer = gateway_client.api.network.get_all_peers()[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "cb63a77b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[syft.service.network.routes.HTTPNodeRoute]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "node_peer.node_routes" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "61882e86", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'syft_node_location': ,\n", + " 'syft_client_verify_key': cc82ec7abd5d516e6972e787ddaafe8b04e223228436b09c026be97b80ad6246,\n", + " 'id': None,\n", + " 'host_or_ip': 'syft-dev-node.syft.local',\n", + " 'private': False,\n", + " 'protocol': 'http',\n", + " 'port': 9082,\n", + " 'proxy_target_uid': None,\n", + " 'priority': 1,\n", + " 'rathole_token': 'b95e8d239d563e6fcc3a4f44a5292177e608a7b0b1194e6106adc1998a1b68a1'}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "node_peer.node_routes[0].__dict__" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "fb19dbc6-869b-46dc-92e3-5e75ee6d0b06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'syft_node_location': ,\n", + " 'syft_client_verify_key': 5ed40db70e275e30e9001cda808922c4542c84f7472a106c00158795b9388b0a,\n", + " 'id': None,\n", + " 'host_or_ip': 'host.k3d.internal',\n", + " 'private': False,\n", + " 'protocol': 'http',\n", + " 'port': 9081,\n", + " 'proxy_target_uid': None,\n", + " 'priority': 1,\n", + " 'rathole_token': None}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "domain_client.api.network.get_all_peers()[0].node_routes[0].__dict__" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "32d09a51", + "metadata": {}, + "outputs": [], + "source": [ + "# node_peer.client_with_key(sy.SyftSigningKey.generate())" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b7d9e41d", + "metadata": {}, + "outputs": [], + "source": [ + "# gateway_client.api.network.delete_route(node_peer.verify_key, node_peer.node_routes[1])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "8fa24ec7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + " \"Logo\"\n",\n", + "

Welcome to syft-dev-node

\n", + "
\n", + " URL: http://localhost:9081
Node Type: Gateway
Node Side Type: High Side
Syft Version: 0.8.7-beta.10
\n", + "
\n", + "
\n", + " ⓘ \n", + " This node is run by the library PySyft to learn more about how it works visit\n", + " github.com/OpenMined/PySyft.\n", + "
\n", + "

Commands to Get Started

\n", + " \n", + "
    \n", + " \n", + "
  • <your_client>\n", + " .domains - list domains connected to this gateway
  • \n", + "
  • <your_client>\n", + " .proxy_client_for - get a connection to a listed domain
  • \n", + "
  • <your_client>\n", + " .login - log into the gateway
  • \n", + " \n", + "
\n", + " \n", + "

\n", + " " + ], + "text/plain": [ + ": HTTPConnection: http://localhost:9081>" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gateway_client" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "3a081250-abc3-43a3-9e06-ff0c3a362ebf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

NodePeer List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/markdown": [ + "```python\n", + "class ProxyClient:\n", + " id: str = 37073e9151ce4fa9b665501ec03924c8\n", + "\n", + "```" + ], + "text/plain": [ + "syft.client.gateway_client.ProxyClient" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gateway_client.peers" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "b6fedfe4-9362-47c9-9342-5cf6eacde8ab", + "metadata": {}, + "outputs": [], + "source": [ + "domain_client_proxy = gateway_client.peers[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "f1940e00-0337-4b56-88c2-d70f397a7016", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class HTTPConnection:\n", + " id: str = None\n", + "\n", + "```" + ], + "text/plain": [ + "HTTPConnection: http://localhost:9081" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "domain_client_proxy.connection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "613125c5-6321-4238-852c-ff0cfcd9526a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.1.-1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 51b72095100444a405c6f8ae443c732417980c07 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 10 Jun 2024 18:30:02 +0530 Subject: [PATCH 078/100] remove hagrid and docker compose files --- packages/grid/docker-compose.build.yml | 29 - packages/grid/docker-compose.dev.yml | 63 - packages/grid/docker-compose.pull.yml | 20 - packages/grid/docker-compose.yml | 249 -- packages/hagrid/hagrid/cli.py | 4425 ------------------------ 5 files changed, 4786 deletions(-) delete mode 100644 packages/grid/docker-compose.build.yml delete mode 100644 packages/grid/docker-compose.dev.yml delete mode 100644 packages/grid/docker-compose.pull.yml delete mode 100644 packages/grid/docker-compose.yml delete mode 100644 packages/hagrid/hagrid/cli.py diff --git a/packages/grid/docker-compose.build.yml b/packages/grid/docker-compose.build.yml deleted file mode 100644 index a0175bc762a..00000000000 --- a/packages/grid/docker-compose.build.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: "3.8" -services: - frontend: - build: - context: ${RELATIVE_PATH}./frontend - dockerfile: frontend.dockerfile - target: "${FRONTEND_TARGET:-grid-ui-development}" - - backend: - build: - context: ${RELATIVE_PATH}../ - dockerfile: ./grid/backend/backend.dockerfile - target: "backend" - - seaweedfs: - build: - context: ${RELATIVE_PATH}./seaweedfs - dockerfile: seaweedfs.dockerfile - - worker: - build: - context: ${RELATIVE_PATH}../ - dockerfile: ./grid/backend/backend.dockerfile - target: "backend" - - rathole: - build: - context: ${RELATIVE_PATH}./rathole - dockerfile: rathole.dockerfile diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml deleted file mode 100644 index bcde98488e7..00000000000 --- a/packages/grid/docker-compose.dev.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: "3.8" -services: - proxy: - ports: - - "8080" - command: - - "--api" # admin panel - - "--api.insecure=true" # admin panel no password - - frontend: - volumes: - - ${RELATIVE_PATH}./frontend/src:/app/src - - ${RELATIVE_PATH}./frontend/static:/app/static - - ${RELATIVE_PATH}./frontend/svelte.config.js:/app/svelte.config.js - - ${RELATIVE_PATH}./frontend/tsconfig.json:/app/tsconfig.json - - ${RELATIVE_PATH}./frontend/vite.config.ts:/app/vite.config.ts - environment: - - FRONTEND_TARGET=grid-ui-development - - mongo: - ports: - - "27017" - - backend: - volumes: - - ${RELATIVE_PATH}./backend/grid:/root/app/grid - - ${RELATIVE_PATH}../syft:/root/app/syft - - ${RELATIVE_PATH}./data/package-cache:/root/.cache - environment: - - DEV_MODE=True - stdin_open: true - tty: true - - worker: - volumes: - - ${RELATIVE_PATH}./backend/grid:/root/app/grid - - ${RELATIVE_PATH}../syft:/root/app/syft - - ${RELATIVE_PATH}./data/package-cache:/root/.cache - environment: - - DEV_MODE=True - - WATCHFILES_FORCE_POLLING=true - stdin_open: true - tty: true - - rathole: - volumes: - - ${RELATIVE_PATH}./rathole/:/root/app/ - environment: - - DEV_MODE=True - - APP_PORT=5555 - - APP_LOG_LEVEL=debug - stdin_open: true - tty: true - ports: - - 2333:2333 - - seaweedfs: - volumes: - - ./data/seaweedfs:/data - ports: - - "9333" # admin web port - - "8888" # filer web port - - "8333" # S3 API port diff --git a/packages/grid/docker-compose.pull.yml b/packages/grid/docker-compose.pull.yml deleted file mode 100644 index e68ed03d968..00000000000 --- a/packages/grid/docker-compose.pull.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: "3.8" -services: - seaweedfs: - image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${VERSION-latest}" - - proxy: - image: ${DOCKER_IMAGE_TRAEFIK?Variable not set}:${TRAEFIK_VERSION?Variable not set} - - mongo: - image: "${MONGO_IMAGE}:${MONGO_VERSION}" - - jaeger: - image: jaegertracing/all-in-one:1.37 - - # Temporary fix until we refactor pull, build, launch UI step during hagrid launch - worker: - image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" - - rathole: - image: "${DOCKER_IMAGE_RATHOLE?Variable not set}:${VERSION-latest}" diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml deleted file mode 100644 index 153dfb718ba..00000000000 --- a/packages/grid/docker-compose.yml +++ /dev/null @@ -1,249 +0,0 @@ -version: "3.8" -services: - # docker-host: - # image: qoomon/docker-host - # cap_add: - # - net_admin - # - net_raw - - proxy: - restart: always - hostname: ${NODE_NAME?Variable not set} - image: ${DOCKER_IMAGE_TRAEFIK?Variable not set}:${TRAEFIK_VERSION?Variable not set} - profiles: - - proxy - networks: - - "${TRAEFIK_PUBLIC_NETWORK?Variable not set}" - - default - volumes: - - "./traefik/docker/traefik.yml:/etc/traefik/traefik.yml" - - "./traefik/docker/dynamic.yml:/etc/traefik/conf/dynamic.yml" - environment: - - SERVICE_NAME=proxy - - RELEASE=${RELEASE:-production} - - HOSTNAME=${NODE_NAME?Variable not set} - - HTTP_PORT=${HTTP_PORT} - - HTTPS_PORT=${HTTPS_PORT} - ports: - - "${HTTP_PORT}:81" - extra_hosts: - - "host.docker.internal:host-gateway" - labels: - - "orgs.openmined.syft=this is a syft proxy container" - - # depends_on: - # - "docker-host" - - frontend: - restart: always - image: "${DOCKER_IMAGE_FRONTEND?Variable not set}:${VERSION-latest}" - profiles: - - frontend - depends_on: - - proxy - environment: - - SERVICE_NAME=frontend - - RELEASE=${RELEASE:-production} - - NODE_TYPE=${NODE_TYPE?Variable not set} - - FRONTEND_TARGET=${FRONTEND_TARGET} - - VERSION=${VERSION} - - VERSION_HASH=${VERSION_HASH} - - PORT=80 - - HTTP_PORT=${HTTP_PORT} - - HTTPS_PORT=${HTTPS_PORT}RELOAD - - BACKEND_API_BASE_URL=${BACKEND_API_BASE_URL} - extra_hosts: - - "host.docker.internal:host-gateway" - labels: - - "orgs.openmined.syft=this is a syft frontend container" - - rathole: - restart: always - image: "${DOCKER_IMAGE_RATHOLE?Variable not set}:${VERSION-latest}" - profiles: - - rathole - depends_on: - - proxy - environment: - - SERVICE_NAME=rathole - - APP_LOG_LEVEL=${APP_LOG_LEVEL:-info} - - MODE=${MODE} - - DEV_MODE=${DEV_MODE} - - APP_PORT=${APP_PORT:-5555} - - RATHOLE_PORT=${RATHOLE_PORT:-2333} - extra_hosts: - - "host.docker.internal:host-gateway" - labels: - - "orgs.openmined.syft=this is a syft rathole container" - - worker: - restart: always - image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" - hostname: ${NODE_NAME?Variable not set} - profiles: - - worker - env_file: - - .env - environment: - - SERVICE_NAME=worker - - RELEASE=${RELEASE:-production} - - VERSION=${VERSION} - - VERSION_HASH=${VERSION_HASH} - - NODE_TYPE=${NODE_TYPE?Variable not set} - - NODE_NAME=${NODE_NAME?Variable not set} - - STACK_API_KEY=${STACK_API_KEY} - - PORT=${HTTP_PORT} - - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} - - HTTP_PORT=${HTTP_PORT} - - HTTPS_PORT=${HTTPS_PORT} - - USE_BLOB_STORAGE=${USE_BLOB_STORAGE} - - CONTAINER_HOST=${CONTAINER_HOST} - - TRACE=False # TODO: Trace Mode is set to False, until jaegar is integrated - - JAEGER_HOST=${JAEGER_HOST} - - JAEGER_PORT=${JAEGER_PORT} - - ASSOCIATION_TIMEOUT=${ASSOCIATION_TIMEOUT} - - DEV_MODE=${DEV_MODE} - - QUEUE_PORT=${QUEUE_PORT} - - CREATE_PRODUCER=true - - NODE_SIDE_TYPE=${NODE_SIDE_TYPE} - - ENABLE_WARNINGS=${ENABLE_WARNINGS} - - INMEMORY_WORKERS=True # hardcoding is intentional, since single_container don't share databases - ports: - - "${HTTP_PORT}:${HTTP_PORT}" - volumes: - - credentials-data:/root/data/creds/ - - /var/run/docker.sock:/var/run/docker.sock - extra_hosts: - - "host.docker.internal:host-gateway" - labels: - - "orgs.openmined.syft=this is a syft worker container" - - backend: - restart: always - image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" - profiles: - - backend - depends_on: - - proxy - - mongo - env_file: - - .env - environment: - - SERVICE_NAME=backend - - RELEASE=${RELEASE:-production} - - VERSION=${VERSION} - - VERSION_HASH=${VERSION_HASH} - - NODE_TYPE=${NODE_TYPE?Variable not set} - - NODE_NAME=${NODE_NAME?Variable not set} - - STACK_API_KEY=${STACK_API_KEY} - - PORT=8001 - - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} - - HTTP_PORT=${HTTP_PORT} - - HTTPS_PORT=${HTTPS_PORT} - - USE_BLOB_STORAGE=${USE_BLOB_STORAGE} - - CONTAINER_HOST=${CONTAINER_HOST} - - TRACE=${TRACE} - - JAEGER_HOST=${JAEGER_HOST} - - JAEGER_PORT=${JAEGER_PORT} - - ASSOCIATION_TIMEOUT=${ASSOCIATION_TIMEOUT} - - DEV_MODE=${DEV_MODE} - - DEFAULT_ROOT_EMAIL=${DEFAULT_ROOT_EMAIL} - - DEFAULT_ROOT_PASSWORD=${DEFAULT_ROOT_PASSWORD} - - QUEUE_PORT=${QUEUE_PORT} - - CREATE_PRODUCER=true - - N_CONSUMERS=1 - - INMEMORY_WORKERS=${INMEMORY_WORKERS} - - HOST_GRID_PATH=${PWD} - command: "./grid/start.sh" - network_mode: service:proxy - volumes: - - ${CREDENTIALS_VOLUME}:/root/data/creds/ - - /var/run/docker.sock:/var/run/docker.sock - stdin_open: true - tty: true - labels: - - "orgs.openmined.syft=this is a syft backend container" - - seaweedfs: - profiles: - - blob-storage - depends_on: - - proxy - env_file: - - .env - image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${VERSION-latest}" - environment: - - SWFS_VOLUME_SIZE_LIMIT_MB=${SWFS_VOLUME_SIZE_LIMIT_MB:-1000} - - S3_ROOT_USER=${S3_ROOT_USER:-admin} - - S3_ROOT_PWD=${S3_ROOT_PWD:-admin} - - MOUNT_API_PORT=${MOUNT_API_PORT:-4001} - volumes: - - seaweedfs-data:/data - labels: - - "orgs.openmined.syft=this is a syft seaweedfs container" - - mongo: - image: "${MONGO_IMAGE}:${MONGO_VERSION}" - profiles: - - mongo - restart: always - environment: - - MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME} - - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD} - volumes: - - mongo-data:/data/db - - mongo-config-data:/data/configdb - labels: - - "orgs.openmined.syft=this is a syft mongo container" - - jaeger: - profiles: - - telemetry - image: jaegertracing/all-in-one:1.37 - environment: - - COLLECTOR_ZIPKIN_HOST_PORT=9411 - - COLLECTOR_OTLP_ENABLED=true - extra_hosts: - - "host.docker.internal:host-gateway" - ports: - - "${JAEGER_PORT}:14268" # http collector - - "16686" # ui - # - "6831:6831/udp" - # - "6832:6832/udp" - # - "5778:5778" - # - "4317:4317" - # - "4318:4318" - # - "14250:14250" - # - "14269:14269" - # - "9411:9411" - volumes: - - jaeger-data:/tmp - labels: - - "orgs.openmined.syft=this is a syft jaeger container" - -volumes: - credentials-data: - labels: - orgs.openmined.syft: "this is a syft credentials volume" - seaweedfs-data: - labels: - orgs.openmined.syft: "this is a syft seaweedfs volume" - mongo-data: - labels: - orgs.openmined.syft: "this is a syft mongo volume" - mongo-config-data: - labels: - orgs.openmined.syft: "this is a syft mongo volume" - jaeger-data: - labels: - orgs.openmined.syft: "this is a syft jaeger volume" - -networks: - traefik-public: - # Allow setting it to false for testing - external: ${TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL-true} - labels: - orgs.openmined.syft: "this is a syft traefik public network" - default: - labels: - orgs.openmined.syft: "this is a syft default network" diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py deleted file mode 100644 index 9407e49add8..00000000000 --- a/packages/hagrid/hagrid/cli.py +++ /dev/null @@ -1,4425 +0,0 @@ -# stdlib -from collections import namedtuple -from collections.abc import Callable -from enum import Enum -import json -import os -from pathlib import Path -import platform -from queue import Queue -import re -import shutil -import socket -import stat -import subprocess # nosec -import sys -import tempfile -from threading import Event -from threading import Thread -import time -from typing import Any -from typing import cast -from urllib.parse import urlparse -import webbrowser - -# third party -import click -import requests -import rich -from rich.console import Console -from rich.live import Live -from rich.progress import BarColumn -from rich.progress import Progress -from rich.progress import SpinnerColumn -from rich.progress import TextColumn -from virtualenvapi.manage import VirtualEnvironment - -# relative -from .art import RichEmoji -from .art import hagrid -from .art import quickstart_art -from .auth import AuthCredentials -from .cache import DEFAULT_BRANCH -from .cache import DEFAULT_REPO -from .cache import arg_cache -from .deps import DEPENDENCIES -from .deps import LATEST_BETA_SYFT -from .deps import allowed_hosts -from .deps import check_docker_service_status -from .deps import check_docker_version -from .deps import check_grid_docker -from .deps import gather_debug -from .deps import get_version_string -from .deps import is_windows -from .exceptions import MissingDependency -from .grammar import BadGrammar -from .grammar import GrammarVerb -from .grammar import parse_grammar -from .land import get_land_verb -from .launch import get_launch_verb -from .lib import GIT_REPO -from .lib import GRID_SRC_PATH -from .lib import GRID_SRC_VERSION -from .lib import check_api_metadata -from .lib import check_host -from .lib import check_jupyter_server -from .lib import check_login_page -from .lib import commit_hash -from .lib import docker_desktop_memory -from .lib import find_available_port -from .lib import generate_process_status_table -from .lib import generate_user_table -from .lib import gitpod_url -from .lib import hagrid_root -from .lib import is_gitpod -from .lib import name_tag -from .lib import save_vm_details_as_json -from .lib import update_repo -from .lib import use_branch -from .mode import EDITABLE_MODE -from .parse_template import deployment_dir -from .parse_template import get_template_yml -from .parse_template import manifest_cache_path -from .parse_template import render_templates -from .parse_template import setup_from_manifest_template -from .quickstart_ui import fetch_notebooks_for_url -from .quickstart_ui import fetch_notebooks_from_zipfile -from .quickstart_ui import quickstart_download_notebook -from .rand_sec import generate_sec_random_password -from .stable_version import LATEST_STABLE_SYFT -from .style import RichGroup -from .util import fix_windows_virtualenv_api -from .util import from_url -from .util import shell - -# fix VirtualEnvironment bug in windows -fix_windows_virtualenv_api(VirtualEnvironment) - - -class NodeSideType(Enum): - LOW_SIDE = "low" - HIGH_SIDE = "high" - - -def get_azure_image(short_name: str) -> str: - prebuild_070 = ( - "madhavajay1632269232059:openmined_mj_grid_domain_ubuntu_1:domain_070:latest" - ) - fresh_ubuntu = "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest" - if short_name == "default": - return fresh_ubuntu - elif short_name == "domain_0.7.0": - return prebuild_070 - raise Exception(f"Image name doesn't exist: {short_name}. Try: default or 0.7.0") - - -@click.group(cls=RichGroup) -def cli() -> None: - pass - - -def get_compose_src_path( - node_name: str, - template_location: str | None = None, - **kwargs: Any, -) -> str: - grid_path = GRID_SRC_PATH() - tag = kwargs["tag"] - # Use local compose files if in editable mode and - # template_location is None and (kwargs["dev"] is True or tag is local) - if ( - EDITABLE_MODE - and template_location is None - and (kwargs["dev"] is True or tag == "local") - ): - path = grid_path - else: - path = deployment_dir(node_name) - - os.makedirs(path, exist_ok=True) - return path - - -@click.command( - help="Restore some part of the hagrid installation or deployment to its initial/starting state.", - context_settings={"show_default": True}, -) -@click.argument("location", type=str, nargs=1) -def clean(location: str) -> None: - if location == "library" or location == "volumes": - print("Deleting all Docker volumes in 2 secs (Ctrl-C to stop)") - time.sleep(2) - subprocess.call("docker volume rm $(docker volume ls -q)", shell=True) # nosec - - if location == "containers" or location == "pantry": - print("Deleting all Docker containers in 2 secs (Ctrl-C to stop)") - time.sleep(2) - subprocess.call("docker rm -f $(docker ps -a -q)", shell=True) # nosec - - if location == "images": - print("Deleting all Docker images in 2 secs (Ctrl-C to stop)") - time.sleep(2) - subprocess.call("docker rmi $(docker images -q)", shell=True) # nosec - - -@click.command( - help="Start a new PyGrid domain/network node!", - context_settings={"show_default": True}, -) -@click.argument("args", type=str, nargs=-1) -@click.option( - "--username", - default=None, - required=False, - type=str, - help="Username for provisioning the remote host", -) -@click.option( - "--key-path", - default=None, - required=False, - type=str, - help="Path to the key file for provisioning the remote host", -) -@click.option( - "--password", - default=None, - required=False, - type=str, - help="Password for provisioning the remote host", -) -@click.option( - "--repo", - default=None, - required=False, - type=str, - help="Repo to fetch source from", -) -@click.option( - "--branch", - default=None, - required=False, - type=str, - help="Branch to monitor for updates", -) -@click.option( - "--tail", - is_flag=True, - help="Tail logs on launch", -) -@click.option( - "--headless", - is_flag=True, - help="Start the frontend container", -) -@click.option( - "--cmd", - is_flag=True, - help="Print the cmd without running it", -) -@click.option( - "--jupyter", - is_flag=True, - help="Enable Jupyter Notebooks", -) -@click.option( - "--in-mem-workers", - is_flag=True, - help="Enable InMemory Workers", -) -@click.option( - "--enable-signup", - is_flag=True, - help="Enable Signup for Node", -) -@click.option( - "--build", - is_flag=True, - help="Disable forcing re-build", -) -@click.option( - "--no-provision", - is_flag=True, - help="Disable provisioning VMs", -) -@click.option( - "--node-count", - default=1, - required=False, - type=click.IntRange(1, 250), - help="Number of independent nodes/VMs to launch", -) -@click.option( - "--auth-type", - default=None, - type=click.Choice(["key", "password"], case_sensitive=False), -) -@click.option( - "--ansible-extras", - default="", - type=str, -) -@click.option("--tls", is_flag=True, help="Launch with TLS configuration") -@click.option("--test", is_flag=True, help="Launch with test configuration") -@click.option("--dev", is_flag=True, help="Shortcut for development mode") -@click.option( - "--release", - default="production", - required=False, - type=click.Choice(["production", "staging", "development"], case_sensitive=False), - help="Choose between production and development release", -) -@click.option( - "--deployment-type", - default="container_stack", - required=False, - type=click.Choice(["container_stack", "single_container"], case_sensitive=False), - help="Choose between container_stack and single_container deployment", -) -@click.option( - "--cert-store-path", - default="/home/om/certs", - required=False, - type=str, - help="Remote path to store and load TLS cert and key", -) -@click.option( - "--upload-tls-cert", - default="", - required=False, - type=str, - help="Local path to TLS cert to upload and store at --cert-store-path", -) -@click.option( - "--upload-tls-key", - default="", - required=False, - type=str, - help="Local path to TLS private key to upload and store at --cert-store-path", -) -@click.option( - "--no-blob-storage", - is_flag=True, - help="Disable blob storage", -) -@click.option( - "--image-name", - default=None, - required=False, - type=str, - help="Image to use for the VM", -) -@click.option( - "--tag", - default=None, - required=False, - type=str, - help="Container image tag to use", -) -@click.option( - "--smtp-username", - default=None, - required=False, - type=str, - help="Username used to auth in email server and enable notification via emails", -) -@click.option( - "--smtp-password", - default=None, - required=False, - type=str, - help="Password used to auth in email server and enable notification via emails", -) -@click.option( - "--smtp-port", - default=None, - required=False, - type=str, - help="Port used by email server to send notification via emails", -) -@click.option( - "--smtp-host", - default=None, - required=False, - type=str, - help="Address used by email server to send notification via emails", -) -@click.option( - "--smtp-sender", - default=None, - required=False, - type=str, - help="Sender email used to deliver PyGrid email notifications.", -) -@click.option( - "--build-src", - default=DEFAULT_BRANCH, - required=False, - type=str, - help="Git branch to use for launch / build operations", -) -@click.option( - "--platform", - default=None, - required=False, - type=str, - help="Run docker with a different platform like linux/arm64", -) -@click.option( - "--verbose", - is_flag=True, - help="Show verbose output", -) -@click.option( - "--trace", - required=False, - type=str, - help="Optional: allow trace to be turned on or off", -) -@click.option( - "--template", - required=False, - default=None, - help="Path or URL to manifest template", -) -@click.option( - "--template-overwrite", - is_flag=True, - help="Force re-downloading of template manifest", -) -@click.option( - "--no-health-checks", - is_flag=True, - help="Turn off auto health checks post node launch", -) -@click.option( - "--set-root-email", - default=None, - required=False, - type=str, - help="Set root email of node", -) -@click.option( - "--set-root-password", - default=None, - required=False, - type=str, - help="Set root password of node", -) -@click.option( - "--azure-resource-group", - default=None, - required=False, - type=str, - help="Azure Resource Group", -) -@click.option( - "--azure-location", - default=None, - required=False, - type=str, - help="Azure Resource Group Location", -) -@click.option( - "--azure-size", - default=None, - required=False, - type=str, - help="Azure VM Size", -) -@click.option( - "--azure-username", - default=None, - required=False, - type=str, - help="Azure VM Username", -) -@click.option( - "--azure-key-path", - default=None, - required=False, - type=str, - help="Azure Key Path", -) -@click.option( - "--azure-repo", - default=None, - required=False, - type=str, - help="Azure Source Repo", -) -@click.option( - "--azure-branch", - default=None, - required=False, - type=str, - help="Azure Source Branch", -) -@click.option( - "--render", - is_flag=True, - help="Render Docker Files", -) -@click.option( - "--no-warnings", - is_flag=True, - help="Enable API warnings on the node.", -) -@click.option( - "--low-side", - is_flag=True, - help="Launch a low side node type else a high side node type", -) -@click.option( - "--set-s3-username", - default=None, - required=False, - type=str, - help="Set root username for s3 blob storage", -) -@click.option( - "--set-s3-password", - default=None, - required=False, - type=str, - help="Set root password for s3 blob storage", -) -@click.option( - "--set-volume-size-limit-mb", - default=1024, - required=False, - type=click.IntRange(1024, 50000), - help="Set the volume size limit (in MBs)", -) -@click.option( - "--association-request-auto-approval", - is_flag=True, - help="Enable auto approval of association requests", -) -@click.option( - "--rathole", - is_flag=True, - help="Enable rathole service", -) -def launch(args: tuple[str], **kwargs: Any) -> None: - verb = get_launch_verb() - try: - grammar = parse_grammar(args=args, verb=verb) - verb.load_grammar(grammar=grammar) - except BadGrammar as e: - print(e) - return - - node_name = verb.get_named_term_type(name="node_name") - snake_name = str(node_name.snake_input) - node_type = verb.get_named_term_type(name="node_type") - - # For enclave currently it is only a single container deployment - # This would change when we have side car containers to enclave - if node_type.input == "enclave": - kwargs["deployment_type"] = "single_container" - - compose_src_path = get_compose_src_path( - node_type=node_type, - node_name=snake_name, - template_location=kwargs["template"], - **kwargs, - ) - kwargs["compose_src_path"] = compose_src_path - - try: - update_repo(repo=GIT_REPO(), branch=str(kwargs["build_src"])) - except Exception as e: - print(f"Failed to update repo. {e}") - try: - cmds = create_launch_cmd(verb=verb, kwargs=kwargs) - cmds = [cmds] if isinstance(cmds, str) else cmds - except Exception as e: - print(f"Error: {e}\n\n") - return - - dry_run = bool(kwargs["cmd"]) - - health_checks = not bool(kwargs["no_health_checks"]) - render_only = bool(kwargs["render"]) - - try: - tail = bool(kwargs["tail"]) - verbose = bool(kwargs["verbose"]) - silent = not verbose - if tail: - silent = False - - if render_only: - print( - "Docker Compose Files Rendered: {}".format(kwargs["compose_src_path"]) - ) - return - - execute_commands( - cmds, - dry_run=dry_run, - silent=silent, - compose_src_path=kwargs["compose_src_path"], - node_type=node_type.input, - ) - - host_term = verb.get_named_term_hostgrammar(name="host") - run_health_checks = ( - health_checks and not dry_run and host_term.host == "docker" and silent - ) - - if run_health_checks: - docker_cmds = cast(dict[str, list[str]], cmds) - - # get the first command (cmd1) from docker_cmds which is of the form - # {"": [cmd1, cmd2], "": [cmd3, cmd4]} - (command, *_), *_ = docker_cmds.values() - - match_port = re.search("HTTP_PORT=[0-9]{1,5}", command) - if match_port: - rich.get_console().print( - "\n[bold green]⠋[bold blue] Checking node API [/bold blue]\t" - ) - port = match_port.group().replace("HTTP_PORT=", "") - - check_status("localhost" + ":" + port, node_name=node_name.snake_input) - - rich.get_console().print( - rich.panel.Panel.fit( - f"✨ To view container logs run [bold green]hagrid logs {node_name.snake_input}[/bold green]\t" - ) - ) - - except Exception as e: - print(f"Error: {e}\n\n") - return - - -def check_errors( - line: str, process: subprocess.Popen, cmd_name: str, progress_bar: Progress -) -> None: - task = progress_bar.tasks[0] - if "Error response from daemon: " in line: - if progress_bar: - progress_bar.update( - 0, - description=f"❌ [bold red]{cmd_name}[/bold red] [{task.completed} / {task.total}]", - refresh=True, - ) - progress_bar.update(0, visible=False) - progress_bar.console.clear_live() - progress_bar.console.quiet = True - progress_bar.stop() - console = rich.get_console() - progress_bar.console.quiet = False - console.print(f"\n\n [red] ERROR [/red]: [bold]{line}[/bold]\n") - process.terminate() - raise Exception - - -def check_pulling(line: str, cmd_name: str, progress_bar: Progress) -> None: - task = progress_bar.tasks[0] - if "Pulling" in line and "fs layer" not in line: - progress_bar.update( - 0, - description=f"[bold]{cmd_name} [{task.completed} / {task.total+1}]", - total=task.total + 1, - refresh=True, - ) - if "Pulled" in line: - progress_bar.update( - 0, - description=f"[bold]{cmd_name} [{task.completed + 1} / {task.total}]", - completed=task.completed + 1, - refresh=True, - ) - if progress_bar.finished: - progress_bar.update( - 0, - description=f"✅ [bold green]{cmd_name} [{task.completed} / {task.total}]", - refresh=True, - ) - - -def check_building(line: str, cmd_name: str, progress_bar: Progress) -> None: - load_pattern = re.compile( - r"^#.* load build definition from [A-Za-z0-9]+\.dockerfile$", re.IGNORECASE - ) - build_pattern = re.compile( - r"^#.* naming to docker\.io/openmined/.* done$", re.IGNORECASE - ) - task = progress_bar.tasks[0] - - if load_pattern.match(line): - progress_bar.update( - 0, - description=f"[bold]{cmd_name} [{task.completed} / {task.total +1}]", - total=task.total + 1, - refresh=True, - ) - if build_pattern.match(line): - progress_bar.update( - 0, - description=f"[bold]{cmd_name} [{task.completed+1} / {task.total}]", - completed=task.completed + 1, - refresh=True, - ) - - if progress_bar.finished: - progress_bar.update( - 0, - description=f"✅ [bold green]{cmd_name} [{task.completed} / {task.total}]", - refresh=True, - ) - - -def check_launching(line: str, cmd_name: str, progress_bar: Progress) -> None: - task = progress_bar.tasks[0] - if "Starting" in line: - progress_bar.update( - 0, - description=f" [bold]{cmd_name} [{task.completed} / {task.total+1}]", - total=task.total + 1, - refresh=True, - ) - if "Started" in line: - progress_bar.update( - 0, - description=f" [bold]{cmd_name} [{task.completed + 1} / {task.total}]", - completed=task.completed + 1, - refresh=True, - ) - if progress_bar.finished: - progress_bar.update( - 0, - description=f"✅ [bold green]{cmd_name} [{task.completed} / {task.total}]", - refresh=True, - ) - - -DOCKER_FUNC_MAP = { - "Pulling": check_pulling, - "Building": check_building, - "Launching": check_launching, -} - - -def read_thread_logs( - progress_bar: Progress, process: subprocess.Popen, queue: Queue, cmd_name: str -) -> None: - line = queue.get() - line = str(line, encoding="utf-8").strip() - - if progress_bar: - check_errors(line, process, cmd_name, progress_bar=progress_bar) - DOCKER_FUNC_MAP[cmd_name](line, cmd_name, progress_bar=progress_bar) - - -def create_thread_logs(process: subprocess.Popen) -> Queue: - def enqueue_output(out: Any, queue: Queue) -> None: - for line in iter(out.readline, b""): - queue.put(line) - out.close() - - queue: Queue = Queue() - thread_1 = Thread(target=enqueue_output, args=(process.stdout, queue)) - thread_2 = Thread(target=enqueue_output, args=(process.stderr, queue)) - - thread_1.daemon = True # thread dies with the program - thread_1.start() - thread_2.daemon = True # thread dies with the program - thread_2.start() - return queue - - -def process_cmd( - cmds: list[str], - node_type: str, - dry_run: bool, - silent: bool, - compose_src_path: str, - progress_bar: Progress | None = None, - cmd_name: str = "", -) -> None: - process_list: list = [] - cwd = compose_src_path - - username, password = ( - extract_username_and_pass(cmds[0]) if len(cmds) > 0 else ("-", "-") - ) - # display VM credentials - console = rich.get_console() - credentials = generate_user_table(username=username, password=password) - if credentials: - console.print(credentials) - - for cmd in cmds: - if dry_run: - print(f"\nRunning:\ncd {cwd}\n", hide_password(cmd=cmd)) - continue - - # use powershell if environment is Windows - cmd_to_exec = ["powershell.exe", "-Command", cmd] if is_windows() else cmd - - try: - if len(cmds) > 1: - process = subprocess.Popen( # nosec - cmd_to_exec, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=cwd, - shell=True, - ) - ip_address = extract_host_ip_from_cmd(cmd) - jupyter_token = extract_jupyter_token(cmd) - process_list.append((ip_address, process, jupyter_token)) - else: - display_jupyter_token(cmd) - if silent: - ON_POSIX = "posix" in sys.builtin_module_names - - process = subprocess.Popen( # nosec - cmd_to_exec, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=cwd, - close_fds=ON_POSIX, - shell=True, - ) - - # Creates two threads to get docker stdout and sterr - logs_queue = create_thread_logs(process=process) - - read_thread_logs(progress_bar, process, logs_queue, cmd_name) - while process.poll() != 0: - while not logs_queue.empty(): - # Read stdout and sterr to check errors or update progress bar. - read_thread_logs( - progress_bar, process, logs_queue, cmd_name - ) - else: - if progress_bar: - progress_bar.stop() - - subprocess.run( # nosec - cmd_to_exec, - shell=True, - cwd=cwd, - ) - except Exception as e: - print(f"Failed to run cmd: {cmd}. {e}") - - if dry_run is False and len(process_list) > 0: - # display VM launch status - display_vm_status(process_list) - - # save vm details as json - save_vm_details_as_json(username, password, process_list) - - -def execute_commands( - cmds: list[str] | dict[str, list[str]], - node_type: str, - compose_src_path: str, - dry_run: bool = False, - silent: bool = False, -) -> None: - """Execute the launch commands and display their status in realtime. - - Args: - cmds (list): list of commands to be executed - dry_run (bool, optional): If `True` only displays cmds to be executed. Defaults to False. - """ - console = rich.get_console() - if isinstance(cmds, dict): - console.print("[bold green]⠋[bold blue] Launching Containers [/bold blue]\t") - for cmd_name, cmd in cmds.items(): - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:.2f}% "), - console=console, - auto_refresh=True, - ) as progress: - if silent: - progress.add_task( - f"[bold green]{cmd_name} Images", - total=0, - ) - process_cmd( - cmds=cmd, - node_type=node_type, - dry_run=dry_run, - silent=silent, - compose_src_path=compose_src_path, - progress_bar=progress, - cmd_name=cmd_name, - ) - else: - process_cmd( - cmds=cmds, - node_type=node_type, - dry_run=dry_run, - silent=silent, - compose_src_path=compose_src_path, - ) - - -def display_vm_status(process_list: list) -> None: - """Display the status of the processes being executed on the VM. - - Args: - process_list (list): list of processes executed. - """ - - # Generate the table showing the status of each process being executed - status_table, process_completed = generate_process_status_table(process_list) - - # Render the live table - with Live(status_table, refresh_per_second=1) as live: - # Loop till all processes have not completed executing - while not process_completed: - status_table, process_completed = generate_process_status_table( - process_list - ) - live.update(status_table) # Update the process status table - - -def display_jupyter_token(cmd: str) -> None: - token = extract_jupyter_token(cmd=cmd) - if token is not None: - print(f"Jupyter Token: {token}") - - -def extract_username_and_pass(cmd: str) -> tuple: - # Extract username - matcher = r"--user (.+?) " - username = re.findall(matcher, cmd) - username = username[0] if len(username) > 0 else None - - # Extract password - matcher = r"ansible_ssh_pass='(.+?)'" - password = re.findall(matcher, cmd) - password = password[0] if len(password) > 0 else None - - return username, password - - -def extract_jupyter_token(cmd: str) -> str | None: - matcher = r"jupyter_token='(.+?)'" - token = re.findall(matcher, cmd) - if len(token) == 1: - return token[0] - return None - - -def hide_password(cmd: str) -> str: - try: - matcher = r"ansible_ssh_pass='(.+?)'" - passwords = re.findall(matcher, cmd) - if len(passwords) > 0: - password = passwords[0] - stars = "*" * 4 - cmd = cmd.replace( - f"ansible_ssh_pass='{password}'", f"ansible_ssh_pass='{stars}'" - ) - return cmd - except Exception as e: - print("Failed to hide password.") - raise e - - -def hide_azure_vm_password(azure_cmd: str) -> str: - try: - matcher = r"admin-password '(.+?)'" - passwords = re.findall(matcher, azure_cmd) - if len(passwords) > 0: - password = passwords[0] - stars = "*" * 4 - azure_cmd = azure_cmd.replace( - f"admin-password '{password}'", f"admin-password '{stars}'" - ) - return azure_cmd - except Exception as e: - print("Failed to hide password.") - raise e - - -class QuestionInputError(Exception): - pass - - -class QuestionInputPathError(Exception): - pass - - -class Question: - def __init__( - self, - var_name: str, - question: str, - kind: str, - default: str | None = None, - cache: bool = False, - options: list[str] | None = None, - ) -> None: - self.var_name = var_name - self.question = question - self.default = default - self.kind = kind - self.cache = cache - self.options = options if options is not None else [] - - def validate(self, value: str) -> str: - value = value.strip() - if self.default is not None and value == "": - return self.default - - if self.kind == "path": - value = os.path.expanduser(value) - if not os.path.exists(value): - error = f"{value} is not a valid path." - if self.default is not None: - error += f" Try {self.default}" - raise QuestionInputPathError(f"{error}") - - if self.kind == "yesno": - if value.lower().startswith("y"): - return "y" - elif value.lower().startswith("n"): - return "n" - else: - raise QuestionInputError(f"{value} is not an yes or no answer") - - if self.kind == "options": - if value in self.options: - return value - first_letter = value.lower()[0] - for option in self.options: - if option.startswith(first_letter): - return option - - raise QuestionInputError( - f"{value} is not one of the options: {self.options}" - ) - - if self.kind == "password": - try: - return validate_password(password=value) - except Exception as e: - raise QuestionInputError(f"Invalid password. {e}") - return value - - -def ask(question: Question, kwargs: dict[str, str]) -> str: - if question.var_name in kwargs and kwargs[question.var_name] is not None: - value = kwargs[question.var_name] - else: - if question.default is not None: - value = click.prompt(question.question, type=str, default=question.default) - elif question.var_name == "password": - value = click.prompt( - question.question, type=str, hide_input=True, confirmation_prompt=True - ) - else: - value = click.prompt(question.question, type=str) - - try: - value = question.validate(value=value) - except QuestionInputError as e: - print(e) - return ask(question=question, kwargs=kwargs) - if question.cache: - arg_cache[question.var_name] = value - - return value - - -def fix_key_permission(private_key_path: str) -> None: - key_permission = oct(stat.S_IMODE(os.stat(private_key_path).st_mode)) - chmod_permission = "400" - octal_permission = f"0o{chmod_permission}" - if key_permission != octal_permission: - print( - f"Fixing key permission: {private_key_path}, setting to {chmod_permission}" - ) - try: - os.chmod(private_key_path, int(octal_permission, 8)) - except Exception as e: - print("Failed to fix key permission", e) - raise e - - -def private_to_public_key(private_key_path: str, temp_path: str, username: str) -> str: - # check key permission - fix_key_permission(private_key_path=private_key_path) - output_path = f"{temp_path}/hagrid_{username}_key.pub" - cmd = f"ssh-keygen -f {private_key_path} -y > {output_path}" - try: - subprocess.check_call(cmd, shell=True) # nosec - except Exception as e: - print("failed to make ssh key", e) - raise e - return output_path - - -def check_azure_authed() -> bool: - cmd = "az account show" - try: - subprocess.check_call(cmd, shell=True, stdout=subprocess.DEVNULL) # nosec - return True - except Exception: # nosec - pass - return False - - -def login_azure() -> bool: - cmd = "az login" - try: - subprocess.check_call(cmd, shell=True, stdout=subprocess.DEVNULL) # nosec - return True - except Exception: # nosec - pass - return False - - -def check_azure_cli_installed() -> bool: - try: - result = subprocess.run( # nosec - ["az", "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT - ) - if result.returncode != 0: - raise FileNotFoundError("az not installed") - except Exception: # nosec - msg = "\nYou don't appear to have the Azure CLI installed!!! \n\n\ -Please install it and then retry your command.\ -\n\nInstallation Instructions: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli\n" - raise FileNotFoundError(msg) - - return True - - -def check_gcloud_cli_installed() -> bool: - try: - subprocess.call(["gcloud", "version"]) # nosec - print("Gcloud cli installed!") - except FileNotFoundError: - msg = "\nYou don't appear to have the gcloud CLI tool installed! \n\n\ -Please install it and then retry again.\ -\n\nInstallation Instructions: https://cloud.google.com/sdk/docs/install-sdk \n" - raise FileNotFoundError(msg) - - return True - - -def check_aws_cli_installed() -> bool: - try: - result = subprocess.run( # nosec - ["aws", "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT - ) - if result.returncode != 0: - raise FileNotFoundError("AWS CLI not installed") - except Exception: # nosec - msg = "\nYou don't appear to have the AWS CLI installed! \n\n\ -Please install it and then retry your command.\ -\n\nInstallation Instructions: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html\n" - raise FileNotFoundError(msg) - - return True - - -def check_gcloud_authed() -> bool: - try: - result = subprocess.run( # nosec - ["gcloud", "auth", "print-identity-token"], stdout=subprocess.PIPE - ) - if result.returncode == 0: - return True - except Exception: # nosec - pass - return False - - -def login_gcloud() -> bool: - cmd = "gcloud auth login" - try: - subprocess.check_call(cmd, shell=True, stdout=subprocess.DEVNULL) # nosec - return True - except Exception: # nosec - pass - return False - - -def str_to_bool(bool_str: str | None) -> bool: - result = False - bool_str = str(bool_str).lower() - if bool_str == "true" or bool_str == "1": - result = True - return result - - -ART = str_to_bool(os.environ.get("HAGRID_ART", "True")) - - -def generate_gcloud_key_at_path(key_path: str) -> str: - key_path = os.path.expanduser(key_path) - if os.path.exists(key_path): - raise Exception(f"Can't generate key since path already exists. {key_path}") - else: - # triggers a key check - cmd = "gcloud compute ssh '' --dry-run" - try: - subprocess.check_call(cmd, shell=True) # nosec - except Exception: # nosec - pass - if not os.path.exists(key_path): - raise Exception(f"gcloud failed to generate ssh-key at: {key_path}") - - return key_path - - -def generate_aws_key_at_path(key_path: str, key_name: str) -> str: - key_path = os.path.expanduser(key_path) - if os.path.exists(key_path): - raise Exception(f"Can't generate key since path already exists. {key_path}") - else: - # TODO we need to do differently for powershell. - # Ex: aws ec2 create-key-pair --key-name MyKeyPair --query 'KeyMaterial' - # --output text | out-file -encoding ascii -filepath MyKeyPair.pem - - print(f"Creating AWS key pair with name {key_name} at path {key_path}..") - cmd = f"aws ec2 create-key-pair --key-name {key_name} --query 'KeyMaterial' --output text > {key_path}" - try: - subprocess.check_call(cmd, shell=True) # nosec - subprocess.check_call(f"chmod 400 {key_path}", shell=True) # nosec - except Exception as e: # nosec - print(f"Failed to create key: {e}") - if not os.path.exists(key_path): - raise Exception(f"AWS failed to generate key pair at: {key_path}") - - return key_path - - -def generate_key_at_path(key_path: str) -> str: - key_path = os.path.expanduser(key_path) - if os.path.exists(key_path): - raise Exception(f"Can't generate key since path already exists. {key_path}") - else: - cmd = f"ssh-keygen -N '' -f {key_path}" - try: - subprocess.check_call(cmd, shell=True) # nosec - if not os.path.exists(key_path): - raise Exception(f"Failed to generate ssh-key at: {key_path}") - except Exception as e: - raise e - - return key_path - - -def validate_password(password: str) -> str: - """Validate if the password entered by the user is valid. - - Password length should be between 12 - 123 characters - Passwords must also meet 3 out of the following 4 complexity requirements: - - Have lower characters - - Have upper characters - - Have a digit - - Have a special character - - Args: - password (str): password for the vm - - Returns: - str: password if it is valid - """ - # Validate password length - if len(password) < 12 or len(password) > 123: - raise ValueError("Password length should be between 12 - 123 characters") - - # Valid character types - character_types = { - "upper_case": False, - "lower_case": False, - "digit": False, - "special": False, - } - - for ch in password: - if ch.islower(): - character_types["lower_case"] = True - elif ch.isupper(): - character_types["upper_case"] = True - elif ch.isdigit(): - character_types["digit"] = True - elif ch.isascii(): - character_types["special"] = True - else: - raise ValueError(f"{ch} is not a valid character for password") - - # Validate characters in the password - required_character_type_count = sum( - [int(value) for value in character_types.values()] - ) - - if required_character_type_count >= 3: - return password - - absent_character_types = ", ".join( - char_type for char_type, value in character_types.items() if value is False - ).strip(", ") - - raise ValueError( - f"At least one {absent_character_types} character types must be present" - ) - - -def create_launch_cmd( - verb: GrammarVerb, - kwargs: dict[str, Any], - ignore_docker_version_check: bool | None = False, -) -> str | list[str] | dict[str, list[str]]: - parsed_kwargs: dict[str, Any] = {} - host_term = verb.get_named_term_hostgrammar(name="host") - - host = host_term.host - auth: AuthCredentials | None = None - - tail = bool(kwargs["tail"]) - - parsed_kwargs = {} - - parsed_kwargs["build"] = bool(kwargs["build"]) - - parsed_kwargs["use_blob_storage"] = not bool(kwargs["no_blob_storage"]) - - parsed_kwargs["in_mem_workers"] = bool(kwargs["in_mem_workers"]) - - if parsed_kwargs["use_blob_storage"]: - parsed_kwargs["set_s3_username"] = kwargs["set_s3_username"] - parsed_kwargs["set_s3_password"] = kwargs["set_s3_password"] - parsed_kwargs["set_volume_size_limit_mb"] = kwargs["set_volume_size_limit_mb"] - - parsed_kwargs["association_request_auto_approval"] = str( - kwargs["association_request_auto_approval"] - ) - - parsed_kwargs["node_count"] = ( - int(kwargs["node_count"]) if "node_count" in kwargs else 1 - ) - - if parsed_kwargs["node_count"] > 1 and host not in ["azure"]: - print("\nArgument `node_count` is only supported with `azure`.\n") - else: - # Default to detached mode if running more than one nodes - tail = False if parsed_kwargs["node_count"] > 1 else tail - - headless = bool(kwargs["headless"]) - parsed_kwargs["headless"] = headless - - parsed_kwargs["tls"] = bool(kwargs["tls"]) - parsed_kwargs["enable_rathole"] = bool(kwargs["rathole"]) - parsed_kwargs["test"] = bool(kwargs["test"]) - parsed_kwargs["dev"] = bool(kwargs["dev"]) - - parsed_kwargs["silent"] = not bool(kwargs["verbose"]) - - parsed_kwargs["trace"] = False - if ("trace" not in kwargs or kwargs["trace"] is None) and parsed_kwargs["dev"]: - # default to trace on in dev mode - parsed_kwargs["trace"] = False - elif "trace" in kwargs: - parsed_kwargs["trace"] = str_to_bool(cast(str, kwargs["trace"])) - - parsed_kwargs["release"] = "production" - if "release" in kwargs and kwargs["release"] != "production": - parsed_kwargs["release"] = kwargs["release"] - - # if we use --dev override it - if parsed_kwargs["dev"] is True: - parsed_kwargs["release"] = "development" - - # derive node type - if kwargs["low_side"]: - parsed_kwargs["node_side_type"] = NodeSideType.LOW_SIDE.value - else: - parsed_kwargs["node_side_type"] = NodeSideType.HIGH_SIDE.value - - parsed_kwargs["smtp_username"] = kwargs["smtp_username"] - parsed_kwargs["smtp_password"] = kwargs["smtp_password"] - parsed_kwargs["smtp_port"] = kwargs["smtp_port"] - parsed_kwargs["smtp_host"] = kwargs["smtp_host"] - parsed_kwargs["smtp_sender"] = kwargs["smtp_sender"] - - parsed_kwargs["enable_warnings"] = not kwargs["no_warnings"] - - # choosing deployment type - parsed_kwargs["deployment_type"] = "container_stack" - if "deployment_type" in kwargs and kwargs["deployment_type"] is not None: - parsed_kwargs["deployment_type"] = kwargs["deployment_type"] - - if "cert_store_path" in kwargs: - parsed_kwargs["cert_store_path"] = kwargs["cert_store_path"] - if "upload_tls_cert" in kwargs: - parsed_kwargs["upload_tls_cert"] = kwargs["upload_tls_cert"] - if "upload_tls_key" in kwargs: - parsed_kwargs["upload_tls_key"] = kwargs["upload_tls_key"] - - parsed_kwargs["provision"] = not bool(kwargs["no_provision"]) - - if "image_name" in kwargs and kwargs["image_name"] is not None: - parsed_kwargs["image_name"] = kwargs["image_name"] - else: - parsed_kwargs["image_name"] = "default" - - if parsed_kwargs["dev"] is True: - parsed_kwargs["tag"] = "local" - else: - if "tag" in kwargs and kwargs["tag"] is not None and kwargs["tag"] != "": - parsed_kwargs["tag"] = kwargs["tag"] - else: - parsed_kwargs["tag"] = "latest" - - if "jupyter" in kwargs and kwargs["jupyter"] is not None: - parsed_kwargs["jupyter"] = str_to_bool(cast(str, kwargs["jupyter"])) - else: - parsed_kwargs["jupyter"] = False - - # allows changing docker platform to other cpu architectures like arm64 - parsed_kwargs["platform"] = kwargs["platform"] if "platform" in kwargs else None - - parsed_kwargs["tail"] = tail - - parsed_kwargs["set_root_password"] = ( - kwargs["set_root_password"] if "set_root_password" in kwargs else None - ) - - parsed_kwargs["set_root_email"] = ( - kwargs["set_root_email"] if "set_root_email" in kwargs else None - ) - - parsed_kwargs["template"] = kwargs["template"] if "template" in kwargs else None - parsed_kwargs["template_overwrite"] = bool(kwargs["template_overwrite"]) - - parsed_kwargs["compose_src_path"] = kwargs["compose_src_path"] - - parsed_kwargs["enable_signup"] = str_to_bool(cast(str, kwargs["enable_signup"])) - - # Override template tag with user input tag - if ( - parsed_kwargs["tag"] is not None - and parsed_kwargs["template"] is None - and parsed_kwargs["tag"] not in ["local"] - ): - # third party - from packaging import version - - pattern = r"[0-9].[0-9].[0-9]" - input_tag = parsed_kwargs["tag"] - if ( - not re.match(pattern, input_tag) - and input_tag != "latest" - and input_tag != "beta" - and "b" not in input_tag - ): - raise Exception( - f"Not a valid tag: {parsed_kwargs['tag']}" - + "\nValid tags: latest, beta, beta version(ex: 0.8.2b35),[0-9].[0-9].[0-9]" - ) - - # TODO: we need to redo this so that pypi and docker mappings are in a single - # file inside dev - if parsed_kwargs["tag"] == "latest": - parsed_kwargs["template"] = LATEST_STABLE_SYFT - parsed_kwargs["tag"] = LATEST_STABLE_SYFT - elif parsed_kwargs["tag"] == "beta" or "b" in parsed_kwargs["tag"]: - tag = ( - LATEST_BETA_SYFT - if parsed_kwargs["tag"] == "beta" - else parsed_kwargs["tag"] - ) - - # Currently, manifest_template.yml is only supported for beta versions >= 0.8.2b34 - beta_version = version.parse(tag) - MINIMUM_BETA_VERSION = "0.8.2b34" - if beta_version < version.parse(MINIMUM_BETA_VERSION): - raise Exception( - f"Minimum beta version tag supported is {MINIMUM_BETA_VERSION}" - ) - - # Check if the beta version is available - template_url = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest_template.yml" - response = requests.get(template_url) # nosec - if response.status_code != 200: - raise Exception( - f"Tag {parsed_kwargs['tag']} is not available" - + " \n for download. Please check the available tags at: " - + "\n https://github.com/OpenMined/PySyft/releases" - ) - - parsed_kwargs["template"] = template_url - parsed_kwargs["tag"] = tag - else: - MINIMUM_TAG_VERSION = version.parse("0.8.0") - tag = version.parse(parsed_kwargs["tag"]) - if tag < MINIMUM_TAG_VERSION: - raise Exception( - f"Minimum supported stable tag version is {MINIMUM_TAG_VERSION}" - ) - parsed_kwargs["template"] = parsed_kwargs["tag"] - - if host in ["docker"] and parsed_kwargs["template"] and host is not None: - # Setup the files from the manifest_template.yml - kwargs = setup_from_manifest_template( - host_type=host, - deployment_type=parsed_kwargs["deployment_type"], - template_location=parsed_kwargs["template"], - overwrite=parsed_kwargs["template_overwrite"], - verbose=kwargs["verbose"], - ) - - parsed_kwargs.update(kwargs) - - if host in ["docker"]: - # Check docker service status - if not ignore_docker_version_check: - check_docker_service_status() - - # Check grid docker versions - if not ignore_docker_version_check: - check_grid_docker(display=True, output_in_text=True) - - if not ignore_docker_version_check: - version = check_docker_version() - else: - version = "n/a" - - if version: - # If the user is using docker desktop (OSX/Windows), check to make sure there's enough RAM. - # If the user is using Linux this isn't an issue because Docker scales to the avaialble RAM, - # but on Docker Desktop it defaults to 2GB which isn't enough. - dd_memory = docker_desktop_memory() - if dd_memory < 8192 and dd_memory != -1: - raise Exception( - "You appear to be using Docker Desktop but don't have " - "enough memory allocated. It appears you've configured " - f"Memory:{dd_memory} MB when 8192MB (8GB) is required. " - f"Please open Docker Desktop Preferences panel and set Memory" - f" to 8GB or higher. \n\n" - f"\tOSX Help: https://docs.docker.com/desktop/mac/\n" - f"\tWindows Help: https://docs.docker.com/desktop/windows/\n\n" - f"Then re-run your hagrid command.\n\n" - f"If you see this warning on Linux then something isn't right. " - f"Please file a Github Issue on PySyft's Github.\n\n" - f"Alternatively in case no more memory could be allocated, " - f"you can run hagrid on the cloud with GitPod by visiting " - f"https://gitpod.io/#https://github.com/OpenMined/PySyft." - ) - - if is_windows() and not DEPENDENCIES["wsl"]: - raise Exception( - "You must install wsl2 for Windows to use HAGrid.\n" - "In PowerShell or Command Prompt type:\n> wsl --install\n\n" - "Read more here: https://docs.microsoft.com/en-us/windows/wsl/install" - ) - - return create_launch_docker_cmd( - verb=verb, - docker_version=version, - tail=tail, - kwargs=parsed_kwargs, - silent=parsed_kwargs["silent"], - ) - - elif host in ["azure"]: - check_azure_cli_installed() - - while not check_azure_authed(): - print("You need to log into Azure") - login_azure() - - if DEPENDENCIES["ansible-playbook"]: - resource_group = ask( - question=Question( - var_name="azure_resource_group", - question="What resource group name do you want to use (or create)?", - default=arg_cache["azure_resource_group"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - location = ask( - question=Question( - var_name="azure_location", - question="If this is a new resource group what location?", - default=arg_cache["azure_location"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - size = ask( - question=Question( - var_name="azure_size", - question="What size machine?", - default=arg_cache["azure_size"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - username = ask( - question=Question( - var_name="azure_username", - question="What do you want the username for the VM to be?", - default=arg_cache["azure_username"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - parsed_kwargs["auth_type"] = ask( - question=Question( - var_name="auth_type", - question="Do you want to login with a key or password", - default=arg_cache["auth_type"], - kind="option", - options=["key", "password"], - cache=True, - ), - kwargs=kwargs, - ) - - key_path = None - if parsed_kwargs["auth_type"] == "key": - key_path_question = Question( - var_name="azure_key_path", - question=f"Absolute path of the private key to access {username}@{host}?", - default=arg_cache["azure_key_path"], - kind="path", - cache=True, - ) - try: - key_path = ask( - key_path_question, - kwargs=kwargs, - ) - except QuestionInputPathError as e: - print(e) - key_path = str(e).split("is not a valid path")[0].strip() - - create_key_question = Question( - var_name="azure_key_path", - question=f"Key {key_path} does not exist. Do you want to create it? (y/n)", - default="y", - kind="yesno", - ) - create_key = ask( - create_key_question, - kwargs=kwargs, - ) - if create_key == "y": - key_path = generate_key_at_path(key_path=key_path) - else: - raise QuestionInputError( - "Unable to create VM without a private key" - ) - elif parsed_kwargs["auth_type"] == "password": - auto_generate_password = ask( - question=Question( - var_name="auto_generate_password", - question="Do you want to auto-generate the password? (y/n)", - kind="yesno", - ), - kwargs=kwargs, - ) - if auto_generate_password == "y": # nosec - parsed_kwargs["password"] = generate_sec_random_password(length=16) - elif auto_generate_password == "n": # nosec - parsed_kwargs["password"] = ask( - question=Question( - var_name="password", - question=f"Password for {username}@{host}?", - kind="password", - ), - kwargs=kwargs, - ) - - repo = ask( - Question( - var_name="azure_repo", - question="Repo to fetch source from?", - default=arg_cache["azure_repo"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - branch = ask( - Question( - var_name="azure_branch", - question="Branch to monitor for updates?", - default=arg_cache["azure_branch"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - use_branch(branch=branch) - - password = parsed_kwargs.get("password") - - auth = AuthCredentials( - username=username, key_path=key_path, password=password - ) - - if not auth.valid: - raise Exception(f"Login Credentials are not valid. {auth}") - - return create_launch_azure_cmd( - verb=verb, - resource_group=resource_group, - location=location, - size=size, - username=username, - password=password, - key_path=key_path, - repo=repo, - branch=branch, - auth=auth, - ansible_extras=kwargs["ansible_extras"], - kwargs=parsed_kwargs, - ) - else: - errors = [] - if not DEPENDENCIES["ansible-playbook"]: - errors.append("ansible-playbook") - msg = "\nERROR!!! MISSING DEPENDENCY!!!" - msg += f"\n\nLaunching a Cloud VM requires: {' '.join(errors)}" - msg += "\n\nPlease follow installation instructions: " - msg += "https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#" - msg += "\n\nNote: we've found the 'conda' based installation instructions to work best" - msg += " (e.g. something lke 'conda install -c conda-forge ansible'). " - msg += "The pip based instructions seem to be a bit buggy if you're using a conda environment" - msg += "\n" - raise MissingDependency(msg) - - elif host in ["gcp"]: - check_gcloud_cli_installed() - - while not check_gcloud_authed(): - print("You need to log into Google Cloud") - login_gcloud() - - if DEPENDENCIES["ansible-playbook"]: - project_id = ask( - question=Question( - var_name="gcp_project_id", - question="What PROJECT ID do you want to use?", - default=arg_cache["gcp_project_id"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - zone = ask( - question=Question( - var_name="gcp_zone", - question="What zone do you want your VM in?", - default=arg_cache["gcp_zone"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - machine_type = ask( - question=Question( - var_name="gcp_machine_type", - question="What size machine?", - default=arg_cache["gcp_machine_type"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - username = ask( - question=Question( - var_name="gcp_username", - question="What is your shell username?", - default=arg_cache["gcp_username"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - key_path_question = Question( - var_name="gcp_key_path", - question=f"Private key to access user@{host}?", - default=arg_cache["gcp_key_path"], - kind="path", - cache=True, - ) - try: - key_path = ask( - key_path_question, - kwargs=kwargs, - ) - except QuestionInputPathError as e: - print(e) - key_path = str(e).split("is not a valid path")[0].strip() - - create_key_question = Question( - var_name="gcp_key_path", - question=f"Key {key_path} does not exist. Do you want gcloud to make it? (y/n)", - default="y", - kind="yesno", - ) - create_key = ask( - create_key_question, - kwargs=kwargs, - ) - if create_key == "y": - key_path = generate_gcloud_key_at_path(key_path=key_path) - else: - raise QuestionInputError( - "Unable to create VM without a private key" - ) - - repo = ask( - Question( - var_name="gcp_repo", - question="Repo to fetch source from?", - default=arg_cache["gcp_repo"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - branch = ask( - Question( - var_name="gcp_branch", - question="Branch to monitor for updates?", - default=arg_cache["gcp_branch"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - use_branch(branch=branch) - - auth = AuthCredentials(username=username, key_path=key_path) - - return create_launch_gcp_cmd( - verb=verb, - project_id=project_id, - zone=zone, - machine_type=machine_type, - repo=repo, - auth=auth, - branch=branch, - ansible_extras=kwargs["ansible_extras"], - kwargs=parsed_kwargs, - ) - else: - errors = [] - if not DEPENDENCIES["ansible-playbook"]: - errors.append("ansible-playbook") - msg = "\nERROR!!! MISSING DEPENDENCY!!!" - msg += f"\n\nLaunching a Cloud VM requires: {' '.join(errors)}" - msg += "\n\nPlease follow installation instructions: " - msg += "https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#" - msg += "\n\nNote: we've found the 'conda' based installation instructions to work best" - msg += " (e.g. something lke 'conda install -c conda-forge ansible'). " - msg += "The pip based instructions seem to be a bit buggy if you're using a conda environment" - msg += "\n" - raise MissingDependency(msg) - - elif host in ["aws"]: - check_aws_cli_installed() - - if DEPENDENCIES["ansible-playbook"]: - aws_region = ask( - question=Question( - var_name="aws_region", - question="In what region do you want to deploy the EC2 instance?", - default=arg_cache["aws_region"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - aws_security_group_name = ask( - question=Question( - var_name="aws_security_group_name", - question="Name of the security group to be created?", - default=arg_cache["aws_security_group_name"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - aws_security_group_cidr = ask( - question=Question( - var_name="aws_security_group_cidr", - question="What IP addresses to allow for incoming network traffic? Please use CIDR notation", - default=arg_cache["aws_security_group_cidr"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - ec2_instance_type = ask( - question=Question( - var_name="aws_ec2_instance_type", - question="What EC2 instance type do you want to deploy?", - default=arg_cache["aws_ec2_instance_type"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - aws_key_name = ask( - question=Question( - var_name="aws_key_name", - question="Enter the name of the key pair to use to connect to the EC2 instance", - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - key_path_qn_str = ( - "Please provide the path of the private key to connect to the instance" - ) - key_path_qn_str += " (if it does not exist, this path corresponds to " - key_path_qn_str += "where you want to store the key upon creation)" - key_path_question = Question( - var_name="aws_key_path", - question=key_path_qn_str, - kind="path", - cache=True, - ) - try: - key_path = ask( - key_path_question, - kwargs=kwargs, - ) - except QuestionInputPathError as e: - print(e) - key_path = str(e).split("is not a valid path")[0].strip() - - create_key_question = Question( - var_name="aws_key_path", - question=f"Key {key_path} does not exist. Do you want AWS to make it? (y/n)", - default="y", - kind="yesno", - ) - create_key = ask( - create_key_question, - kwargs=kwargs, - ) - if create_key == "y": - key_path = generate_aws_key_at_path( - key_path=key_path, key_name=aws_key_name - ) - else: - raise QuestionInputError( - "Unable to create EC2 instance without key" - ) - - repo = ask( - Question( - var_name="aws_repo", - question="Repo to fetch source from?", - default=arg_cache["aws_repo"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - branch = ask( - Question( - var_name="aws_branch", - question="Branch to monitor for updates?", - default=arg_cache["aws_branch"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - use_branch(branch=branch) - - username = arg_cache["aws_ec2_instance_username"] - auth = AuthCredentials(username=username, key_path=key_path) - - return create_launch_aws_cmd( - verb=verb, - region=aws_region, - ec2_instance_type=ec2_instance_type, - security_group_name=aws_security_group_name, - aws_security_group_cidr=aws_security_group_cidr, - key_path=key_path, - key_name=aws_key_name, - repo=repo, - branch=branch, - ansible_extras=kwargs["ansible_extras"], - kwargs=parsed_kwargs, - ami_id=arg_cache["aws_image_id"], - username=username, - auth=auth, - ) - - else: - errors = [] - if not DEPENDENCIES["ansible-playbook"]: - errors.append("ansible-playbook") - msg = "\nERROR!!! MISSING DEPENDENCY!!!" - msg += f"\n\nLaunching a Cloud VM requires: {' '.join(errors)}" - msg += "\n\nPlease follow installation instructions: " - msg += "https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#" - msg += "\n\nNote: we've found the 'conda' based installation instructions to work best" - msg += " (e.g. something lke 'conda install -c conda-forge ansible'). " - msg += "The pip based instructions seem to be a bit buggy if you're using a conda environment" - msg += "\n" - raise MissingDependency(msg) - else: - if DEPENDENCIES["ansible-playbook"]: - if host != "localhost": - parsed_kwargs["username"] = ask( - question=Question( - var_name="username", - question=f"Username for {host} with sudo privledges?", - default=arg_cache["username"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - parsed_kwargs["auth_type"] = ask( - question=Question( - var_name="auth_type", - question="Do you want to login with a key or password", - default=arg_cache["auth_type"], - kind="option", - options=["key", "password"], - cache=True, - ), - kwargs=kwargs, - ) - if parsed_kwargs["auth_type"] == "key": - parsed_kwargs["key_path"] = ask( - question=Question( - var_name="key_path", - question=f"Private key to access {parsed_kwargs['username']}@{host}?", - default=arg_cache["key_path"], - kind="path", - cache=True, - ), - kwargs=kwargs, - ) - elif parsed_kwargs["auth_type"] == "password": - parsed_kwargs["password"] = ask( - question=Question( - var_name="password", - question=f"Password for {parsed_kwargs['username']}@{host}?", - kind="password", - ), - kwargs=kwargs, - ) - - parsed_kwargs["repo"] = ask( - question=Question( - var_name="repo", - question="Repo to fetch source from?", - default=arg_cache["repo"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - parsed_kwargs["branch"] = ask( - Question( - var_name="branch", - question="Branch to monitor for updates?", - default=arg_cache["branch"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - auth = None - if host != "localhost": - if parsed_kwargs["auth_type"] == "key": - auth = AuthCredentials( - username=parsed_kwargs["username"], - key_path=parsed_kwargs["key_path"], - ) - else: - auth = AuthCredentials( - username=parsed_kwargs["username"], - key_path=parsed_kwargs["password"], - ) - if not auth.valid: - raise Exception(f"Login Credentials are not valid. {auth}") - parsed_kwargs["ansible_extras"] = kwargs["ansible_extras"] - return create_launch_custom_cmd(verb=verb, auth=auth, kwargs=parsed_kwargs) - else: - errors = [] - if not DEPENDENCIES["ansible-playbook"]: - errors.append("ansible-playbook") - raise MissingDependency( - f"Launching a Custom VM requires: {' '.join(errors)}" - ) - - host_options = ", ".join(allowed_hosts) - raise MissingDependency( - f"Launch requires a correct host option, try: {host_options}" - ) - - -def pull_command(cmd: str, kwargs: dict[str, Any]) -> list[str]: - pull_cmd = str(cmd) - if kwargs["release"] == "production": - pull_cmd += " --file docker-compose.yml" - else: - pull_cmd += " --file docker-compose.pull.yml" - pull_cmd += " pull --ignore-pull-failures" # ignore missing version from Dockerhub - return [pull_cmd] - - -def build_command(cmd: str) -> list[str]: - build_cmd = str(cmd) - build_cmd += " --file docker-compose.build.yml" - build_cmd += " build" - return [build_cmd] - - -def deploy_command(cmd: str, tail: bool, dev_mode: bool) -> list[str]: - up_cmd = str(cmd) - up_cmd += " --file docker-compose.dev.yml" if dev_mode else "" - up_cmd += " up" - if not tail: - up_cmd += " -d" - return [up_cmd] - - -def create_launch_docker_cmd( - verb: GrammarVerb, - docker_version: str, - kwargs: dict[str, Any], - tail: bool = True, - silent: bool = False, -) -> dict[str, list[str]]: - host_term = verb.get_named_term_hostgrammar(name="host") - node_name = verb.get_named_term_type(name="node_name") - node_type = verb.get_named_term_type(name="node_type") - - snake_name = str(node_name.snake_input) - tag = name_tag(name=str(node_name.input)) - - if ART and not silent: - hagrid() - - print( - "Launching a PyGrid " - + str(node_type.input).capitalize() - + " node on port " - + str(host_term.free_port) - + "!\n" - ) - - version_string = kwargs["tag"] - version_hash = "dockerhub" - build = kwargs["build"] - - # if in development mode, generate a version_string which is either - # the one you inputed concatenated with -dev or the contents of the VERSION file - version = GRID_SRC_VERSION() - if "release" in kwargs and kwargs["release"] == "development": - # force version to have -dev at the end in dev mode - # during development we can use the latest beta version - if version_string is None: - version_string = version[0] - version_string += "-dev" - version_hash = version[1] - build = True - else: - # whereas if in production mode and tag == "local" use the local VERSION file - # or if its not set somehow, which should never happen, use stable - # otherwise use the kwargs["tag"] from above - - # during production the default would be stable - if version_string == "local": - # this can be used in VMs in production to auto update from src - version_string = version[0] - version_hash = version[1] - build = True - elif version_string is None: - version_string = "latest" - - if platform.uname().machine.lower() in ["x86_64", "amd64"]: - docker_platform = "linux/amd64" - else: - docker_platform = "linux/arm64" - - if "platform" in kwargs and kwargs["platform"] is not None: - docker_platform = kwargs["platform"] - - if kwargs["template"]: - _, template_hash = get_template_yml(kwargs["template"]) - template_dir = manifest_cache_path(template_hash) - template_grid_dir = f"{template_dir}/packages/grid" - else: - template_grid_dir = GRID_SRC_PATH() - - compose_src_path = kwargs["compose_src_path"] - if not compose_src_path: - compose_src_path = get_compose_src_path( - node_type=node_type, - node_name=snake_name, - template_location=kwargs["template"], - **kwargs, - ) - - default_env = f"{template_grid_dir}/default.env" - if not os.path.exists(default_env): - # old path - default_env = f"{template_grid_dir}/.env" - default_envs = {} - with open(default_env) as f: - for line in f.readlines(): - if "=" in line: - parts = line.strip().split("=") - key = parts[0] - value = "" - if len(parts) > 1: - value = parts[1] - default_envs[key] = value - - single_container_mode = kwargs["deployment_type"] == "single_container" - in_mem_workers = kwargs.get("in_mem_workers") - smtp_username = kwargs.get("smtp_username") - smtp_sender = kwargs.get("smtp_sender") - smtp_password = kwargs.get("smtp_password") - smtp_port = kwargs.get("smtp_port") - if smtp_port is None or smtp_port == "": - smtp_port = int(default_envs["SMTP_PORT"]) - smtp_host = kwargs.get("smtp_host") - - print(" - NAME: " + str(snake_name)) - print(" - TEMPLATE DIR: " + template_grid_dir) - if compose_src_path: - print(" - COMPOSE SOURCE: " + compose_src_path) - print(" - RELEASE: " + f'{kwargs["node_side_type"]}-{kwargs["release"]}') - print(" - DEPLOYMENT:", kwargs["deployment_type"]) - print(" - ARCH: " + docker_platform) - print(" - TYPE: " + str(node_type.input)) - print(" - DOCKER_TAG: " + version_string) - if version_hash != "dockerhub": - print(" - GIT_HASH: " + version_hash) - print(" - HAGRID_VERSION: " + get_version_string()) - if EDITABLE_MODE: - print(" - HAGRID_REPO_SHA: " + commit_hash()) - print(" - PORT: " + str(host_term.free_port)) - print(" - DOCKER COMPOSE: " + docker_version) - print(" - IN-MEMORY WORKERS: " + str(in_mem_workers)) - print("\n") - - use_blob_storage = ( - False - if str(node_type.input) in ["network", "gateway"] - else bool(kwargs["use_blob_storage"]) - ) - - enable_rathole = bool(kwargs.get("enable_rathole")) or str(node_type.input) in [ - "network", - "gateway", - ] - - # use a docker volume - host_path = "credentials-data" - - # # in development use a folder mount - # if kwargs.get("release", "") == "development": - # RELATIVE_PATH = "" - # # if EDITABLE_MODE: - # # RELATIVE_PATH = "../" - # # we might need to change this for the hagrid template mode - # host_path = f"{RELATIVE_PATH}./data/storage/{snake_name}" - - rathole_mode = ( - "client" if enable_rathole and str(node_type.input) in ["domain"] else "server" - ) - - envs = { - "RELEASE": "production", - "COMPOSE_DOCKER_CLI_BUILD": 1, - "DOCKER_BUILDKIT": 1, - "HTTP_PORT": int(host_term.free_port), - "HTTPS_PORT": int(host_term.free_port_tls), - "TRAEFIK_TAG": str(tag), - "NODE_NAME": str(snake_name), - "NODE_TYPE": str(node_type.input), - "TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL": "False", - "VERSION": version_string, - "VERSION_HASH": version_hash, - "USE_BLOB_STORAGE": str(use_blob_storage), - "FRONTEND_TARGET": "grid-ui-production", - "STACK_API_KEY": str( - generate_sec_random_password(length=48, special_chars=False) - ), - "CREDENTIALS_VOLUME": host_path, - "NODE_SIDE_TYPE": kwargs["node_side_type"], - "SINGLE_CONTAINER_MODE": single_container_mode, - "INMEMORY_WORKERS": in_mem_workers, - } - - if smtp_host and smtp_port and smtp_username and smtp_password: - envs["SMTP_HOST"] = smtp_host - envs["SMTP_PORT"] = smtp_port - envs["SMTP_USERNAME"] = smtp_username - envs["SMTP_PASSWORD"] = smtp_password - envs["EMAIL_SENDER"] = smtp_sender - - if "trace" in kwargs and kwargs["trace"] is True: - envs["TRACE"] = "True" - envs["JAEGER_HOST"] = "host.docker.internal" - envs["JAEGER_PORT"] = int( - find_available_port(host="localhost", port=14268, search=True) - ) - - if "association_request_auto_approval" in kwargs: - envs["ASSOCIATION_REQUEST_AUTO_APPROVAL"] = kwargs[ - "association_request_auto_approval" - ] - - if "enable_warnings" in kwargs: - envs["ENABLE_WARNINGS"] = kwargs["enable_warnings"] - - if "platform" in kwargs and kwargs["platform"] is not None: - envs["DOCKER_DEFAULT_PLATFORM"] = docker_platform - - if "tls" in kwargs and kwargs["tls"] is True and len(kwargs["cert_store_path"]) > 0: - envs["TRAEFIK_TLS_CERTS"] = kwargs["cert_store_path"] - - if ( - "tls" in kwargs - and kwargs["tls"] is True - and "test" in kwargs - and kwargs["test"] is True - ): - envs["IGNORE_TLS_ERRORS"] = "True" - - if "test" in kwargs and kwargs["test"] is True: - envs["SWFS_VOLUME_SIZE_LIMIT_MB"] = "100" # GitHub CI is small - - if kwargs.get("release", "") == "development": - envs["RABBITMQ_MANAGEMENT"] = "-management" - - # currently we only have a domain frontend for dev mode - if kwargs.get("release", "") == "development" and ( - str(node_type.input) not in ["network", "gateway"] - ): - envs["FRONTEND_TARGET"] = "grid-ui-development" - - if "set_root_password" in kwargs and kwargs["set_root_password"] is not None: - envs["DEFAULT_ROOT_PASSWORD"] = kwargs["set_root_password"] - - if "set_root_email" in kwargs and kwargs["set_root_email"] is not None: - envs["DEFAULT_ROOT_EMAIL"] = kwargs["set_root_email"] - - if "set_s3_username" in kwargs and kwargs["set_s3_username"] is not None: - envs["S3_ROOT_USER"] = kwargs["set_s3_username"] - - if "set_s3_password" in kwargs and kwargs["set_s3_password"] is not None: - envs["S3_ROOT_PWD"] = kwargs["set_s3_password"] - - if ( - "set_volume_size_limit_mb" in kwargs - and kwargs["set_volume_size_limit_mb"] is not None - ): - envs["SWFS_VOLUME_SIZE_LIMIT_MB"] = kwargs["set_volume_size_limit_mb"] - - if "release" in kwargs: - envs["RELEASE"] = kwargs["release"] - - if "enable_signup" in kwargs: - envs["ENABLE_SIGNUP"] = kwargs["enable_signup"] - - if enable_rathole: - envs["MODE"] = rathole_mode - - cmd = "" - args = [] - for k, v in envs.items(): - if is_windows(): - # powershell envs - quoted = f"'{v}'" if not isinstance(v, int) else v - args.append(f"$env:{k}={quoted}") - else: - args.append(f"{k}={v}") - if is_windows(): - cmd += "; ".join(args) - cmd += "; " - else: - cmd += " ".join(args) - - cmd += " docker compose -p " + snake_name - - # new docker compose regression work around - # default_env = os.path.expanduser("~/.hagrid/app/.env") - - default_envs.update(envs) - - # env file path - env_file_path = compose_src_path + "/.env" - - # Render templates if creating stack from the manifest_template.yml - if kwargs["template"] and host_term.host is not None: - # If release is development, update relative path - # if EDITABLE_MODE: - # default_envs["RELATIVE_PATH"] = "../" - - render_templates( - node_name=snake_name, - deployment_type=kwargs["deployment_type"], - template_location=kwargs["template"], - env_vars=default_envs, - host_type=host_term.host, - ) - - try: - env_file = "" - for k, v in default_envs.items(): - env_file += f"{k}={v}\n" - - with open(env_file_path, "w") as f: - f.write(env_file) - - # cmd += f" --env-file {env_file_path}" - except Exception: # nosec - pass - - if single_container_mode: - cmd += " --profile worker" - else: - cmd += " --profile backend" - cmd += " --profile proxy" - cmd += " --profile mongo" - - if str(node_type.input) in ["network", "gateway"]: - cmd += " --profile network" - - if use_blob_storage: - cmd += " --profile blob-storage" - - if enable_rathole: - cmd += " --profile rathole" - - # no frontend container so expect bad gateway on the / route - if not bool(kwargs["headless"]): - cmd += " --profile frontend" - - if "trace" in kwargs and kwargs["trace"]: - cmd += " --profile telemetry" - - final_commands = {} - final_commands["Pulling"] = pull_command(cmd, kwargs) - - cmd += " --file docker-compose.yml" - if "tls" in kwargs and kwargs["tls"] is True: - cmd += " --file docker-compose.tls.yml" - if "test" in kwargs and kwargs["test"] is True: - cmd += " --file docker-compose.test.yml" - - if build: - my_build_command = build_command(cmd) - final_commands["Building"] = my_build_command - - dev_mode = kwargs.get("dev", False) - final_commands["Launching"] = deploy_command(cmd, tail, dev_mode) - return final_commands - - -def create_launch_vagrant_cmd(verb: GrammarVerb) -> str: - host_term = verb.get_named_term_hostgrammar(name="host") - node_name = verb.get_named_term_type(name="node_name") - node_type = verb.get_named_term_type(name="node_type") - - snake_name = str(node_name.snake_input) - - if ART: - hagrid() - - print( - "Launching a " - + str(node_type.input) - + " PyGrid node on port " - + str(host_term.port) - + "!\n" - ) - - print(" - TYPE: " + str(node_type.input)) - print(" - NAME: " + str(snake_name)) - print(" - PORT: " + str(host_term.port)) - # print(" - VAGRANT: " + "1") - # print(" - VIRTUALBOX: " + "1") - print("\n") - - cmd = "" - cmd += 'ANSIBLE_ARGS="' - cmd += f"-e 'node_name={snake_name}'" - cmd += f"-e 'node_type={node_type.input}'" - cmd += '" ' - cmd += "vagrant up --provision" - cmd = "cd " + GRID_SRC_PATH() + ";" + cmd - return cmd - - -def get_or_make_resource_group(resource_group: str, location: str = "westus") -> None: - cmd = f"az group show --resource-group {resource_group}" - exists = True - try: - subprocess.check_call(cmd, shell=True) # nosec - except Exception: # nosec - # group doesn't exist so lets create it - exists = False - - if not exists: - cmd = f"az group create -l {location} -n {resource_group}" - try: - print(f"Creating resource group.\nRunning: {cmd}") - subprocess.check_call(cmd, shell=True) # nosec - except Exception as e: - raise Exception( - f"Unable to create resource group {resource_group} @ {location}. {e}" - ) - - -def extract_host_ip(stdout: bytes) -> str | None: - output = stdout.decode("utf-8") - - try: - j = json.loads(output) - if "publicIpAddress" in j: - return str(j["publicIpAddress"]) - except Exception: # nosec - matcher = r'publicIpAddress":\s+"(.+)"' - ips = re.findall(matcher, output) - if len(ips) > 0: - return ips[0] - - return None - - -def get_vm_host_ips(node_name: str, resource_group: str) -> list | None: - cmd = f"az vm list-ip-addresses -g {resource_group} --query " - cmd += f""""[?starts_with(virtualMachine.name, '{node_name}')]""" - cmd += '''.virtualMachine.network.publicIpAddresses[0].ipAddress"''' - output = subprocess.check_output(cmd, shell=True) # nosec - try: - host_ips = json.loads(output) - return host_ips - except Exception as e: - print(f"Failed to extract ips: {e}") - - return None - - -def is_valid_ip(host_or_ip: str) -> bool: - matcher = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" - ips = re.findall(matcher, host_or_ip.strip()) - if len(ips) == 1: - return True - return False - - -def extract_host_ip_gcp(stdout: bytes) -> str | None: - output = stdout.decode("utf-8") - - try: - matcher = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" - ips = re.findall(matcher, output) - if len(ips) == 2: - return ips[1] - except Exception: # nosec - pass - - return None - - -def extract_host_ip_from_cmd(cmd: str) -> str | None: - try: - matcher = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" - ips = re.findall(matcher, cmd) - if ips: - return ips[0] - except Exception: # nosec - pass - - return None - - -def check_ip_for_ssh( - host_ip: str, timeout: int = 600, wait_time: int = 5, silent: bool = False -) -> bool: - if not silent: - print(f"Checking VM at {host_ip} is up") - checks = int(timeout / wait_time) # 10 minutes in 5 second chunks - first_run = True - while checks > 0: - checks -= 1 - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(wait_time) - result = sock.connect_ex((host_ip, 22)) - sock.close() - if result == 0: - if not silent: - print(f"VM at {host_ip} is up!") - return True - else: - if first_run: - if not silent: - print("Waiting for VM to start", end="", flush=True) - first_run = False - else: - if not silent: - print(".", end="", flush=True) - except Exception: # nosec - pass - return False - - -def create_aws_security_group( - security_group_name: str, region: str, snake_name: str -) -> str: - sg_description = f"{snake_name} security group" - create_cmd = f"aws ec2 create-security-group --group-name {security_group_name} " - create_cmd += f'--region {region} --description "{sg_description}" ' - sg_output = subprocess.check_output( # nosec - create_cmd, - shell=True, - ) - sg_output_dict = json.loads(sg_output) - if "GroupId" in sg_output_dict: - return sg_output_dict["GroupId"] - - return "" - - -def open_port_aws( - security_group_name: str, port_no: int, cidr: str, region: str -) -> None: - cmd = f"aws ec2 authorize-security-group-ingress --group-name {security_group_name} --protocol tcp " - cmd += f"--port {port_no} --cidr {cidr} --region {region}" - subprocess.check_call( # nosec - cmd, - shell=True, - ) - - -def extract_instance_ids_aws(stdout: bytes) -> list: - output = stdout.decode("utf-8") - output_dict = json.loads(output) - instance_ids: list = [] - if "Instances" in output_dict: - for ec2_instance_metadata in output_dict["Instances"]: - if "InstanceId" in ec2_instance_metadata: - instance_ids.append(ec2_instance_metadata["InstanceId"]) - - return instance_ids - - -def get_host_ips_given_instance_ids( - instance_ids: list, timeout: int = 600, wait_time: int = 10 -) -> list: - checks = int(timeout / wait_time) # 10 minutes in 10 second chunks - instance_ids_str = " ".join(instance_ids) - cmd = f"aws ec2 describe-instances --instance-ids {instance_ids_str}" - cmd += " --query 'Reservations[*].Instances[*].{StateName:State.Name,PublicIpAddress:PublicIpAddress}'" - cmd += " --output json" - while checks > 0: - checks -= 1 - time.sleep(wait_time) - desc_ec2_output = subprocess.check_output(cmd, shell=True) # nosec - instances_output_json = json.loads(desc_ec2_output.decode("utf-8")) - host_ips: list = [] - all_instances_running = True - for reservation in instances_output_json: - for instance_metadata in reservation: - if instance_metadata["StateName"] != "running": - all_instances_running = False - break - else: - host_ips.append(instance_metadata["PublicIpAddress"]) - if all_instances_running: - return host_ips - # else, wait another wait_time seconds and try again - - return [] - - -def make_aws_ec2_instance( - ami_id: str, ec2_instance_type: str, key_name: str, security_group_name: str -) -> list: - # From the docs: "For security groups in a nondefault VPC, you must specify the security group ID". - # Right now, since we're using default VPC, we can use security group name instead of ID. - - ebs_size = 200 # gb - cmd = f"aws ec2 run-instances --image-id {ami_id} --count 1 --instance-type {ec2_instance_type} " - cmd += f"--key-name {key_name} --security-groups {security_group_name} " - tmp_cmd = rf"[{{\"DeviceName\":\"/dev/sdf\",\"Ebs\":{{\"VolumeSize\":{ebs_size},\"DeleteOnTermination\":false}}}}]" - cmd += f'--block-device-mappings "{tmp_cmd}"' - - host_ips: list = [] - try: - print(f"Creating EC2 instance.\nRunning: {cmd}") - create_ec2_output = subprocess.check_output(cmd, shell=True) # nosec - instance_ids = extract_instance_ids_aws(create_ec2_output) - host_ips = get_host_ips_given_instance_ids(instance_ids=instance_ids) - except Exception as e: - print("failed", e) - - if not (host_ips): - raise Exception("Failed to create EC2 instance(s) or get public ip(s)") - - return host_ips - - -def create_launch_aws_cmd( - verb: GrammarVerb, - region: str, - ec2_instance_type: str, - security_group_name: str, - aws_security_group_cidr: str, - key_name: str, - key_path: str, - ansible_extras: str, - kwargs: dict[str, Any], - repo: str, - branch: str, - ami_id: str, - username: str, - auth: AuthCredentials, -) -> list[str]: - node_name = verb.get_named_term_type(name="node_name") - snake_name = str(node_name.snake_input) - create_aws_security_group(security_group_name, region, snake_name) - open_port_aws( - security_group_name=security_group_name, - port_no=80, - cidr=aws_security_group_cidr, - region=region, - ) # HTTP - open_port_aws( - security_group_name=security_group_name, - port_no=443, - cidr=aws_security_group_cidr, - region=region, - ) # HTTPS - open_port_aws( - security_group_name=security_group_name, - port_no=22, - cidr=aws_security_group_cidr, - region=region, - ) # SSH - if kwargs["jupyter"]: - open_port_aws( - security_group_name=security_group_name, - port_no=8888, - cidr=aws_security_group_cidr, - region=region, - ) # Jupyter - - host_ips = make_aws_ec2_instance( - ami_id=ami_id, - ec2_instance_type=ec2_instance_type, - key_name=key_name, - security_group_name=security_group_name, - ) - - launch_cmds: list[str] = [] - - for host_ip in host_ips: - # get old host - host_term = verb.get_named_term_hostgrammar(name="host") - - # replace - host_term.parse_input(host_ip) - verb.set_named_term_type(name="host", new_term=host_term) - - if not bool(kwargs["provision"]): - print("Skipping automatic provisioning.") - print("VM created with:") - print(f"IP: {host_ip}") - print(f"Key: {key_path}") - print("\nConnect with:") - print(f"ssh -i {key_path} {username}@{host_ip}") - - else: - extra_kwargs = { - "repo": repo, - "branch": branch, - "ansible_extras": ansible_extras, - } - kwargs.update(extra_kwargs) - - # provision - host_up = check_ip_for_ssh(host_ip=host_ip) - if not host_up: - print(f"Warning: {host_ip} ssh not available yet") - launch_cmd = create_launch_custom_cmd(verb=verb, auth=auth, kwargs=kwargs) - launch_cmds.append(launch_cmd) - - return launch_cmds - - -def make_vm_azure( - node_name: str, - resource_group: str, - username: str, - password: str | None, - key_path: str | None, - size: str, - image_name: str, - node_count: int, -) -> list: - disk_size_gb = "200" - try: - temp_dir = tempfile.TemporaryDirectory() - public_key_path = ( - private_to_public_key( - private_key_path=key_path, temp_path=temp_dir.name, username=username - ) - if key_path - else None - ) - except Exception: # nosec - temp_dir.cleanup() - - authentication_type = "ssh" if key_path else "password" - cmd = f"az vm create -n {node_name} -g {resource_group} --size {size} " - cmd += f"--image {image_name} --os-disk-size-gb {disk_size_gb} " - cmd += f"--public-ip-sku Standard --authentication-type {authentication_type} --admin-username {username} " - cmd += f"--ssh-key-values {public_key_path} " if public_key_path else "" - cmd += f"--admin-password '{password}' " if password else "" - cmd += f"--count {node_count} " if node_count > 1 else "" - - host_ips: list | None = [] - try: - print(f"Creating vm.\nRunning: {hide_azure_vm_password(cmd)}") - subprocess.check_output(cmd, shell=True) # nosec - host_ips = get_vm_host_ips(node_name=node_name, resource_group=resource_group) - except Exception as e: - print("failed", e) - finally: - temp_dir.cleanup() - - if not host_ips: - raise Exception("Failed to create vm or get VM public ip") - - try: - # clean up temp public key - if public_key_path: - os.unlink(public_key_path) - except Exception: # nosec - pass - - return host_ips - - -def open_port_vm_azure( - resource_group: str, node_name: str, port_name: str, port: int, priority: int -) -> None: - cmd = f"az network nsg rule create --resource-group {resource_group} " - cmd += f"--nsg-name {node_name}NSG --name {port_name} --destination-port-ranges {port} --priority {priority}" - try: - print(f"Creating {port_name} {port} ngs rule.\nRunning: {cmd}") - output = subprocess.check_call(cmd, shell=True) # nosec - print("output", output) - pass - except Exception as e: - print("failed", e) - - -def create_project(project_id: str) -> None: - cmd = f"gcloud projects create {project_id} --set-as-default" - try: - print(f"Creating project.\nRunning: {cmd}") - subprocess.check_call(cmd, shell=True) # nosec - except Exception as e: - print("failed", e) - - print("create project complete") - - -def create_launch_gcp_cmd( - verb: GrammarVerb, - project_id: str, - zone: str, - machine_type: str, - ansible_extras: str, - kwargs: dict[str, Any], - repo: str, - branch: str, - auth: AuthCredentials, -) -> str: - # create project if it doesn't exist - create_project(project_id) - # vm - node_name = verb.get_named_term_type(name="node_name") - kebab_name = str(node_name.kebab_input) - disk_size_gb = "200" - host_ip = make_gcp_vm( - vm_name=kebab_name, - project_id=project_id, - zone=zone, - machine_type=machine_type, - disk_size_gb=disk_size_gb, - ) - - # get old host - host_term = verb.get_named_term_hostgrammar(name="host") - - host_up = check_ip_for_ssh(host_ip=host_ip) - if not host_up: - raise Exception(f"Something went wrong launching the VM at IP: {host_ip}.") - - if not bool(kwargs["provision"]): - print("Skipping automatic provisioning.") - print("VM created with:") - print(f"IP: {host_ip}") - print(f"User: {auth.username}") - print(f"Key: {auth.key_path}") - print("\nConnect with:") - print(f"ssh -i {auth.key_path} {auth.username}@{host_ip}") - sys.exit(0) - - # replace - host_term.parse_input(host_ip) - verb.set_named_term_type(name="host", new_term=host_term) - - extra_kwargs = { - "repo": repo, - "branch": branch, - "auth_type": "key", - "ansible_extras": ansible_extras, - } - kwargs.update(extra_kwargs) - - # provision - return create_launch_custom_cmd(verb=verb, auth=auth, kwargs=kwargs) - - -def make_gcp_vm( - vm_name: str, project_id: str, zone: str, machine_type: str, disk_size_gb: str -) -> str: - create_cmd = "gcloud compute instances create" - network_settings = "network=default,network-tier=PREMIUM" - maintenance_policy = "MIGRATE" - scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/service.management.readonly", - "https://www.googleapis.com/auth/trace.append", - ] - tags = "http-server,https-server" - disk_image = "projects/ubuntu-os-cloud/global/images/ubuntu-2204-jammy-v20230429" - disk = ( - f"auto-delete=yes,boot=yes,device-name={vm_name},image={disk_image}," - + f"mode=rw,size={disk_size_gb},type=pd-ssd" - ) - security_flags = ( - "--no-shielded-secure-boot --shielded-vtpm " - + "--shielded-integrity-monitoring --reservation-affinity=any" - ) - - cmd = ( - f"{create_cmd} {vm_name} " - + f"--project={project_id} " - + f"--zone={zone} " - + f"--machine-type={machine_type} " - + f"--create-disk={disk} " - + f"--network-interface={network_settings} " - + f"--maintenance-policy={maintenance_policy} " - + f"--scopes={','.join(scopes)} --tags={tags} " - + f"{security_flags}" - ) - - host_ip = None - try: - print(f"Creating vm.\nRunning: {cmd}") - output = subprocess.check_output(cmd, shell=True) # nosec - host_ip = extract_host_ip_gcp(stdout=output) - except Exception as e: - print("failed", e) - - if host_ip is None: - raise Exception("Failed to create vm or get VM public ip") - - return host_ip - - -def create_launch_azure_cmd( - verb: GrammarVerb, - resource_group: str, - location: str, - size: str, - username: str, - password: str | None, - key_path: str | None, - repo: str, - branch: str, - auth: AuthCredentials, - ansible_extras: str, - kwargs: dict[str, Any], -) -> list[str]: - get_or_make_resource_group(resource_group=resource_group, location=location) - - node_count = kwargs.get("node_count", 1) - print("Total VMs to create: ", node_count) - - # vm - node_name = verb.get_named_term_type(name="node_name") - snake_name = str(node_name.snake_input) - image_name = get_azure_image(kwargs["image_name"]) - host_ips = make_vm_azure( - snake_name, - resource_group, - username, - password, - key_path, - size, - image_name, - node_count, - ) - - # open port 80 - open_port_vm_azure( - resource_group=resource_group, - node_name=snake_name, - port_name="HTTP", - port=80, - priority=500, - ) - - # open port 443 - open_port_vm_azure( - resource_group=resource_group, - node_name=snake_name, - port_name="HTTPS", - port=443, - priority=501, - ) - - if kwargs["jupyter"]: - # open port 8888 - open_port_vm_azure( - resource_group=resource_group, - node_name=snake_name, - port_name="Jupyter", - port=8888, - priority=502, - ) - - launch_cmds: list[str] = [] - - for host_ip in host_ips: - # get old host - host_term = verb.get_named_term_hostgrammar(name="host") - - # replace - host_term.parse_input(host_ip) - verb.set_named_term_type(name="host", new_term=host_term) - - if not bool(kwargs["provision"]): - print("Skipping automatic provisioning.") - print("VM created with:") - print(f"Name: {snake_name}") - print(f"IP: {host_ip}") - print(f"User: {username}") - print(f"Password: {password}") - print(f"Key: {key_path}") - print("\nConnect with:") - if kwargs["auth_type"] == "key": - print(f"ssh -i {key_path} {username}@{host_ip}") - else: - print(f"ssh {username}@{host_ip}") - else: - extra_kwargs = { - "repo": repo, - "branch": branch, - "ansible_extras": ansible_extras, - } - kwargs.update(extra_kwargs) - - # provision - host_up = check_ip_for_ssh(host_ip=host_ip) - if not host_up: - print(f"Warning: {host_ip} ssh not available yet") - launch_cmd = create_launch_custom_cmd(verb=verb, auth=auth, kwargs=kwargs) - launch_cmds.append(launch_cmd) - - return launch_cmds - - -def create_ansible_land_cmd( - verb: GrammarVerb, auth: AuthCredentials | None, kwargs: dict[str, Any] -) -> str: - try: - host_term = verb.get_named_term_hostgrammar(name="host") - print("Landing PyGrid node on port " + str(host_term.port) + "!\n") - - print(" - PORT: " + str(host_term.port)) - print("\n") - - grid_path = GRID_SRC_PATH() - playbook_path = grid_path + "/ansible/site.yml" - ansible_cfg_path = grid_path + "/ansible.cfg" - auth = cast(AuthCredentials, auth) - - if not os.path.exists(playbook_path): - print(f"Can't find playbook site.yml at: {playbook_path}") - cmd = f"ANSIBLE_CONFIG={ansible_cfg_path} ansible-playbook " - if host_term.host == "localhost": - cmd += "--connection=local " - cmd += f"-i {host_term.host}, {playbook_path}" - if host_term.host != "localhost" and kwargs["auth_type"] == "key": - cmd += f" --private-key {auth.key_path} --user {auth.username}" - elif host_term.host != "localhost" and kwargs["auth_type"] == "password": - cmd += f" -c paramiko --user {auth.username}" - - ANSIBLE_ARGS = {"install": "false"} - - if host_term.host != "localhost" and kwargs["auth_type"] == "password": - ANSIBLE_ARGS["ansible_ssh_pass"] = kwargs["password"] - - if host_term.host == "localhost": - ANSIBLE_ARGS["local"] = "true" - - if "ansible_extras" in kwargs and kwargs["ansible_extras"] != "": - options = kwargs["ansible_extras"].split(",") - for option in options: - parts = option.strip().split("=") - if len(parts) == 2: - ANSIBLE_ARGS[parts[0]] = parts[1] - - for k, v in ANSIBLE_ARGS.items(): - cmd += f" -e \"{k}='{v}'\"" - - cmd = "cd " + grid_path + ";" + cmd - return cmd - except Exception as e: - print(f"Failed to construct custom deployment cmd: {cmd}. {e}") - raise e - - -def create_launch_custom_cmd( - verb: GrammarVerb, auth: AuthCredentials | None, kwargs: dict[str, Any] -) -> str: - try: - host_term = verb.get_named_term_hostgrammar(name="host") - node_name = verb.get_named_term_type(name="node_name") - node_type = verb.get_named_term_type(name="node_type") - # source_term = verb.get_named_term_type(name="source") - - snake_name = str(node_name.snake_input) - - if ART: - hagrid() - - print( - "Launching a " - + str(node_type.input) - + " PyGrid node on port " - + str(host_term.port) - + "!\n" - ) - - print(" - TYPE: " + str(node_type.input)) - print(" - NAME: " + str(snake_name)) - print(" - PORT: " + str(host_term.port)) - print("\n") - - grid_path = GRID_SRC_PATH() - playbook_path = grid_path + "/ansible/site.yml" - ansible_cfg_path = grid_path + "/ansible.cfg" - auth = cast(AuthCredentials, auth) - - if not os.path.exists(playbook_path): - print(f"Can't find playbook site.yml at: {playbook_path}") - cmd = f"ANSIBLE_CONFIG={ansible_cfg_path} ansible-playbook " - if host_term.host == "localhost": - cmd += "--connection=local " - cmd += f"-i {host_term.host}, {playbook_path}" - if host_term.host != "localhost" and kwargs["auth_type"] == "key": - cmd += f" --private-key {auth.key_path} --user {auth.username}" - elif host_term.host != "localhost" and kwargs["auth_type"] == "password": - cmd += f" -c paramiko --user {auth.username}" - - version_string = kwargs["tag"] - if version_string is None: - version_string = "local" - - ANSIBLE_ARGS = { - "node_type": node_type.input, - "node_name": snake_name, - "github_repo": kwargs["repo"], - "repo_branch": kwargs["branch"], - "docker_tag": version_string, - } - - if host_term.host != "localhost" and kwargs["auth_type"] == "password": - ANSIBLE_ARGS["ansible_ssh_pass"] = kwargs["password"] - - if host_term.host == "localhost": - ANSIBLE_ARGS["local"] = "true" - - if "node_side_type" in kwargs: - ANSIBLE_ARGS["node_side_type"] = kwargs["node_side_type"] - - if kwargs["tls"] is True: - ANSIBLE_ARGS["tls"] = "true" - - if "release" in kwargs: - ANSIBLE_ARGS["release"] = kwargs["release"] - - if "set_root_email" in kwargs and kwargs["set_root_email"] is not None: - ANSIBLE_ARGS["root_user_email"] = kwargs["set_root_email"] - - if "set_root_password" in kwargs and kwargs["set_root_password"] is not None: - ANSIBLE_ARGS["root_user_password"] = kwargs["set_root_password"] - - if ( - kwargs["tls"] is True - and "cert_store_path" in kwargs - and len(kwargs["cert_store_path"]) > 0 - ): - ANSIBLE_ARGS["cert_store_path"] = kwargs["cert_store_path"] - - if ( - kwargs["tls"] is True - and "upload_tls_key" in kwargs - and len(kwargs["upload_tls_key"]) > 0 - ): - ANSIBLE_ARGS["upload_tls_key"] = kwargs["upload_tls_key"] - - if ( - kwargs["tls"] is True - and "upload_tls_cert" in kwargs - and len(kwargs["upload_tls_cert"]) > 0 - ): - ANSIBLE_ARGS["upload_tls_cert"] = kwargs["upload_tls_cert"] - - if kwargs["jupyter"] is True: - ANSIBLE_ARGS["jupyter"] = "true" - ANSIBLE_ARGS["jupyter_token"] = generate_sec_random_password( - length=48, upper_case=False, special_chars=False - ) - - if "ansible_extras" in kwargs and kwargs["ansible_extras"] != "": - options = kwargs["ansible_extras"].split(",") - for option in options: - parts = option.strip().split("=") - if len(parts) == 2: - ANSIBLE_ARGS[parts[0]] = parts[1] - - # if mode == "deploy": - # ANSIBLE_ARGS["deploy"] = "true" - - for k, v in ANSIBLE_ARGS.items(): - cmd += f" -e \"{k}='{v}'\"" - - cmd = "cd " + grid_path + ";" + cmd - return cmd - except Exception as e: - print(f"Failed to construct custom deployment cmd: {cmd}. {e}") - raise e - - -def create_land_cmd(verb: GrammarVerb, kwargs: dict[str, Any]) -> str: - host_term = verb.get_named_term_hostgrammar(name="host") - host = host_term.host if host_term.host is not None else "" - - if host in ["docker"]: - target = verb.get_named_term_grammar("node_name").input - prune_volumes: bool = kwargs.get("prune_vol", False) - - if target == "all": - # land all syft nodes - if prune_volumes: - land_cmd = "docker rm `docker ps --filter label=orgs.openmined.syft -q` --force " - land_cmd += "&& docker volume rm " - land_cmd += "$(docker volume ls --filter label=orgs.openmined.syft -q)" - return land_cmd - else: - return "docker rm `docker ps --filter label=orgs.openmined.syft -q` --force" - - version = check_docker_version() - if version: - return create_land_docker_cmd(verb=verb, prune_volumes=prune_volumes) - - elif host == "localhost" or is_valid_ip(host): - parsed_kwargs = {} - if DEPENDENCIES["ansible-playbook"]: - if host != "localhost": - parsed_kwargs["username"] = ask( - question=Question( - var_name="username", - question=f"Username for {host} with sudo privledges?", - default=arg_cache["username"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - parsed_kwargs["auth_type"] = ask( - question=Question( - var_name="auth_type", - question="Do you want to login with a key or password", - default=arg_cache["auth_type"], - kind="option", - options=["key", "password"], - cache=True, - ), - kwargs=kwargs, - ) - if parsed_kwargs["auth_type"] == "key": - parsed_kwargs["key_path"] = ask( - question=Question( - var_name="key_path", - question=f"Private key to access {parsed_kwargs['username']}@{host}?", - default=arg_cache["key_path"], - kind="path", - cache=True, - ), - kwargs=kwargs, - ) - elif parsed_kwargs["auth_type"] == "password": - parsed_kwargs["password"] = ask( - question=Question( - var_name="password", - question=f"Password for {parsed_kwargs['username']}@{host}?", - kind="password", - ), - kwargs=kwargs, - ) - - auth = None - if host != "localhost": - if parsed_kwargs["auth_type"] == "key": - auth = AuthCredentials( - username=parsed_kwargs["username"], - key_path=parsed_kwargs["key_path"], - ) - else: - auth = AuthCredentials( - username=parsed_kwargs["username"], - key_path=parsed_kwargs["password"], - ) - if not auth.valid: - raise Exception(f"Login Credentials are not valid. {auth}") - parsed_kwargs["ansible_extras"] = kwargs["ansible_extras"] - return create_ansible_land_cmd(verb=verb, auth=auth, kwargs=parsed_kwargs) - else: - errors = [] - if not DEPENDENCIES["ansible-playbook"]: - errors.append("ansible-playbook") - raise MissingDependency( - f"Launching a Custom VM requires: {' '.join(errors)}" - ) - - host_options = ", ".join(allowed_hosts) - raise MissingDependency( - f"Launch requires a correct host option, try: {host_options}" - ) - - -def create_land_docker_cmd(verb: GrammarVerb, prune_volumes: bool = False) -> str: - """ - Create docker `land` command to remove containers when a node's name is specified - """ - node_name = verb.get_named_term_type(name="node_name") - snake_name = str(node_name.snake_input) - - path = GRID_SRC_PATH() - env_var = ";export $(cat .env | sed 's/#.*//g' | xargs);" - - cmd = "" - cmd += "docker compose" - cmd += ' --file "docker-compose.yml"' - cmd += ' --project-name "' + snake_name + '"' - cmd += " down --remove-orphans" - - if prune_volumes: - cmd += ( - f' && docker volume rm $(docker volume ls --filter name="{snake_name}" -q)' - ) - - cmd += f" && docker rm $(docker ps --filter name={snake_name} -q) --force" - - cmd = "cd " + path + env_var + cmd - return cmd - - -@click.command( - help="Stop a running PyGrid domain/network node.", - context_settings={"show_default": True}, -) -@click.argument("args", type=str, nargs=-1) -@click.option( - "--cmd", - is_flag=True, - help="Print the cmd without running it", -) -@click.option( - "--ansible-extras", - default="", - type=str, -) -@click.option( - "--build-src", - default=DEFAULT_BRANCH, - required=False, - type=str, - help="Git branch to use for launch / build operations", -) -@click.option( - "--silent", - is_flag=True, - help="Suppress extra outputs", -) -@click.option( - "--force", - is_flag=True, - help="Bypass the prompt during hagrid land", -) -@click.option( - "--prune-vol", - is_flag=True, - help="Prune docker volumes after land.", -) -def land(args: tuple[str], **kwargs: Any) -> None: - verb = get_land_verb() - silent = bool(kwargs["silent"]) - force = bool(kwargs["force"]) - try: - grammar = parse_grammar(args=args, verb=verb) - verb.load_grammar(grammar=grammar) - except BadGrammar as e: - print(e) - return - - try: - update_repo(repo=GIT_REPO(), branch=str(kwargs["build_src"])) - except Exception as e: - print(f"Failed to update repo. {e}") - - try: - cmd = create_land_cmd(verb=verb, kwargs=kwargs) - except Exception as e: - print(f"{e}") - return - - target = verb.get_named_term_grammar("node_name").input - - if not force: - _land_domain = ask( - Question( - var_name="_land_domain", - question=f"Are you sure you want to land {target} (y/n)", - kind="yesno", - ), - kwargs={}, - ) - - grid_path = GRID_SRC_PATH() - - if force or _land_domain == "y": - if not bool(kwargs["cmd"]): - if not silent: - print("Running: \n", cmd) - try: - if silent: - process = subprocess.Popen( # nosec - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=grid_path, - shell=True, - ) - process.communicate() - - print(f"HAGrid land {target} complete!") - else: - subprocess.call(cmd, shell=True, cwd=grid_path) # nosec - except Exception as e: - print(f"Failed to run cmd: {cmd}. {e}") - else: - print("Hagrid land aborted.") - - -cli.add_command(launch) -cli.add_command(land) -cli.add_command(clean) - - -@click.command( - help="Show HAGrid debug information", context_settings={"show_default": True} -) -@click.argument("args", type=str, nargs=-1) -def debug(args: tuple[str], **kwargs: Any) -> None: - debug_info = gather_debug() - print("\n\nWhen reporting bugs, please copy everything between the lines.") - print("==================================================================\n") - print(json.dumps(debug_info)) - print("\n=================================================================\n\n") - - -cli.add_command(debug) - - -DEFAULT_HEALTH_CHECKS = ["host", "UI (βeta)", "api", "ssh", "jupyter"] -HEALTH_CHECK_FUNCTIONS = { - "host": check_host, - "UI (βeta)": check_login_page, - "api": check_api_metadata, - "ssh": check_ip_for_ssh, - "jupyter": check_jupyter_server, -} - -HEALTH_CHECK_ICONS = { - "host": "🔌", - "UI (βeta)": "🖱", - "api": "⚙️", - "ssh": "🔐", - "jupyter": "📗", -} - -HEALTH_CHECK_URLS = { - "host": "{ip_address}", - "UI (βeta)": "http://{ip_address}/login", - "api": "http://{ip_address}/api/v2/openapi.json", - "ssh": "hagrid ssh {ip_address}", - "jupyter": "http://{ip_address}:8888", -} - - -def check_host_health(ip_address: str, keys: list[str]) -> dict[str, bool]: - status = {} - for key in keys: - func: Callable = HEALTH_CHECK_FUNCTIONS[key] # type: ignore - status[key] = func(ip_address, silent=True) - return status - - -def icon_status(status: bool) -> str: - return "✅" if status else "❌" - - -def get_health_checks(ip_address: str) -> tuple[bool, list[list[str]]]: - keys = list(DEFAULT_HEALTH_CHECKS) - if "localhost" in ip_address: - new_keys = [] - for key in keys: - if key not in ["host", "jupyter", "ssh"]: - new_keys.append(key) - keys = new_keys - - health_status = check_host_health(ip_address=ip_address, keys=keys) - complete_status = all(health_status.values()) - - # find port from ip_address - try: - port = int(ip_address.split(":")[1]) - except Exception: - # default to 80 - port = 80 - - # url to display based on running environment - display_url = gitpod_url(port).split("//")[1] if is_gitpod() else ip_address - - # figure out how to add this back? - # console.print("[bold magenta]Checking host:[/bold magenta]", ip_address, ":mage:") - table_contents = [] - for key, value in health_status.items(): - table_contents.append( - [ - HEALTH_CHECK_ICONS[key], - key, - HEALTH_CHECK_URLS[key].replace("{ip_address}", display_url), - icon_status(value), - ] - ) - - return complete_status, table_contents - - -def create_check_table( - table_contents: list[list[str]], time_left: int = 0 -) -> rich.table.Table: - table = rich.table.Table() - table.add_column("PyGrid", style="magenta") - table.add_column("Info", justify="left", overflow="fold") - time_left_str = "" if time_left == 0 else str(time_left) - table.add_column(time_left_str, justify="left") - for row in table_contents: - table.add_row(row[1], row[2], row[3]) - return table - - -def get_host_name(container_name: str, by_suffix: str) -> str: - # Assumption we always get proxy containers first. - # if users have old docker compose versios. - # the container names are _ instead of - - # canada_proxy_1 instead of canada-proxy-1 - try: - host_name = container_name[0 : container_name.find(by_suffix) - 1] # noqa: E203 - except Exception: - host_name = "" - return host_name - - -def get_docker_status( - ip_address: str, node_name: str | None -) -> tuple[bool, tuple[str, str]]: - url = from_url(ip_address) - port = url[2] - network_container = ( - shell( - "docker ps --format '{{.Names}} {{.Ports}}' | " + f"grep '0.0.0.0:{port}'" - ) - .strip() - .split(" ")[0] - ) - - # Second conditional handle the case when internal port of worker container - # matches with host port of launched Domain/Network Container - if not network_container or (node_name and node_name not in network_container): - # check if it is a worker container and an internal port was passed - worker_containers_output: str = shell( - "docker ps --format '{{.Names}} {{.Ports}}' | " + f"grep '{port}/tcp'" - ).strip() - if not worker_containers_output or not node_name: - return False, ("", "") - - # If there are worker containers with an internal port - # fetch the worker container with the launched worker name - worker_containers = worker_containers_output.split("\n") - for worker_container in worker_containers: - container_name = worker_container.split(" ")[0] - if node_name in container_name: - network_container = container_name - break - else: - # If the worker container is not created yet - return False, ("", "") - - if "proxy" in network_container: - host_name = get_host_name(network_container, by_suffix="proxy") - - backend_containers = shell( - "docker ps --format '{{.Names}}' | grep 'backend' " - ).split() - - _backend_exists = False - for container in backend_containers: - if host_name in container and "stream" not in container: - _backend_exists = True - break - if not _backend_exists: - return False, ("", "") - - node_type = "Domain" - - # TODO: Identify if node_type is Gateway - # for container in headscale_containers: - # if host_name in container: - # node_type = "Gateway" - # break - - return True, (host_name, node_type) - else: - # health check for worker node - host_name = get_host_name(network_container, by_suffix="worker") - return True, (host_name, "Worker") - - -def get_syft_install_status(host_name: str, node_type: str) -> bool: - container_search = "backend" if node_type != "Worker" else "worker" - search_containers = shell( - "docker ps --format '{{.Names}}' | " + f"grep '{container_search}' " - ).split() - - context_container = None - for container in search_containers: - # stream keyword is for our old container stack - if host_name in container and "stream" not in container: - context_container = container - break - - if not context_container: - print(f"❌ {container_search} Docker Stack for: {host_name} not found") - exit(0) - else: - container_log = shell(f"docker logs {context_container}") - if "Application startup complete" not in container_log: - return False - return True - - -@click.command( - help="Check health of an IP address/addresses or a resource group", - context_settings={"show_default": True}, -) -@click.argument("ip_addresses", type=str, nargs=-1) -@click.option( - "--timeout", - default=300, - help="Timeout for hagrid check command", -) -@click.option( - "--verbose", - is_flag=True, - help="Refresh output", -) -def check( - ip_addresses: list[str], verbose: bool = False, timeout: int | str = 300 -) -> None: - check_status(ip_addresses=ip_addresses, silent=not verbose, timeout=timeout) - - -def _check_status( - ip_addresses: str | list[str], - silent: bool = True, - signal: Event | None = None, - node_name: str | None = None, -) -> None: - OK_EMOJI = RichEmoji("white_heavy_check_mark").to_str() - # Check if ip_addresses is str, then convert to list - if ip_addresses and isinstance(ip_addresses, str): - ip_addresses = [ip_addresses] - console = Console() - node_info = None - if len(ip_addresses) == 0: - headers = {"User-Agent": "curl/7.79.1"} - print("Detecting External IP...") - ip_res = requests.get("https://ifconfig.co", headers=headers) # nosec - ip_address = ip_res.text.strip() - ip_addresses = [ip_address] - - if len(ip_addresses) == 1: - ip_address = ip_addresses[0] - status, table_contents = get_health_checks(ip_address=ip_address) - table = create_check_table(table_contents=table_contents) - max_timeout = 600 - if not status: - table = create_check_table( - table_contents=table_contents, time_left=max_timeout - ) - if silent: - with console.status("Gathering Node information") as console_status: - console_status.update( - "[bold orange_red1]Waiting for Container Creation" - ) - docker_status, node_info = get_docker_status(ip_address, node_name) - while not docker_status: - docker_status, node_info = get_docker_status( - ip_address, node_name - ) - time.sleep(1) - if ( - signal and signal.is_set() - ): # Stop execution if timeout is triggered - return - console.print( - f"{OK_EMOJI} {node_info[0]} {node_info[1]} Containers Created" - ) - - console_status.update("[bold orange_red1]Starting Backend") - syft_install_status = get_syft_install_status( - node_info[0], node_info[1] - ) - while not syft_install_status: - syft_install_status = get_syft_install_status( - node_info[0], node_info[1] - ) - time.sleep(1) - # Stop execution if timeout is triggered - if signal and signal.is_set(): - return - console.print(f"{OK_EMOJI} Backend") - console.print(f"{OK_EMOJI} Startup Complete") - - status, table_contents = get_health_checks(ip_address) - table = create_check_table( - table_contents=table_contents, time_left=max_timeout - ) - else: - while not status: - # Stop execution if timeout is triggered - if signal is not None and signal.is_set(): - return - with Live( - table, refresh_per_second=2, screen=True, auto_refresh=False - ) as live: - max_timeout -= 1 - if max_timeout % 5 == 0: - status, table_contents = get_health_checks(ip_address) - table = create_check_table( - table_contents=table_contents, time_left=max_timeout - ) - live.update(table) - if status: - break - time.sleep(1) - - # TODO: Create new health checks table for Worker Container - if (node_info and node_info[1] != "Worker") or not node_info: - console.print(table) - else: - for ip_address in ip_addresses: - _, table_contents = get_health_checks(ip_address) - table = create_check_table(table_contents=table_contents) - console.print(table) - - -def check_status( - ip_addresses: str | list[str], - silent: bool = True, - timeout: int | str = 300, - node_name: str | None = None, -) -> None: - timeout = int(timeout) - # third party - from rich import print - - signal = Event() - - t = Thread( - target=_check_status, - kwargs={ - "ip_addresses": ip_addresses, - "silent": silent, - "signal": signal, - "node_name": node_name, - }, - ) - t.start() - t.join(timeout=timeout) - - if t.is_alive(): - signal.set() - t.join() - - print(f"Hagrid check command timed out after: {timeout} seconds 🕛") - print( - "Please try increasing the timeout or kindly check the docker containers for error logs." - ) - print("You can view your container logs using the following tool:") - print("Tool: [link=https://ctop.sh]Ctop[/link]") - print("Video Explanation: https://youtu.be/BJhlCxerQP4 \n") - - -cli.add_command(check) - - -# add Hagrid info to the cli -@click.command(help="Show HAGrid info", context_settings={"show_default": True}) -def version() -> None: - print(f"HAGRID_VERSION: {get_version_string()}") - if EDITABLE_MODE: - print(f"HAGRID_REPO_SHA: {commit_hash()}") - - -cli.add_command(version) - - -def run_quickstart( - url: str | None = None, - syft: str = "latest", - reset: bool = False, - quiet: bool = False, - pre: bool = False, - test: bool = False, - repo: str = DEFAULT_REPO, - branch: str = DEFAULT_BRANCH, - commit: str | None = None, - python: str | None = None, - zip_file: str | None = None, -) -> None: - try: - quickstart_art() - directory = os.path.expanduser("~/.hagrid/quickstart/") - confirm_reset = None - if reset: - if not quiet: - confirm_reset = click.confirm( - "This will create a new quickstart virtualenv and reinstall Syft and " - "Jupyter. Are you sure you want to continue?" - ) - else: - confirm_reset = True - if confirm_reset is False: - return - - if reset and confirm_reset or not os.path.isdir(directory): - quickstart_setup( - directory=directory, - syft_version=syft, - reset=reset, - pre=pre, - python=python, - ) - downloaded_files = [] - if zip_file: - downloaded_files = fetch_notebooks_from_zipfile( - zip_file, - directory=directory, - reset=reset, - ) - elif url: - downloaded_files = fetch_notebooks_for_url( - url=url, - directory=directory, - reset=reset, - repo=repo, - branch=branch, - commit=commit, - ) - else: - file_path = add_intro_notebook(directory=directory, reset=reset) - downloaded_files.append(file_path) - - if len(downloaded_files) == 0: - raise Exception(f"Unable to find files at: {url}") - file_path = sorted(downloaded_files)[0] - - # add virtualenv path - environ = os.environ.copy() - os_bin_path = "Scripts" if is_windows() else "bin" - venv_dir = directory + ".venv" - environ["PATH"] = venv_dir + os.sep + os_bin_path + os.pathsep + environ["PATH"] - jupyter_binary = "jupyter.exe" if is_windows() else "jupyter" - - if is_windows(): - env_activate_cmd = ( - "(Powershell): " - + "cd " - + venv_dir - + "; " - + os_bin_path - + os.sep - + "activate" - ) - else: - env_activate_cmd = ( - "(Linux): source " + venv_dir + os.sep + os_bin_path + "/activate" - ) - - print(f"To activate your virtualenv {env_activate_cmd}") - - try: - allow_browser = " --no-browser" if is_gitpod() else "" - cmd = ( - venv_dir - + os.sep - + os_bin_path - + os.sep - + f"{jupyter_binary} lab{allow_browser} --ip 0.0.0.0 --notebook-dir={directory} {file_path}" - ) - if test: - jupyter_path = venv_dir + os.sep + os_bin_path + os.sep + jupyter_binary - if not os.path.exists(jupyter_path): - print(f"Failed to install Jupyter in path: {jupyter_path}") - sys.exit(1) - print(f"Jupyter exists at: {jupyter_path}. CI Test mode exiting.") - sys.exit(0) - - disable_toolbar_extension = ( - venv_dir - + os.sep - + os_bin_path - + os.sep - + f"{jupyter_binary} labextension disable @jupyterlab/cell-toolbar-extension" - ) - - subprocess.run( # nosec - disable_toolbar_extension.split(" "), cwd=directory, env=environ - ) - - ON_POSIX = "posix" in sys.builtin_module_names - - def enqueue_output(out: Any, queue: Queue) -> None: - for line in iter(out.readline, b""): - queue.put(line) - out.close() - - proc = subprocess.Popen( # nosec - cmd.split(" "), - cwd=directory, - env=environ, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=ON_POSIX, - ) - queue: Queue = Queue() - thread_1 = Thread(target=enqueue_output, args=(proc.stdout, queue)) - thread_2 = Thread(target=enqueue_output, args=(proc.stderr, queue)) - thread_1.daemon = True # thread dies with the program - thread_1.start() - thread_2.daemon = True # thread dies with the program - thread_2.start() - - display_url = None - console = rich.get_console() - - # keepn reading the queue of stdout + stderr - while True: - try: - if not display_url: - # try to read the line and extract a jupyter url: - with console.status( - "Starting Jupyter service" - ) as console_status: - line = queue.get() - display_url = extract_jupyter_url(line.decode("utf-8")) - if display_url: - display_jupyter_url(url_parts=display_url) - console_status.stop() - except KeyboardInterrupt: - proc.kill() # make sure jupyter gets killed - sys.exit(1) - except Exception: # nosec - pass # nosec - except KeyboardInterrupt: - proc.kill() # make sure jupyter gets killed - sys.exit(1) - except Exception as e: - print(f"Error running quickstart: {e}") - raise e - - -@click.command( - help="Launch a Syft + Jupyter Session with a Notebook URL / Path", - context_settings={"show_default": True}, -) -@click.argument("url", type=str, required=False) -@click.option( - "--reset", - is_flag=True, - default=False, - help="Force hagrid quickstart to setup a fresh virtualenv", -) -@click.option( - "--syft", - default="latest", - help="Choose a syft version or just use latest", -) -@click.option( - "--quiet", - is_flag=True, - help="Silence confirmation prompts", -) -@click.option( - "--pre", - is_flag=True, - help="Install pre-release versions of syft", -) -@click.option( - "--python", - default=None, - help="Specify the path to which python to use", -) -@click.option( - "--test", - is_flag=True, - help="CI Test Mode, don't hang on Jupyter", -) -@click.option( - "--repo", - default=DEFAULT_REPO, - help="Choose a repo to fetch the notebook from or just use OpenMined/PySyft", -) -@click.option( - "--branch", - default=DEFAULT_BRANCH, - help="Choose a branch to fetch from or just use dev", -) -@click.option( - "--commit", - help="Choose a specific commit to fetch the notebook from", -) -def quickstart_cli( - url: str | None = None, - syft: str = "latest", - reset: bool = False, - quiet: bool = False, - pre: bool = False, - test: bool = False, - repo: str = DEFAULT_REPO, - branch: str = DEFAULT_BRANCH, - commit: str | None = None, - python: str | None = None, -) -> None: - return run_quickstart( - url=url, - syft=syft, - reset=reset, - quiet=quiet, - pre=pre, - test=test, - repo=repo, - branch=branch, - commit=commit, - python=python, - ) - - -cli.add_command(quickstart_cli, "quickstart") - - -def display_jupyter_url(url_parts: tuple[str, str, int]) -> None: - url = url_parts[0] - if is_gitpod(): - parts = urlparse(url) - query = getattr(parts, "query", "") - url = gitpod_url(port=url_parts[2]) + "?" + query - - console = rich.get_console() - - tick_emoji = RichEmoji("white_heavy_check_mark").to_str() - link_emoji = RichEmoji("link").to_str() - - console.print( - f"[bold white]{tick_emoji} Jupyter Server is running at:\n{link_emoji} [bold blue]{url}\n" - + "[bold white]Use Control-C to stop this server and shut down all kernels.", - new_line_start=True, - ) - - # if is_gitpod(): - # open_browser_with_url(url=url) - - -def open_browser_with_url(url: str) -> None: - webbrowser.open(url) - - -def extract_jupyter_url(line: str) -> tuple[str, str, int] | None: - jupyter_regex = r"^.*(http.*127.*)" - try: - matches = re.match(jupyter_regex, line) - if matches is not None: - url = matches.group(1).strip() - parts = urlparse(url) - host_or_ip_parts = parts.netloc.split(":") - # netloc is host:port - port = 8888 - if len(host_or_ip_parts) > 1: - port = int(host_or_ip_parts[1]) - host_or_ip = host_or_ip_parts[0] - return (url, host_or_ip, port) - except Exception as e: - print("failed to parse jupyter url", e) - return None - - -def quickstart_setup( - directory: str, - syft_version: str, - reset: bool = False, - pre: bool = False, - python: str | None = None, -) -> None: - console = rich.get_console() - OK_EMOJI = RichEmoji("white_heavy_check_mark").to_str() - - try: - with console.status( - "[bold blue]Setting up Quickstart Environment" - ) as console_status: - os.makedirs(directory, exist_ok=True) - virtual_env_dir = os.path.abspath(directory + ".venv/") - if reset and os.path.exists(virtual_env_dir): - shutil.rmtree(virtual_env_dir) - env = VirtualEnvironment(virtual_env_dir, python=python) - console.print( - f"{OK_EMOJI} Created Virtual Environment {RichEmoji('evergreen_tree').to_str()}" - ) - - # upgrade pip - console_status.update("[bold blue]Installing pip") - env.install("pip", options=["-U"]) - console.print(f"{OK_EMOJI} pip") - - # upgrade packaging - console_status.update("[bold blue]Installing packaging") - env.install("packaging", options=["-U"]) - console.print(f"{OK_EMOJI} packaging") - - # Install jupyter lab - console_status.update("[bold blue]Installing Jupyter Lab") - env.install("jupyterlab") - env.install("ipywidgets") - console.print(f"{OK_EMOJI} Jupyter Lab") - - # Install hagrid - if EDITABLE_MODE: - local_hagrid_dir = Path( - os.path.abspath(Path(hagrid_root()) / "../hagrid") - ) - console_status.update( - f"[bold blue]Installing HAGrid in Editable Mode: {str(local_hagrid_dir)}" - ) - env.install("-e " + str(local_hagrid_dir)) - console.print( - f"{OK_EMOJI} HAGrid in Editable Mode: {str(local_hagrid_dir)}" - ) - else: - console_status.update("[bold blue]Installing hagrid") - env.install("hagrid", options=["-U"]) - console.print(f"{OK_EMOJI} HAGrid") - except Exception as e: - print(e) - raise e - - -def add_intro_notebook(directory: str, reset: bool = False) -> str: - filenames = ["00-quickstart.ipynb", "01-install-wizard.ipynb"] - - files = os.listdir(directory) - try: - files.remove(".venv") - except Exception: # nosec - pass - - existing = 0 - for file in files: - if file in filenames: - existing += 1 - - if existing != len(filenames) or reset: - if EDITABLE_MODE: - local_src_dir = Path(os.path.abspath(Path(hagrid_root()) / "../../")) - for filename in filenames: - file_path = os.path.abspath(f"{directory}/{filename}") - shutil.copyfile( - local_src_dir / f"notebooks/quickstart/{filename}", - file_path, - ) - else: - for filename in filenames: - url = ( - "https://raw.githubusercontent.com/OpenMined/PySyft/dev/" - + f"notebooks/quickstart/{filename}" - ) - file_path, _, _ = quickstart_download_notebook( - url=url, directory=directory, reset=reset - ) - if arg_cache["install_wizard_complete"]: - filename = filenames[0] - else: - filename = filenames[1] - return os.path.abspath(f"{directory}/{filename}") - - -@click.command(help="Walk the Path", context_settings={"show_default": True}) -@click.argument("zip_file", type=str, default="padawan.zip", metavar="ZIPFILE") -def dagobah(zip_file: str) -> None: - if not os.path.exists(zip_file): - for text in ( - f"{zip_file} does not exists.", - "Please specify the path to the zip file containing the notebooks.", - "hagrid dagobah [ZIPFILE]", - ): - print(text, file=sys.stderr) - sys.exit(1) - - return run_quickstart(zip_file=zip_file) - - -cli.add_command(dagobah) - - -def ssh_into_remote_machine( - host_ip: str, - username: str, - auth_type: str, - private_key_path: str | None, - cmd: str = "", -) -> None: - """Access or execute command on the remote machine. - - Args: - host_ip (str): ip address of the VM - private_key_path (str): private key of the VM - username (str): username on the VM - cmd (str, optional): Command to execute on the remote machine. Defaults to "". - """ - try: - if auth_type == "key": - subprocess.call( # nosec - ["ssh", "-i", f"{private_key_path}", f"{username}@{host_ip}", cmd] - ) - elif auth_type == "password": - subprocess.call(["ssh", f"{username}@{host_ip}", cmd]) # nosec - except Exception as e: - raise e - - -@click.command( - help="SSH into the IP address or a resource group", - context_settings={"show_default": True}, -) -@click.argument("ip_address", type=str) -@click.option( - "--cmd", - type=str, - required=False, - default="", - help="Optional: command to execute on the remote machine.", -) -def ssh(ip_address: str, cmd: str) -> None: - kwargs: dict = {} - key_path: str | None = None - - if check_ip_for_ssh(ip_address, timeout=10, silent=False): - username = ask( - question=Question( - var_name="azure_username", - question="What is the username for the VM?", - default=arg_cache["azure_username"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - auth_type = ask( - question=Question( - var_name="auth_type", - question="Do you want to login with a key or password", - default=arg_cache["auth_type"], - kind="option", - options=["key", "password"], - cache=True, - ), - kwargs=kwargs, - ) - - if auth_type == "key": - key_path = ask( - question=Question( - var_name="azure_key_path", - question="Absolute path to the private key of the VM?", - default=arg_cache["azure_key_path"], - kind="string", - cache=True, - ), - kwargs=kwargs, - ) - - # SSH into the remote and execute the command - ssh_into_remote_machine( - host_ip=ip_address, - username=username, - auth_type=auth_type, - private_key_path=key_path, - cmd=cmd, - ) - - -cli.add_command(ssh) - - -# Add hagrid logs command to the CLI -@click.command( - help="Get the logs of the HAGrid node", context_settings={"show_default": True} -) -@click.argument("domain_name", type=str) -def logs(domain_name: str) -> None: # nosec - container_ids = ( - subprocess.check_output( # nosec - f"docker ps -qf name=^{domain_name}-*", shell=True - ) - .decode("utf-8") - .split() - ) - Container = namedtuple("Container", "id name logs") - container_names = [] - for container in container_ids: - container_name = ( - subprocess.check_output( # nosec - "docker inspect --format '{{.Name}}' " + container, shell=True - ) - .decode("utf-8") - .strip() - .replace("/", "") - ) - log_command = "docker logs -f " + container_name - container_names.append( - Container(id=container, name=container_name, logs=log_command) - ) - # Generate a table of the containers and their logs with Rich - table = rich.table.Table(title="Container Logs") - table.add_column("Container ID", justify="center", style="cyan", no_wrap=True) - table.add_column("Container Name", justify="right", style="cyan", no_wrap=True) - table.add_column("Log Command", justify="right", style="cyan", no_wrap=True) - for container in container_names: # type: ignore - table.add_row(container.id, container.name, container.logs) # type: ignore - console = rich.console.Console() - console.print(table) - # Print instructions on how to view the logs - console.print( - rich.panel.Panel( - long_string, - title="How to view logs", - border_style="white", - expand=False, - padding=1, - highlight=True, - ) - ) - - -long_string = ( - "ℹ [bold green]To view the live logs of a container,copy the log command and paste it into your terminal.[/bold green]\n" # noqa: E501 - + "\n" - + "ℹ [bold green]The logs will be streamed to your terminal until you exit the command.[/bold green]\n" - + "\n" - + "ℹ [bold green]To exit the logs, press CTRL+C.[/bold green]\n" - + "\n" - + "🚨 The [bold white]backend,backend_stream & celery[/bold white] [bold green]containers are the most important to monitor for debugging.[/bold green]\n" # noqa: E501 - + "\n" - + " [bold white]--------------- Ctop 🦾 -------------------------[/bold white]\n" - + "\n" - + "🧠 To learn about using [bold white]ctop[/bold white] to monitor your containers,visit https://www.youtube.com/watch?v=BJhlCxerQP4n \n" # noqa: E501 - + "\n" - + " [bold white]----------------- How to view this. 🙂 ---------------[/bold white]\n" - + "\n" - + """ℹ [bold green]To view this panel again, run the command [bold white]hagrid logs {{NODE_NAME}}[/bold white] [/bold green]\n""" # noqa: E501 - + "\n" - + """🚨 NODE_NAME above is the name of your Hagrid deployment,without the curly braces. E.g hagrid logs canada [bold green]\n""" # noqa: E501 - + "\n" - + " [bold green]HAPPY DEBUGGING! 🐛🐞🦗🦟🦠🦠🦠[/bold green]\n " -) - -cli.add_command(logs) From 2df1656944aefb630f328e0799a538b1c1a04475 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 10 Jun 2024 18:31:06 +0530 Subject: [PATCH 079/100] lint notebook --- notebooks/Experimental/Network.ipynb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/notebooks/Experimental/Network.ipynb b/notebooks/Experimental/Network.ipynb index 88240421fb3..d12b95f7129 100644 --- a/notebooks/Experimental/Network.ipynb +++ b/notebooks/Experimental/Network.ipynb @@ -7,6 +7,7 @@ "metadata": {}, "outputs": [], "source": [ + "# syft absolute\n", "import syft as sy" ] }, @@ -37,7 +38,9 @@ } ], "source": [ - "gateway_client = sy.login(url=\"http://localhost\", port=9081, email=\"info@openmined.org\", password=\"changethis\")" + "gateway_client = sy.login(\n", + " url=\"http://localhost\", port=9081, email=\"info@openmined.org\", password=\"changethis\"\n", + ")" ] }, { @@ -67,7 +70,9 @@ } ], "source": [ - "domain_client = sy.login(url=\"http://localhost\", port=9082, email=\"info@openmined.org\", password=\"changethis\")" + "domain_client = sy.login(\n", + " url=\"http://localhost\", port=9082, email=\"info@openmined.org\", password=\"changethis\"\n", + ")" ] }, { From cec992b993c5fbac0a7226d01d3f9f19d60608d1 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 10 Jun 2024 18:32:45 +0530 Subject: [PATCH 080/100] remove output from Network notebook --- notebooks/Experimental/Network.ipynb | 7888 +------------------------- 1 file changed, 28 insertions(+), 7860 deletions(-) diff --git a/notebooks/Experimental/Network.ipynb b/notebooks/Experimental/Network.ipynb index d12b95f7129..7a1f3f257dc 100644 --- a/notebooks/Experimental/Network.ipynb +++ b/notebooks/Experimental/Network.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "bd9a2226-3e53-4f27-9213-75a8c3ff9176", "metadata": {}, "outputs": [], @@ -13,30 +13,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "fddf8d07-d154-4284-a27b-d74e35d3f851", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - }, - { - "data": { - "text/html": [ - "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" - ], - "text/plain": [ - "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "gateway_client = sy.login(\n", " url=\"http://localhost\", port=9081, email=\"info@openmined.org\", password=\"changethis\"\n", @@ -45,30 +25,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "8f7b106d-b784-45d8-b54d-4ce2de2da453", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - }, - { - "data": { - "text/html": [ - "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" - ], - "text/plain": [ - "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "domain_client = sy.login(\n", " url=\"http://localhost\", port=9082, email=\"info@openmined.org\", password=\"changethis\"\n", @@ -77,2556 +37,27 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "ff504949-620d-4e26-beee-0d39e0e502eb", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: Connected domain 'syft-dev-node' to gateway 'syft-dev-node'. Routes Exchanged

" - ], - "text/plain": [ - "SyftSuccess: Connected domain 'syft-dev-node' to gateway 'syft-dev-node'. Routes Exchanged" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "domain_client.connect_to_gateway(gateway_client, reverse_tunnel=True)" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "ba7bc71a-4e6a-4429-9588-7b3d0ed19e27", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

Request List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "gateway_client.api.services.request" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "5b4984e1-331e-4fd8-b012-768fc613f48a", "metadata": {}, "outputs": [], @@ -2636,2525 +67,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "90dc44bd", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

NodePeer List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "[syft.service.network.node_peer.NodePeer]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "node_peers = gateway_client.api.network.get_all_peers()\n", "node_peers" @@ -5162,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "8c06aaa6-4157-42d1-959f-9d47722a3420", "metadata": {}, "outputs": [], @@ -5172,88 +88,37 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "cb63a77b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[syft.service.network.routes.HTTPNodeRoute]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "node_peer.node_routes" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "61882e86", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'syft_node_location': ,\n", - " 'syft_client_verify_key': cc82ec7abd5d516e6972e787ddaafe8b04e223228436b09c026be97b80ad6246,\n", - " 'id': None,\n", - " 'host_or_ip': 'syft-dev-node.syft.local',\n", - " 'private': False,\n", - " 'protocol': 'http',\n", - " 'port': 9082,\n", - " 'proxy_target_uid': None,\n", - " 'priority': 1,\n", - " 'rathole_token': 'b95e8d239d563e6fcc3a4f44a5292177e608a7b0b1194e6106adc1998a1b68a1'}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "node_peer.node_routes[0].__dict__" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "fb19dbc6-869b-46dc-92e3-5e75ee6d0b06", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'syft_node_location': ,\n", - " 'syft_client_verify_key': 5ed40db70e275e30e9001cda808922c4542c84f7472a106c00158795b9388b0a,\n", - " 'id': None,\n", - " 'host_or_ip': 'host.k3d.internal',\n", - " 'private': False,\n", - " 'protocol': 'http',\n", - " 'port': 9081,\n", - " 'proxy_target_uid': None,\n", - " 'priority': 1,\n", - " 'rathole_token': None}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "domain_client.api.network.get_all_peers()[0].node_routes[0].__dict__" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "32d09a51", "metadata": {}, "outputs": [], @@ -5263,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "b7d9e41d", "metadata": {}, "outputs": [], @@ -5273,2706 +138,27 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "8fa24ec7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "
\n", - " \"Logo\"\n",\n", - "

Welcome to syft-dev-node

\n", - "
\n", - " URL: http://localhost:9081
Node Type: Gateway
Node Side Type: High Side
Syft Version: 0.8.7-beta.10
\n", - "
\n", - "
\n", - " ⓘ \n", - " This node is run by the library PySyft to learn more about how it works visit\n", - " github.com/OpenMined/PySyft.\n", - "
\n", - "

Commands to Get Started

\n", - " \n", - "
    \n", - " \n", - "
  • <your_client>\n", - " .domains - list domains connected to this gateway
  • \n", - "
  • <your_client>\n", - " .proxy_client_for - get a connection to a listed domain
  • \n", - "
  • <your_client>\n", - " .login - log into the gateway
  • \n", - " \n", - "
\n", - " \n", - "

\n", - " " - ], - "text/plain": [ - ": HTTPConnection: http://localhost:9081>" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "gateway_client" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "3a081250-abc3-43a3-9e06-ff0c3a362ebf", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

NodePeer List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/markdown": [ - "```python\n", - "class ProxyClient:\n", - " id: str = 37073e9151ce4fa9b665501ec03924c8\n", - "\n", - "```" - ], - "text/plain": [ - "syft.client.gateway_client.ProxyClient" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "gateway_client.peers" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "b6fedfe4-9362-47c9-9342-5cf6eacde8ab", "metadata": {}, "outputs": [], @@ -7982,28 +168,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "f1940e00-0337-4b56-88c2-d70f397a7016", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "class HTTPConnection:\n", - " id: str = None\n", - "\n", - "```" - ], - "text/plain": [ - "HTTPConnection: http://localhost:9081" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "domain_client_proxy.connection" ] From 0db775f4969c02eb126543fc83d459d6f750dc06 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 11 Jun 2024 15:58:42 +0530 Subject: [PATCH 081/100] ignore prefix for rathole in blob store path --- packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index 748dfee78c4..705c153442a 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -43,7 +43,7 @@ data: - "web" service: "backend" blob-storage: - rule: "PathPrefix(`/blob`)" + rule: "PathPrefix(`/blob`) && !PathPrefix(`/rathole`)" entryPoints: - "web" service: "seaweedfs" From abcc997a4de04843679ba917e968d460f9b7297e Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Sun, 16 Jun 2024 22:40:32 +0530 Subject: [PATCH 082/100] add a stream upload API for blob storage - integrate proxy uid to stream upload data from client to domain via gateway --- packages/syft/src/syft/client/client.py | 30 ++++++++++++++++ packages/syft/src/syft/node/node.py | 3 +- packages/syft/src/syft/node/routes.py | 35 ++++++++++++++++++- .../src/syft/protocol/protocol_version.json | 12 +++++++ .../src/syft/store/blob_storage/seaweedfs.py | 17 ++++++--- 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index dc3fa447d58..0969c6cf4d9 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -4,6 +4,7 @@ # stdlib import base64 from collections.abc import Callable +from collections.abc import Generator from collections.abc import Iterator from copy import deepcopy from enum import Enum @@ -216,6 +217,35 @@ def _make_get( return response.content + def _make_put( + self, path: str, data: bytes | Generator, stream: bool = False + ) -> Response: + headers = {} + url = self.url + + if self.rathole_token: + url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) + headers = {"Host": self.url.host_or_ip} + + url = url.with_path(path) + response = self.session.put( + str(url), + verify=verify_tls(), + proxies={}, + data=data, + headers=headers, + stream=stream, + ) + if response.status_code != 200: + raise requests.ConnectionError( + f"Failed to fetch {url}. Response returned with code {response.status_code}" + ) + + # upgrade to tls if available + self.url = upgrade_tls(self.url, response) + + return response + def _make_post( self, path: str, diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 1e2c00c6f24..15508cdf184 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -117,6 +117,7 @@ from ..store.blob_storage import BlobStorageConfig from ..store.blob_storage.on_disk import OnDiskBlobStorageClientConfig from ..store.blob_storage.on_disk import OnDiskBlobStorageConfig +from ..store.blob_storage.seaweedfs import SeaweedFSBlobDeposit from ..store.dict_document_store import DictStoreConfig from ..store.document_store import StoreConfig from ..store.linked_obj import LinkedObject @@ -1181,7 +1182,7 @@ def forward_message( # relative from ..store.blob_storage import BlobRetrievalByURL - if isinstance(result, BlobRetrievalByURL): + if isinstance(result, BlobRetrievalByURL | SeaweedFSBlobDeposit): result.proxy_node_uid = peer.id return result diff --git a/packages/syft/src/syft/node/routes.py b/packages/syft/src/syft/node/routes.py index f32aa5c3bd9..8b91786af2c 100644 --- a/packages/syft/src/syft/node/routes.py +++ b/packages/syft/src/syft/node/routes.py @@ -1,6 +1,7 @@ # stdlib import base64 import binascii +from collections.abc import AsyncGenerator from typing import Annotated # third party @@ -62,7 +63,7 @@ def _get_node_connection(peer_uid: UID) -> NodeConnection: return connection @router.get("/stream/{peer_uid}/{url_path}/", name="stream") - async def stream(peer_uid: str, url_path: str) -> StreamingResponse: + async def stream_download(peer_uid: str, url_path: str) -> StreamingResponse: try: url_path_parsed = base64.urlsafe_b64decode(url_path.encode()).decode() except binascii.Error: @@ -79,6 +80,38 @@ async def stream(peer_uid: str, url_path: str) -> StreamingResponse: return StreamingResponse(stream_response, media_type="text/event-stream") + async def read_request_body_in_chunks( + request: Request, + ) -> AsyncGenerator[bytes, None]: + async for chunk in request.stream(): + yield chunk + + @router.put("/stream/{peer_uid}/{url_path}/", name="stream") + async def stream_upload(peer_uid: str, url_path: str, request: Request) -> Response: + try: + url_path_parsed = base64.urlsafe_b64decode(url_path.encode()).decode() + except binascii.Error: + raise HTTPException(404, "Invalid `url_path`.") + + data = await request.body() + + peer_uid_parsed = UID.from_string(peer_uid) + + try: + peer_connection = _get_node_connection(peer_uid_parsed) + url = peer_connection.to_blob_route(url_path_parsed) + + print("Url on stream", url.path) + response = peer_connection._make_put(url.path, data=data, stream=True) + except requests.RequestException: + raise HTTPException(404, "Failed to upload data to domain") + + return Response( + content=response.content, + headers=response.headers, + media_type="application/octet-stream", + ) + @router.get( "/", name="healthcheck", diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 98c6b4576ba..02ca42bc84d 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -308,6 +308,18 @@ "hash": "9e7e3700a2f7b1a67f054efbcb31edc71bbf358c469c85ed7760b81233803bac", "action": "add" } + }, + "SeaweedFSBlobDeposit": { + "3": { + "version": 3, + "hash": "05e61e6328b085b738e5d41c0781d87852d44d218894cb3008f5be46e337f6d8", + "action": "remove" + }, + "4": { + "version": 4, + "hash": "f475543ed5e0066ca09c0dfd8c903e276d4974519e9958473d8141f8d446c881", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 1d88fedda37..b3667e61251 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -37,7 +37,8 @@ from ...types.blob_storage import SeaweedSecureFilePathLocation from ...types.blob_storage import SecureFilePathLocation from ...types.grid_url import GridURL -from ...types.syft_object import SYFT_OBJECT_VERSION_3 +from ...types.syft_object import SYFT_OBJECT_VERSION_4 +from ...types.uid import UID from ...util.constants import DEFAULT_TIMEOUT MAX_QUEUE_SIZE = 100 @@ -49,10 +50,11 @@ @serializable() class SeaweedFSBlobDeposit(BlobDeposit): __canonical_name__ = "SeaweedFSBlobDeposit" - __version__ = SYFT_OBJECT_VERSION_3 + __version__ = SYFT_OBJECT_VERSION_4 urls: list[GridURL] size: int + proxy_node_uid: UID | None = None def write(self, data: BytesIO) -> SyftSuccess | SyftError: # relative @@ -87,9 +89,14 @@ def write(self, data: BytesIO) -> SyftSuccess | SyftError: start=1, ): if api is not None and api.connection is not None: - blob_url = api.connection.to_blob_route( - url.url_path, host=url.host_or_ip - ) + if self.proxy_node_uid is None: + blob_url = api.connection.to_blob_route( + url.url_path, host=url.host_or_ip + ) + else: + blob_url = api.connection.stream_via( + self.proxy_node_uid, url.url_path + ) else: blob_url = url From c335d5374cf10b178a52bb8cf545e64c882a01dc Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 24 Jun 2024 18:57:19 +0530 Subject: [PATCH 083/100] add a test for reverse tunnel - cleanup rathole start script - use slim version of python in rathole dockerfile --- .../rathole/rathole-statefulset.yaml | 4 +- packages/grid/helm/syft/values.yaml | 1 + packages/grid/helm/values.dev.yaml | 1 + packages/grid/rathole/rathole.dockerfile | 32 +---------- packages/grid/rathole/start.sh | 54 +++++++++++++------ tests/integration/network/gateway_test.py | 53 ++++++++++++++++++ 6 files changed, 97 insertions(+), 48 deletions(-) diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index 0f07516e352..c441991ef47 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -38,8 +38,8 @@ spec: env: - name: SERVICE_NAME value: "rathole" - - name: APP_LOG_LEVEL - value: {{ .Values.rathole.appLogLevel | quote }} + - name: LOG_LEVEL + value: {{ .Values.rathole.logLevel | quote }} - name: MODE value: {{ .Values.rathole.mode | quote }} - name: DEV_MODE diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 8cd78c68e89..32f52b3bd25 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -240,6 +240,7 @@ rathole: # Extra environment vars env: null enabled: true + logLevel: info port: 2333 mode: client diff --git a/packages/grid/helm/values.dev.yaml b/packages/grid/helm/values.dev.yaml index 97ccbe4dfc7..713948f4a9b 100644 --- a/packages/grid/helm/values.dev.yaml +++ b/packages/grid/helm/values.dev.yaml @@ -49,6 +49,7 @@ proxy: rathole: enabled: "true" + logLevel: "trace" # attestation: # enabled: true # resourcesPreset: null diff --git a/packages/grid/rathole/rathole.dockerfile b/packages/grid/rathole/rathole.dockerfile index 42d147527c7..1dee47a9411 100644 --- a/packages/grid/rathole/rathole.dockerfile +++ b/packages/grid/rathole/rathole.dockerfile @@ -10,9 +10,10 @@ RUN git clone -b v${RATHOLE_VERSION} https://github.com/rapiz1/rathole WORKDIR /rathole RUN cargo build --locked --release --features ${FEATURES:-default} -FROM python:${PYTHON_VERSION}-bookworm +FROM python:${PYTHON_VERSION}-slim-bookworm ARG RATHOLE_VERSION ENV MODE="client" +ENV LOG_LEVEL="info" RUN apt update && apt install -y netcat-openbsd vim rsync COPY --from=build /rathole/target/release/rathole /app/rathole @@ -23,32 +24,3 @@ EXPOSE 2333/udp EXPOSE 2333 CMD ["sh", "-c", "/app/start.sh"] - - -# build and run a fake domain to simulate a normal http container service -# docker build -f domain.dockerfile . -t domain -# docker run --name domain1 -it -d -p 8080:8000 domain - - - -# check the web server is running on 8080 -# curl localhost:8080 - -# build and run the rathole container -# docker build -f rathole.dockerfile . -t rathole - -# run the rathole server -# docker run --add-host host.docker.internal:host-gateway --name rathole-server -it -p 8001:8001 -p 8002:8002 -p 2333:2333 -e MODE=server rathole - -# check nothing is on port 8001 yet -# curl localhost:8001 - -# run the rathole client -# docker run --add-host host.docker.internal:host-gateway --name rathole-client -it -e MODE=client rathole - -# try port 8001 now -# curl localhost:8001 - -# add another client and edit the server.toml and client.toml for port 8002 - - diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh index 0e708908836..b1af50597fc 100755 --- a/packages/grid/rathole/start.sh +++ b/packages/grid/rathole/start.sh @@ -1,30 +1,52 @@ #!/usr/bin/env bash + MODE=${MODE:-server} +RUST_LOG=${LOG_LEVEL:-trace} -cp -L -r -f /conf/* conf/ +# Copy configuration files +copy_config() { + cp -L -r -f /conf/* conf/ +} -if [[ $MODE == "server" ]]; then - RUST_LOG=trace /app/rathole conf/server.toml & -elif [[ $MODE = "client" ]]; then +# Start the server +start_server() { + RUST_LOG=$RUST_LOG /app/rathole conf/server.toml & +} + +# Start the client +start_client() { while true; do - RUST_LOG=trace /app/rathole conf/client.toml + RUST_LOG=$RUST_LOG /app/rathole conf/client.toml status=$? if [ $status -eq 0 ]; then - break + break else - echo "Failed to load client.toml, retrying in 5 seconds..." - sleep 10 + echo "Failed to load client.toml, retrying in 5 seconds..." + sleep 10 fi done & +} + +# Reload configuration every 10 seconds +reload_config() { + echo "Starting configuration reload loop..." + while true; do + copy_config + sleep 10 + done +} + +# Make an initial copy of the configuration +copy_config + +if [[ $MODE == "server" ]]; then + start_server +elif [[ $MODE == "client" ]]; then + start_client else echo "RATHOLE MODE is set to an invalid value. Exiting." + exit 1 fi -# reload config every 10 seconds -while true -do - # Execute your script here - cp -L -r -f /conf/* conf/ - # Sleep for 10 seconds - sleep 10 -done \ No newline at end of file +# Start the configuration reload in the background to keep the configuration up to date +reload_config diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index 9c42a9e9687..1e09c2a775e 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -912,3 +912,56 @@ def test_peer_health_check(set_env_var, gateway_port: int, domain_1_port: int) - # Remove existing peers assert isinstance(_remove_existing_peers(domain_client), SyftSuccess) assert isinstance(_remove_existing_peers(gateway_client), SyftSuccess) + + +def test_reverse_tunnel_connection(domain_1_port: int, gateway_port: int): + # login to the domain and gateway + + gateway_client: GatewayClient = sy.login( + port=gateway_port, email="info@openmined.org", password="changethis" + ) + domain_client: DomainClient = sy.login( + port=domain_1_port, email="info@openmined.org", password="changethis" + ) + + res = gateway_client.settings.allow_association_request_auto_approval(enable=False) + + # Try removing existing peers just to make sure + _remove_existing_peers(domain_client) + _remove_existing_peers(gateway_client) + + # connecting the domain to the gateway + result = domain_client.connect_to_gateway(gateway_client, reverse_tunnel=True) + + assert isinstance(result, Request) + assert isinstance(result.changes[0], AssociationRequestChange) + + assert len(domain_client.peers) == 1 + + # Domain's peer is a gateway and vice-versa + domain_peer = domain_client.peers[0] + assert domain_peer.node_type == NodeType.GATEWAY + assert domain_peer.node_routes[0].rathole_token is None + assert len(gateway_client.peers) == 0 + + gateway_client_root = gateway_client.login( + email="info@openmined.org", password="changethis" + ) + res = gateway_client_root.api.services.request.get_all()[-1].approve() + assert not isinstance(res, SyftError) + + time.sleep(90) + + gateway_peers = gateway_client.api.services.network.get_all_peers() + assert len(gateway_peers) == 1 + assert len(gateway_peers[0].node_routes) == 1 + assert gateway_peers[0].node_routes[0].rathole_token is not None + + proxy_domain_client = gateway_client.peers[0] + + assert isinstance(proxy_domain_client, DomainClient) + assert isinstance(domain_peer, NodePeer) + assert gateway_client.name == domain_peer.name + assert domain_client.name == proxy_domain_client.name + + assert not isinstance(proxy_domain_client.datasets.get_all(), SyftError) From 8e88b48592757e2c230f488e5ad10e64f1363a13 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 25 Jun 2024 22:41:33 +0530 Subject: [PATCH 084/100] modulizer rathole config builder to a seperate class - have a more generalize class for reverse tunnel service - define methods to remove config from rathole client and server toml and configmaps --- .../service/network/association_request.py | 3 +- .../syft/service/network/network_service.py | 49 +++----- ...e_service.py => rathole_config_builder.py} | 105 ++++++++++++++++-- .../src/syft/service/network/rathole_toml.py | 13 +++ .../service/network/reverse_tunnel_service.py | 40 +++++++ 5 files changed, 165 insertions(+), 45 deletions(-) rename packages/syft/src/syft/service/network/{rathole_service.py => rathole_config_builder.py} (66%) create mode 100644 packages/syft/src/syft/service/network/reverse_tunnel_service.py diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index 043bdbc101e..262692e283c 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -66,7 +66,8 @@ def _run( and self.remote_peer.latest_added_route == rathole_route ) - # If the remote peer is added via rathole, we don't need to ping the peer + # If the remote peer is added via rathole, we skip ping to peer + # and add the peer to the rathole server if add_rathole_route: network_service.rathole_service.add_host_to_server(self.remote_peer) else: diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index bbbdc5a002c..9419e479348 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -56,7 +56,7 @@ from .association_request import AssociationRequestChange from .node_peer import NodePeer from .node_peer import NodePeerUpdate -from .rathole_service import RatholeService +from .reverse_tunnel_service import ReverseTunnelService from .routes import HTTPNodeRoute from .routes import NodeRoute from .routes import NodeRouteType @@ -69,7 +69,7 @@ REVERSE_TUNNEL_RATHOLE_ENABLED = "REVERSE_TUNNEL_RATHOLE_ENABLED" -def get_rathole_enabled() -> bool: +def reverse_tunnel_enabled() -> bool: return str_to_bool(get_env(REVERSE_TUNNEL_RATHOLE_ENABLED, "false")) @@ -164,8 +164,8 @@ class NetworkService(AbstractService): def __init__(self, store: DocumentStore) -> None: self.store = store self.stash = NetworkStash(store=store) - if get_rathole_enabled(): - self.rathole_service = RatholeService() + if reverse_tunnel_enabled(): + self.rtunnel_service = ReverseTunnelService() @service_method( path="network.exchange_credentials_with", @@ -188,7 +188,9 @@ def exchange_credentials_with( # Step 1: Validate the Route self_node_peer = self_node_route.validate_with_context(context=context) - if reverse_tunnel: + if reverse_tunnel and not reverse_tunnel_enabled(): + return SyftError(message="Reverse tunneling is not enabled on this node.") + elif reverse_tunnel: _rathole_route = self_node_peer.node_routes[-1] _rathole_route.rathole_token = generate_token() _rathole_route.host_or_ip = f"{self_node_peer.name}.syft.local" @@ -257,9 +259,10 @@ def exchange_credentials_with( return SyftError(message="Failed to update route information.") # Step 5: Save rathole config to enable reverse tunneling - if reverse_tunnel and get_rathole_enabled(): - self._add_reverse_tunneling_config_for_peer( - self_node_peer=self_node_peer, remote_node_route=remote_node_route + if reverse_tunnel and reverse_tunnel_enabled(): + self.rtunnel_service.set_client_config( + self_node_peer=self_node_peer, + remote_node_route=remote_node_route, ) return ( @@ -268,32 +271,6 @@ def exchange_credentials_with( else remote_res ) - def _add_reverse_tunneling_config_for_peer( - self, - self_node_peer: NodePeer, - remote_node_route: NodeRoute, - ) -> None: - rathole_route = self_node_peer.get_rathole_route() - if not rathole_route: - raise Exception( - "Failed to exchange routes via . " - + f"Peer: {self_node_peer} has no rathole route: {rathole_route}" - ) - - remote_url = GridURL( - host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port - ) - rathole_remote_addr = remote_url.as_container_host() - - remote_addr = rathole_remote_addr.url_no_protocol - - self.rathole_service.add_host_to_client( - peer_name=self_node_peer.name, - peer_id=str(self_node_peer.id), - rathole_token=rathole_route.rathole_token, - remote_addr=remote_addr, - ) - @service_method(path="network.add_peer", name="add_peer", roles=GUEST_ROLE_LEVEL) def add_peer( self, @@ -509,12 +486,12 @@ def update_peer( node_side_type = cast(NodeType, context.node.node_type) if node_side_type.value == NodeType.GATEWAY.value: rathole_route = peer.get_rathole_route() - self.rathole_service.add_host_to_server(peer) if rathole_route else None + self.rtunnel_service.set_server_config(peer) if rathole_route else None else: self_node_peer: NodePeer = context.node.settings.to(NodePeer) rathole_route = self_node_peer.get_rathole_route() ( - self._add_reverse_tunneling_config_for_peer( + self.rtunnel_service.set_client_config( self_node_peer=self_node_peer, remote_node_route=peer.pick_highest_priority_route(), ) diff --git a/packages/syft/src/syft/service/network/rathole_service.py b/packages/syft/src/syft/service/network/rathole_config_builder.py similarity index 66% rename from packages/syft/src/syft/service/network/rathole_service.py rename to packages/syft/src/syft/service/network/rathole_config_builder.py index afdf48503d7..a847b6fcbde 100644 --- a/packages/syft/src/syft/service/network/rathole_service.py +++ b/packages/syft/src/syft/service/network/rathole_config_builder.py @@ -21,7 +21,7 @@ PROXY_CONFIG_MAP = "proxy-config" -class RatholeService: +class RatholeConfigBuilder: def __init__(self) -> None: self.k8rs_client = get_kr8s_client() @@ -39,7 +39,7 @@ def add_host_to_server(self, peer: NodePeer) -> None: if not rathole_route: raise Exception(f"Peer: {peer} has no rathole route: {rathole_route}") - random_port = self.get_random_port() + random_port = self._get_random_port() peer_id = cast(UID, peer.id) @@ -78,9 +78,43 @@ def add_host_to_server(self, peer: NodePeer) -> None: KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) # Add the peer info to the proxy config map - self.add_dynamic_addr_to_rathole(config) + self._add_dynamic_addr_to_rathole(config) - def get_random_port(self) -> int: + def remove_host_from_server(self, peer_id: str, server_name: str) -> None: + """Remove a host from the rathole server toml file. + + Args: + peer_id (str): The id of the peer to be removed. + server_name (str): The name of the peer to be removed. + + Returns: + None + """ + + rathole_config_map = KubeUtils.get_configmap( + client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP + ) + + if rathole_config_map is None: + raise Exception("Rathole config map not found.") + + client_filename = RatholeServerToml.filename + + toml_str = rathole_config_map.data[client_filename] + + rathole_toml = RatholeServerToml(toml_str=toml_str) + + rathole_toml.remove_config(peer_id) + + data = {client_filename: rathole_toml.toml_str} + + # Update the rathole config map + KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) + + # Remove the peer info from the proxy config map + self._remove_dynamic_addr_from_rathole(server_name) + + def _get_random_port(self) -> int: """Get a random port number.""" return secrets.randbits(15) @@ -120,7 +154,32 @@ def add_host_to_client( # Update the rathole config map KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) - def add_dynamic_addr_to_rathole( + def remove_host_from_client(self, peer_id: str) -> None: + """Remove a host from the rathole client toml file.""" + + rathole_config_map = KubeUtils.get_configmap( + client=self.k8rs_client, name=RATHOLE_TOML_CONFIG_MAP + ) + + if rathole_config_map is None: + raise Exception("Rathole config map not found.") + + client_filename = RatholeClientToml.filename + + toml_str = rathole_config_map.data[client_filename] + + rathole_toml = RatholeClientToml(toml_str=toml_str) + + rathole_toml.remove_config(peer_id) + + rathole_toml.clear_remote_addr() + + data = {client_filename: rathole_toml.toml_str} + + # Update the rathole config map + KubeUtils.update_configmap(config_map=rathole_config_map, patch={"data": data}) + + def _add_dynamic_addr_to_rathole( self, config: RatholeConfig, entrypoint: str = "web" ) -> None: """Add a port to the rathole proxy config map.""" @@ -166,9 +225,39 @@ def add_dynamic_addr_to_rathole( patch={"data": {"rathole-dynamic.yml": yaml.safe_dump(rathole_proxy)}}, ) - self.expose_port_on_rathole_service(config.server_name, config.local_addr_port) + self._expose_port_on_rathole_service(config.server_name, config.local_addr_port) + + def _remove_dynamic_addr_from_rathole(self, server_name: str) -> None: + """Remove a port from the rathole proxy config map.""" + + rathole_proxy_config_map = KubeUtils.get_configmap( + self.k8rs_client, RATHOLE_PROXY_CONFIG_MAP + ) + + if rathole_proxy_config_map is None: + raise Exception("Rathole proxy config map not found.") + + rathole_proxy = rathole_proxy_config_map.data["rathole-dynamic.yml"] + + if not rathole_proxy: + return + + rathole_proxy = yaml.safe_load(rathole_proxy) + + if server_name in rathole_proxy["http"]["routers"]: + del rathole_proxy["http"]["routers"][server_name] + + if server_name in rathole_proxy["http"]["services"]: + del rathole_proxy["http"]["services"][server_name] + + KubeUtils.update_configmap( + config_map=rathole_proxy_config_map, + patch={"data": {"rathole-dynamic.yml": yaml.safe_dump(rathole_proxy)}}, + ) + + self._remove_port_on_rathole_service(server_name) - def expose_port_on_rathole_service(self, port_name: str, port: int) -> None: + def _expose_port_on_rathole_service(self, port_name: str, port: int) -> None: """Expose a port on the rathole service.""" rathole_service = KubeUtils.get_service(self.k8rs_client, "rathole") @@ -199,7 +288,7 @@ def expose_port_on_rathole_service(self, port_name: str, port: int) -> None: rathole_service.patch(config) - def remove_port_on_rathole_service(self, port_name: str) -> None: + def _remove_port_on_rathole_service(self, port_name: str) -> None: """Remove a port from the rathole service.""" rathole_service = KubeUtils.get_service(self.k8rs_client, "rathole") diff --git a/packages/syft/src/syft/service/network/rathole_toml.py b/packages/syft/src/syft/service/network/rathole_toml.py index e5fe17b59e9..8ded821279e 100644 --- a/packages/syft/src/syft/service/network/rathole_toml.py +++ b/packages/syft/src/syft/service/network/rathole_toml.py @@ -53,6 +53,19 @@ def set_remote_addr(self, remote_host: str) -> None: self.save(toml) + def clear_remote_addr(self) -> None: + """Clear the remote address from the client toml file.""" + + toml = self.read() + + # Clear the remote address + if "client" not in toml: + return + + toml["client"]["remote_addr"] = "" + + self.save(toml) + def add_config(self, config: RatholeConfig) -> None: """Add a new config to the toml file.""" diff --git a/packages/syft/src/syft/service/network/reverse_tunnel_service.py b/packages/syft/src/syft/service/network/reverse_tunnel_service.py new file mode 100644 index 00000000000..99783649a44 --- /dev/null +++ b/packages/syft/src/syft/service/network/reverse_tunnel_service.py @@ -0,0 +1,40 @@ +# relative +from ...types.grid_url import GridURL +from .node_peer import NodePeer +from .rathole_config_builder import RatholeConfigBuilder +from .routes import NodeRoute + + +class ReverseTunnelService: + def __init__(self) -> None: + self.builder = RatholeConfigBuilder() + + def set_client_config( + self, + self_node_peer: NodePeer, + remote_node_route: NodeRoute, + ) -> None: + rathole_route = self_node_peer.get_rathole_route() + if not rathole_route: + raise Exception( + "Failed to exchange routes via . " + + f"Peer: {self_node_peer} has no rathole route: {rathole_route}" + ) + + remote_url = GridURL( + host_or_ip=remote_node_route.host_or_ip, port=remote_node_route.port + ) + rathole_remote_addr = remote_url.as_container_host() + + remote_addr = rathole_remote_addr.url_no_protocol + + self.builder.add_host_to_client( + peer_name=self_node_peer.name, + peer_id=str(self_node_peer.id), + rathole_token=rathole_route.rathole_token, + remote_addr=remote_addr, + ) + + def set_server_config(self, remote_peer: NodePeer) -> None: + rathole_route = remote_peer.get_rathole_route() + self.builder.add_host_to_server(remote_peer) if rathole_route else None From 2be7e689d45da98f3169274d100029862d961720 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 26 Jun 2024 00:34:26 +0530 Subject: [PATCH 085/100] integrate configmap deletion --- .../service/network/association_request.py | 5 ++--- .../syft/service/network/network_service.py | 18 ++++++++++++++++++ .../service/network/reverse_tunnel_service.py | 8 ++++++++ tests/integration/network/gateway_test.py | 4 ++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index 262692e283c..bdbe8dc6cbc 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -66,10 +66,9 @@ def _run( and self.remote_peer.latest_added_route == rathole_route ) - # If the remote peer is added via rathole, we skip ping to peer - # and add the peer to the rathole server + # If the remote peer is added via reverse tunnel, we skip ping to peer if add_rathole_route: - network_service.rathole_service.add_host_to_server(self.remote_peer) + network_service.rtunnel_service.set_server_config(self.remote_peer) else: # Pinging the remote peer to verify the connection try: diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 9419e479348..b442f3b83b4 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -512,6 +512,24 @@ def delete_peer_by_id( self, context: AuthedServiceContext, uid: UID ) -> SyftSuccess | SyftError: """Delete Node Peer""" + retrieve_result = self.stash.get_by_uid(context.credentials, uid) + if err := retrieve_result.is_err(): + return SyftError( + message=f"Failed to retrieve peer with UID {uid}: {retrieve_result.err()}." + ) + peer_to_delete = cast(NodePeer, retrieve_result.ok()) + + node_side_type = cast(NodeType, context.node.node_type) + if node_side_type.value == NodeType.GATEWAY.value: + rathole_route = peer_to_delete.get_rathole_route() + ( + self.rtunnel_service.clear_server_config(peer_to_delete) + if rathole_route + else None + ) + + # TODO: Handle the case when peer is deleted from domain node + result = self.stash.delete_by_uid(context.credentials, uid) if err := result.is_err(): return SyftError(message=f"Failed to delete peer with UID {uid}: {err}.") diff --git a/packages/syft/src/syft/service/network/reverse_tunnel_service.py b/packages/syft/src/syft/service/network/reverse_tunnel_service.py index 99783649a44..bb80c56f401 100644 --- a/packages/syft/src/syft/service/network/reverse_tunnel_service.py +++ b/packages/syft/src/syft/service/network/reverse_tunnel_service.py @@ -38,3 +38,11 @@ def set_client_config( def set_server_config(self, remote_peer: NodePeer) -> None: rathole_route = remote_peer.get_rathole_route() self.builder.add_host_to_server(remote_peer) if rathole_route else None + + def clear_client_config(self, self_node_peer: NodePeer) -> None: + self.builder.remove_host_from_client(str(self_node_peer.id)) + + def clear_server_config(self, remote_peer: NodePeer) -> None: + self.builder.remove_host_from_server( + str(remote_peer.id), server_name=remote_peer.name + ) diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index 1e09c2a775e..e8f36be2ff2 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -965,3 +965,7 @@ def test_reverse_tunnel_connection(domain_1_port: int, gateway_port: int): assert domain_client.name == proxy_domain_client.name assert not isinstance(proxy_domain_client.datasets.get_all(), SyftError) + + # Try removing existing peers just to make sure + _remove_existing_peers(gateway_client) + _remove_existing_peers(domain_client) From d4074b00be6611b1f36da4a7afbfc3280368e8e8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 26 Jun 2024 12:20:32 +0530 Subject: [PATCH 086/100] fix protocol version for HttpConnection --- .../src/syft/protocol/protocol_version.json | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 9ff9826a945..14a8214bc61 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -274,9 +274,14 @@ } }, "HTTPConnection": { + "2": { + "version": 2, + "hash": "68409295f8916ceb22a8cf4abf89f5e4bcff0d75dc37e16ede37250ada28df59", + "action": "remove" + }, "3": { "version": 3, - "hash": "54b452bb4ab76691ac1e704b62e7bcec740850fea00805145259b37973ecd0f4", + "hash": "cac31ba98bdcc42c0717555a0918d0c8aef0d2235f892a2d86dceff09930fb88", "action": "add" } }, @@ -354,6 +359,54 @@ "hash": "ba9ebb04cc3e8b3ae3302fd42a67e47261a0a330bae5f189d8f4819cf2804711", "action": "add" } + }, + "PythonConnection": { + "2": { + "version": 2, + "hash": "eb479c671fc112b2acbedb88bc5624dfdc9592856c04c22c66410f6c863e1708", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "1084c85a59c0436592530b5fe9afc2394088c8d16faef2b19fdb9fb83ff0f0e2", + "action": "add" + } + }, + "HTTPNodeRoute": { + "2": { + "version": 2, + "hash": "2134ea812f7c6ea41522727ae087245c4b1195ffbad554db638070861cd9eb1c", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "9e7e3700a2f7b1a67f054efbcb31edc71bbf358c469c85ed7760b81233803bac", + "action": "add" + } + }, + "PythonNodeRoute": { + "2": { + "version": 2, + "hash": "3eca5767ae4a8fbe67744509e58c6d9fb78f38fa0a0f7fcf5960ab4250acc1f0", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "1bc413ec7c1d498ec945878e21e00affd9bd6d53b564b1e10e52feb09f177d04", + "action": "add" + } + }, + "SeaweedFSBlobDeposit": { + "3": { + "version": 3, + "hash": "05e61e6328b085b738e5d41c0781d87852d44d218894cb3008f5be46e337f6d8", + "action": "remove" + }, + "4": { + "version": 4, + "hash": "f475543ed5e0066ca09c0dfd8c903e276d4974519e9958473d8141f8d446c881", + "action": "add" + } } } } From f1745379f1b1e463fe722c9e86eb78a9313c351e Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 26 Jun 2024 15:57:24 +0530 Subject: [PATCH 087/100] fix url path in _make_get --- packages/syft/src/syft/client/client.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index d770ff73280..7d7f5be95d1 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -207,6 +207,9 @@ def session(self) -> Session: def _make_get( self, path: str, params: dict | None = None, stream: bool = False ) -> bytes | Iterable: + if params is None: + return self._make_get_no_params(path, stream=stream) + url = self.url if self.rathole_token: @@ -216,9 +219,6 @@ def _make_get( url = url.with_path(path) - if params is None: - return self._make_get_no_params(path) - url = self.url.with_path(path) response = self.session.get( str(url), headers=self.headers, @@ -239,8 +239,15 @@ def _make_get( @cached(cache=TTLCache(maxsize=128, ttl=300)) def _make_get_no_params(self, path: str, stream: bool = False) -> bytes | Iterable: - print(path) - url = self.url.with_path(path) + url = self.url + + if self.rathole_token: + self.headers = {} if self.headers is None else self.headers + url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) + self.headers["Host"] = self.url.host_or_ip + + url = url.with_path(path) + response = self.session.get( str(url), headers=self.headers, From 2f5554cbf07d5acf498ccdeb83b269433b36c08b Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 26 Jun 2024 16:08:58 +0530 Subject: [PATCH 088/100] deprecate HttpConnectionV2 --- packages/syft/src/syft/client/client.py | 12 ----- .../src/syft/protocol/protocol_version.json | 48 +++++++++++++++++++ 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 7d7f5be95d1..9540a50fb7a 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -51,7 +51,6 @@ from ..service.user.user_roles import ServiceRole from ..service.user.user_service import UserService from ..types.grid_url import GridURL -from ..types.syft_object import SYFT_OBJECT_VERSION_2 from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.uid import UID from ..util.telemetry import instrument @@ -131,17 +130,6 @@ class Routes(Enum): STREAM = f"{API_PATH}/stream" -@serializable(attrs=["proxy_target_uid", "url"]) -class HTTPConnectionV2(NodeConnection): - __canonical_name__ = "HTTPConnection" - __version__ = SYFT_OBJECT_VERSION_2 - - url: GridURL - proxy_target_uid: UID | None = None - routes: type[Routes] = Routes - session_cache: Session | None = None - - @serializable(attrs=["proxy_target_uid", "url", "rathole_token"]) class HTTPConnection(NodeConnection): __canonical_name__ = "HTTPConnection" diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 766da3a3326..c08ea39b7b2 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -378,6 +378,54 @@ "hash": "1b9bd1d3d096abab5617c2ff597b4c80751f686d16482a2cff4efd8741b84d53", "action": "add" } + }, + "PythonConnection": { + "2": { + "version": 2, + "hash": "eb479c671fc112b2acbedb88bc5624dfdc9592856c04c22c66410f6c863e1708", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "1084c85a59c0436592530b5fe9afc2394088c8d16faef2b19fdb9fb83ff0f0e2", + "action": "add" + } + }, + "HTTPNodeRoute": { + "2": { + "version": 2, + "hash": "2134ea812f7c6ea41522727ae087245c4b1195ffbad554db638070861cd9eb1c", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "9e7e3700a2f7b1a67f054efbcb31edc71bbf358c469c85ed7760b81233803bac", + "action": "add" + } + }, + "PythonNodeRoute": { + "2": { + "version": 2, + "hash": "3eca5767ae4a8fbe67744509e58c6d9fb78f38fa0a0f7fcf5960ab4250acc1f0", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "1bc413ec7c1d498ec945878e21e00affd9bd6d53b564b1e10e52feb09f177d04", + "action": "add" + } + }, + "SeaweedFSBlobDeposit": { + "3": { + "version": 3, + "hash": "05e61e6328b085b738e5d41c0781d87852d44d218894cb3008f5be46e337f6d8", + "action": "remove" + }, + "4": { + "version": 4, + "hash": "f475543ed5e0066ca09c0dfd8c903e276d4974519e9958473d8141f8d446c881", + "action": "add" + } } } } From 5d5ac71403440c8465328cf5547618c7f1eacd3a Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 26 Jun 2024 16:21:57 +0530 Subject: [PATCH 089/100] ignore security on hardcoded binding for rathole config --- .../syft/src/syft/service/network/rathole_config_builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/network/rathole_config_builder.py b/packages/syft/src/syft/service/network/rathole_config_builder.py index a847b6fcbde..90f499ec237 100644 --- a/packages/syft/src/syft/service/network/rathole_config_builder.py +++ b/packages/syft/src/syft/service/network/rathole_config_builder.py @@ -19,6 +19,7 @@ RATHOLE_TOML_CONFIG_MAP = "rathole-config" RATHOLE_PROXY_CONFIG_MAP = "proxy-config-dynamic" PROXY_CONFIG_MAP = "proxy-config" +DEFAULT_LOCAL_ADDR_HOST = "0.0.0.0" # nosec class RatholeConfigBuilder: @@ -46,7 +47,7 @@ def add_host_to_server(self, peer: NodePeer) -> None: config = RatholeConfig( uuid=peer_id.to_string(), secret_token=rathole_route.rathole_token, - local_addr_host="0.0.0.0", + local_addr_host=DEFAULT_LOCAL_ADDR_HOST, local_addr_port=random_port, server_name=peer.name, ) From fbcfe599b288c4149950124d87b2a5579779c907 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 27 Jun 2024 14:09:30 +0530 Subject: [PATCH 090/100] mark reverse tunnel with network marker --- tests/integration/network/gateway_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index e8f36be2ff2..07a5e360f31 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -112,6 +112,10 @@ def test_domain_connect_to_gateway( _remove_existing_peers(domain_client) _remove_existing_peers(gateway_client) + # Disable automatic acceptance of association requests + res = gateway_client.settings.allow_association_request_auto_approval(enable=False) + assert isinstance(res, SyftSuccess) + # connecting the domain to the gateway result = domain_client.connect_to_gateway(gateway_client) assert isinstance(result, Request) @@ -914,6 +918,7 @@ def test_peer_health_check(set_env_var, gateway_port: int, domain_1_port: int) - assert isinstance(_remove_existing_peers(gateway_client), SyftSuccess) +@pytest.mark.network def test_reverse_tunnel_connection(domain_1_port: int, gateway_port: int): # login to the domain and gateway From c9be64ecd921f204d91c921823b629dc23cc56c6 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 27 Jun 2024 14:58:14 +0530 Subject: [PATCH 091/100] remove use of route id for route deletion --- .../syft/service/network/network_service.py | 40 +++++-------------- .../src/syft/service/network/node_peer.py | 15 +------ tests/integration/network/gateway_test.py | 2 +- 3 files changed, 12 insertions(+), 45 deletions(-) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 7deb8775686..4a39a1edf98 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -625,6 +625,8 @@ def add_route( message=f"The route already exists between '{context.node.name}' and " f"peer '{remote_node_peer.name}'." ) + + remote_node_peer.update_route(route=route) # update the peer in the store with the updated routes peer_update = NodePeerUpdate( id=remote_node_peer.id, node_routes=remote_node_peer.node_routes @@ -646,8 +648,7 @@ def delete_route_on_peer( self, context: AuthedServiceContext, peer: NodePeer, - route: NodeRoute | None = None, - route_id: UID | None = None, + route: NodeRoute, ) -> SyftSuccess | SyftError | SyftInfo: """ Delete the route on the remote peer. @@ -656,7 +657,6 @@ def delete_route_on_peer( context (AuthedServiceContext): The authentication context for the service. peer (NodePeer): The peer for which the route will be deleted. route (NodeRoute): The route to be deleted. - route_id (UID): The UID of the route to be deleted. Returns: SyftSuccess: If the route is successfully deleted. @@ -664,17 +664,6 @@ def delete_route_on_peer( SyftInfo: If there is only one route left for the peer and the admin chose not to remove it """ - if route is None and route_id is None: - return SyftError( - message="Either `route` or `route_id` arg must be provided" - ) - - if route and route_id and route.id != route_id: - return SyftError( - message=f"Both `route` and `route_id` are provided, but " - f"route's id ({route.id}) and route_id ({route_id}) do not match" - ) - # creates a client on the remote node based on the credentials # of the current node's client remote_client = peer.client_with_context(context=context) @@ -688,7 +677,6 @@ def delete_route_on_peer( result = remote_client.api.services.network.delete_route( peer_verify_key=context.credentials, route=route, - route_id=route_id, called_by_peer=True, ) return result @@ -701,7 +689,6 @@ def delete_route( context: AuthedServiceContext, peer_verify_key: SyftVerifyKey, route: NodeRoute | None = None, - route_id: UID | None = None, called_by_peer: bool = False, ) -> SyftSuccess | SyftError | SyftInfo: """ @@ -713,7 +700,6 @@ def delete_route( context (AuthedServiceContext): The authentication context for the service. peer_verify_key (SyftVerifyKey): The verify key of the remote node peer. route (NodeRoute): The route to be deleted. - route_id (UID): The UID of the route to be deleted. called_by_peer (bool): The flag to indicate that it's called by a remote peer. Returns: @@ -755,20 +741,12 @@ def delete_route( f"'{remote_node_peer.node_routes[0].id}' was not deleted." ) - if route: - result = remote_node_peer.delete_route(route=route) - return_message = ( - f"Route '{str(route)}' with id '{route.id}' to peer " - f"{remote_node_peer.node_type.value} '{remote_node_peer.name}' " - f"was deleted for {str(context.node.node_type)} '{context.node.name}'." - ) - if route_id: - result = remote_node_peer.delete_route(route_id=route_id) - return_message = ( - f"Route with id '{route_id}' to peer " - f"{remote_node_peer.node_type.value} '{remote_node_peer.name}' " - f"was deleted for {str(context.node.node_type)} '{context.node.name}'." - ) + result = remote_node_peer.delete_route(route=route) + return_message = ( + f"Route '{str(route)}' to peer " + f"{remote_node_peer.node_type.value} '{remote_node_peer.name}' " + f"was deleted for {str(context.node.node_type)} '{context.node.name}'." + ) if isinstance(result, SyftError): return result diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 357a8d17967..e3f14481f36 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -291,28 +291,17 @@ def get_rathole_route(self) -> HTTPNodeRoute | None: return route return None - def delete_route( - self, route: NodeRouteType | None = None, route_id: UID | None = None - ) -> SyftError | None: + def delete_route(self, route: NodeRouteType) -> SyftError | None: """ Deletes a route from the peer's route list. Takes O(n) where is n is the number of routes in self.node_routes. Args: route (NodeRouteType): The route to be deleted; - route_id (UID): The id of the route to be deleted; Returns: - SyftError: If deleting failed + SyftError: If failing to delete node route """ - if route_id: - try: - self.node_routes = [r for r in self.node_routes if r.id != route_id] - except Exception as e: - return SyftError( - message=f"Error deleting route with id {route_id}. Exception: {e}" - ) - if route: try: self.node_routes = [r for r in self.node_routes if r != route] diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index 07a5e360f31..ef8b090a8a6 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -618,7 +618,7 @@ def test_delete_route_on_peer( # gateway delete the routes for the domain res = gateway_client.api.services.network.delete_route_on_peer( - peer=domain_peer, route_id=new_route.id + peer=domain_peer, route=new_route ) assert isinstance(res, SyftSuccess) gateway_peer = domain_client.peers[0] From 7609728a7080a400c7d57b2373e10b139477d7fa Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 27 Jun 2024 16:19:09 +0530 Subject: [PATCH 092/100] refactor reverse tunnel config logic to a single method --- .../service/network/association_request.py | 5 ++- .../syft/service/network/network_service.py | 40 +++++++++++++------ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index bdbe8dc6cbc..a910302005e 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -68,7 +68,10 @@ def _run( # If the remote peer is added via reverse tunnel, we skip ping to peer if add_rathole_route: - network_service.rtunnel_service.set_server_config(self.remote_peer) + network_service.set_reverse_tunnel_config( + context=context, + remote_node_peer=self.remote_peer, + ) else: # Pinging the remote peer to verify the connection try: diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 4a39a1edf98..4aa48481df2 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -261,9 +261,10 @@ def exchange_credentials_with( # Step 5: Save rathole config to enable reverse tunneling if reverse_tunnel and reverse_tunnel_enabled(): - self.rtunnel_service.set_client_config( + self.set_reverse_tunnel_config( + context=context, self_node_peer=self_node_peer, - remote_node_route=remote_node_route, + remote_node_peer=remote_node_peer, ) return ( @@ -484,26 +485,41 @@ def update_peer( peer = result.ok() - node_side_type = cast(NodeType, context.node.node_type) - if node_side_type.value == NodeType.GATEWAY.value: - rathole_route = peer.get_rathole_route() - self.rtunnel_service.set_server_config(peer) if rathole_route else None + self.set_reverse_tunnel_config(context=context, remote_node_peer=peer) + return SyftSuccess( + message=f"Peer '{result.ok().name}' information successfully updated." + ) + + def set_reverse_tunnel_config( + self, + context: AuthedServiceContext, + remote_node_peer: NodePeer, + self_node_peer: NodePeer | None = None, + ) -> None: + node_type = cast(NodeType, context.node.node_type) + if node_type.value == NodeType.GATEWAY.value: + rathole_route = remote_node_peer.get_rathole_route() + ( + self.rtunnel_service.set_server_config(remote_node_peer) + if rathole_route + else None + ) else: - self_node_peer: NodePeer = context.node.settings.to(NodePeer) + self_node_peer = ( + context.node.settings.to(NodePeer) + if self_node_peer is None + else self_node_peer + ) rathole_route = self_node_peer.get_rathole_route() ( self.rtunnel_service.set_client_config( self_node_peer=self_node_peer, - remote_node_route=peer.pick_highest_priority_route(), + remote_node_route=remote_node_peer.pick_highest_priority_route(), ) if rathole_route else None ) - return SyftSuccess( - message=f"Peer '{result.ok().name}' information successfully updated." - ) - @service_method( path="network.delete_peer_by_id", name="delete_peer_by_id", From e6f31f0e3a9d35391e8a7dff6c583e4826101b17 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 28 Jun 2024 11:25:23 +0530 Subject: [PATCH 093/100] rename rathole to reverse tunnel in syft application - rename rathole path and token to rtunnel --- packages/grid/backend/grid/core/config.py | 4 +-- packages/grid/devspace.yaml | 12 ++++---- .../backend/backend-statefulset.yaml | 6 ++-- .../syft/templates/proxy/proxy-configmap.yaml | 8 ++--- .../templates/rathole/rathole-configmap.yaml | 4 +-- .../rathole/rathole-statefulset.yaml | 30 ++++++++----------- packages/grid/helm/syft/values.yaml | 6 ++-- packages/grid/helm/values.dev.yaml | 5 ++-- packages/syft/src/syft/client/client.py | 18 +++++------ .../service/network/association_request.py | 12 ++++---- .../syft/service/network/network_service.py | 30 +++++++++---------- .../src/syft/service/network/node_peer.py | 4 +-- .../syft/src/syft/service/network/rathole.py | 2 +- .../service/network/rathole_config_builder.py | 8 ++--- .../service/network/reverse_tunnel_service.py | 6 ++-- .../syft/src/syft/service/network/routes.py | 4 +-- packages/syft/src/syft/util/util.py | 2 +- 17 files changed, 78 insertions(+), 83 deletions(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 33d65719fe8..b6f5ddf9067 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -155,8 +155,8 @@ def get_emails_enabled(self) -> Self: ASSOCIATION_REQUEST_AUTO_APPROVAL: bool = str_to_bool( os.getenv("ASSOCIATION_REQUEST_AUTO_APPROVAL", "False") ) - REVERSE_TUNNEL_RATHOLE_ENABLED: bool = str_to_bool( - os.getenv("REVERSE_TUNNEL_RATHOLE_ENABLED", "false") + REVERSE_TUNNEL_ENABLED: bool = str_to_bool( + os.getenv("REVERSE_TUNNEL_ENABLED", "false") ) model_config = SettingsConfigDict(case_sensitive=True) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index d1c6facde50..7e37e705214 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -86,7 +86,7 @@ deployments: version: dev-${DEVSPACE_TIMESTAMP} node: type: domain # required for the gateway profile - rathole: + reverse_tunnel: mode: client dev: @@ -125,12 +125,12 @@ dev: - path: ../syft:/root/app/syft ssh: localPort: 3480 - rathole: + reverse_tunnel: labelSelector: app.kubernetes.io/name: syft - app.kubernetes.io/component: rathole + app.kubernetes.io/component: reverse_tunnel ports: - - port: "2333" # rathole + - port: "2333" # reverse_tunnel profiles: - name: dev-low @@ -158,7 +158,7 @@ profiles: # Patch mode to server - op: replace - path: deployments.syft.helm.values.rathole.mode + path: deployments.syft.helm.values.reverse_tunnel.mode value: server # Port Re-Mapping @@ -179,7 +179,7 @@ profiles: # Mongo - op: replace - path: dev.rathole.ports[0].port + path: dev.reverse_tunnel.ports[0].port value: 2334:2333 - name: gcp diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index e08db908fea..e5dd285985f 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -91,10 +91,10 @@ spec: - name: ASSOCIATION_REQUEST_AUTO_APPROVAL value: {{ .Values.node.associationRequestAutoApproval | quote }} {{- end }} - {{- if .Values.rathole.enabled }} + {{- if .Values.reverse_tunnel.enabled }} - name: RATHOLE_PORT - value: {{ .Values.rathole.port | quote }} - - name: REVERSE_TUNNEL_RATHOLE_ENABLED + value: {{ .Values.reverse_tunnel.port | quote }} + - name: REVERSE_TUNNEL_ENABLED value: "true" {{- end }} # MongoDB diff --git a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml index 705c153442a..1bcdff49876 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-configmap.yaml @@ -28,22 +28,22 @@ data: - url: "http://rathole:2333" routers: rathole: - rule: "PathPrefix(`/`) && Headers(`Upgrade`, `websocket`) && !PathPrefix(`/rathole`)" + rule: "PathPrefix(`/`) && Headers(`Upgrade`, `websocket`) && !PathPrefix(`/rtunnel`)" entryPoints: - "web" service: "rathole" frontend: - rule: "PathPrefix(`/`) && !PathPrefix(`/rathole`)" + rule: "PathPrefix(`/`) && !PathPrefix(`/rtunnel`)" entryPoints: - "web" service: "frontend" backend: - rule: "(PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)) && !PathPrefix(`/rathole`)" + rule: "(PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`)) && !PathPrefix(`/rtunnel`)" entryPoints: - "web" service: "backend" blob-storage: - rule: "PathPrefix(`/blob`) && !PathPrefix(`/rathole`)" + rule: "PathPrefix(`/blob`) && !PathPrefix(`/rtunnel`)" entryPoints: - "web" service: "seaweedfs" diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index cd5453f1ea2..3bebf40cffc 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -7,7 +7,7 @@ metadata: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: rathole data: - {{- if eq .Values.rathole.mode "server" }} + {{- if eq .Values.reverse_tunnel.mode "server" }} server.toml: | [server] bind_addr = "0.0.0.0:2333" @@ -22,7 +22,7 @@ data: bind_addr = "0.0.0.0:8001" {{- end }} - {{- if eq .Values.rathole.mode "client" }} + {{- if eq .Values.reverse_tunnel.mode "client" }} client.toml: | [client] remote_addr = "0.0.0.0:2333" diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index c441991ef47..dfa068218d6 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -20,36 +20,30 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: rathole - {{- if .Values.rathole.podLabels }} - {{- toYaml .Values.rathole.podLabels | nindent 8 }} + {{- if .Values.reverse_tunnel.podLabels }} + {{- toYaml .Values.reverse_tunnel.podLabels | nindent 8 }} {{- end }} - {{- if .Values.rathole.podAnnotations }} - annotations: {{- toYaml .Values.rathole.podAnnotations | nindent 8 }} + {{- if .Values.reverse_tunnel.podAnnotations }} + annotations: {{- toYaml .Values.reverse_tunnel.podAnnotations | nindent 8 }} {{- end }} spec: - {{- if .Values.rathole.nodeSelector }} - nodeSelector: {{- .Values.rathole.nodeSelector | toYaml | nindent 8 }} + {{- if .Values.reverse_tunnel.nodeSelector }} + nodeSelector: {{- .Values.reverse_tunnel.nodeSelector | toYaml | nindent 8 }} {{- end }} containers: - name: rathole image: {{ .Values.global.registry }}/openmined/grid-rathole:{{ .Values.global.version }} imagePullPolicy: Always - resources: {{ include "common.resources.set" (dict "resources" .Values.rathole.resources "preset" .Values.rathole.resourcesPreset) | nindent 12 }} + resources: {{ include "common.resources.set" (dict "resources" .Values.reverse_tunnel.resources "preset" .Values.reverse_tunnel.resourcesPreset) | nindent 12 }} env: - - name: SERVICE_NAME - value: "rathole" - name: LOG_LEVEL - value: {{ .Values.rathole.logLevel | quote }} + value: {{ .Values.reverse_tunnel.logLevel | quote }} - name: MODE - value: {{ .Values.rathole.mode | quote }} - - name: DEV_MODE - value: {{ .Values.rathole.devMode | quote }} - - name: APP_PORT - value: {{ .Values.rathole.appPort | quote }} + value: {{ .Values.reverse_tunnel.mode | quote }} - name: RATHOLE_PORT - value: {{ .Values.rathole.ratholePort | quote }} - {{- if .Values.rathole.env }} - {{- toYaml .Values.rathole.env | nindent 12 }} + value: {{ .Values.reverse_tunnel.port | quote }} + {{- if .Values.reverse_tunnel.env }} + {{- toYaml .Values.reverse_tunnel.env | nindent 12 }} {{- end }} ports: - name: rathole-port diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index e6c5cb51be2..8e225b94599 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -238,11 +238,11 @@ ingress: # ================================================================================= -rathole: +reverse_tunnel: # Extra environment vars env: null - enabled: true - logLevel: info + enabled: false + logLevel: "info" port: 2333 mode: client diff --git a/packages/grid/helm/values.dev.yaml b/packages/grid/helm/values.dev.yaml index 713948f4a9b..e9c32543c0b 100644 --- a/packages/grid/helm/values.dev.yaml +++ b/packages/grid/helm/values.dev.yaml @@ -47,9 +47,10 @@ proxy: resourcesPreset: null resources: null -rathole: - enabled: "true" +reverse_tunnel: + enabled: true logLevel: "trace" + # attestation: # enabled: true # resourcesPreset: null diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 9540a50fb7a..f153fe88f41 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -117,7 +117,7 @@ def forward_message_to_proxy( API_PATH = "/api/v2" DEFAULT_PYGRID_PORT = 80 DEFAULT_PYGRID_ADDRESS = f"http://localhost:{DEFAULT_PYGRID_PORT}" -INTERNAL_PROXY_TO_RATHOLE = "http://proxy:80/rathole/" +INTERNAL_PROXY_TO_RATHOLE = "http://proxy:80/rtunnel/" class Routes(Enum): @@ -130,7 +130,7 @@ class Routes(Enum): STREAM = f"{API_PATH}/stream" -@serializable(attrs=["proxy_target_uid", "url", "rathole_token"]) +@serializable(attrs=["proxy_target_uid", "url", "rtunnel_token"]) class HTTPConnection(NodeConnection): __canonical_name__ = "HTTPConnection" __version__ = SYFT_OBJECT_VERSION_3 @@ -140,7 +140,7 @@ class HTTPConnection(NodeConnection): routes: type[Routes] = Routes session_cache: Session | None = None headers: dict[str, str] | None = None - rathole_token: str | None = None + rtunnel_token: str | None = None @field_validator("url", mode="before") @classmethod @@ -158,7 +158,7 @@ def with_proxy(self, proxy_target_uid: UID) -> Self: return HTTPConnection( url=self.url, proxy_target_uid=proxy_target_uid, - rathole_token=self.rathole_token, + rtunnel_token=self.rtunnel_token, ) def stream_via(self, proxy_uid: UID, url_path: str) -> GridURL: @@ -200,7 +200,7 @@ def _make_get( url = self.url - if self.rathole_token: + if self.rtunnel_token: self.headers = {} if self.headers is None else self.headers url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) self.headers["Host"] = self.url.host_or_ip @@ -229,7 +229,7 @@ def _make_get( def _make_get_no_params(self, path: str, stream: bool = False) -> bytes | Iterable: url = self.url - if self.rathole_token: + if self.rtunnel_token: self.headers = {} if self.headers is None else self.headers url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) self.headers["Host"] = self.url.host_or_ip @@ -261,7 +261,7 @@ def _make_put( ) -> Response: url = self.url - if self.rathole_token: + if self.rtunnel_token: url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) self.headers = {} if self.headers is None else self.headers self.headers["Host"] = self.url.host_or_ip @@ -293,7 +293,7 @@ def _make_post( ) -> bytes: url = self.url - if self.rathole_token: + if self.rtunnel_token: url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) self.headers = {} if self.headers is None else self.headers self.headers["Host"] = self.url.host_or_ip @@ -408,7 +408,7 @@ def register(self, new_user: UserCreate) -> SyftSigningKey: def make_call(self, signed_call: SignedSyftAPICall) -> Any | SyftError: msg_bytes: bytes = _serialize(obj=signed_call, to_bytes=True) - if self.rathole_token: + if self.rtunnel_token: api_url = GridURL.from_url(INTERNAL_PROXY_TO_RATHOLE) api_url = api_url.with_path(self.routes.ROUTE_API_CALL.value) self.headers = {} if self.headers is None else self.headers diff --git a/packages/syft/src/syft/service/network/association_request.py b/packages/syft/src/syft/service/network/association_request.py index a910302005e..a13dd5085d3 100644 --- a/packages/syft/src/syft/service/network/association_request.py +++ b/packages/syft/src/syft/service/network/association_request.py @@ -59,15 +59,15 @@ def _run( ) network_stash = network_service.stash - # Check if remote peer to be added is via rathole - rathole_route = self.remote_peer.get_rathole_route() - add_rathole_route = ( - rathole_route is not None - and self.remote_peer.latest_added_route == rathole_route + # Check if remote peer to be added is via reverse tunnel + rtunnel_route = self.remote_peer.get_rtunnel_route() + add_rtunnel_route = ( + rtunnel_route is not None + and self.remote_peer.latest_added_route == rtunnel_route ) # If the remote peer is added via reverse tunnel, we skip ping to peer - if add_rathole_route: + if add_rtunnel_route: network_service.set_reverse_tunnel_config( context=context, remote_node_peer=self.remote_peer, diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 4aa48481df2..35de9bfaf91 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -67,11 +67,11 @@ NodeTypePartitionKey = PartitionKey(key="node_type", type_=NodeType) OrderByNamePartitionKey = PartitionKey(key="name", type_=str) -REVERSE_TUNNEL_RATHOLE_ENABLED = "REVERSE_TUNNEL_RATHOLE_ENABLED" +REVERSE_TUNNEL_ENABLED = "REVERSE_TUNNEL_ENABLED" def reverse_tunnel_enabled() -> bool: - return str_to_bool(get_env(REVERSE_TUNNEL_RATHOLE_ENABLED, "false")) + return str_to_bool(get_env(REVERSE_TUNNEL_ENABLED, "false")) @serializable() @@ -192,10 +192,10 @@ def exchange_credentials_with( if reverse_tunnel and not reverse_tunnel_enabled(): return SyftError(message="Reverse tunneling is not enabled on this node.") elif reverse_tunnel: - _rathole_route = self_node_peer.node_routes[-1] - _rathole_route.rathole_token = generate_token() - _rathole_route.host_or_ip = f"{self_node_peer.name}.syft.local" - self_node_peer.node_routes[-1] = _rathole_route + _rtunnel_route = self_node_peer.node_routes[-1] + _rtunnel_route.rtunnel_token = generate_token() + _rtunnel_route.host_or_ip = f"{self_node_peer.name}.syft.local" + self_node_peer.node_routes[-1] = _rtunnel_route if isinstance(self_node_peer, SyftError): return self_node_peer @@ -259,7 +259,7 @@ def exchange_credentials_with( ) return SyftError(message="Failed to update route information.") - # Step 5: Save rathole config to enable reverse tunneling + # Step 5: Save config to enable reverse tunneling if reverse_tunnel and reverse_tunnel_enabled(): self.set_reverse_tunnel_config( context=context, @@ -498,10 +498,10 @@ def set_reverse_tunnel_config( ) -> None: node_type = cast(NodeType, context.node.node_type) if node_type.value == NodeType.GATEWAY.value: - rathole_route = remote_node_peer.get_rathole_route() + rtunnel_route = remote_node_peer.get_rtunnel_route() ( self.rtunnel_service.set_server_config(remote_node_peer) - if rathole_route + if rtunnel_route else None ) else: @@ -510,13 +510,13 @@ def set_reverse_tunnel_config( if self_node_peer is None else self_node_peer ) - rathole_route = self_node_peer.get_rathole_route() + rtunnel_route = self_node_peer.get_rtunnel_route() ( self.rtunnel_service.set_client_config( self_node_peer=self_node_peer, remote_node_route=remote_node_peer.pick_highest_priority_route(), ) - if rathole_route + if rtunnel_route else None ) @@ -538,10 +538,10 @@ def delete_peer_by_id( node_side_type = cast(NodeType, context.node.node_type) if node_side_type.value == NodeType.GATEWAY.value: - rathole_route = peer_to_delete.get_rathole_route() + rtunnel_route = peer_to_delete.get_rtunnel_route() ( self.rtunnel_service.clear_server_config(peer_to_delete) - if rathole_route + if rtunnel_route else None ) @@ -955,7 +955,7 @@ def from_grid_url(context: TransformContext) -> TransformContext: context.output["private"] = False context.output["proxy_target_uid"] = context.obj.proxy_target_uid context.output["priority"] = 1 - context.output["rathole_token"] = context.obj.rathole_token + context.output["rtunnel_token"] = context.obj.rtunnel_token return context @@ -995,7 +995,7 @@ def node_route_to_http_connection( return HTTPConnection( url=url, proxy_target_uid=obj.proxy_target_uid, - rathole_token=obj.rathole_token, + rtunnel_token=obj.rtunnel_token, ) diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index e3f14481f36..23c1ffc7057 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -285,9 +285,9 @@ def guest_client(self) -> SyftClient: def proxy_from(self, client: SyftClient) -> SyftClient: return client.proxy_to(self) - def get_rathole_route(self) -> HTTPNodeRoute | None: + def get_rtunnel_route(self) -> HTTPNodeRoute | None: for route in self.node_routes: - if hasattr(route, "rathole_token") and route.rathole_token: + if hasattr(route, "rtunnel_token") and route.rtunnel_token: return route return None diff --git a/packages/syft/src/syft/service/network/rathole.py b/packages/syft/src/syft/service/network/rathole.py index d311bfa9c2b..4fd0be445b1 100644 --- a/packages/syft/src/syft/service/network/rathole.py +++ b/packages/syft/src/syft/service/network/rathole.py @@ -36,7 +36,7 @@ def from_peer(cls, peer: NodePeer) -> Self: return cls( uuid=peer.id, - secret_token=peer.rathole_token, + secret_token=peer.rtunnel_token, local_addr_host=high_priority_route.host_or_ip, local_addr_port=high_priority_route.port, server_name=peer.name, diff --git a/packages/syft/src/syft/service/network/rathole_config_builder.py b/packages/syft/src/syft/service/network/rathole_config_builder.py index 90f499ec237..f134468a120 100644 --- a/packages/syft/src/syft/service/network/rathole_config_builder.py +++ b/packages/syft/src/syft/service/network/rathole_config_builder.py @@ -36,7 +36,7 @@ def add_host_to_server(self, peer: NodePeer) -> None: None """ - rathole_route = peer.get_rathole_route() + rathole_route = peer.get_rtunnel_route() if not rathole_route: raise Exception(f"Peer: {peer} has no rathole route: {rathole_route}") @@ -46,7 +46,7 @@ def add_host_to_server(self, peer: NodePeer) -> None: config = RatholeConfig( uuid=peer_id.to_string(), - secret_token=rathole_route.rathole_token, + secret_token=rathole_route.rtunnel_token, local_addr_host=DEFAULT_LOCAL_ADDR_HOST, local_addr_port=random_port, server_name=peer.name, @@ -120,13 +120,13 @@ def _get_random_port(self) -> int: return secrets.randbits(15) def add_host_to_client( - self, peer_name: str, peer_id: str, rathole_token: str, remote_addr: str + self, peer_name: str, peer_id: str, rtunnel_token: str, remote_addr: str ) -> None: """Add a host to the rathole client toml file.""" config = RatholeConfig( uuid=peer_id, - secret_token=rathole_token, + secret_token=rtunnel_token, local_addr_host="proxy", local_addr_port=80, server_name=peer_name, diff --git a/packages/syft/src/syft/service/network/reverse_tunnel_service.py b/packages/syft/src/syft/service/network/reverse_tunnel_service.py index bb80c56f401..36d5b14e151 100644 --- a/packages/syft/src/syft/service/network/reverse_tunnel_service.py +++ b/packages/syft/src/syft/service/network/reverse_tunnel_service.py @@ -14,7 +14,7 @@ def set_client_config( self_node_peer: NodePeer, remote_node_route: NodeRoute, ) -> None: - rathole_route = self_node_peer.get_rathole_route() + rathole_route = self_node_peer.get_rtunnel_route() if not rathole_route: raise Exception( "Failed to exchange routes via . " @@ -31,12 +31,12 @@ def set_client_config( self.builder.add_host_to_client( peer_name=self_node_peer.name, peer_id=str(self_node_peer.id), - rathole_token=rathole_route.rathole_token, + rtunnel_token=rathole_route.rtunnel_token, remote_addr=remote_addr, ) def set_server_config(self, remote_peer: NodePeer) -> None: - rathole_route = remote_peer.get_rathole_route() + rathole_route = remote_peer.get_rtunnel_route() self.builder.add_host_to_server(remote_peer) if rathole_route else None def clear_client_config(self, self_node_peer: NodePeer) -> None: diff --git a/packages/syft/src/syft/service/network/routes.py b/packages/syft/src/syft/service/network/routes.py index b1ff68f6b72..ca5ea04c999 100644 --- a/packages/syft/src/syft/service/network/routes.py +++ b/packages/syft/src/syft/service/network/routes.py @@ -96,7 +96,7 @@ class HTTPNodeRoute(SyftObject, NodeRoute): port: int = 80 proxy_target_uid: UID | None = None priority: int = 1 - rathole_token: str | None = None + rtunnel_token: str | None = None def __eq__(self, other: Any) -> bool: if not isinstance(other, HTTPNodeRoute): @@ -109,7 +109,7 @@ def __hash__(self) -> int: + hash(self.port) + hash(self.protocol) + hash(self.proxy_target_uid) - + hash(self.rathole_token) + + hash(self.rtunnel_token) ) def __str__(self) -> str: diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index eedc61173a0..d8098f55e1d 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -920,7 +920,7 @@ def get_dev_mode() -> bool: def generate_token() -> str: - return hashlib.sha256(secrets.token_bytes(16)).hexdigest() + return secrets.token_hex(64) def sanitize_html(html: str) -> str: From b393b7a80c92c1de6bcf6a676a61da102871affc Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 28 Jun 2024 11:46:18 +0530 Subject: [PATCH 094/100] rename reverse_tunnel to rtunnel in values and values.dev yaml --- packages/grid/devspace.yaml | 10 ++++---- .../backend/backend-service-account.yaml | 4 ++-- .../backend/backend-statefulset.yaml | 4 ++-- .../templates/rathole/rathole-configmap.yaml | 8 ++----- .../rathole/rathole-statefulset.yaml | 24 +++++++++---------- packages/grid/helm/syft/values.yaml | 2 +- packages/grid/helm/values.dev.yaml | 2 +- packages/grid/rathole/start.sh | 13 ++++++++-- 8 files changed, 36 insertions(+), 31 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 7e37e705214..41e0ac00aa4 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -86,7 +86,7 @@ deployments: version: dev-${DEVSPACE_TIMESTAMP} node: type: domain # required for the gateway profile - reverse_tunnel: + rathole: mode: client dev: @@ -125,10 +125,10 @@ dev: - path: ../syft:/root/app/syft ssh: localPort: 3480 - reverse_tunnel: + rathole: labelSelector: app.kubernetes.io/name: syft - app.kubernetes.io/component: reverse_tunnel + app.kubernetes.io/component: rathole ports: - port: "2333" # reverse_tunnel @@ -158,7 +158,7 @@ profiles: # Patch mode to server - op: replace - path: deployments.syft.helm.values.reverse_tunnel.mode + path: deployments.syft.helm.values.rtunnel.mode value: server # Port Re-Mapping @@ -179,7 +179,7 @@ profiles: # Mongo - op: replace - path: dev.reverse_tunnel.ports[0].port + path: dev.rtunnel.ports[0].port value: 2334:2333 - name: gcp diff --git a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml index 76d70afee70..ee4634fc45f 100644 --- a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml @@ -26,10 +26,10 @@ metadata: app.kubernetes.io/component: backend rules: - apiGroups: [""] - resources: ["pods", "configmaps", "secrets", "services"] + resources: ["pods", "configmaps", "secrets"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] - apiGroups: [""] - resources: ["pods/log"] + resources: ["pods/log", "services"] verbs: ["get", "list", "watch"] - apiGroups: ["batch"] resources: ["jobs"] diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index e5dd285985f..b86916d6bde 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -91,9 +91,9 @@ spec: - name: ASSOCIATION_REQUEST_AUTO_APPROVAL value: {{ .Values.node.associationRequestAutoApproval | quote }} {{- end }} - {{- if .Values.reverse_tunnel.enabled }} + {{- if .Values.rtunnel.enabled }} - name: RATHOLE_PORT - value: {{ .Values.reverse_tunnel.port | quote }} + value: {{ .Values.rtunnel.port | quote }} - name: REVERSE_TUNNEL_ENABLED value: "true" {{- end }} diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index 3bebf40cffc..ae506486ce4 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -7,7 +7,7 @@ metadata: {{- include "common.labels" . | nindent 4 }} app.kubernetes.io/component: rathole data: - {{- if eq .Values.reverse_tunnel.mode "server" }} + {{- if eq .Values.rtunnel.mode "server" }} server.toml: | [server] bind_addr = "0.0.0.0:2333" @@ -16,13 +16,9 @@ data: type = "websocket" [server.transport.websocket] tls = false - - [server.services.domain] - token = "domain-specific-rathole-secret" - bind_addr = "0.0.0.0:8001" {{- end }} - {{- if eq .Values.reverse_tunnel.mode "client" }} + {{- if eq .Values.rtunnel.mode "client" }} client.toml: | [client] remote_addr = "0.0.0.0:2333" diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index dfa068218d6..b1286f6a1f4 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -20,30 +20,30 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: rathole - {{- if .Values.reverse_tunnel.podLabels }} - {{- toYaml .Values.reverse_tunnel.podLabels | nindent 8 }} + {{- if .Values.rtunnel.podLabels }} + {{- toYaml .Values.rtunnel.podLabels | nindent 8 }} {{- end }} - {{- if .Values.reverse_tunnel.podAnnotations }} - annotations: {{- toYaml .Values.reverse_tunnel.podAnnotations | nindent 8 }} + {{- if .Values.rtunnel.podAnnotations }} + annotations: {{- toYaml .Values.rtunnel.podAnnotations | nindent 8 }} {{- end }} spec: - {{- if .Values.reverse_tunnel.nodeSelector }} - nodeSelector: {{- .Values.reverse_tunnel.nodeSelector | toYaml | nindent 8 }} + {{- if .Values.rtunnel.nodeSelector }} + nodeSelector: {{- .Values.rtunnel.nodeSelector | toYaml | nindent 8 }} {{- end }} containers: - name: rathole image: {{ .Values.global.registry }}/openmined/grid-rathole:{{ .Values.global.version }} imagePullPolicy: Always - resources: {{ include "common.resources.set" (dict "resources" .Values.reverse_tunnel.resources "preset" .Values.reverse_tunnel.resourcesPreset) | nindent 12 }} + resources: {{ include "common.resources.set" (dict "resources" .Values.rtunnel.resources "preset" .Values.rtunnel.resourcesPreset) | nindent 12 }} env: - name: LOG_LEVEL - value: {{ .Values.reverse_tunnel.logLevel | quote }} + value: {{ .Values.rtunnel.logLevel | quote }} - name: MODE - value: {{ .Values.reverse_tunnel.mode | quote }} + value: {{ .Values.rtunnel.mode | quote }} - name: RATHOLE_PORT - value: {{ .Values.reverse_tunnel.port | quote }} - {{- if .Values.reverse_tunnel.env }} - {{- toYaml .Values.reverse_tunnel.env | nindent 12 }} + value: {{ .Values.rtunnel.port | quote }} + {{- if .Values.rtunnel.env }} + {{- toYaml .Values.rtunnel.env | nindent 12 }} {{- end }} ports: - name: rathole-port diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 8e225b94599..b5372d8e857 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -238,7 +238,7 @@ ingress: # ================================================================================= -reverse_tunnel: +rtunnel: # Extra environment vars env: null enabled: false diff --git a/packages/grid/helm/values.dev.yaml b/packages/grid/helm/values.dev.yaml index e9c32543c0b..493850cbb67 100644 --- a/packages/grid/helm/values.dev.yaml +++ b/packages/grid/helm/values.dev.yaml @@ -47,7 +47,7 @@ proxy: resourcesPreset: null resources: null -reverse_tunnel: +rtunnel: enabled: true logLevel: "trace" diff --git a/packages/grid/rathole/start.sh b/packages/grid/rathole/start.sh index b1af50597fc..87111ac8c9f 100755 --- a/packages/grid/rathole/start.sh +++ b/packages/grid/rathole/start.sh @@ -8,9 +8,18 @@ copy_config() { cp -L -r -f /conf/* conf/ } -# Start the server +# Start the server and reload until healthy start_server() { - RUST_LOG=$RUST_LOG /app/rathole conf/server.toml & + while true; do + RUST_LOG=$RUST_LOG /app/rathole conf/server.toml + status=$? + if [ $status -eq 0 ]; then + break + else + echo "Server failed to start, retrying in 5 seconds..." + sleep 5 + fi + done & } # Start the client From 30f1132082ad14b8cf3c4e5618bcce9a2639a160 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 28 Jun 2024 12:44:11 +0530 Subject: [PATCH 095/100] revert backend account to have permission to patch services - rename reference of rathole_token to rtunnel_token in gateway test --- packages/grid/devspace.yaml | 2 +- .../helm/syft/templates/backend/backend-service-account.yaml | 4 ++-- tests/integration/network/gateway_test.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 41e0ac00aa4..dc0d2dbd142 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -86,7 +86,7 @@ deployments: version: dev-${DEVSPACE_TIMESTAMP} node: type: domain # required for the gateway profile - rathole: + rtunnel: mode: client dev: diff --git a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml index ee4634fc45f..76d70afee70 100644 --- a/packages/grid/helm/syft/templates/backend/backend-service-account.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-service-account.yaml @@ -26,10 +26,10 @@ metadata: app.kubernetes.io/component: backend rules: - apiGroups: [""] - resources: ["pods", "configmaps", "secrets"] + resources: ["pods", "configmaps", "secrets", "services"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] - apiGroups: [""] - resources: ["pods/log", "services"] + resources: ["pods/log"] verbs: ["get", "list", "watch"] - apiGroups: ["batch"] resources: ["jobs"] diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index ef8b090a8a6..d64ab32fdca 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -946,7 +946,7 @@ def test_reverse_tunnel_connection(domain_1_port: int, gateway_port: int): # Domain's peer is a gateway and vice-versa domain_peer = domain_client.peers[0] assert domain_peer.node_type == NodeType.GATEWAY - assert domain_peer.node_routes[0].rathole_token is None + assert domain_peer.node_routes[0].rtunnel_token is None assert len(gateway_client.peers) == 0 gateway_client_root = gateway_client.login( @@ -960,7 +960,7 @@ def test_reverse_tunnel_connection(domain_1_port: int, gateway_port: int): gateway_peers = gateway_client.api.services.network.get_all_peers() assert len(gateway_peers) == 1 assert len(gateway_peers[0].node_routes) == 1 - assert gateway_peers[0].node_routes[0].rathole_token is not None + assert gateway_peers[0].node_routes[0].rtunnel_token is not None proxy_domain_client = gateway_client.peers[0] From ad31f5bbc924bc3a8b880767cb68dbc2907641a5 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 28 Jun 2024 13:35:20 +0530 Subject: [PATCH 096/100] force enable proxy in case of gateways --- packages/grid/devspace.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index dc0d2dbd142..b961d17da26 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -88,6 +88,8 @@ deployments: type: domain # required for the gateway profile rtunnel: mode: client + proxy: + enabled: true # required for the gateway profile dev: mongo: @@ -161,6 +163,11 @@ profiles: path: deployments.syft.helm.values.rtunnel.mode value: server + # Enable proxy for gateway + - op: replace + path: deployments.syft.helm.values.proxy.enabled + value: true + # Port Re-Mapping # Mongo - op: replace @@ -177,7 +184,7 @@ profiles: path: dev.backend.containers.backend-container.ssh.localPort value: 3481 - # Mongo + # Reverse tunnel port - op: replace path: dev.rtunnel.ports[0].port value: 2334:2333 From a95a815098e7f792a4885df8d35bca1c565f14e8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 28 Jun 2024 13:57:54 +0530 Subject: [PATCH 097/100] start rathole pod, config, and service if rtunnel flag is enabled --- .../grid/helm/syft/templates/rathole/rathole-configmap.yaml | 2 ++ .../grid/helm/syft/templates/rathole/rathole-service.yaml | 2 ++ .../grid/helm/syft/templates/rathole/rathole-statefulset.yaml | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml index ae506486ce4..77f2bec4c4b 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-configmap.yaml @@ -1,3 +1,4 @@ +{{- if .Values.rtunnel.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -28,3 +29,4 @@ data: [client.transport.websocket] tls = false {{- end }} +{{- end }} diff --git a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml index 01fa305ac77..087da2256e6 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-service.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-service.yaml @@ -1,3 +1,4 @@ +{{- if .Values.rtunnel.enabled }} apiVersion: v1 kind: Service metadata: @@ -15,3 +16,4 @@ spec: port: 2333 targetPort: 2333 protocol: TCP +{{- end }} \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml index b1286f6a1f4..86d39b51551 100644 --- a/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml +++ b/packages/grid/helm/syft/templates/rathole/rathole-statefulset.yaml @@ -1,3 +1,4 @@ +{{- if .Values.rtunnel.enabled }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -75,4 +76,5 @@ spec: - ReadWriteOnce resources: requests: - storage: 10Mi \ No newline at end of file + storage: 10Mi +{{- end }} \ No newline at end of file From feaa0d4a8db0c8ad01812e5128546063b42b47df Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Fri, 28 Jun 2024 14:07:03 +0530 Subject: [PATCH 098/100] add retry to test_delete_route_on_peer for flakyness --- tests/integration/network/gateway_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/network/gateway_test.py b/tests/integration/network/gateway_test.py index d64ab32fdca..48cddf8ce4c 100644 --- a/tests/integration/network/gateway_test.py +++ b/tests/integration/network/gateway_test.py @@ -573,6 +573,7 @@ def test_add_route_on_peer(set_env_var, gateway_port: int, domain_1_port: int) - @pytest.mark.network +@pytest.mark.flaky(reruns=2, reruns_delay=2) def test_delete_route_on_peer( set_env_var, gateway_port: int, domain_1_port: int ) -> None: From e95af15c8b8789fac5c4880655fd9b080f69c630 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sun, 30 Jun 2024 14:56:43 +0530 Subject: [PATCH 099/100] fix incorrect path --- .../syft/src/syft/service/network/rathole_config_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/network/rathole_config_builder.py b/packages/syft/src/syft/service/network/rathole_config_builder.py index f134468a120..15d1e8fb7eb 100644 --- a/packages/syft/src/syft/service/network/rathole_config_builder.py +++ b/packages/syft/src/syft/service/network/rathole_config_builder.py @@ -211,7 +211,7 @@ def _add_dynamic_addr_to_rathole( proxy_rule = ( f"Host(`{config.server_name}.syft.local`) || " - f"HostHeader(`{config.server_name}.syft.local`) && PathPrefix(`/rathole`)" + f"HostHeader(`{config.server_name}.syft.local`) && PathPrefix(`/rtunnel`)" ) rathole_proxy["http"]["routers"][config.server_name] = { From c2ab1029579cb98fb763ac4fd9996242957bfac0 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sun, 30 Jun 2024 17:24:45 +0530 Subject: [PATCH 100/100] fix charts --- packages/grid/devspace.yaml | 126 +++++++++--------- .../dev/base.yaml} | 29 ++-- .../grid/helm/examples/dev/domain.tunnel.yaml | 11 ++ packages/grid/helm/examples/dev/enclave.yaml | 8 ++ packages/grid/helm/examples/dev/gateway.yaml | 14 ++ packages/grid/helm/syft/values.yaml | 8 +- packages/grid/helm/values.dev.high.yaml | 48 ------- packages/grid/helm/values.dev.low.yaml | 48 ------- tox.ini | 21 ++- 9 files changed, 133 insertions(+), 180 deletions(-) rename packages/grid/helm/{values.dev.yaml => examples/dev/base.yaml} (74%) create mode 100644 packages/grid/helm/examples/dev/domain.tunnel.yaml create mode 100644 packages/grid/helm/examples/dev/enclave.yaml create mode 100644 packages/grid/helm/examples/dev/gateway.yaml delete mode 100644 packages/grid/helm/values.dev.high.yaml delete mode 100644 packages/grid/helm/values.dev.low.yaml diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index b961d17da26..8bbf3487daf 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -60,14 +60,6 @@ images: context: ./seaweedfs tags: - dev-${DEVSPACE_TIMESTAMP} - rathole: - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_RATHOLE}" - buildKit: - args: ["--platform", "linux/${PLATFORM}"] - dockerfile: ./rathole/rathole.dockerfile - context: ./rathole - tags: - - dev-${DEVSPACE_TIMESTAMP} # This is a list of `deployments` that DevSpace can create for this project deployments: @@ -76,20 +68,16 @@ deployments: releaseName: syft-dev chart: name: ./helm/syft - # anything that does not need devspace $env vars should go in values.dev.yaml - valuesFiles: - - ./helm/syft/values.yaml - - ./helm/values.dev.yaml + # values that need to be templated go here values: global: registry: ${CONTAINER_REGISTRY} version: dev-${DEVSPACE_TIMESTAMP} - node: - type: domain # required for the gateway profile - rtunnel: - mode: client - proxy: - enabled: true # required for the gateway profile + node: {} + # anything that does not need templating should go in helm/examples/dev/base.yaml + # or profile specific values files + valuesFiles: + - ./helm/examples/dev/base.yaml dev: mongo: @@ -127,69 +115,86 @@ dev: - path: ../syft:/root/app/syft ssh: localPort: 3480 - rathole: - labelSelector: - app.kubernetes.io/name: syft - app.kubernetes.io/component: rathole - ports: - - port: "2333" # reverse_tunnel profiles: - - name: dev-low + - name: domain-low + description: "Deploy a low-side domain" patches: - op: add path: deployments.syft.helm.values.node value: side: low - - name: dev-high + + - name: domain-tunnel + description: "Deploy a domain with tunneling enabled" patches: + # enable rathole image - op: add - path: deployments.syft.helm.values.node + path: images value: - side: high + rathole: + image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_RATHOLE}" + buildKit: + args: ["--platform", "linux/${PLATFORM}"] + dockerfile: ./rathole/rathole.dockerfile + context: ./rathole + tags: + - dev-${DEVSPACE_TIMESTAMP} + # use rathole client-specific chart values + - op: add + path: deployments.syft.helm.valuesFiles + value: ./helm/examples/dev/domain.tunnel.yaml - name: gateway + description: "Deploy a Gateway Node with tunnel enabled" patches: - - op: replace - path: deployments.syft.helm.values.node.type - value: "gateway" + # enable rathole image + - op: add + path: images + value: + rathole: + image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_RATHOLE}" + buildKit: + args: ["--platform", "linux/${PLATFORM}"] + dockerfile: ./rathole/rathole.dockerfile + context: ./rathole + tags: + - dev-${DEVSPACE_TIMESTAMP} + # enable rathole `devspace dev` config + - op: add + path: dev + value: + rathole: + labelSelector: + app.kubernetes.io/name: syft + app.kubernetes.io/component: rathole + ports: + - port: "2333" + # use gateway-specific chart values + - op: add + path: deployments.syft.helm.valuesFiles + value: ./helm/examples/dev/gateway.yaml + # remove unused images - op: remove path: images.seaweedfs - op: remove path: dev.seaweedfs - - # Patch mode to server - - op: replace - path: deployments.syft.helm.values.rtunnel.mode - value: server - - # Enable proxy for gateway - - op: replace - path: deployments.syft.helm.values.proxy.enabled - value: true - # Port Re-Mapping - # Mongo - op: replace path: dev.mongo.ports[0].port value: 27018:27017 - - # Backend - op: replace path: dev.backend.ports[0].port value: 5679:5678 - - # Backend Container SSH - op: replace path: dev.backend.containers.backend-container.ssh.localPort value: 3481 - - # Reverse tunnel port - op: replace path: dev.rtunnel.ports[0].port value: 2334:2333 - name: gcp + description: "Deploy a high-side domain on GCP" patches: - op: replace path: deployments.syft.helm.valuesFiles @@ -197,6 +202,7 @@ profiles: - ./helm/examples/gcp/gcp.high.yaml - name: gcp-low + description: "Deploy a low-side domain on GCP" patches: - op: replace path: deployments.syft.helm.valuesFiles @@ -204,6 +210,7 @@ profiles: - ./helm/examples/gcp/gcp.low.yaml - name: azure + description: "Deploy a high-side domain on AKS" patches: - op: replace path: deployments.syft.helm.valuesFiles @@ -211,11 +218,9 @@ profiles: - ./helm/examples/azure/azure.high.yaml - name: enclave + description: "Deploy an enclave node" patches: - - op: replace - path: deployments.syft.helm.values.node.type - value: "enclave" - + # enable image build for enclave-attestation - op: add path: images value: @@ -233,29 +238,20 @@ profiles: enclave-attestation: sync: - path: ./enclave/attestation/server:/app/server - + # use gateway-specific chart values - op: add - path: deployments.syft.helm.values - value: - attestation: - enabled: true - + path: deployments.syft.helm.valuesFiles + value: ./helm/examples/dev/enclave.yaml # Port Re-Mapping - # Mongo - op: replace path: dev.mongo.ports[0].port value: 27019:27017 - - # Backend - op: replace path: dev.backend.ports[0].port value: 5680:5678 - - # Backend Container SSH - op: replace path: dev.backend.containers.backend-container.ssh.localPort value: 3482 - - op: replace path: dev.seaweedfs.ports value: diff --git a/packages/grid/helm/values.dev.yaml b/packages/grid/helm/examples/dev/base.yaml similarity index 74% rename from packages/grid/helm/values.dev.yaml rename to packages/grid/helm/examples/dev/base.yaml index 493850cbb67..b81e4847cd8 100644 --- a/packages/grid/helm/values.dev.yaml +++ b/packages/grid/helm/examples/dev/base.yaml @@ -1,15 +1,9 @@ -# Helm chart values used for development and testing -# Can be used through `helm install -f values.dev.yaml` or devspace `valuesFiles` +# Base Helm chart values used for development and testing +# Can be used through `helm install -f packages/grid/helm/examples/dev/base.yaml` or devspace `valuesFiles` global: randomizedSecrets: false -registry: - resourcesPreset: null - resources: null - - storageSize: "5Gi" - node: rootEmail: info@openmined.org associationRequestAutoApproval: true @@ -44,14 +38,21 @@ frontend: resources: null proxy: + enabled: true + resourcesPreset: null resources: null -rtunnel: +registry: enabled: true - logLevel: "trace" -# attestation: -# enabled: true -# resourcesPreset: null -# resources: null + resourcesPreset: null + resources: null + + storageSize: "5Gi" + +rtunnel: + enabled: false + +attestation: + enabled: false diff --git a/packages/grid/helm/examples/dev/domain.tunnel.yaml b/packages/grid/helm/examples/dev/domain.tunnel.yaml new file mode 100644 index 00000000000..cec2e97cc6e --- /dev/null +++ b/packages/grid/helm/examples/dev/domain.tunnel.yaml @@ -0,0 +1,11 @@ +# Values for deploying a domain with a reverse tunnel server in client-mode +# Patched on top of patch `base.yaml` + +# Proxy is required for the tunnel to work +proxy: + enabled: true + +rtunnel: + enabled: true + mode: client + logLevel: debug diff --git a/packages/grid/helm/examples/dev/enclave.yaml b/packages/grid/helm/examples/dev/enclave.yaml new file mode 100644 index 00000000000..2951da06b05 --- /dev/null +++ b/packages/grid/helm/examples/dev/enclave.yaml @@ -0,0 +1,8 @@ +# Values for deploying an enclave +# Patched on top of patch `base.yaml` + +node: + type: enclave + +attestation: + enabled: true diff --git a/packages/grid/helm/examples/dev/gateway.yaml b/packages/grid/helm/examples/dev/gateway.yaml new file mode 100644 index 00000000000..e0916c98c21 --- /dev/null +++ b/packages/grid/helm/examples/dev/gateway.yaml @@ -0,0 +1,14 @@ +# Values for deploying a gateway with a reverse tunnel server +# Patched on top of patch `base.yaml` + +node: + type: gateway + +# Proxy is required for the tunnel to work +proxy: + enabled: true + +rtunnel: + enabled: true + mode: server + logLevel: debug diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index b5372d8e857..377bd763c54 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -134,6 +134,7 @@ proxy: registry: enabled: true + # Extra environment vars env: null @@ -239,14 +240,15 @@ ingress: rtunnel: - # Extra environment vars - env: null enabled: false - logLevel: "info" + logLevel: info port: 2333 mode: client + # Extra environment vars + env: null + # Pod labels & annotations podLabels: null podAnnotations: null diff --git a/packages/grid/helm/values.dev.high.yaml b/packages/grid/helm/values.dev.high.yaml deleted file mode 100644 index 9a0e266704a..00000000000 --- a/packages/grid/helm/values.dev.high.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Helm chart values used for development and testing -# Can be used through `helm install -f values.dev.yaml` or devspace `valuesFiles` - -global: - randomizedSecrets: false - -registry: - resourcesPreset: null - resources: null - - storageSize: "5Gi" - -node: - rootEmail: info@openmined.org - side: high - - resourcesPreset: 2xlarge - resources: null - - defaultWorkerPool: - count: 1 - podLabels: null - podAnnotations: null - - secret: - defaultRootPassword: changethis - -mongo: - resourcesPreset: null - resources: null - - secret: - rootPassword: example - -seaweedfs: - resourcesPreset: null - resources: null - - secret: - s3RootPassword: admin - -frontend: - resourcesPreset: null - resources: null - -proxy: - resourcesPreset: null - resources: null diff --git a/packages/grid/helm/values.dev.low.yaml b/packages/grid/helm/values.dev.low.yaml deleted file mode 100644 index 7e5de1a68f2..00000000000 --- a/packages/grid/helm/values.dev.low.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Helm chart values used for development and testing -# Can be used through `helm install -f values.dev.yaml` or devspace `valuesFiles` - -global: - randomizedSecrets: false - -registry: - resourcesPreset: null - resources: null - - storageSize: "5Gi" - -node: - rootEmail: info@openmined.org - side: low - - resourcesPreset: 2xlarge - resources: null - - defaultWorkerPool: - count: 1 - podLabels: null - podAnnotations: null - - secret: - defaultRootPassword: changethis - -mongo: - resourcesPreset: null - resources: null - - secret: - rootPassword: example - -seaweedfs: - resourcesPreset: null - resources: null - - secret: - s3RootPassword: admin - -frontend: - resourcesPreset: null - resources: null - -proxy: - resourcesPreset: null - resources: null diff --git a/tox.ini b/tox.ini index 6fdda86e25c..7955837a2ae 100644 --- a/tox.ini +++ b/tox.ini @@ -471,7 +471,7 @@ commands = # Creating test-domain-1 cluster on port 9082 bash -c '\ - export CLUSTER_NAME=${DOMAIN_CLUSTER_NAME} CLUSTER_HTTP_PORT=9082 && \ + export CLUSTER_NAME=${DOMAIN_CLUSTER_NAME} CLUSTER_HTTP_PORT=9082 DEVSPACE_PROFILE=domain-tunnel && \ tox -e dev.k8s.start && \ tox -e dev.k8s.deploy' @@ -874,6 +874,23 @@ commands = bash -c 'devspace cleanup images --kube-context k3d-${CLUSTER_NAME} --no-warn --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 || true' bash -c 'kubectl --context k3d-${CLUSTER_NAME} delete namespace syft --now=true || true' +[testenv:dev.k8s.render] +description = Dump devspace rendered chargs for debugging. Save in `packages/grid/out.render` +changedir = {toxinidir}/packages/grid +passenv = HOME, USER, DEVSPACE_PROFILE +setenv= + OUTPUT_DIR = {env:OUTPUT_DIR:./.devspace/rendered} +allowlist_externals = + bash +commands = + bash -c '\ + if [[ -n "${DEVSPACE_PROFILE}" ]]; then export DEVSPACE_PROFILE="-p ${DEVSPACE_PROFILE}"; fi && \ + rm -rf ${OUTPUT_DIR} && \ + mkdir -p ${OUTPUT_DIR} && \ + echo "profile: $DEVSPACE_PROFILE" && \ + devspace print ${DEVSPACE_PROFILE} > ${OUTPUT_DIR}/config.txt && \ + devspace deploy --render --skip-build --no-warn ${DEVSPACE_PROFILE} --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 > ${OUTPUT_DIR}/chart.yaml' + [testenv:dev.k8s.launch.gateway] description = Launch a single gateway on K8s passenv = HOME, USER @@ -888,7 +905,7 @@ commands = tox -e dev.k8s.{posargs:deploy} [testenv:dev.k8s.launch.domain] -description = Launch a single domain on K8s +description = Launch a single domain on K8s passenv = HOME, USER setenv= CLUSTER_NAME = {env:CLUSTER_NAME:test-domain-1}