diff --git a/geth/accounts.py b/geth/accounts.py index 84921669..2a75e1e8 100644 --- a/geth/accounts.py +++ b/geth/accounts.py @@ -9,7 +9,7 @@ ) -def get_accounts(data_dir, **geth_kwargs): +def get_accounts(data_dir, docker_container=None, **geth_kwargs): """ Returns all geth accounts as tuple of hex encoded strings @@ -17,11 +17,32 @@ def get_accounts(data_dir, **geth_kwargs): ... ('0x...', '0x...') """ command, proc = spawn_geth( - dict(data_dir=data_dir, suffix_args=["account", "list"], **geth_kwargs) + dict(data_dir=data_dir, suffix_args=["account", "list"], **geth_kwargs), docker_container=docker_container ) - stdoutdata, stderrdata = proc.communicate() - if proc.returncode: + if docker_container is not None: + print("proc returned: ", proc) + exitcode = proc.exit_code + + stderrdata, stdoutdata = "", "" + + if exitcode != 0: + stderrdata = proc.output + else: + stdoutdata = proc.output + + print("stdoutdata: ", stdoutdata) + print("stderrdata: ", stderrdata) + print("exitcode: ", exitcode) + + condition = exitcode != 0 + + else: + stdoutdata, stderrdata = proc.communicate() + condition = proc.returncode != 0 + exitcode = proc.returncode + + if condition: if "no keys in store" in stderrdata.decode("utf-8"): return tuple() else: @@ -29,7 +50,7 @@ def get_accounts(data_dir, **geth_kwargs): format_error_message( "Error trying to list accounts", command, - proc.returncode, + exitcode, stdoutdata, stderrdata, ) diff --git a/geth/chain.py b/geth/chain.py index 40eb61b1..9a695496 100644 --- a/geth/chain.py +++ b/geth/chain.py @@ -14,11 +14,39 @@ ) -def get_live_data_dir(): +def get_live_data_dir(docker=False, docker_geth_version=None): """ `py-geth` needs a base directory to store it's chain data. By default this is the directory that `geth` uses as it's `datadir`. """ + + if docker: + if docker_geth_version is None: + raise ValueError( + "Must specify `docker_geth_version` when using `docker=True`" + ) + + if not docker_geth_version.startswith("v"): + docker_geth_version = f"v{docker_geth_version}" + + data_dir = os.path.expanduser( + os.path.join( + "~", + ".py-geth", + docker_geth_version, + ".ethereum", + ) + ) + + # check if the docker data dir exists + if not os.path.exists(data_dir): + raise ValueError( + "The docker data dir does not exist." + f" Are you sure that your volumes have been mounted at {data_dir}?" + ) + + return data_dir + if sys.platform == "darwin": data_dir = os.path.expanduser( os.path.join( @@ -54,7 +82,6 @@ def get_live_data_dir(): ) return data_dir - def get_ropsten_data_dir(): return os.path.abspath( os.path.expanduser( @@ -70,12 +97,11 @@ def get_default_base_dir(): return get_live_data_dir() -def get_chain_data_dir(base_dir, name): +def get_chain_data_dir(base_dir, name, docker=False): data_dir = os.path.abspath(os.path.join(base_dir, name)) ensure_path_exists(data_dir) return data_dir - def get_genesis_file_path(data_dir): return os.path.join(data_dir, "genesis.json") diff --git a/geth/install.py b/geth/install.py index dca49918..59263e39 100644 --- a/geth/install.py +++ b/geth/install.py @@ -9,6 +9,8 @@ import sys import tarfile +from geth.utils.docker import image_fix + V1_11_0 = "v1.11.0" V1_11_1 = "v1.11.1" V1_11_2 = "v1.11.2" @@ -376,11 +378,18 @@ def install_from_source_code_release(identifier): V1_13_8: install_v1_13_8, V1_13_9: install_v1_13_9, V1_13_10: install_v1_13_10, - }, + } } +def install_geth(identifier=None, platform=None, docker=False, docker_install_version=None): + if docker: + # for testing purposes + image_fix(docker_install_version=docker_install_version) + return + + if identifier is None: + raise ValueError("Must specify a geth version to install if not using docker") -def install_geth(identifier, platform=None): if platform is None: platform = get_platform() @@ -402,11 +411,15 @@ def install_geth(identifier, platform=None): if __name__ == "__main__": try: - identifier = sys.argv[1] + identifier: str = sys.argv[1] + docker_option: bool = False + if len(sys.argv) > 2: + docker_option = sys.argv[2] == "docker" + except IndexError: print( "Invocation error. Should be invoked as `python -m geth.install `" # noqa: E501 ) sys.exit(1) - install_geth(identifier) + install_geth(identifier, docker=docker_option) diff --git a/geth/process.py b/geth/process.py index 77e0458a..b1232886 100644 --- a/geth/process.py +++ b/geth/process.py @@ -1,10 +1,13 @@ import logging import os +import docker as dockerlib import socket import subprocess import time import warnings +from geth.utils.docker import cleanup_chaindata, start_container, stop_container, verify_and_get_tag + try: from urllib.request import ( URLError, @@ -52,6 +55,7 @@ class BaseGethProcess(object): _proc = None + container = None def __init__( self, @@ -59,28 +63,63 @@ def __init__( stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + docker=False, + geth_version_docker=None, ): self.geth_kwargs = geth_kwargs - self.command = construct_popen_command(**geth_kwargs) + self.command = construct_popen_command(**geth_kwargs, docker=docker) self.stdin = stdin self.stdout = stdout self.stderr = stderr + self.docker = docker + self.client: dockerlib.DockerClient = None + self.container: dockerlib.models.containers.Container = None + self.geth_version_docker = geth_version_docker + if self.docker: + # exposing for easier testing + self.client = dockerlib.from_env() is_running = False def start(self): if self.is_running: raise ValueError("Already running") + + if self.docker: + self.start_docker() + self.is_running = True logger.info("Launching geth: %s", " ".join(self.command)) - self.proc = subprocess.Popen( - self.command, - stdin=self.stdin, - stdout=self.stdout, - stderr=self.stderr, + + # i will let self.proc be empty if docker is True + if not self.docker: + self.proc = subprocess.Popen( + self.command, + stdin=self.stdin, + stdout=self.stdout, + stderr=self.stderr, + ) + + def start_docker(self): + if self.geth_version_docker is None: + # default to latest + self.geth_version_docker = "latest" + + # check if image exists + image_name = verify_and_get_tag(self.geth_version_docker) + + if self.geth_version_docker == "latest": + self.geth_version_docker = image_name.split(":")[1].split("-")[0] + + self.container = start_container( + image_name, + commands=self.command, ) + def cleanup_docker_chain_data(self): + cleanup_chaindata(self.geth_version_docker) + def __enter__(self): self.start() return self @@ -88,9 +127,13 @@ def __enter__(self): def stop(self): if not self.is_running: raise ValueError("Not running") + + if self.docker: + stop_container(self.container) - if self.proc.poll() is None: - kill_proc(self.proc) + if not self.docker: + if self.proc.poll() is None: + kill_proc(self.proc) self.is_running = False @@ -107,7 +150,8 @@ def is_stopped(self): @property def accounts(self): - return get_accounts(**self.geth_kwargs) + print("data_dir: ", self.data_dir) + return get_accounts(data_dir=self.data_dir, **self.geth_kwargs, docker_container=self.container) @property def rpc_enabled(self): @@ -201,18 +245,18 @@ def wait_for_dag(self, timeout=0): class MainnetGethProcess(BaseGethProcess): - def __init__(self, geth_kwargs=None): + def __init__(self, geth_kwargs=None, docker=False): if geth_kwargs is None: geth_kwargs = {} if "data_dir" in geth_kwargs: raise ValueError("You cannot specify `data_dir` for a MainnetGethProcess") - super(MainnetGethProcess, self).__init__(geth_kwargs) + super(MainnetGethProcess, self).__init__(geth_kwargs, docker=docker) @property def data_dir(self): - return get_live_data_dir() + return get_live_data_dir(docker=self.docker, docker_geth_version=self.geth_version_docker) class LiveGethProcess(MainnetGethProcess): diff --git a/geth/utils/docker.py b/geth/utils/docker.py new file mode 100644 index 00000000..abd92c69 --- /dev/null +++ b/geth/utils/docker.py @@ -0,0 +1,216 @@ +import os +import docker +import shutil +import logging +import requests +from typing import List + +client = docker.from_env() + +logger = logging.getLogger(__name__) + +# the philosophy of this module is that, for now we will only support +# one geth container running at a time for simplicity, for a single version +# multiple geth containers can be unstable and unpredictable for now + +# and right now, we will only be supporting linux/unix systems for simplicity + +def map_architecture(architecture: str): + architecture_mapping = { + "x86_64": "amd64", + "armv7l": "arm", + "arm64": "arm64", + "aarch64": "arm64", + "amd64": "amd64" + } + + if architecture not in architecture_mapping: + raise ValueError(f"Unknown architecture: {architecture}") + + return architecture_mapping[architecture] + +# returns the latest version of geth +def verify_and_get_tag(docker_install_version=None) -> str: + # if docker_install_version="latest", return latest tag + print("Version specified: ", docker_install_version) + + # check all folders initialised in ~/.py-geth that start with "v" + path = os.path.join(os.path.expanduser("~"), ".py-geth") + if os.path.exists(path): # and docker_install_version is None or docker_install_version == "latest": + print(f"Checking for geth versions in {path}") + listed = os.listdir(path) + for folder in listed: + if folder.startswith("v"): + if docker_install_version == "latest" or docker_install_version is None: + docker_install_version = folder + + if (docker_install_version in folder or folder in docker_install_version): + docker_install_version = folder + # read folder/.docker_tag + tag_path = os.path.join(path, folder, ".docker_tag") + if os.path.exists(tag_path): + with open(tag_path, "r") as f: + tag = f.read() + return tag + print(f"Warning: Unable to find .docker_tag in {tag_path}") + logger.warning(f"verify_and_get_tag - Unable to find .docker_tag in {tag_path}") + + print("Querying GitHub API for latest geth version") + + GITHUB_API = "https://api.github.com/repos/ethereum/go-ethereum/" + + if docker_install_version is None: + docker_install_version = "latest" + else: + docker_install_version = f"{docker_install_version}" + + RELEASES_API = GITHUB_API + "releases/" + + release_url = f"{RELEASES_API}{docker_install_version}" + + r = requests.get(release_url) + if r.status_code == 404: + raise ValueError(f"Unable to find docker install version: {docker_install_version} from URL: {release_url}") + elif r.status_code != 200: + raise ValueError(f"Unexpected status code while checking for geth versions: {r.status_code}") + + release_data = r.json() + if docker_install_version == "latest": + docker_install_version = release_data.get("tag_name") + commit_tag = release_data.get("target_commitish") + + if docker_install_version is None or commit_tag is None: + raise ValueError(f"Unable to find docker install version/commit tag: {docker_install_version}/{commit_tag}") + + # detect arm or amd64 + arc = os.uname().machine + architecture = map_architecture(arc) + + # check if image ethereum/client-go:{docker_install_version}-{architecture} exists + repository = "ethereum/client-go" + tag = f"{docker_install_version}-{architecture}" + + # check if tag exists on docker hub + image_url = f"https://hub.docker.com/v2/repositories/{repository}/tags/{tag}" + r = requests.head(image_url) + if r.status_code != 200: + raise ValueError(f"Unable to find docker image {tag} from URL: {image_url}") + + total_image_tag = f"{repository}:{tag}" + + return total_image_tag + +# return image tag (useful for external use) +# just in case, "latest" was given +def image_fix(docker_install_version=None, docker_image_tag=None) -> str: + tag = docker_image_tag + if tag is None: + # get the latest version of geth + tag = verify_and_get_tag(docker_install_version=docker_install_version) + + # check if image exists + try: + client.images.get(tag) + logger.info(f"Image already exists: {tag}") + except docker.errors.ImageNotFound: + logger.info(f"Pulling image: {tag}") + try: + client.images.pull(tag) + except docker.errors.APIError as e: + raise ValueError(f"Unable to pull image: {tag}") from e + + # create folder with geth version in ~/.py-geth + geth_version = tag.split(":")[1].split("-")[0] + + ethereum_path = os.path.join(os.path.expanduser("~"), ".py-geth", geth_version, ".ethereum") + tag_path = os.path.join(os.path.expanduser("~"), ".py-geth", geth_version, ".docker_tag") + + if not os.path.exists(ethereum_path): + os.makedirs(ethereum_path) + + if not os.path.exists(tag_path): + with open(tag_path, "w+") as f: + f.write(tag) + + return tag + +def stop_container(container: docker.models.containers.Container): + container.stop() + container.remove() + +# returns a list of all containers using image_name +def image_to_containers(image_name: str) -> List[docker.models.containers.Container]: + if image_name == "latest": + image_name = verify_and_get_tag() + + try: + client.images.get(image_name) + except docker.errors.ImageNotFound: + return [] + + containers = client.containers.list( + all=True, + filters={ + "ancestor": image_name, + } + ) + + if len(containers) == 0: + return [] + else: + return containers + +def fix_containers(image_name: str): + containers = image_to_containers(image_name) + for container in containers: + container.stop() + container.remove() + +def cleanup_chaindata(version): + if version == "latest" or None: + raise ValueError("Cannot cleanup chaindata for latest/None version") + + if not version.startswith("v"): + version = f"v{version}" + + path = os.path.join(os.path.expanduser("~"), ".py-geth", version, ".ethereum") + if os.path.exists(path): + logger.info(f"Cleaning up chaindata for version {version}") + shutil.rmtree(path) + + +# image must be existing +# this function assumes that image_name has +# the version number in it as it's tag +def start_container(image_name: str, commands: List[str] = []): + # check if image exists + try: + client.images.get(image_name) + except docker.errors.ImageNotFound as e: + raise ValueError("Image not found") from e + + image_version_with_arc = image_name.split(":")[1] + image_version = image_version_with_arc.split("-")[0] + + ethereum_path = os.path.join(os.path.expanduser("~"), ".py-geth", image_version, ".ethereum") + + if not os.path.exists(ethereum_path): + os.makedirs(ethereum_path) + + fix_containers(image_name) + + # build container with image_name + # and mount ethereum_path to /root/.ethereum + container = client.containers.run( + image_name, + detach=True, + volumes={ + ethereum_path: { + "bind": "/root/.ethereum", + "mode": "rw" + } + }, + command=" ".join(commands) + ) + + return container diff --git a/geth/wrapper.py b/geth/wrapper.py index 4647e6c3..519bd656 100644 --- a/geth/wrapper.py +++ b/geth/wrapper.py @@ -94,11 +94,9 @@ def construct_test_chain_kwargs(**overrides): return overrides - def get_geth_binary_path(): return os.environ.get("GETH_BINARY", "geth") - class CommandBuilder: def __init__(self): self.command = [] @@ -148,11 +146,15 @@ def construct_popen_command( tx_pool_price_limit=None, cache=None, gcmode=None, + docker=False ): if geth_executable is None: geth_executable = get_geth_binary_path() - if not is_executable_available(geth_executable): + if docker: + geth_executable = " " + + if not is_executable_available(geth_executable) and not docker: raise ValueError( "No geth executable found. Please ensure geth is installed and " "available on your PATH or use the GETH_BINARY environment variable" @@ -165,7 +167,7 @@ def construct_popen_command( ) builder = CommandBuilder() - if nice and is_nice_available(): + if nice and is_nice_available() and not docker: builder.extend(("nice", "-n", "20")) builder.append(geth_executable) @@ -201,7 +203,10 @@ def construct_popen_command( builder.extend(("--ws.api", ws_api)) if data_dir is not None: - builder.extend(("--datadir", data_dir)) + if docker: + builder.extend(("--datadir", "/root/.ethereum")) + else: + builder.extend(("--datadir", data_dir)) if max_peers is not None: builder.extend(("--maxpeers", max_peers)) @@ -325,15 +330,28 @@ def geth_wrapper(**geth_kwargs): def spawn_geth( - geth_kwargs, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE + geth_kwargs, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, docker_container=None ): - command = construct_popen_command(**geth_kwargs) + docker_opt = False + if docker_container is not None: + docker_opt = True - proc = subprocess.Popen( - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - ) + command = construct_popen_command(**geth_kwargs, docker=docker_opt) + + if docker_opt: + # execute command in docker + + command_str = " ".join(command) + + command = "/usr/local/bin/geth" + command_str + print("Executing command: ", command) + proc = docker_container.exec_run(command, stdin=True, stdout=True, stderr=True) + else: + proc = subprocess.Popen( + command, + stdin=stdin, + stdout=stdout, + stderr=stderr, + ) return command, proc diff --git a/setup.py b/setup.py index a8318391..f6494dfd 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,8 @@ py_modules=["geth"], install_requires=[ "semantic-version>=2.6.0", + "requests>=2.28.2", + "docker>=6.0.1", ], python_requires=">=3.8, <4", extras_require=extras_require,