Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closes #50: Geth docker addition #180

Draft
wants to merge 39 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
6060860
init: basic template dockerfile
0x0elliot Jan 13, 2024
fccf518
init: basic first draft for install
0x0elliot Jan 13, 2024
e5681a4
progress: ready for first draft
0x0elliot Jan 13, 2024
70efd11
progress: ready for first draft
0x0elliot Jan 13, 2024
00c4c74
progress: making docker install independent of my machine
0x0elliot Jan 13, 2024
f250552
progress: adding image building support
0x0elliot Jan 13, 2024
abd6b79
progress: cleanup
0x0elliot Jan 13, 2024
a5d2d9e
progress: building works (?)
0x0elliot Jan 13, 2024
9457c91
progress: docker build fix
0x0elliot Jan 13, 2024
31883c1
progress: docker build fix
0x0elliot Jan 13, 2024
fc42ad8
progress: image building works!
0x0elliot Jan 13, 2024
3ed4e39
progress: basic typing
0x0elliot Jan 13, 2024
c1650bb
deps: adding dependency for necessary modules
0x0elliot Jan 13, 2024
91ab275
progress: refactoring the image & container workflow
0x0elliot Jan 14, 2024
d328a56
progress: refactoring the image & container workflow (1)
0x0elliot Jan 14, 2024
b0e6e53
progress: removing Dockerfile.template (not needed anymore)
0x0elliot Jan 14, 2024
ee661b8
progress: create folders for volume in ~/.pygeth
0x0elliot Jan 14, 2024
e7d1297
progress: adding docker to setup.py
0x0elliot Jan 14, 2024
7d0f460
progress: more utility, making things make sense
0x0elliot Jan 14, 2024
424b94f
progress: more utility, making things make sense (1)
0x0elliot Jan 14, 2024
817608e
draft: making start(docker=True) work
0x0elliot Jan 14, 2024
cf4e496
draft: plugging it in MainnetGethProcess
0x0elliot Jan 14, 2024
7c98866
draft: plugging it in MainnetGethProcess (1)
0x0elliot Jan 14, 2024
e956cb9
draft: plugging it in MainnetGethProcess (2)
0x0elliot Jan 14, 2024
37d759e
draft: plugging it in MainnetGethProcess (3)
0x0elliot Jan 14, 2024
6cd66e2
draft: plugging it in MainnetGethProcess (4)
0x0elliot Jan 14, 2024
d16fe1b
draft: plugging it in MainnetGethProcess (5)
0x0elliot Jan 14, 2024
458be9b
draft: making stop() work
0x0elliot Jan 14, 2024
873542d
progress: working on self.data_dir for main
0x0elliot Jan 20, 2024
88e26b1
progress: working on self.data_dir for main
0x0elliot Jan 20, 2024
8718dbe
progress: cleaning up stuff
0x0elliot Jan 20, 2024
bdf31c7
progress: less chances of getting rate-limited
0x0elliot Jan 20, 2024
aecfbfb
progress: less chances of getting rate-limited
0x0elliot Jan 20, 2024
67a93f9
progress: less chances of getting rate-limited and more dynamic pick ups
0x0elliot Jan 20, 2024
52364b3
progress: less chances of getting rate-limited and more dynamic pick ups
0x0elliot Jan 20, 2024
1b21eb3
progress: making states easier to manage. less requests
0x0elliot Jan 20, 2024
dfdefaf
progress: fixing self.data_dir
0x0elliot Jan 20, 2024
8831eac
progress: saving progress
0x0elliot Jan 21, 2024
85a78ba
progress: saving progress
0x0elliot Jan 21, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions geth/accounts.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,27 +9,48 @@
)


def get_accounts(data_dir, **geth_kwargs):
def get_accounts(data_dir, docker_container=None, **geth_kwargs):
"""
Returns all geth accounts as tuple of hex encoded strings

>>> geth_accounts()
... ('0x...', '0x...')
"""
command, proc = spawn_geth(
dict(data_dir=data_dir, suffix_args=["account", "list"], **geth_kwargs)
dict(data_dir=data_dir, suffix_args=["account", "list"], **geth_kwargs), docker_container=docker_container
)
stdoutdata, stderrdata = proc.communicate()

if proc.returncode:
if docker_container is not None:
print("proc returned: ", proc)
exitcode = proc.exit_code

stderrdata, stdoutdata = "", ""

if exitcode != 0:
stderrdata = proc.output
else:
stdoutdata = proc.output

print("stdoutdata: ", stdoutdata)
print("stderrdata: ", stderrdata)
print("exitcode: ", exitcode)

condition = exitcode != 0

else:
stdoutdata, stderrdata = proc.communicate()
condition = proc.returncode != 0
exitcode = proc.returncode

if condition:
if "no keys in store" in stderrdata.decode("utf-8"):
return tuple()
else:
raise ValueError(
format_error_message(
"Error trying to list accounts",
command,
proc.returncode,
exitcode,
stdoutdata,
stderrdata,
)
Expand Down
34 changes: 30 additions & 4 deletions geth/chain.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,39 @@
)


def get_live_data_dir():
def get_live_data_dir(docker=False, docker_geth_version=None):
"""
`py-geth` needs a base directory to store it's chain data. By default this is
the directory that `geth` uses as it's `datadir`.
"""

if docker:
if docker_geth_version is None:
raise ValueError(
"Must specify `docker_geth_version` when using `docker=True`"
)

if not docker_geth_version.startswith("v"):
docker_geth_version = f"v{docker_geth_version}"

data_dir = os.path.expanduser(
os.path.join(
"~",
".py-geth",
docker_geth_version,
".ethereum",
)
)

# check if the docker data dir exists
if not os.path.exists(data_dir):
raise ValueError(
"The docker data dir does not exist."
f" Are you sure that your volumes have been mounted at {data_dir}?"
)

return data_dir

if sys.platform == "darwin":
data_dir = os.path.expanduser(
os.path.join(
Expand Down Expand Up @@ -54,7 +82,6 @@ def get_live_data_dir():
)
return data_dir


def get_ropsten_data_dir():
return os.path.abspath(
os.path.expanduser(
Expand All @@ -70,12 +97,11 @@ def get_default_base_dir():
return get_live_data_dir()


def get_chain_data_dir(base_dir, name):
def get_chain_data_dir(base_dir, name, docker=False):
data_dir = os.path.abspath(os.path.join(base_dir, name))
ensure_path_exists(data_dir)
return data_dir


def get_genesis_file_path(data_dir):
return os.path.join(data_dir, "genesis.json")

Expand Down
21 changes: 17 additions & 4 deletions geth/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import sys
import tarfile

from geth.utils.docker import image_fix

V1_11_0 = "v1.11.0"
V1_11_1 = "v1.11.1"
V1_11_2 = "v1.11.2"
Expand Down Expand Up @@ -376,11 +378,18 @@ def install_from_source_code_release(identifier):
V1_13_8: install_v1_13_8,
V1_13_9: install_v1_13_9,
V1_13_10: install_v1_13_10,
},
}
}

def install_geth(identifier=None, platform=None, docker=False, docker_install_version=None):
if docker:
# for testing purposes
image_fix(docker_install_version=docker_install_version)
return

if identifier is None:
raise ValueError("Must specify a geth version to install if not using docker")

def install_geth(identifier, platform=None):
if platform is None:
platform = get_platform()

Expand All @@ -402,11 +411,15 @@ def install_geth(identifier, platform=None):

if __name__ == "__main__":
try:
identifier = sys.argv[1]
identifier: str = sys.argv[1]
docker_option: bool = False
if len(sys.argv) > 2:
docker_option = sys.argv[2] == "docker"

except IndexError:
print(
"Invocation error. Should be invoked as `python -m geth.install <release-tag>`" # noqa: E501
)
sys.exit(1)

install_geth(identifier)
install_geth(identifier, docker=docker_option)
68 changes: 56 additions & 12 deletions geth/process.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import logging
import os
import docker as dockerlib
import socket
import subprocess
import time
import warnings

from geth.utils.docker import cleanup_chaindata, start_container, stop_container, verify_and_get_tag

try:
from urllib.request import (
URLError,
Expand Down Expand Up @@ -52,45 +55,85 @@

class BaseGethProcess(object):
_proc = None
container = None

def __init__(
self,
geth_kwargs,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
docker=False,
geth_version_docker=None,
):
self.geth_kwargs = geth_kwargs
self.command = construct_popen_command(**geth_kwargs)
self.command = construct_popen_command(**geth_kwargs, docker=docker)
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.docker = docker
self.client: dockerlib.DockerClient = None
self.container: dockerlib.models.containers.Container = None
self.geth_version_docker = geth_version_docker
if self.docker:
# exposing for easier testing
self.client = dockerlib.from_env()

is_running = False

def start(self):
if self.is_running:
raise ValueError("Already running")

if self.docker:
self.start_docker()

self.is_running = True

logger.info("Launching geth: %s", " ".join(self.command))
self.proc = subprocess.Popen(
self.command,
stdin=self.stdin,
stdout=self.stdout,
stderr=self.stderr,

# i will let self.proc be empty if docker is True
if not self.docker:
self.proc = subprocess.Popen(
self.command,
stdin=self.stdin,
stdout=self.stdout,
stderr=self.stderr,
)

def start_docker(self):
if self.geth_version_docker is None:
# default to latest
self.geth_version_docker = "latest"

# check if image exists
image_name = verify_and_get_tag(self.geth_version_docker)

if self.geth_version_docker == "latest":
self.geth_version_docker = image_name.split(":")[1].split("-")[0]

self.container = start_container(
image_name,
commands=self.command,
)

def cleanup_docker_chain_data(self):
cleanup_chaindata(self.geth_version_docker)

def __enter__(self):
self.start()
return self

def stop(self):
if not self.is_running:
raise ValueError("Not running")

if self.docker:
stop_container(self.container)

if self.proc.poll() is None:
kill_proc(self.proc)
if not self.docker:
if self.proc.poll() is None:
kill_proc(self.proc)

self.is_running = False

Expand All @@ -107,7 +150,8 @@ def is_stopped(self):

@property
def accounts(self):
return get_accounts(**self.geth_kwargs)
print("data_dir: ", self.data_dir)
return get_accounts(data_dir=self.data_dir, **self.geth_kwargs, docker_container=self.container)

@property
def rpc_enabled(self):
Expand Down Expand Up @@ -201,18 +245,18 @@ def wait_for_dag(self, timeout=0):


class MainnetGethProcess(BaseGethProcess):
def __init__(self, geth_kwargs=None):
def __init__(self, geth_kwargs=None, docker=False):
if geth_kwargs is None:
geth_kwargs = {}

if "data_dir" in geth_kwargs:
raise ValueError("You cannot specify `data_dir` for a MainnetGethProcess")

super(MainnetGethProcess, self).__init__(geth_kwargs)
super(MainnetGethProcess, self).__init__(geth_kwargs, docker=docker)

@property
def data_dir(self):
return get_live_data_dir()
return get_live_data_dir(docker=self.docker, docker_geth_version=self.geth_version_docker)


class LiveGethProcess(MainnetGethProcess):
Expand Down
Loading