Merge remote-tracking branch 'upstream/master'

CHANGELOG.md

@@ -15,13 +15,18 @@ directory (`cd ~/eth2-docker` by default):
 * !! If coming from Prysm alpha.29 or earlier, make changes as per notes for [v0.1.7](#v017-2020-10-15)
 * `sudo docker-compose up -d eth2`
 
+## v0.1.8.6 2020-11-16
+
+* Nethermind added as eth1 option, thanks to adrienlac. Not stable in testing.
+* First attempt at binary option for all but eth2.0-deposit-cli
+
 ## v0.1.8.5 2020-11-11
 
 * Added option to run eth1 node exposed to the host on RPC port
 
 ## v0.1.8.4 2020-11-08
 
-* Updated grafana image to change all occurrences of `job="beacon"` to `jon=beacon_node` in the metanull dashboard.
+* Updated grafana image to change all occurrences of `job="beacon"` to `job=beacon_node` in the metanull dashboard.
 * Updated grafana image to rename prysm dashboard titles.
 
 ## v0.1.8.3 2020-11-07

README.md

@@ -1,4 +1,4 @@
-# eth2-docker v0.1.8.5
+# eth2-docker v0.1.8.6
 
 Unofficial and experimental docker build instructions for eth2 clients
 
@@ -10,8 +10,8 @@ Without their previous work, this project would not exist.
 
 ## Supported clients
 
-This project builds clients from official source repositories. A similar workflow for
-binary images is a TODO.
+This project can build from official docker images or from official source repositories. In most cases,
+binary is the default.
 
 Currently supported clients:
 - Lighthouse
@@ -21,8 +21,8 @@ Currently supported clients:
 
 Currently supported optional components:
 - geth, local eth1 node.
-- openethereum, local eth1 node
-- nethermind, local eth1 node
+- openethereum, local eth1 node - testing only, DB corruption observed on mainnet
+- nethermind, local eth1 node - testing only, issues with API calls
 > Use one of the local eth1 node options or a 3rd-party provider of eth1 chain data to "feed"
 > your eth2 beacon node, so you can [propose](https://ethos.dev/beacon-chain/) blocks.
 - slasher, Running slasher is optional, but helps secure the chain and may result in additional earnings.
@@ -111,7 +111,9 @@ Import the validator key(s) to the validator client:
 `sudo docker-compose run --rm validator-import`
 
 > #### Prysm-specific
-> - You will be asked to provide a wallet directory. Use `/var/lib/prysm`.
+> - You will be asked whether you will be using the Web UI to import keys.
+> Answer "y"es only if you are testing Prysm's experimental Web UI via
+> `prysm-web.yml`
 > - You will be asked to provide a "New wallet password", independent of the
 >   keystore password. 
 > - If you choose not to store the wallet password with the validator,

SETUP.md

@@ -48,8 +48,8 @@ Please choose:
   * Nimbus
 * Your source of eth1 data
   * geth
-  * openethereum (NB: **DB corruption observed**, use for testing only for now)
-  * nethermind
+  * openethereum - testing only, DB corruption observed
+  * nethermind - testing only, API calls not stable
   * 3rd-party
 * Whether to run a slasher (experimental for Prysm)
 * Whether to run a grafana dashboard for monitoring
@@ -98,8 +98,8 @@ openethereum with `:` between the file names.
 - `teku-base.yml` - Teku
 - `nimbus-base.yml` - Nimbus
 - `geth.yml` - local geth eth1 chain node
-- `oe.yml` - local openethereum eth1 chain node - for test use only, DB corruption was observed on the ethereum chain
-- 'nm.yml' - local nethermind eth1 chain node
+- `oe.yml` - local openethereum eth1 chain node - testing only, DB corruption observed
+- `nm.yml` - local nethermind eth1 chain node - testing only, API calls not stable
 - `shared-eth1.yml` - makes the RPC port of the eth1 node available from the host, for using the eth1 node with other nodes or with Metamask. **Not encrypted**, do not expose to Internet.
 - `prysm-slasher.yml` - Prysm experimental Slasher which helps secure the chain and may result in additional earnings. The experimental slasher can lead to missed attestations do to the additional resource demand.
 - `lh-grafana.yml` - grafana dashboard for Lighthouse

default.env

@@ -21,8 +21,8 @@ LOG_LEVEL=info
 # Change to empty (nothing after first =) to disable that functionality
 NIM_METRICS=NIMFLAGS="-d:insecure"
 # Network to use for eth2. If using main net, set to mainnet.
-NETWORK=medalla
-# Network to use for eth1. If using OpenEthereum and main net, set to ethereum.
+NETWORK=pyrmont
+# Network to use for eth1. If using OpenEthereum and main net, set to ethereum. For Nethermind and main net, set to mainnet. Not in use for Geth.
 ETH1_NETWORK=goerli
 # If using Geth as the eth1 node and main net, comment out.
 GETH1_NETWORK=--goerli
@@ -40,37 +40,72 @@ TEKU_PORT=9000
 GRAFANA_PORT=3000
 # ETH1 RPC port, important when using shared-eth1.yml
 ETH1_RPC_PORT=8545
+# ETH1 WS port, used with Nimbus
+ETH1_WS_PORT=8546
 # ETH1 RPC CORS open to all. Insecure but allows for easy testing
 ETH1_RPC_CORS=all
 # ETH1 RPC CORS, here for Metamask extension - was not functional in testing
 #ETH1_RPC_CORS="chrome-extension://nkbihfbeogaeaoehlefnkodbefgpgknn moz-extension://f3a66ea7-df4d-4731-80db-de45a4abebd6"
 
 
-# These variables are not likely to require adjustment
+# You can set specific version targets and choose
+# binary or source builds below. 
+# Doing so is entirely optional, and arguably advanced
+# use.
 
-GETH_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+LH_SRC_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+LH_DOCKER_TAG=v0.3.4 # eventually stable or stable-latest when sigp introduce that
+# Change to Dockerfile.source to build from source
+LH_DOCKERFILE=Dockerfile.binary
+LH_USER=lighthouse
+
+GETH_SRC_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+GETH_DOCKER_TAG=stable
+# Change to Dockerfile.source to build from source
+GETH_DOCKERFILE=Dockerfile.binary
 GETH_USER=geth
 
-# Build target dev until #74 has been pulled into main. At that point, switch to release tracking
-OE_BUILD_TARGET=dev
+# Build target dev/nightly until #74 has been pulled into main. At that point, switch to release tracking
+OE_SRC_BUILD_TARGET=dev
 #OE_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+OE_DOCKER_TAG=nightly
+#OE_DOCKER_TAG=latest
+# Change to Dockerfile.source to build from source
+OE_DOCKERFILE=Dockerfile.binary
+#Used only for source build; binary build defaults to this
 OE_USER=openethereum
 
-NM_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+NM_SRC_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+NM_DOCKER_TAG=latest
+# Change to Dockerfile.source to build from source
+NM_DOCKERFILE=Dockerfile.binary
 NM_USER=nethermind
 
-LH_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
-LH_USER=lighthouse
-
-PRYSM_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+PRYSM_SRC_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+PRYSM_DOCKER_TAG=stable
+# Change to Dockerfile.source to build from source
+PRYSM_DOCKERFILE=Dockerfile.binary
 PRYSM_USER=prysm
 
-TEKU_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+# Recent build failure when following release tag. Switch back to release tag when that has been resolved by Teku team
+TEKU_SRC_BUILD_TARGET=master
+#TEKU_SRC_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+TEKU_DOCKER_TAG=latest
+# Change to Dockerfile.source to build from source
+TEKU_DOCKERFILE=Dockerfile.binary
+#Used only for source build; binary build defaults to this
 TEKU_USER=teku
 
-NIM_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+NIM_SRC_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+NIM_DOCKER_TAG=devel # eventually stable or stable-latest when statusteam introduce that
+# Binary images at docker hub are not for public consumption and won't run. They'd need
+# to be built with NIMFLAGS="-d:disableMarchNative" to work. This can become binary
+# once statusteam create docker images meant for public use.
+NIM_DOCKERFILE=Dockerfile.source
 NIM_USER=nimbus
 
-DEPCLI_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+DEPCLI_SRC_BUILD_TARGET=$(git describe --tags $(git rev-list --tags --max-count=1))
+# Does not (yet?) exist on Docker hub
+DEPCLI_DOCKER_TAG=nonesuch
 DEPCLI_USER=depcli
 DEPCLI_LOCALDIR=./.eth2

geth.yml

@@ -1,17 +1,19 @@
-version: "3"
+version: "3.4"
 services:
   eth1:
     restart: "${RESTART}"
     user: ${LOCAL_UID}:${LOCAL_UID}
     build:
       context: ./geth
+      dockerfile: ${GETH_DOCKERFILE}
       args:
-        - BUILD_TARGET=${GETH_BUILD_TARGET} 
+        - BUILD_TARGET=${GETH_SRC_BUILD_TARGET} 
         - USER=${GETH_USER}
         - UID=${LOCAL_UID}
+        - DOCKER_TAG=${GETH_DOCKER_TAG}
     image: geth
     volumes:
-      - eth1-data:/var/lib/goethereum
+      - geth-eth1-data:/var/lib/goethereum
     ports:
       - ${ETH1_PORT}:${ETH1_PORT}/tcp
       - ${ETH1_PORT}:${ETH1_PORT}/udp
@@ -33,6 +35,14 @@ services:
       - ${ETH1_PORT}
       - --rpcport
       - ${ETH1_RPC_PORT}
+      - --ws
+      - --ws.addr
+      - 0.0.0.0
+      - --ws.port
+      - ${ETH1_WS_PORT}
+      - --ws.api
+      - eth,net
+      - --ws.origins=*
     command: ${GETH1_NETWORK}
   beacon:
     depends_on:
@@ -41,4 +51,4 @@ services:
     depends_on:
       - eth1
 volumes:
-  eth1-data:
+  geth-eth1-data:

geth/Dockerfile.binary

@@ -0,0 +1,28 @@
+ARG DOCKER_TAG
+
+FROM ethereum/client-go:${DOCKER_TAG}
+
+# Unused, this is here to avoid build time complaints
+ARG BUILD_TARGET
+
+ARG USER
+ARG UID
+
+# See https://stackoverflow.com/a/55757473/12429735RUN
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    "${USER}"
+
+RUN mkdir -p /var/lib/goethereum && chown ${USER}:${USER} /var/lib/goethereum
+
+# Use an unprivileged user.
+USER ${USER}:${USER}
+
+#EXPOSE 8545 30303 30303/udp
+ENTRYPOINT ["geth", "--nousb", "--http", "--http.addr", "0.0.0.0", "--http.vhosts", "*", "--http.api", "eth,net", "--ipcdisable", "--datadir", "/var/lib/goethereum"]
+CMD ["--goerli"]

geth/Dockerfile → geth/Dockerfile.source

@@ -1,6 +1,9 @@
 # Build Geth in a stock Go build container
 FROM golang:alpine as builder
 
+# Unused, this is here to avoid build time complaints
+ARG DOCKER_TAG
+
 ARG BUILD_TARGET
 ARG USER
 ARG UID

geth/README.md

@@ -1,6 +1,8 @@
 Docker container for go-ethereum geth
 Configured for goerli testnet, see CMD and override as desired
 
+Use Dockerfile.sourcebuild to compile from source
+
 This creates a statically compiled geth, in a scratch container, for minimal attack surface
 
 Pass BUILD_TARGET, USER and UID during build if you are not using docker-compose

lh-base.yml

@@ -1,14 +1,15 @@
-version: "3"
+version: "3.4"
 services:
   beacon:
     restart: "${RESTART}"
     user: ${LOCAL_UID}:${LOCAL_UID}
     build:
       context: ./lighthouse
       args:
-        - BUILD_TARGET=${LH_BUILD_TARGET}
+        - SRC_BUILD_TARGET=${LH_SRC_BUILD_TARGET}
         - USER=${LH_USER}
         - UID=${LOCAL_UID}
+        - DOCKER_TAG=${LH_DOCKER_TAG}
     image: lighthouse
     volumes:
       - lhbeacon-data:/var/lib/lighthouse
@@ -38,9 +39,10 @@ services:
     build:
       context: ./lighthouse
       args:
-        - BUILD_TARGET=${LH_BUILD_TARGET}
+        - SRC_BUILD_TARGET=${LH_SRC_BUILD_TARGET}
         - USER=${LH_USER}
         - UID=${LOCAL_UID}
+        - DOCKER_TAG=${LH_DOCKER_TAG}
     image: lighthouse
     volumes:
       - lhvalidator-data:/var/lib/lighthouse
@@ -65,9 +67,10 @@ services:
     build:
       context: ./lighthouse
       args:
-        - BUILD_TARGET=${LH_BUILD_TARGET}
+        - SRC_BUILD_TARGET=${LH_SRC_BUILD_TARGET}
         - USER=${LH_USER}
         - UID=${LOCAL_UID}
+        - DOCKER_TAG=${LH_DOCKER_TAG}
     volumes:
       - lhvalidator-data:/var/lib/lighthouse
       - ${DEPCLI_LOCALDIR}/validator_keys:/var/lib/lighthouse/validator_keys
@@ -91,9 +94,10 @@ services:
     build:
       context: ./lighthouse
       args:
-        - BUILD_TARGET=${LH_BUILD_TARGET}
+        - SRC_BUILD_TARGET=${LH_SRC_BUILD_TARGET}
         - USER=${LH_USER}
         - UID=${LOCAL_UID}
+        - DOCKER_TAG=${LH_DOCKER_TAG}
     volumes:
       - lhvalidator-data:/var/lib/lighthouse
       - ${DEPCLI_LOCALDIR}/validator_keys:/var/lib/lighthouse/validator_keys
@@ -119,9 +123,10 @@ services:
     build:
       context: ./lighthouse
       args:
-        - BUILD_TARGET=${LH_BUILD_TARGET}
+        - SRC_BUILD_TARGET=${LH_SRC_BUILD_TARGET}
         - USER=${LH_USER}
         - UID=${LOCAL_UID}
+        - DOCKER_TAG=${LH_DOCKER_TAG}
     volumes:
       - lhvalidator-data:/var/lib/lighthouse
       - ${DEPCLI_LOCALDIR}:/interchange
@@ -140,7 +145,7 @@ services:
     build:
       context: ./eth2.0-deposit-cli
       args:
-        - BUILD_TARGET=${DEPCLI_BUILD_TARGET}
+        - BUILD_TARGET=${DEPCLI_SRC_BUILD_TARGET}
         - USER=${DEPCLI_USER}
         - UID=${LOCAL_UID}
     image: eth2.0-deposit-cli
@@ -162,7 +167,7 @@ services:
     build:
       context: ./eth2.0-deposit-cli
       args:
-        - BUILD_TARGET=${DEPCLI_BUILD_TARGET}
+        - BUILD_TARGET=${DEPCLI_SRC_BUILD_TARGET}
         - USER=${DEPCLI_USER}
         - UID=${LOCAL_UID}
     image: eth2.0-deposit-cli

lh-grafana.yml

@@ -1,4 +1,4 @@
-version: "3"
+version: "3.4"
 services:
   beacon:
     expose:

lighthouse/Dockerfile.binary

@@ -0,0 +1,33 @@
+ARG DOCKER_TAG
+
+FROM sigp/lighthouse:${DOCKER_TAG}
+
+# Unused, this is here to avoid build time complaints
+ARG BUILD_TARGET
+
+ARG USER
+ARG UID
+
+# See https://stackoverflow.com/a/55757473/12429735RUN
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    "${USER}"
+
+# Create data mount point with permissions
+RUN mkdir -p /var/lib/lighthouse && chown ${USER}:${USER} /var/lib/lighthouse && chmod 700 /var/lib/lighthouse
+# Script to query whether all validator key passwords are the same
+COPY ./query-lh-validator-passwords.sh /usr/local/bin
+# Use an unprivileged user.
+USER ${USER}:${USER}
+
+# For voluntary exit
+ENV KEYSTORE=nonesuch
+
+#EXPOSE 5052 9000 9000/udp
+ENTRYPOINT ["lighthouse", "bn", "--datadir", "/var/lib/lighthouse/", "--http", "--http-address", "0.0.0.0"]
+CMD ["--testnet", "medalla", "--eth1-endpoint", "http://geth:8545"]