Teku custom testnet

lodestar-vc-only.yml

@@ -30,6 +30,7 @@ services:
       - GRAFFITI=${GRAFFITI:-}
       - DEFAULT_GRAFFITI=${DEFAULT_GRAFFITI:-false}
       - WEB3SIGNER=${WEB3SIGNER}
+      - NETWORK=${NETWORK}
     volumes:
       - lsvalidator-data:/var/lib/lodestar/validators
       - /etc/localtime:/etc/localtime:ro
@@ -61,8 +62,6 @@ services:
       - 0.0.0.0
       - --logLevel
       - ${LOG_LEVEL}
-      - --network
-      - ${NETWORK}
       - --suggestedFeeRecipient
       - ${FEE_RECIPIENT}
       - --force

lodestar.yml

@@ -35,6 +35,7 @@ services:
       - BEACON_STATS_MACHINE=${BEACON_STATS_MACHINE}
       - CL_EXTRAS=${CL_EXTRAS:-}
       - ARCHIVE_NODE=${ARCHIVE_NODE:-}
+      - NETWORK=${NETWORK}
     ports:
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp
@@ -68,8 +69,6 @@ services:
       - ${CL_MAX_PEER_COUNT:-50}
       - --logLevel
       - ${LOG_LEVEL}
-      - --network
-      - ${NETWORK}
       - --suggestedFeeRecipient
       - ${FEE_RECIPIENT}
 

lodestar/Dockerfile.binary

@@ -5,7 +5,7 @@ FROM chainsafe/lodestar:${DOCKER_TAG}
 # Here only to avoid build-time errors
 ARG BUILD_TARGET
 
-RUN apk update && apk add --no-cache ca-certificates tzdata bash su-exec && rm -rf /var/cache/apk/*
+RUN apk update && apk add --no-cache ca-certificates tzdata bash su-exec git && rm -rf /var/cache/apk/*
 
 ARG USER=lsconsensus
 ARG UID=10002

lodestar/Dockerfile.source

@@ -14,7 +14,7 @@ RUN bash -c "cd .. && rm -rf app && git clone https://github.com/ChainSafe/lodes
 
 FROM node:20-alpine
 
-RUN apk update && apk add --no-cache ca-certificates tzdata bash su-exec && rm -rf /var/cache/apk/*
+RUN apk update && apk add --no-cache ca-certificates tzdata bash su-exec git && rm -rf /var/cache/apk/*
 
 ARG USER=lsconsensus
 ARG UID=10002

lodestar/docker-entrypoint.sh

@@ -29,6 +29,32 @@ if [[ -O "/var/lib/lodestar/consensus/ee-secret/jwtsecret" ]]; then
   chmod 666 /var/lib/lodestar/consensus/ee-secret/jwtsecret
 fi
 
+if [[ "${NETWORK}" =~ ^https?:// ]]; then
+  echo "Custom testnet at ${NETWORK}"
+  repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
+  branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
+  config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
+  echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
+  # For want of something more amazing, let's just fail if git fails to pull this
+  set -e
+  if [ ! -d "/var/lib/lodestar/consensus/testnet/${config_dir}" ]; then
+    mkdir -p /var/lib/lodestar/consensus/testnet
+    cd /var/lib/lodestar/consensus/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+  fi
+  bootnodes="$(paste -s -d, "/var/lib/lodestar/consensus/testnet/${config_dir}/bootstrap_nodes.txt")"
+  set +e
+  __network="--paramsFile=/var/lib/lodestar/consensus/testnet/${config_dir}/config.yaml --genesisStateFile=/var/lib/lodestar/consensus/testnet/${config_dir}/genesis.ssz \
+--bootnodes=${bootnodes} --network.connectToDiscv5Bootnodes --chain.trustedSetup=/var/lib/lodestar/consensus/testnet/${config_dir}/trusted_setup.txt \
+--rest.namespace=*"
+else
+  __network="--network ${NETWORK}"
+fi
+
 # Check whether we should use MEV Boost
 if [ "${MEV_BOOST}" = "true" ]; then
   __mev_boost="--builder --builder.urls=${MEV_NODE:-http://mev-boost:18550}"
@@ -60,4 +86,4 @@ fi
 
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-exec "$@" ${__mev_boost} ${__beacon_stats} ${__rapid_sync} ${CL_EXTRAS}
+exec "$@" ${__network} ${__mev_boost} ${__beacon_stats} ${__rapid_sync} ${CL_EXTRAS}

nethermind.yml

@@ -25,6 +25,7 @@ services:
       - EL_EXTRAS=${EL_EXTRAS:-}
       - ARCHIVE_NODE=${ARCHIVE_NODE:-}
       - AUTOPRUNE_NM=${AUTOPRUNE_NM:-true}
+      - NETWORK=${NETWORK}
     volumes:
       - nm-eth1-data:/var/lib/nethermind
       - /etc/localtime:/etc/localtime:ro
@@ -67,8 +68,6 @@ services:
       - 0.0.0.0
       - --JsonRpc.EnginePort
       - ${EE_PORT:-8551}
-      - --JsonRpc.EnabledModules
-      - "Web3,Eth,Subscribe,Net,Health,Parity,Proof,Trace,TxPool"
       - --JsonRpc.AdditionalRpcUrls=http://127.0.0.1:1337|http|admin
       - --JsonRpc.JwtSecretFile=/var/lib/nethermind/ee-secret/jwtsecret
 # Temporary Prysm workaround. Remove when no longer needed

nethermind/Dockerfile.binary

@@ -27,7 +27,7 @@ RUN adduser \
 
 RUN set -eux; \
 	apt-get update; \
-	DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y ca-certificates gosu tzdata wget; \
+	DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y ca-certificates gosu tzdata wget git; \
 	rm -rf /var/lib/apt/lists/*; \
 # verify that the binary works
         gosu nobody true

nethermind/Dockerfile.dev

@@ -27,7 +27,7 @@ RUN adduser \
 
 RUN set -eux; \
 	apt-get update; \
-	DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y ca-certificates gosu tzdata; \
+	DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y ca-certificates gosu tzdata git; \
 	rm -rf /var/lib/apt/lists/*; \
 # verify that the binary works
         gosu nobody true

nethermind/Dockerfile.source

@@ -21,7 +21,7 @@ RUN bash -c "\
 
 FROM mcr.microsoft.com/dotnet/aspnet:7.0
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install libsnappy-dev libc6-dev libc6 ca-certificates gosu tzdata wget && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install libsnappy-dev libc6-dev libc6 ca-certificates gosu tzdata wget git && rm -rf /var/lib/apt/lists/*
 
 ARG USER=nethermind
 ARG UID=10001

nethermind/docker-entrypoint.sh

@@ -37,20 +37,23 @@ if [[ "${NETWORK}" =~ ^https?:// ]]; then
   branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
   config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
   echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
-  # For want of something more amazing, let's just fail if git fails to pull this
-  set -e
-  mkdir /var/lib/nethermind/testnet-conf
-  cd /var/lib/nethermind/testnet-conf
-  git init
-  git remote add origin "${repo}"
-  git config core.sparseCheckout true
-  echo "${config_dir}" > .git/info/sparse-checkout
-  git pull origin "${branch}"
-  set +e
-  bootnodes="$(paste -s -d, bootnode.txt)"
-  __network="--config none.cfg --Init.ChainSpecPath=/var/lib/nethermind/testnet-conf/chainspec.json --Discovery.Bootnodes=${bootnodes}"
+  if [ ! -d "/var/lib/nethermind/testnet/${config_dir}" ]; then
+    # For want of something more amazing, let's just fail if git fails to pull this
+    set -e
+    mkdir -p /var/lib/nethermind/testnet
+    cd /var/lib/nethermind/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+    set +e
+  fi
+  bootnodes="$(paste -s -d, "/var/lib/nethermind/testnet/${config_dir}/bootnode.txt")"
+  __network="--config none.cfg --Init.ChainSpecPath=/var/lib/nethermind/testnet/${config_dir}/chainspec.json --Discovery.Bootnodes=${bootnodes} \
+--JsonRpc.EnabledModules=Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin --Pruning.Mode=None --Init.IsMining=false"
 else
-  __network="--config ${NETWORK}"
+  __network="--config ${NETWORK} --JsonRpc.EnabledModules Web3,Eth,Subscribe,Net,Health,Parity,Proof,Trace,TxPool"
 fi
 
 __memtotal=$(awk '/MemTotal/ {printf "%d", int($2/1024/1024)}' /proc/meminfo)

teku-cl-only.yml

@@ -42,6 +42,7 @@ services:
       - DEFAULT_GRAFFITI=true
       - WEB3SIGNER=false
       - EMBEDDED_VC=false
+      - NETWORK=${NETWORK}
     ports:
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp
@@ -55,7 +56,6 @@ services:
       - /opt/teku/bin/teku
       - --data-path=/var/lib/teku
       - --log-destination=CONSOLE
-      - --network=${NETWORK}
       - --ee-endpoint=${EL_NODE}
       - --ee-jwt-secret-file
       - /var/lib/teku/ee-secret/jwtsecret

teku.yml

@@ -42,6 +42,7 @@ services:
       - DEFAULT_GRAFFITI=${DEFAULT_GRAFFITI:-false}
       - WEB3SIGNER=${WEB3SIGNER:-false}
       - EMBEDDED_VC=true
+      - NETWORK=${NETWORK}
     ports:
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp
       - ${HOST_IP:-}${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp
@@ -55,7 +56,6 @@ services:
       - /opt/teku/bin/teku
       - --data-path=/var/lib/teku
       - --log-destination=CONSOLE
-      - --network=${NETWORK}
       - --ee-endpoint=${EL_NODE}
       - --ee-jwt-secret-file
       - /var/lib/teku/ee-secret/jwtsecret

teku/Dockerfile.binary

@@ -14,7 +14,7 @@ RUN groupmod -g "${UID}" ${USER} && usermod -u "${UID}" -g "${UID}" ${USER}
 
 RUN set -eux; \
         apt-get update; \
-        DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y gosu ca-certificates tzdata; \
+        DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y gosu ca-certificates tzdata git; \
         rm -rf /var/lib/apt/lists/*; \
 # verify that the binary works
         gosu nobody true

teku/Dockerfile.source

@@ -27,6 +27,7 @@ ARG UID=10002
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y --no-install-recommends \
   ca-certificates \
   tzdata \
+  git \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 

teku/docker-entrypoint.sh

@@ -40,7 +40,7 @@ fi
 # Check whether we should rapid sync
 if [ -n "${RAPID_SYNC_URL:+x}" ]; then
     if [ "${ARCHIVE_NODE}" = "true" ]; then
-        echo "Besu archive node cannot use checkpoint sync: Syncing from genesis."
+        echo "Teku archive node cannot use checkpoint sync: Syncing from genesis."
         __rapid_sync=""
     else
         __rapid_sync="--initial-state=${RAPID_SYNC_URL}/eth/v2/debug/beacon/states/finalized"
@@ -50,6 +50,36 @@ else
     __rapid_sync=""
 fi
 
+if [[ "${NETWORK}" =~ ^https?:// ]]; then
+  echo "Custom testnet at ${NETWORK}"
+  repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
+  branch=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f1)
+  config_dir=$(awk -F'/tree/' '{print $2}' <<< "${NETWORK}" | cut -d'/' -f2-)
+  echo "This appears to be the ${repo} repo, branch ${branch} and config directory ${config_dir}."
+  # For want of something more amazing, let's just fail if git fails to pull this
+  set -e
+  if [ ! -d "/var/lib/teku/testnet/${config_dir}" ]; then
+    mkdir -p /var/lib/teku/testnet
+    cd /var/lib/teku/testnet
+    git init --initial-branch="${branch}"
+    git remote add origin "${repo}"
+    git config core.sparseCheckout true
+    echo "${config_dir}" > .git/info/sparse-checkout
+    git pull origin "${branch}"
+  fi
+  bootnodes="$(paste -s -d, "/var/lib/teku/testnet/${config_dir}/bootstrap_nodes.txt")"
+  set +e
+  if [ -z "${__rapid_sync}" ]; then
+    __rapid_sync="--initial-state=/var/lib/teku/testnet/${config_dir}/genesis.ssz"
+  fi
+  __network="--network=/var/lib/teku/testnet/${config_dir}/config.yaml --p2p-discovery-bootnodes=${bootnodes} \
+--data-storage-non-canonical-blocks-enabled=true --Xlog-include-p2p-warnings-enabled \
+--metrics-block-timing-tracking-enabled --Xmetrics-blob-sidecars-storage-enabled=true --Xtrusted-setup=/var/lib/teku/testnet/${config_dir}/trusted_setup.txt \
+--Xpeer-rate-limit=100000 --Xpeer-request-limit=1000"
+else
+  __network="--network=${NETWORK}"
+fi
+
 # Check whether we should use MEV Boost
 if [ "${MEV_BOOST}" = "true" ]; then
   __mev_boost="--validators-builder-registration-default-enabled --builder-endpoint=${MEV_NODE:-http://mev-boost:18550}"
@@ -75,7 +105,7 @@ else
 fi
 
 if [ "${ARCHIVE_NODE}" = "true" ]; then
-  echo "Besu archive node without pruning"
+  echo "Teku archive node without pruning"
   __prune="--data-storage-mode=ARCHIVE"
 else
   __prune="--data-storage-mode=MINIMAL"
@@ -100,9 +130,9 @@ fi
 if [ "${DEFAULT_GRAFFITI}" = "true" ]; then
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${CL_EXTRAS} ${VC_EXTRAS}
+  exec "$@" ${__network} ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${CL_EXTRAS} ${VC_EXTRAS}
 else
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-  exec "$@" "--validators-graffiti=${GRAFFITI}" ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${CL_EXTRAS} ${VC_EXTRAS}
+  exec "$@" ${__network} "--validators-graffiti=${GRAFFITI}" ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${CL_EXTRAS} ${VC_EXTRAS}
 fi