Archive-Node-API support. (#6)

o1-labs · Oct 19, 2023 · 279d46f · 279d46f
1 parent 29b91dc
commit 279d46f
Show file tree

Hide file tree

Showing 7 changed files with 88 additions and 39 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -11,43 +11,27 @@ jobs:
       - name: Install and cache Hetzner Cloud CLI
         uses: 3bit/setup-hcloud@v2
       - name: Create mina-build-system cloud VM
-        run: hcloud server create --name mina-build-system --datacenter fsn1-dc14 --primary-ipv4 35577977 --image 121535411 --type cpx31 --ssh-key app@localhost --start-after-create
+        run: hcloud server create --name mina-build-system --datacenter fsn1-dc14 --primary-ipv4 35577977 --image 132107836 --type cpx31 --ssh-key app@localhost --start-after-create
         env:
           HCLOUD_TOKEN: ${{ secrets.HETZNER_API_TOKEN }}
       - name: Sleep for 120 seconds
         run: sleep 120s
         shell: bash
-      - name: Update remote repository
+      - name: Update remote repositories
         uses: appleboy/[email protected]
         with:
           host: ${{ secrets.MINA_BUILD_SYSTEM_IP }}
           username: ${{ secrets.MINA_BUILD_SYSTEM_SSH_USER_NAME }}
           key: ${{ secrets.MINA_BUILD_SYSTEM_SSH_PRIVATE_KEY }}
-          script: bash -ilc "cd ${HOME}/projects/o1labs/mina-lightnet-docker && git stash && git pull && git reset && git clean -f && git checkout . && git submodule sync && git submodule update --recursive --init"
-          command_timeout: 170m
-      - name: Update GoEnv
-        uses: appleboy/[email protected]
-        with:
-          host: ${{ secrets.MINA_BUILD_SYSTEM_IP }}
-          username: ${{ secrets.MINA_BUILD_SYSTEM_SSH_USER_NAME }}
-          key: ${{ secrets.MINA_BUILD_SYSTEM_SSH_PRIVATE_KEY }}
-          script: bash -ilc "cd ${HOME}/.goenv && git fetch --all && git pull"
-          command_timeout: 170m
-      - name: Install Go 1.19.12
-        uses: appleboy/[email protected]
-        with:
-          host: ${{ secrets.MINA_BUILD_SYSTEM_IP }}
-          username: ${{ secrets.MINA_BUILD_SYSTEM_SSH_USER_NAME }}
-          key: ${{ secrets.MINA_BUILD_SYSTEM_SSH_PRIVATE_KEY }}
-          script: bash -ilc "goenv install --skip-existing 1.19.12 && goenv global 1.19.12"
+          script: bash -ilc "${HOME}/projects/git-pull-all.sh"
           command_timeout: 170m
       - name: Build and publish Docker images (major branches)
         uses: appleboy/[email protected]
         with:
           host: ${{ secrets.MINA_BUILD_SYSTEM_IP }}
           username: ${{ secrets.MINA_BUILD_SYSTEM_SSH_USER_NAME }}
           key: ${{ secrets.MINA_BUILD_SYSTEM_SSH_PRIVATE_KEY }}
-          script: bash -ilc "${HOME}/projects/o1labs/mina-lightnet-docker/scripts/build-all.sh ${HOME}/projects/o1labs/mina ${HOME}/projects/o1labs/mina-lightnet-docker ${HOME}/projects/p42/mina-accounts-manager o1labs \"develop berkeley rampup\""
+          script: bash -ilc "${HOME}/projects/o1labs/mina-lightnet-docker/scripts/build-all.sh ${HOME}/projects/o1labs/mina ${HOME}/projects/o1labs/Archive-Node-API ${HOME}/projects/o1labs/mina-lightnet-docker ${HOME}/projects/p42/mina-accounts-manager o1labs \"develop berkeley rampup\""
           command_timeout: 170m
       - name: Sleep for 15 seconds
         run: sleep 15s

diff --git a/DockerHub.md b/DockerHub.md
@@ -27,6 +27,8 @@ Connection to the container's PostgreSQL can be used for other needs, including
   postgresql://postgres:postgres@localhost:5432/archive
   ```
 
+If no Mina Archive Node will run inside the container, then [Archive-Node-API](https://github.com/o1-labs/Archive-Node-API) won't be available.
+
 ## Mina accounts manager
 
 The Mina accounts manager helper tool, provided with the Docker images, automates how users retrieve account information.  
@@ -46,7 +48,7 @@ Before the Mina network can be used in your job or jobs steps, it must reach the
 ### Single Node
 
 ```shell
-docker run -it --env NETWORK_TYPE="single-node" --env PROOF_LEVEL="none" -p 3085:3085 -p 5432:5432 -p 8080:8080 -p 8181:8181 o1labs/mina-local-network:rampup-latest-lightnet
+docker run -it --env NETWORK_TYPE="single-node" --env PROOF_LEVEL="none" -p 3085:3085 -p 5432:5432 -p 8080:8080 -p 8181:8181 -p 8282:8282 o1labs/mina-local-network:rampup-latest-lightnet
 ```
 
 #### Single Node network properties
@@ -58,7 +60,7 @@ docker run -it --env NETWORK_TYPE="single-node" --env PROOF_LEVEL="none" -p 3085
 - ~815-850 MB of RAM consumption after initial spike and if stays alive during less than 2 hours ~= 1/2 epoch
 - The startup and sync time is ~1-2 minutes
 
-#### Single Node Logs
+#### Single Node logs
 
 By default, logs produced by the Mina processes will be redirected into the files located by the following path pattern inside the container:
 
@@ -74,7 +76,7 @@ docker run ... --mount "type=bind,source=/tmp,dst=/root/logs" ...
 
 It is especially useful if you want to keep the logs after the container is stopped and deleted. For example when used in CI/CD pipelines.
 
-[GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idservices) example:
+#### [GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idservices) example
 
 ```yaml
 ...
@@ -110,10 +112,18 @@ jobs:
           retention-days: 5
 ```
 
+#### Single Node ports reference
+
+- **3085**: Mina Daemon GraphQL endpoint
+- **5432**: PostgreSQL RDBMS
+- **8080**: NGINX reverse proxy against Mina Daemon GraphQL endpoint
+- **8181**: Mina Accounts Manager
+- **8282**: Archive-Node-API
+
 ### Multi-Node
 
 ```shell
-docker run -it --env NETWORK_TYPE="multi-node" --env PROOF_LEVEL="none" -p 4001:4001 -p 4006:4006 -p 5001:5001 -p 5432:5432 -p 6001:6001 -p 8080:8080 -p 8181:8181 o1labs/mina-local-network:rampup-latest-lightnet
+docker run -it --env NETWORK_TYPE="multi-node" --env PROOF_LEVEL="none" -p 4001:4001 -p 4006:4006 -p 5001:5001 -p 5432:5432 -p 6001:6001 -p 8080:8080 -p 8181:8181 -p 8282:8282 o1labs/mina-local-network:rampup-latest-lightnet
 ```
 
 #### Multi-Node network properties
@@ -125,14 +135,25 @@ docker run -it --env NETWORK_TYPE="multi-node" --env PROOF_LEVEL="none" -p 4001:
 - ~5.5-6+ GB of RAM consumption
 - The startup and sync time is ~4-6 minutes
 
-#### Multi-Node Logs
+#### Multi-Node logs
 
 By default, logs produced by the Mina processes will be redirected into the files located by the following path pattern inside the container:
 
 ```shell
 /root/mina-local-network-2-1-1/nodes/**/logs/*.log
 ```
 
+#### Multi-Node ports reference
+
+- **4001**: Whale #1 Mina Daemon GraphQL endpoint
+- **4006**: Whale #2 Mina Daemon GraphQL endpoint
+- **5001**: Fish #1 Mina Daemon GraphQL endpoint
+- **5432**: PostgreSQL RDBMS
+- **6001**: Follower #1 Mina Daemon GraphQL endpoint
+- **8080**: NGINX reverse proxy against Whale #2 Mina Daemon GraphQL endpoint
+- **8181**: Mina Accounts Manager
+- **8282**: Archive-Node-API
+
 ## Image tags anatomy
 
 Several image tags are available for download, like:

diff --git a/configuration/Dockerfile b/configuration/Dockerfile
@@ -2,6 +2,17 @@ FROM postgres:14-bookworm
 
 RUN apt update && apt install -y --no-install-recommends python3 python3-pip jq wget curl psmisc libpq-dev libjemalloc-dev libffi-dev nginx netcat-traditional && apt clean && rm -rf /var/lib/apt/lists/*
 
+ENV NODE_VERSION=20.8.1
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+ENV NVM_DIR=/root/.nvm
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && npm install -g npm
+ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
+RUN node --version
+RUN npm --version
+
 COPY mina-local-network-2-1-1/ /root/.mina-network/mina-local-network-2-1-1/
 COPY nginx.conf /root/
 COPY spinup-testnet.sh /root/
@@ -12,6 +23,7 @@ COPY zkapp_tables.sql /root/
 COPY mina.exe /root/
 COPY archive.exe /root/
 COPY libp2p_helper /root/
+COPY Archive-Node-API /root/
 # COPY accounts-manager /root/
 
 WORKDIR /root
@@ -31,6 +43,7 @@ EXPOSE 5432
 EXPOSE 6001
 EXPOSE 8080
 # EXPOSE 8181
+EXPOSE 8282
 
 ENV RUN_ARCHIVE_NODE="true"
 ENV NETWORK_TYPE="single-node"

diff --git a/configuration/archive-node-api.env b/configuration/archive-node-api.env
@@ -0,0 +1,12 @@
+PORT=8282
+CORS_ORIGIN="*"
+
+PG_CONN="postgresql://postgres:postgres@localhost:5432/archive"
+
+ENABLE_GRAPHIQL="true"
+ENABLE_INTROSPECTION="true"
+
+ENABLE_LOGGING="true"
+_ENABLE_JAEGER="false"
+JAEGER_SERVICE_NAME="archive-api"
+JAEGER_ENDPOINT='http://localhost:14268/api/traces'
diff --git a/scripts/build-all.sh b/scripts/build-all.sh
@@ -3,8 +3,8 @@
 # Exit script when commands fail
 # set -e
 
-if [[ $# -lt 5 ]]; then
-  echo "Usage: $0 <Mina repository root path> <Docker image building scripts repository root path> <Mina-Accounts-Manager repository root path> <Docker Hub user name> <Target branches>"
+if [[ $# -lt 6 ]]; then
+  echo "Usage: $0 <Mina repository root path> <Archive-Node-API repository root path> <Docker image building scripts repository root path> <Mina-Accounts-Manager repository root path> <Docker Hub user name> <Target branches>"
   exit 1
 fi
 
@@ -14,10 +14,11 @@ START=$(date +%s)
 CURRENT_DIR="$(pwd)"
 
 MINA_REPO_DIR=${1}
-DOCKER_IMAGE_BUILDING_SCRIPTS_REPO_DIR=${2}
-MINA_ACCOUNTS_MANAGER_REPO_DIR=${3}
-DOCKER_HUB_USER_NAME=${4}
-TARGET_BRANCHES=(${5})
+ARCHIVE_NODE_API_REPO_DIR=${2}
+DOCKER_IMAGE_BUILDING_SCRIPTS_REPO_DIR=${3}
+MINA_ACCOUNTS_MANAGER_REPO_DIR=${4}
+DOCKER_HUB_USER_NAME=${5}
+TARGET_BRANCHES=(${6})
 
 cd ${MINA_REPO_DIR}
 cd ../
@@ -95,14 +96,14 @@ for TARGET_BRANCH in "${TARGET_BRANCHES[@]}"; do
   echo ""
   buildMina "devnet" false
   cd ${DOCKER_IMAGE_BUILDING_SCRIPTS_REPO_DIR}
-  ./scripts/build-image.sh ${HOME}/projects/o1labs/mina full ${DOCKER_HUB_USER_NAME} ${BRANCH_NAME}-latest-devnet ${MINA_ACCOUNTS_MANAGER_REPO_DIR}/build/native/nativeCompile/accounts-manager
+  ./scripts/build-image.sh ${MINA_REPO_DIR} ${ARCHIVE_NODE_API_REPO_DIR} full ${DOCKER_HUB_USER_NAME} ${BRANCH_NAME}-latest-devnet ${MINA_ACCOUNTS_MANAGER_REPO_DIR}/build/native/nativeCompile/accounts-manager
   gitPullAll && gitPullAll
   echo ""
   echo "[INFO] For Lightnet dune profile..."
   echo ""
   buildMina "lightnet" false
   cd ${DOCKER_IMAGE_BUILDING_SCRIPTS_REPO_DIR}
-  ./scripts/build-image.sh ${HOME}/projects/o1labs/mina none ${DOCKER_HUB_USER_NAME} ${BRANCH_NAME}-latest-lightnet ${MINA_ACCOUNTS_MANAGER_REPO_DIR}/build/native/nativeCompile/accounts-manager
+  ./scripts/build-image.sh ${MINA_REPO_DIR} ${ARCHIVE_NODE_API_REPO_DIR} none ${DOCKER_HUB_USER_NAME} ${BRANCH_NAME}-latest-lightnet ${MINA_ACCOUNTS_MANAGER_REPO_DIR}/build/native/nativeCompile/accounts-manager
 done
 
 END=$(date +%s)

diff --git a/scripts/build-image.sh b/scripts/build-image.sh
@@ -3,7 +3,7 @@
 # Exit script when commands fail
 set -e
 
-if [[ $# -lt 4 ]]; then
+if [[ $# -lt 5 ]]; then
   echo "Usage: $0 <Mina repository root path> <Proof level> <Docker Hub user name> <Docker Hub image tag> [<Accounts-Manager binary path>]"
   exit 1
 fi
@@ -12,14 +12,16 @@ START=$(date +%s)
 CURRENT_DIR="$(pwd)"
 
 MINA_REPO_DIR=${1}
-PROOF_LEVEL=${2}
-DOCKER_HUB_USER_NAME=${3}
-DOCKER_HUB_IMAGE_TAG=${4}
+ARCHIVE_NODE_API_REPO_DIR=${2}
+PROOF_LEVEL=${3}
+DOCKER_HUB_USER_NAME=${4}
+DOCKER_HUB_IMAGE_TAG=${5}
 TMP_FOLDER=$(mktemp -d)
 KEYS_LOCATION_TARGETS=(${TMP_FOLDER}/mina-local-network-2-1-1/nodes/fish_0/wallets/store/ ${TMP_FOLDER}/mina-local-network-2-1-1/nodes/node_0/wallets/store/ ${TMP_FOLDER}/mina-local-network-2-1-1/nodes/seed/wallets/store/ ${TMP_FOLDER}/mina-local-network-2-1-1/nodes/snark_coordinator/wallets/store/ ${TMP_FOLDER}/mina-local-network-2-1-1/nodes/snark_workers/worker_0/wallets/store/ ${TMP_FOLDER}/mina-local-network-2-1-1/nodes/whale_0/wallets/store/ ${TMP_FOLDER}/mina-local-network-2-1-1/nodes/whale_1/wallets/store/)
 
 echo ""
 echo "[INFO] Mina repository root: ${MINA_REPO_DIR}"
+echo "[INFO] Archive-Node-API repository root: ${ARCHIVE_NODE_API_REPO_DIR}"
 echo "[INFO] Proof level: ${PROOF_LEVEL}"
 echo "[INFO] Docker Hub user name: ${DOCKER_HUB_USER_NAME}"
 echo "[INFO] Docker Hub image tag: ${DOCKER_HUB_IMAGE_TAG}"
@@ -43,8 +45,10 @@ cp -r ${MINA_REPO_DIR}/src/app/libp2p_helper/result/bin/libp2p_helper ${TMP_FOLD
 cp -r ${MINA_REPO_DIR}/_build/default/src/app/archive/archive.exe ${TMP_FOLDER}/
 # cp -r ${MINA_REPO_DIR}/_build/default/src/app/logproc/logproc.exe ${TMP_FOLDER}/
 # cp -r ${MINA_REPO_DIR}/_build/default/src/app/zkapp_test_transaction/zkapp_test_transaction.exe ${TMP_FOLDER}/
-if [[ $# -eq 5 ]]; then
-  cp -r ${5} ${TMP_FOLDER}/
+cp -r ${ARCHIVE_NODE_API_REPO_DIR} ${TMP_FOLDER}
+cp -r ./configuration/archive-node-api.env ${TMP_FOLDER}/Archive-Node-API/.env
+if [[ $# -eq 6 ]]; then
+  cp -r ${6} ${TMP_FOLDER}/
   perl -i -p -e 's~# COPY accounts-manager /root/~COPY accounts-manager /root/~g' ${TMP_FOLDER}/Dockerfile
   perl -i -p -e 's~# RUN chmod \+x accounts-manager~RUN chmod \+x accounts-manager~g' ${TMP_FOLDER}/Dockerfile
   perl -i -p -e 's~# EXPOSE 8181~EXPOSE 8181~g' ${TMP_FOLDER}/Dockerfile

diff --git a/scripts/spinup-testnet.sh b/scripts/spinup-testnet.sh
@@ -8,6 +8,7 @@ trap "killall background" EXIT
 
 ARCHIVE_NODE_PORT=3086
 RDBMS_PORT=5432
+ARCHIVE_NODE_API_PORT=8282
 LEDGER_FOLDER="$(pwd)/.mina-network/mina-local-network-2-1-1"
 GENESIS_LEDGER_CONFIG_FILE=${LEDGER_FOLDER}/daemon.json
 ACCOUNTS_MANAGER_EXE="$(pwd)/accounts-manager"
@@ -36,6 +37,18 @@ prepare-rdbms() {
   echo ""
 }
 
+build-start-archive-node-api() {
+  echo ""
+  echo "Building and starting the Archive-Node-API service..."
+  echo ""
+  cd ${HOME}/Archive-Node-API/
+  rm -rf node_modules/ && npm install && npm run build
+  npm run start &
+  cd ../
+  wait-for-service ${ARCHIVE_NODE_API_PORT}
+  echo ""
+}
+
 nginx-reload() {
   GRAPHQL_PORT=${1}
 
@@ -55,6 +68,7 @@ done
 
 if [[ $RUN_ARCHIVE_NODE == "true" ]]; then
   prepare-rdbms
+  build-start-archive-node-api
 fi
 
 if [ -f "${ACCOUNTS_MANAGER_EXE}" ]; then