Add PGAdmin & SubQuery Node Hot-Reload & DevEx

* Refactor Docker files and scripts, update package.json and remove environment files
* Add PGAdmin services
* Add nodemon for hot-reloading and clean-up scripts
* Add warnings for hot-reload in production and improve Dockerfile
* Chore: update `cosmjs` submodule
* Chore: ensure `.env` file exists when building
* Refactor: use `error_log` function

---------

Co-authored-by: Bryan White <[email protected]>

.dockerignore

@@ -0,0 +1,25 @@
+# git
+.github
+.gitignore
+.gitmodules
+.git
+
+# root and vendor modules
+node_modules
+*/**/node_modules
+
+# generated code
+dist
+src/types
+project.yaml
+
+# dotenv files
+.env*
+
+# not needed
+docs
+.run
+*.md
+*/**/.md
+*/**/.png
+*/**/.jpg

.env.sample

@@ -0,0 +1,34 @@
+# Postgres
+POSTGRES_VERSION=16-alpine
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+
+# Pgadmin4
+# NOTE: if you modify POSTGRES_USER/PASSWORD/DB then auto-imported server will not match and you need to setup
+# your own with the modified values on the pgadmin website after first login.
+PGADMIN_VERSION=8.8
+PGADMIN_DEFAULT_EMAIL=[email protected]
+PGADMIN_DEFAULT_PASSWORD=admin
+PGADMIN_LISTEN_PORT=5050
+
+# Subquery Node
+NODE_ENV=develop
+# changing this value requires rebuilding the image
+WATCH=true
+# for development purposes is better reduce the amount of worker due to the number of logs that many
+# workers will produce, so if you will or need to move faster, you can safely increase this number to
+# any number that match with the number of CPUs
+WORKERS=1
+BATCH_SIZE=10
+DB_SCHEMA=app
+START_BLOCK=1
+CHAIN_ID=poktroll
+# LocalNet
+ENDPOINT=http://proxy:26657
+# Testnet
+# ENDPOINT=https://testnet-validated-validator-rpc.poktroll.com
+
+# Subquery Graphql Engine
+SUBQUERY_GRAPHQL_ENGINE_VERSION=latest
+SUBQUERY_GRAPHQL_ENGINE_PORT=3000

.gitignore

@@ -28,7 +28,7 @@ install-state.gz
 
 # Generated files
 target/
-dist/
+dist
 src/types
 project.yaml
 
@@ -58,8 +58,8 @@ Thumbs.db
 *.wmv
 
 .data
-dist
 
 # ENV local files
-.env.local
-.env.develop.local
+.env
+.env*
+!.env.sample

.run/Localdev.run.xml

@@ -0,0 +1,9 @@
+<!-- This file help anyone that uses IntelliJ IDEs to been able to run the project locally -->
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Localdev" type="NodeJSConfigurationType" application-parameters="-f=./ --db-schema=app --workers=4 --batch-size=30 --unfinalized-blocks=true" path-to-js-file="vendor/subql-cosmos/packages/node/bin/run" working-dir="$PROJECT_DIR$">
+    <envs>
+      <env name="NODE_ENV" value="develop" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>

README.md

@@ -28,42 +28,40 @@ yarn install
 
 ### 3. Generate types
 
-Types will need to be regenerated any time the graphql.schema is changed.
+Types will need to be regenerated any time the `graphql.schema` is changed.
 
 ```shell
 yarn run codegen
 ```
 
-### 4. Build
-
-The project will need to be rebuilt any time any TypeScript source files are changed, including when types are regenerated.
-
-_NB: Also runs `codegen`._
+### 4. Run
 
+Create a copy of `.env.sample` as `.env` which is required on the following commands.
+You can modify `.env` as you wish. IMPORTANT: if you change WATCH value, you need to rebuild the image.
 ```shell
-# Builds for poktroll testnet.
-yarn run build
-
-# OR
-# Builds for poktroll localnet.
-yarn run build:develop
+cp .env.sample .env
 ```
 
-### 5. Run locally
-
 Build & start:
 ```shell
+# Run this only if indexing poktroll localnet. This will allows subquery-node to connect with the poktroll validator
+# leave this open in a separated terminal
+yarn docker:tunnel
+
+# Then build docker and start
 yarn run docker:build
+# By default subquery-node has WATCH=true and NODE_ENV=develop 
+# which mean that any change to code/schema/dotenv files will reload it and will be using .env.develop file
 yarn run docker:start
-
-# If indexing poktroll localnet, after starting the docker container, run the
-# following to expose the poktroll validator to the subquery-node:
-yarn docker:tunnel
 ```
 
-Stop & clean up (delete postgres data):
+Stop (without deleted data)
 ```shell
 yarn run docker:stop
+```
+
+Or Stop & clean up (delete postgres data):
+```shell
 yarn run docker:clean
 ```
 
@@ -125,7 +123,7 @@ yarn db:dump:schema -- -t blocks -t messages
 ##### When `pg_dump` is insufficient
 
 In some cases, `pg_dump` may not export a relevant DB object; for example, enums.
-In these cases it is necessary to manually extract the relevant data from the DB and incorporate it into the initial_schema.sql file.
+In these cases, it is necessary to manually extract the relevant data from the DB and incorporate it into the initial_schema.sql file.
 
 In the case of **enums**, the following queries expose the relevant records, the results from which can be re-written as a COPY or INSERT statements into the respective tables from which they came:
 

docker-compose.yml

@@ -3,21 +3,41 @@ services:
     build:
       context: .
       dockerfile: ./docker/pg-Dockerfile
+      args:
+        POSTGRES_VERSION: ${POSTGRES_VERSION}
     ports:
       - 5432:5432
     volumes:
       - postgres_data:/var/lib/postgresql/data
     environment:
-      POSTGRES_PASSWORD: postgres
+      POSTGRES_PASSWORD: ${POSTGRES_USER}
+      POSTGRES_USER: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 5s
       retries: 5
 
-# Note: The proxy service is only necessary if you are running a Shannon
+  pgadmin:
+    image: dpage/pgadmin4:${PGADMIN_VERSION}
+    container_name: pgadmin
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD}
+      PGADMIN_LISTEN_PORT: 5050
+    volumes:
+      - ./docker/pgadmin/servers.json:/pgadmin4/servers.json
+    depends_on:
+      postgres:
+        condition: service_healthy
+    ports:
+      - ${PGADMIN_LISTEN_PORT}:5050
+
+# Note: The proxy service is only necessary if you're running a Shannon
 # localnet and are unable to use the docker network in bridge mode.
-# To use the proxy service you must run the following command on the host
+# To use the proxy service, you must run the following command on the host
 # to establish a reverse proxy such that docker services in the composition
 # will be able to port forward to the localnet services exposed on the host:
 # ssh -o StrictHostKeyChecking=no -N -R 26657:localhost:26657 proxyuser@localhost -p 2222
@@ -26,6 +46,8 @@ services:
       context: .
       dockerfile: ./docker/proxy.dockerfile
     environment:
+      # This values doesn't deserve the time and setup needed to use with .env.docker file
+      # they are referenced on package.json script which doesn't load .env.docker file
       - USER_NAME=proxyuser
       - USER_PASSWORD=proxypass
     ports:
@@ -37,24 +59,46 @@ services:
     build:
       context: .
       dockerfile: ./docker/node.dockerfile
+      args:
+        NODE_ENV: ${NODE_ENV:-production}
+        # anything that is not "true" will avoid start nodemon
+        # changing this value requires rebuilding the image
+        WATCH: ${WATCH:-false}
+        ENDPOINT: ${ENDPOINT}
+        CHAIN_ID: ${CHAIN_ID:-poktroll}
     depends_on:
       "postgres":
         condition: service_healthy
     restart: unless-stopped
     environment:
-      DB_USER: postgres
-      DB_PASS: postgres
-      DB_DATABASE: postgres
+      # database connection
+      DB_USER: ${POSTGRES_USER}
+      DB_PASS: ${POSTGRES_PASSWORD}
+      DB_DATABASE: ${POSTGRES_DB}
       DB_HOST: postgres
       DB_PORT: 5432
+      # node configuration
+      NODE_ENV: ${NODE_ENV:-production}
+      # changing this value requires rebuilding the image
+      WATCH: ${WATCH:-false}
+      WORKERS: ${WORKERS:-1}
+      BATCH_SIZE: ${BATCH_SIZE:-30}
+      DB_SCHEMA: ${DB_SCHEMA:-poktroll}
+      ### this below only avoids you need to rebuild because are injected on project.yaml with the entrypoint
+      ### if you're using WATCH=true they will be overridden by the rebuild process
+      START_BLOCK: ${START_BLOCK:-1}
+      ENDPOINT: ${ENDPOINT:-http://proxy:26657}
+      CHAIN_ID: ${CHAIN_ID:-poktroll}
+      # this will manage which dotenv file will be used on every reload
     volumes:
-      - ./:/app
+      # Replaced ./:/app with individual tracked files to avoid the generated code go outside the container
+      - ./schema.graphql:/home/app/schema.graphql
+      - ./project.ts:/home/app/project.ts
+      - ./proto:/home/app/proto
+      - ./src:/home/app/src
+      - ignore_types:/home/app/src/types
     command:
       - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests
-      - -f=/app
-      - --db-schema=app
-      - --workers=4
-      - --batch-size=30
       - --unfinalized-blocks=true
     healthcheck:
       test: ["CMD", "curl", "-f", "http://subquery-node:3000/ready"]
@@ -63,26 +107,30 @@ services:
       retries: 10
 
   graphql-engine:
-    image: subquerynetwork/subql-query:latest
+    image: subquerynetwork/subql-query:${SUBQUERY_GRAPHQL_ENGINE_VERSION}
     ports:
-      - 3000:3000
+      - ${SUBQUERY_GRAPHQL_ENGINE_PORT}:3000
     depends_on:
       "postgres":
         condition: service_healthy
       "subquery-node":
         condition: service_healthy
     restart: always
     environment:
-      DB_USER: postgres
-      DB_PASS: postgres
-      DB_DATABASE: postgres
+      DB_USER: ${POSTGRES_USER}
+      DB_PASS: ${POSTGRES_PASSWORD}
+      DB_DATABASE: ${POSTGRES_DB}
       DB_HOST: postgres
       DB_PORT: 5432
     command:
-      - --name=app
+      - --name=${DB_SCHEMA:-poktroll}
       - --playground
       - --indexer=http://subquery-node:3000
 
 volumes:
+  ignore_dist:
+    driver: local
+  ignore_types:
+    driver: local
   postgres_data:
-    driver: local
+    driver: local