HARMONY-1583: Add easy start/stop of services for dev (#456)

* HARMONY-1583: Add easy start/stop of services for dev * HARMONY-1583: Add logs directories to git repo
nasa · Sep 8, 2023 · 8ae90dc · 8ae90dc
1 parent bc2fd22
commit 8ae90dc
Show file tree

Hide file tree

Showing 20 changed files with 141 additions and 35 deletions.
diff --git a/app/routers/router.ts b/app/routers/router.ts
@@ -263,6 +263,11 @@ export default function router({ skipEarthdataLogin = 'false' }: RouterConfig):
   result.get('/stac/:jobId', asyncHandler(getStacCatalog));
   result.get('/stac/:jobId/:itemIndex', asyncHandler(getStacItem));
 
+  // Kubernetes readiness probe for Harmony in a Box
+  result.get('/readiness', async (_req, res, _next: Function): Promise<void> => {
+    res.send('OK');
+  });
+
   result.get('/*', () => { throw new NotFoundError('The requested page was not found.'); });
   result.post('/*', () => { throw new NotFoundError('The requested POST page was not found.'); });
   return result;

diff --git a/bin/bootstrap-harmony b/bin/bootstrap-harmony
@@ -23,6 +23,9 @@ else
     source .env
 fi
 
+# Used to decide whether or not to run harmony in k8s
+export LOCAL_DEV
+
 # run localstack, postgresql, and harmony in kubernetes
 ./bin/start-all
 

diff --git a/bin/deploy-services b/bin/deploy-services
@@ -19,30 +19,33 @@ fi
 
 . ./bin/create-k8s-config-maps-and-secrets
 
-# create the work scheduler
-if [ "$USE_SERVICE_QUEUES" = true ]; then
-  file="kubernetes-services/work-scheduler/config/service-template.yaml"
+if [ ! "$LOCAL_DEV" = true ]; then
+
+  # create the work scheduler
+  if [ "$USE_SERVICE_QUEUES" = true ]; then
+    file="kubernetes-services/work-scheduler/config/service-template.yaml"
+    if [ ! -f "$file" ]; then
+      echo "work scheduler template was not found."
+      exit 1
+    fi
+    envsubst < $file | kubectl apply -f - -n harmony
+  fi
+
+  # create the work updaters
+  file="kubernetes-services/work-updater/config/service-template-large.yaml"
   if [ ! -f "$file" ]; then
-    echo "work scheduler template was not found."
+    echo "large work updater template was not found."
+    exit 1
+  fi
+  envsubst < $file | kubectl apply -f - -n harmony
+  file="kubernetes-services/work-updater/config/service-template-small.yaml"
+  if [ ! -f "$file" ]; then
+    echo "small work updater template was not found."
     exit 1
   fi
   envsubst < $file | kubectl apply -f - -n harmony
 fi
 
-# create the work updaters
-file="kubernetes-services/work-updater/config/service-template-large.yaml"
-if [ ! -f "$file" ]; then
-  echo "large work updater template was not found."
-  exit 1
-fi
-envsubst < $file | kubectl apply -f - -n harmony
-file="kubernetes-services/work-updater/config/service-template-small.yaml"
-if [ ! -f "$file" ]; then
-  echo "small work updater template was not found."
-  exit 1
-fi
-envsubst < $file | kubectl apply -f - -n harmony
-
 export LOCALSTACK_HOST=localhost
 
 # create the query-cmr service

diff --git a/bin/start-all b/bin/start-all
@@ -3,7 +3,10 @@
 # start postgress, localstack, and harmony in the harmony namespace in the local kubernetes cluster
 
 ./bin/start-postgres-localstack
-./bin/start-harmony
-echo ''
-echo 'Run `bin/reload-services-config` to restart harmony.'
-echo 'Run `kubectl delete namespace harmony` to remove all local resources'
+if [ ! "$LOCAL_DEV" = true ]; then
+  echo 'Running harmony in kubernetes'
+  ./bin/start-harmony
+  echo 'Run `bin/reload-services-config` to restart harmony.'
+fi
+
+echo 'Run `./bin/stop-harmony-and-services` to remove all local resources'
diff --git a/bin/start-dev-services b/bin/start-dev-services
@@ -0,0 +1,42 @@
+#!/bin/bash
+# create internal services for Harmony development
+
+env_save=$(export -p)
+set -a
+source "lib/util/env-defaults"
+source "env-defaults"
+source ".env"
+set +a
+eval "$env_save"
+
+# run harmony
+echo "Starting Harmony server..."
+npm run start-dev-fast > logs/server.log 2>&1 &
+
+# create the work scheduler
+echo "Starting the work scheduler..."
+pushd kubernetes-services/work-scheduler > /dev/null
+export PORT=5001  
+npm run start-dev-fast > logs/work-scheduler.log 2>&1 &
+popd > /dev/null
+
+sleep 5
+
+# create the work updaters
+export LOCALSTACK_HOST=localhost
+echo "Starting the large job work updater..."
+pushd kubernetes-services/work-updater > /dev/null
+export PORT=5002
+export DEBUG_PORT=9202
+export WORK_ITEM_UPDATE_QUEUE_TYPE=large
+npm run start-dev-fast > logs/work-update-large.log 2>&1 &
+
+sleep 5
+
+echo "Starting the small job work updater..."
+export PORT=5003
+export DEBUG_PORT=9203
+export WORK_ITEM_UPDATE_QUEUE_TYPE=small
+npm run start-dev-fast > logs/work-update-small.log 2>&1 &
+
+popd > /dev/null
diff --git a/bin/start-harmony b/bin/start-harmony
@@ -11,6 +11,8 @@ set +a
 eval "$env_save"
 
 envsubst < ./config/harmony-k8s.yaml | kubectl apply -f - -n harmony > /dev/null
+# harmony takes a while to start up, so we will do other things before we try to set up
+# port forwarding
 bin/port-forward start harmony 3000:3000
 
 harmony_pod=$(kubectl get pods -n harmony -l app=harmony | grep -v NAME | awk '{print $1;}')

diff --git a/bin/stop-dev-services b/bin/stop-dev-services
@@ -0,0 +1,10 @@
+#!/bin/bash
+# stop any services running in development mode
+
+for DEBUG_PORT in 9200 9201 9202 9203
+do
+  for PID in $( ps -ef | grep node | grep "inspect=127.0.0.1:$DEBUG_PORT" | awk '{print $2}')
+  do
+    kill -9 $PID
+  done
+done
diff --git a/bin/stop-harmony-and-services b/bin/stop-harmony-and-services
@@ -12,4 +12,13 @@ fi
 bin/port-forward stop harmony
 bin/port-forward stop localstack
 bin/port-forward stop postgres
-kubectl delete ns harmony
+kubectl delete ns harmony
+
+# stop any services running in development mode
+for DEBUG_PORT in 9200 9201 9202 9203
+do
+  for PID in $( ps -ef | grep node | grep "inspect=127.0.0.1:$DEBUG_PORT" | awk '{print $2}')
+  do
+    kill -9 $PID
+  done
+done
diff --git a/bin/update-harmony b/bin/update-harmony
@@ -77,8 +77,8 @@ while read -r line; do
   all_images+=( "$line" )
 done < <(sort <(echo "${referenced_images[*]}") <(echo "${pulled_images[*]}") | uniq -d)
 
-# always reload the harmony image, query-cmr and service-runner images
-all_images+=( "harmonyservices/harmony:latest" "harmonyservices/query-cmr:latest" "harmonyservices/service-runner:latest" )
+# always reload the harmony image, query-cmr, work-updater, work-scheudler, and service-runner images
+all_images+=( "harmonyservices/harmony:latest" "harmonyservices/query-cmr:latest" "harmonyservices/service-runner:latest" "harmonyservices/work-updater:latest" "harmony-services/work-scheduler:latest" )
 
 for image in ${all_images[@]}; do
   echo "${image}"

diff --git a/config/harmony-k8s.yaml b/config/harmony-k8s.yaml
@@ -28,6 +28,10 @@ spec:
           ports:
             - containerPort: 3000
             - containerPort: 3001
+          readinessProbe:
+            httpGet:
+              path: /readiness
+              port: 3000
 ---
 apiVersion: v1
 kind: Service

diff --git a/docs/guides/develop.md b/docs/guides/develop.md
@@ -116,6 +116,8 @@ WORK_ITEM_UPDATE_QUEUE_URL=http://localhost:4566/queue/work-item-update-queue
 LARGE_WORK_ITEM_UPDATE_QUEUE_URL=http://localhost:4566/queue/large-work-item-update-queue
 BACKEND_HOST=host.docker.internal
 CALLBACK_URL_ROOT=http://host.docker.internal:3001
+LOCAL_DEV=true
+
 ```
 
 Linux
@@ -125,6 +127,7 @@ WORK_ITEM_UPDATE_QUEUE_URL=http://localhost:4566/queue/work-item-update-queue
 LARGE_WORK_ITEM_UPDATE_QUEUE_URL=http://localhost:4566/queue/large-work-item-update-queue
 BACKEND_HOST=localhost
 CALLBACK_URL_ROOT=http://localhost:3001
+LOCAL_DEV=true
 ```
 
 ### (minikube only) Configuring the callback URL for backend services
@@ -137,21 +140,44 @@ minikube ssh grep host.minikube.internal /etc/hosts | cut -f1
 
 This should print out an IP address. Use this in your .env file to specify the `CALLBACK_URL_ROOT` value, e.g., `CALLBACK_URL_ROOT=http://192.168.65.2:4001`.
 
-### Set Up and Run Postgres and Localstack
+## Run Harmony and Services
 
-In development Harmony uses [Localstack](https://github.com/localstack/localstack) to avoid allocating AWS resources. Postgres is also installed (but not used by default).
+Harmony and the services can be run using the following:
 
 ```
-$ ./bin/start-postgres-localstack
+./bin/bootstrap-harmony
+./bin/start-dev-services
 ```
 
-This will install Postgres and Localstack and forward their ports to localhost. It will take a few minutes the first time you run it. You will know when it has completed when it prints
+NOTE: You must set `LOCAL_DEV=true` before running these to prevent `bootstrap-harmony` from
+starting harmony and its support services in kubernetes.
+
+The provider services along with postgresql and localstack will now be running in kubernetes,
+while Harmony and its support services will be running as local Node.js processes. Each process
+has a specific port and debug port as shown in the following table:
+
+| Process              | Port | Debug Port |
+|----------------------|------|------------|
+| harmony              | 3000 | 9200       |
+| work-scheduler       | 5001 | 9201       |
+| work-updater (large) | 5002 | 9202       |
+| work-updater (small) | 5003 | 9203       |
+
+## Stopping Harmony and Services
 
+The services running in kubernetes can be stopped using the following (this will also delete 
+the `harmony` namespace):
+
+```
+./bin/stop-harmony-and-services
+```  
+
+The Node.js processes for Harmony and its support services can be stopped using the following:
 ```
-Localstack has started at http://localhost:4566/
-Postgres has started at localhost:5432
+./bin/stop-dev-services
 ```
 
+
 ## Add A Service
 
 Clone the Harmony service example repository into a peer directory of the main Harmony repo

diff --git a/kubernetes-services/work-scheduler/logs/.gitkeep b/kubernetes-services/work-scheduler/logs/.gitkeep
diff --git a/kubernetes-services/work-scheduler/package.json b/kubernetes-services/work-scheduler/package.json
@@ -6,7 +6,7 @@
   "scripts": {
     "start": "ts-node app/server.ts",
     "start-dev": "strict-npm-engines && ts-node-dev --no-notify -r tsconfig-paths/register --watch app/views --respawn app/server",
-    "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:9231 app/server",
+    "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:${DEBUG_PORT:-9201} app/server",
     "test": "eslint --ext .ts . && nyc mocha && better-npm-audit audit",
     "test-fast": "TS_NODE_TRANSPILE_ONLY=true mocha",
     "lint": "eslint --ext .ts .",

diff --git a/kubernetes-services/work-updater/logs/.gitkeep b/kubernetes-services/work-updater/logs/.gitkeep
diff --git a/kubernetes-services/work-updater/package.json b/kubernetes-services/work-updater/package.json
@@ -6,7 +6,7 @@
   "scripts": {
     "start": "ts-node app/server.ts",
     "start-dev": "strict-npm-engines && ts-node-dev --no-notify -r tsconfig-paths/register --watch app/views --respawn app/server",
-    "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:9231 app/server",
+    "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:${DEBUG_PORT:-9202} app/server",
     "test": "eslint --ext .ts . && nyc mocha && better-npm-audit audit",
     "test-fast": "TS_NODE_TRANSPILE_ONLY=true mocha",
     "lint": "eslint --ext .ts .",

diff --git a/logs/.gitkeep b/logs/.gitkeep
diff --git a/package.json b/package.json
@@ -10,14 +10,13 @@
     "coverage": "nyc mocha",
     "start": "NODE_OPTIONS=--max-old-space-size=3096 ts-node -r tsconfig-paths/register --require './app/tracing.ts' app/server.ts",
     "start-dev": "strict-npm-engines && ts-node-dev --no-notify -r tsconfig-paths/register --watch app/views,public/js --respawn app/server",
-    "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:9229 app/server",
+    "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:${DEBUG_PORT:-9200} app/server",
     "update-dev": "npm install && lerna run build && bin/restart-services",
     "copy-files": "copyfiles 'app/schemas/**/*' 'app/templates/**/*' 'app/frontends/templates/**/*' 'config/**/*' 'db/**/*' 'public/**/*' built",
     "watch-debug": "TS_NODE_TRANSPILE_ONLY=true nodemon --inspect=5858 -e ts,tsx,html --exec node -r tsconfig-paths/register -r ts-node/register app/server.ts",
     "postinstall": "if test \"$NODE_ENV\" != \"production\" && test \"$LERNA\" != \"false\" ; then lerna exec -- npm install; fi",
     "build": "docker build -t harmonyservices/harmony:${VERSION:-latest} .",
     "build-all": "docker build -t harmonyservices/harmony:${VERSION:-latest} . && lerna exec -- npm run build",
-    "build-all-services": "lerna exec -- npm run build",
     "build-all-m1": "lerna exec -- npm run build-m1",
     "build-notebooks-image": "docker build --platform linux/amd64 -t harmonyservices/notebooks:${VERSION:-latest} -f Dockerfile-notebooks .",
     "run-notebooks-image": "source .env && docker run -i -t --platform linux/amd64 --init -p 8888:8888 --env NB_EDL_USERNAME=\"${NB_EDL_USERNAME}\" --env NB_EDL_PASSWORD=\"${NB_EDL_PASSWORD}\" harmonyservices/notebooks:${VERSION:-latest}",

diff --git a/tasks/giovanni-adapter/logs/.gitkeep b/tasks/giovanni-adapter/logs/.gitkeep
diff --git a/tasks/query-cmr/logs/.gitkeep b/tasks/query-cmr/logs/.gitkeep
diff --git a/tasks/service-runner/logs/.gitkeep b/tasks/service-runner/logs/.gitkeep