HARMONY-1538: Changes to fix docker image builds

nasa · Oct 9, 2023 · c6615e4 · c6615e4
1 parent 6d6b3f5
commit c6615e4
Show file tree

Hide file tree

Showing 30 changed files with 83 additions and 85 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,5 +1,4 @@
 node_modules
-services
 .env
 db/*.sqlite3
 logs

diff --git a/.eslintrc.yml b/.eslintrc.yml
@@ -69,7 +69,3 @@ overrides:
       # the preferred method described in the Mocha docs
       prefer-arrow-callback: "off"
       func-names: "off"
-  - files: [ "tasks/**/*" ]
-    rules:
-      import/no-extraneous-dependencies: ["error"]
-      node/no-unpublished-import: ["off"]
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -40,7 +40,7 @@ jobs:
           # build and publish harmony
           VERSION="${VERSION}" npm run build
           VERSION="${VERSION}" npm run publish
-          # build and publish images for all tasks
+          # build and publish images for all services
           VERSION="${VERSION}" lerna run build
           VERSION="${VERSION}" lerna run publish
 
diff --git a/bin/start-harmony b/bin/start-harmony
@@ -5,7 +5,7 @@
 
 env_save=$(export -p)
 set -a
-source "env-defaults"
+source "services/harmony/env-defaults"
 source ".env"
 set +a
 eval "$env_save"

diff --git a/kubernetes-services/README.md b/kubernetes-services/README.md
@@ -11,7 +11,7 @@ the services.
 ## Creating a Task Service
 
 The easiest way to create a new task service is to copy the `work-scheduler` directory to a new
-directory under this (kubernetes-services) folder, then modify the code to do what you want.
+directory under this (services) folder, then modify the code to do what you want.
 `work-scheduler` is a small `Nodes.js` `Express` app that uses the `Worker` class/pattern as
 defined in `app/workers.ts`. The `Express` server provides a health check endpoint for kubernetes
 while the `Worker` performs the real work of the application.
@@ -21,7 +21,7 @@ can scrape it to drive an HPA, but that is beyond the scope of this discussion.
 
 The specific steps to create a new service are
 
-1. `cd kubernetes-services`
+1. `cd services`
 2. `cp work-scheduler <new service>`
 3. Modify the code in the `app` and `test` directories to implement/test your service.
 4. Modify the `Dockerfile`, `env-defaults`, and `config/service-template.yml` files as well as

diff --git a/package.json b/package.json
@@ -8,19 +8,19 @@
     "test-fast": "lerna run test-fast",
     "test-bail": "lerna run test-bail",
     "coverage": "nyc mocha",
-    "start": "NODE_OPTIONS=--max-old-space-size=3096 ts-node -r tsconfig-paths/register --require './app/tracing.ts' app/server.ts",
+    "start": "cd services/harmony && NODE_OPTIONS=--max-old-space-size=3096 ts-node -r tsconfig-paths/register --require './app/tracing.ts' app/server.ts",
     "start-dev": "strict-npm-engines && ts-node-dev --no-notify -r tsconfig-paths/register --watch app/views,public/js --respawn app/server",
     "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:${DEBUG_PORT:-9200} app/server",
     "update-dev": "npm install && lerna run build && bin/restart-services",
     "copy-files": "copyfiles 'app/schemas/**/*' 'app/templates/**/*' 'app/frontends/templates/**/*' 'config/**/*' 'db/**/*' 'public/**/*' built",
     "watch-debug": "TS_NODE_TRANSPILE_ONLY=true nodemon --inspect=5858 -e ts,tsx,html --exec node -r tsconfig-paths/register -r ts-node/register app/server.ts",
     "postinstall": "if test \"$NODE_ENV\" != \"production\" && test \"$LERNA\" != \"false\" ; then lerna exec -- npm install; fi",
-    "build": "docker build -t harmonyservices/harmony:${VERSION:-latest} .",
+    "build": "lerna run build --scope=harmony",
     "build-all": "lerna exec -- npm run build",
     "build-all-m1": "lerna exec -- npm run build-m1",
     "build-notebooks-image": "docker build --platform linux/amd64 -t harmonyservices/notebooks:${VERSION:-latest} -f Dockerfile-notebooks .",
     "run-notebooks-image": "source .env && docker run -i -t --platform linux/amd64 --init -p 8888:8888 --env NB_EDL_USERNAME=\"${NB_EDL_USERNAME}\" --env NB_EDL_PASSWORD=\"${NB_EDL_PASSWORD}\" harmonyservices/notebooks:${VERSION:-latest}",
-    "publish": "docker push harmonyservices/harmony:${VERSION:-latest}",
+    "publish": "lerna run publish --scope=harmony",
     "compare-services": "SKIP_ENV_VALIDATION=true ts-node scripts/service-comparison.ts",
     "better-audit": "better-npm-audit audit"
   },

diff --git a/packages/util/env.ts b/packages/util/env.ts
@@ -36,7 +36,7 @@ const envDefaults = dotenv.parse(fs.readFileSync(path.resolve(__dirname, 'env-de
 export let envOverrides = {};
 if (process.env.NODE_ENV !== 'test') {
   try {
-    envOverrides = dotenv.parse(fs.readFileSync('.env'));
+    envOverrides = dotenv.parse(fs.readFileSync('../../.env'));
   } catch (e) {
     logger.warn('Could not parse environment overrides from .env file');
     logger.warn(e.message);

diff --git a/services/giovanni-adapter/Dockerfile b/services/giovanni-adapter/Dockerfile
@@ -4,16 +4,16 @@ RUN apk update
 RUN apk add bash vim curl git
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /giovanni-adapter/tasks/giovanni-adapter
+RUN mkdir -p /giovanni-adapter/services/giovanni-adapter
 RUN mkdir -p /tmp/metadata
 
-COPY package.json package-lock.json /giovanni-adapter/tasks/giovanni-adapter/
+COPY package.json package-lock.json /giovanni-adapter/services/giovanni-adapter/
 COPY built /giovanni-adapter/
-COPY config/giovanni-datafield.json /giovanni-adapter/tasks/giovanni-adapter/config
-WORKDIR /giovanni-adapter/tasks/giovanni-adapter
+COPY config/giovanni-datafield.json /giovanni-adapter/services/giovanni-adapter/config
+WORKDIR /giovanni-adapter/services/giovanni-adapter
 RUN npm ci
 WORKDIR /giovanni-adapter
 # This symlink is needed to make module loading work correctly
-RUN ln -s tasks/giovanni-adapter/node_modules .
+RUN ln -s services/giovanni-adapter/node_modules .
 
 ENTRYPOINT [ "node", "app/cli.js" ]
diff --git a/services/giovanni-adapter/bin/prebuild b/services/giovanni-adapter/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/harmony
+copyfiles -u 2 ../harmony/env-defaults built/prebuild/harmony
diff --git a/services/harmony/Dockerfile b/services/harmony/Dockerfile
@@ -2,12 +2,13 @@ ARG BASE_IMAGE=node:16-buster
 FROM $BASE_IMAGE
 RUN apt update && apt-get -y install sqlite3 python3 python3-pip python3-setuptools
 RUN pip3 install --upgrade pip awscli awscli-local
-RUN mkdir -p /harmony
-COPY package.json package-lock.json lerna.json /harmony/
+RUN mkdir -p /harmony/services/harmony
+COPY built /harmony
 RUN chown node -R /harmony
 USER node
-WORKDIR /harmony
+WORKDIR /harmony/packages/util
+RUN env NODE_ENV=production npm ci
+WORKDIR /harmony/services/harmony
 RUN env NODE_ENV=production npm ci
 RUN npm install sqlite3 --save
-COPY . /harmony/
-ENTRYPOINT [ "npm", "run", "start" ]
+ENTRYPOINT [ "npm", "run", "start" ]
diff --git a/services/harmony/bin/prebuild b/services/harmony/bin/prebuild
@@ -5,5 +5,8 @@
 
 rimraf built/*
 copyfiles -u 2 -e "../../db/test.sqlite3" "../../db/**/*" built
+# need to copy the db directory into the servcies/harmony directory as well to make knex migrations work because knex is stupid
+copyfiles -u 2 -e "../../db/test.sqlite3" "../../db/**/*" built/services/harmony
+copyfiles -u 2 "../../bin/start-harmony-in-container" built
 copyfiles -u 4 -e "../../packages/util/node_modules/**/*" -e "../../packages/util/built/**/*" -e "../../packages/util/test/**/*" -e "../../packages/util/coverage/**/*" "../../packages/util/**/*"  built/packages/util
 copyfiles -e "./test/**/*" -e "./node_modules/**/*" -e "./coverage/**/*" -e "./built/**/*" -e "./bin/**/*" -e "./fixtures/**/*" "./**/*" built/services/harmony
diff --git a/services/harmony/env-defaults b/services/harmony/env-defaults
@@ -378,7 +378,7 @@ GIOVANNI_ADAPTER_REQUESTS_CPU=128m
 GIOVANNI_ADAPTER_REQUESTS_MEMORY=128Mi
 GIOVANNI_ADAPTER_LIMITS_CPU=128m
 GIOVANNI_ADAPTER_LIMITS_MEMORY=512Mi
-GIOVANNI_ADAPTER_INVOCATION_ARGS='node tasks/giovanni-adapter/app/cli'
+GIOVANNI_ADAPTER_INVOCATION_ARGS='node services/giovanni-adapter/app/cli'
 GIOVANNI_ADAPTER_SERVICE_QUEUE_URLS='["harmonyservices/giovanni-adapter:latest,http://localstack:4566/queue/giovanni-adapter.fifo"]'
 
 # The services to deploy locally. A comma-separated list of services that the bin/deploy-services

diff --git a/services/harmony/package.json b/services/harmony/package.json
@@ -13,10 +13,8 @@
     "start-dev-fast": "TS_NODE_TRANSPILE_ONLY=true ts-node-dev --no-notify -r tsconfig-paths/register --respawn --inspect=127.0.0.1:${DEBUG_PORT:-9200} app/server",
     "copy-files": "copyfiles 'app/schemas/**/*' 'app/templates/**/*' 'app/frontends/templates/**/*' 'config/**/*' 'db/**/*' 'public/**/*' built",
     "watch-debug": "TS_NODE_TRANSPILE_ONLY=true nodemon --inspect=5858 -e ts,tsx,html --exec node -r tsconfig-paths/register -r ts-node/register app/server.ts",
+    "prebuild": "./bin/prebuild",
     "build": "docker build -t harmonyservices/harmony:${VERSION:-latest} .",
-    "build-all": "docker build -t harmonyservices/harmony:${VERSION:-latest} . && lerna exec -- npm run build",
-    "build-all-m1": "lerna exec -- npm run build-m1",
-    "build-notebooks-image": "docker build --platform linux/amd64 -t harmonyservices/notebooks:${VERSION:-latest} -f Dockerfile-notebooks .",
     "run-notebooks-image": "source .env && docker run -i -t --platform linux/amd64 --init -p 8888:8888 --env NB_EDL_USERNAME=\"${NB_EDL_USERNAME}\" --env NB_EDL_PASSWORD=\"${NB_EDL_PASSWORD}\" harmonyservices/notebooks:${VERSION:-latest}",
     "publish": "docker push harmonyservices/harmony:${VERSION:-latest}",
     "compare-services": "SKIP_ENV_VALIDATION=true ts-node scripts/service-comparison.ts"

diff --git a/services/query-cmr/Dockerfile b/services/query-cmr/Dockerfile
@@ -4,17 +4,17 @@ RUN apk update
 RUN apk add bash vim curl git
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /query-cmr/tasks/query-cmr
+RUN mkdir -p /query-cmr/services/query-cmr
 RUN mkdir -p /tmp/metadata
 
-COPY package.json package-lock.json /query-cmr/tasks/query-cmr/
+COPY package.json package-lock.json /query-cmr/services/query-cmr/
 COPY built /query-cmr/
-WORKDIR /query-cmr/tasks/query-cmr
+WORKDIR /query-cmr/services/query-cmr
 RUN npm update
 RUN npm ci
 WORKDIR /query-cmr
 # This symlink is needed to make module loading work correctly
-RUN ln -s tasks/query-cmr/node_modules .
-WORKDIR /query-cmr/tasks/query-cmr
+RUN ln -s services/query-cmr/node_modules .
+WORKDIR /query-cmr/services/query-cmr
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/query-cmr/bin/prebuild b/services/query-cmr/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/harmony
+copyfiles -u 2 ../harmony/env-defaults built/services/harmony
diff --git a/services/service-runner/Dockerfile b/services/service-runner/Dockerfile
@@ -3,15 +3,15 @@ FROM node:16-alpine
 RUN apk add bash vim curl git
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /service-runner/tasks/service-runner
+RUN mkdir -p /service-runner/services/service-runner
 RUN mkdir -p /tmp/metadata
-COPY env-defaults package.json package-lock.json /service-runner/tasks/service-runner/
+COPY env-defaults package.json package-lock.json /service-runner/services/service-runner/
 COPY built /service-runner/
-WORKDIR /service-runner/tasks/service-runner
+WORKDIR /service-runner/services/service-runner
 RUN npm ci
 WORKDIR /service-runner
 # This symlink is needed to make module loading work correctly
-RUN ln -s tasks/service-runner/node_modules .
-WORKDIR /service-runner/tasks/service-runner
+RUN ln -s services/service-runner/node_modules .
+WORKDIR /service-runner/services/service-runner
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/service-runner/bin/prebuild b/services/service-runner/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/harmony
+copyfiles -u 2 ../harmony/env-defaults built/services/harmony
diff --git a/services/service-runner/package.json b/services/service-runner/package.json
@@ -16,6 +16,7 @@
     "build": "tsc && docker build --tag harmonyservices/service-runner:${VERSION:-latest} .",
     "build-m1": "tsc && docker build --platform linux/amd64 --tag harmonyservices/service-runner:${VERSION:-latest} .",
     "publish": "docker push harmonyservices/service-runner:${VERSION:-latest}",
+    "push-image": "./bin/push-image",
     "better-audit": "better-npm-audit audit"
   },
   "engines": {
@@ -71,4 +72,4 @@
     "ts-node": "^10.4.0",
     "typescript": "^5.1.6"
   }
-}
+}
diff --git a/services/work-failer/Dockerfile b/services/work-failer/Dockerfile
@@ -3,14 +3,14 @@ FROM node:16-buster
 RUN apt-get update && apt-get -y install postgresql
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /work-failer/kubernetes-services/work-failer
-COPY env-defaults package.json package-lock.json /work-failer/kubernetes-services/work-failer/
+RUN mkdir -p /work-failer/services/work-failer
+COPY env-defaults package.json package-lock.json /work-failer/services/work-failer/
 COPY built /work-failer/
-WORKDIR /work-failer/kubernetes-services/work-failer
+WORKDIR /work-failer/services/work-failer
 RUN npm ci
 WORKDIR /work-failer
 # This symlink is needed to make module loading work correctly
-RUN ln -s kubernetes-services/work-failer/node_modules .
-WORKDIR /work-failer/kubernetes-services/work-failer
+RUN ln -s services/work-failer/node_modules .
+WORKDIR /work-failer/services/work-failer
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/work-failer/Dockerfile.mac b/services/work-failer/Dockerfile.mac
@@ -4,14 +4,14 @@ RUN apk update
 RUN apk add bash vim curl git python3 postgresql-client make gcc g++ libc-dev libpq-dev
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /work-failer/kubernetes-services/work-failer
-COPY env-defaults package.json package-lock.json /work-failer/kubernetes-services/work-failer/
+RUN mkdir -p /work-failer/services/work-failer
+COPY env-defaults package.json package-lock.json /work-failer/services/work-failer/
 COPY built /work-failer/
-WORKDIR /work-failer/kubernetes-services/work-failer
+WORKDIR /work-failer/services/work-failer
 RUN npm ci
 WORKDIR /work-failer
 # This symlink is needed to make module loading work correctly
-RUN ln -s kubernetes-services/work-failer/node_modules .
-WORKDIR /work-failer/kubernetes-services/work-failer
+RUN ln -s services/work-failer/node_modules .
+WORKDIR /work-failer/services/work-failer
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/work-failer/bin/prebuild b/services/work-failer/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/harmony
+copyfiles -u 2 ../harmony/env-defaults built/services/harmony
diff --git a/services/work-reaper/Dockerfile b/services/work-reaper/Dockerfile
@@ -3,14 +3,14 @@ FROM node:16-buster
 RUN apt-get update && apt-get -y install postgresql
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /work-reaper/kubernetes-services/work-reaper
-COPY env-defaults package.json package-lock.json /work-reaper/kubernetes-services/work-reaper/
+RUN mkdir -p /work-reaper/services/work-reaper
+COPY env-defaults package.json package-lock.json /work-reaper/services/work-reaper/
 COPY built /work-reaper/
-WORKDIR /work-reaper/kubernetes-services/work-reaper
+WORKDIR /work-reaper/services/work-reaper
 RUN npm ci
 WORKDIR /work-reaper
 # This symlink is needed to make module loading work correctly
-RUN ln -s kubernetes-services/work-reaper/node_modules .
-WORKDIR /work-reaper/kubernetes-services/work-reaper
+RUN ln -s services/work-reaper/node_modules .
+WORKDIR /work-reaper/services/work-reaper
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/work-reaper/Dockerfile.mac b/services/work-reaper/Dockerfile.mac
@@ -4,14 +4,14 @@ RUN apk update
 RUN apk add bash vim curl git python3 postgresql-client make gcc g++ libc-dev libpq-dev
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /work-reaper/kubernetes-services/work-reaper
-COPY env-defaults package.json package-lock.json /work-reaper/kubernetes-services/work-reaper/
+RUN mkdir -p /work-reaper/services/work-reaper
+COPY env-defaults package.json package-lock.json /work-reaper/services/work-reaper/
 COPY built /work-reaper/
-WORKDIR /work-reaper/kubernetes-services/work-reaper
+WORKDIR /work-reaper/services/work-reaper
 RUN npm ci
 WORKDIR /work-reaper
 # This symlink is needed to make module loading work correctly
-RUN ln -s kubernetes-services/work-reaper/node_modules .
-WORKDIR /work-reaper/kubernetes-services/work-reaper
+RUN ln -s services/work-reaper/node_modules .
+WORKDIR /work-reaper/services/work-reaper
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/work-reaper/bin/prebuild b/services/work-reaper/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/harmony
+copyfiles -u 2 ../harmony/env-defaults built/services/harmony
diff --git a/services/work-scheduler/Dockerfile b/services/work-scheduler/Dockerfile
@@ -3,15 +3,15 @@ FROM node:16-buster
 RUN apt-get update && apt-get -y install postgresql
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /work-scheduler/kubernetes-services/work-scheduler
+RUN mkdir -p /work-scheduler/services/work-scheduler
 
-COPY env-defaults package.json package-lock.json /work-scheduler/kubernetes-services/work-scheduler/
+COPY env-defaults package.json package-lock.json /work-scheduler/services/work-scheduler/
 COPY built /work-scheduler/
-WORKDIR /work-scheduler/kubernetes-services/work-scheduler
+WORKDIR /work-scheduler/services/work-scheduler
 RUN npm ci
 WORKDIR /work-scheduler
 # This symlink is needed to make module loading work correctly
-RUN ln -s kubernetes-services/work-scheduler/node_modules .
-WORKDIR /work-scheduler/kubernetes-services/work-scheduler
+RUN ln -s services/work-scheduler/node_modules .
+WORKDIR /work-scheduler/services/work-scheduler
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/work-scheduler/Dockerfile.mac b/services/work-scheduler/Dockerfile.mac
@@ -4,15 +4,15 @@ RUN apk update
 RUN apk add bash vim curl git python3 postgresql-client make gcc g++ libc-dev libpq-dev
 RUN git config --global url."https://".insteadOf ssh://
 
-RUN mkdir -p /work-scheduler/kubernetes-services/work-scheduler
+RUN mkdir -p /work-scheduler/services/work-scheduler
 
-COPY env-defaults package.json package-lock.json /work-scheduler/kubernetes-services/work-scheduler/
+COPY env-defaults package.json package-lock.json /work-scheduler/services/work-scheduler/
 COPY built /work-scheduler/
-WORKDIR /work-scheduler/kubernetes-services/work-scheduler
+WORKDIR /work-scheduler/services/work-scheduler
 RUN npm ci
 WORKDIR /work-scheduler
 # This symlink is needed to make module loading work correctly
-RUN ln -s kubernetes-services/work-scheduler/node_modules .
-WORKDIR /work-scheduler/kubernetes-services/work-scheduler
+RUN ln -s services/work-scheduler/node_modules .
+WORKDIR /work-scheduler/services/work-scheduler
 
 ENTRYPOINT [ "node", "app/server.js"]
diff --git a/services/work-scheduler/bin/prebuild b/services/work-scheduler/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/harmony
+copyfiles -u 2 ../harmony/env-defaults built/services/harmony