HARMONY-1618: Speed up building all images when only a code change is…

… being made.
nasa · Dec 11, 2023 · 0411827 · 0411827
1 parent 71f4fbd
commit 0411827
Show file tree

Hide file tree

Showing 14 changed files with 63 additions and 30 deletions.
diff --git a/Dockerfile-notebooks b/Dockerfile-notebooks
@@ -6,12 +6,13 @@ RUN pip install jupyter
 RUN pip install rasterio OWSLib GDAL matplotlib netCDF4 numpy
 RUN pip install s3fs zarr
 
-RUN mkdir -p notebooks
-RUN mkdir -p notebooks/tmp
+RUN mkdir -p notebooks notebooks/tmp notebooks/notebook_helpers
+
+COPY ./docs/notebook_helpers/requirements.txt /notebooks/notebook_helpers/
 WORKDIR /notebooks
+RUN pip install -r notebook_helpers/requirements.txt
+
 COPY ./docs .
 COPY ./config /config
 
-RUN pip install -r notebook_helpers/requirements.txt
-
 ENTRYPOINT notebook_helpers/run-jupyter-docker
diff --git a/services/giovanni-adapter/Dockerfile b/services/giovanni-adapter/Dockerfile
@@ -8,10 +8,11 @@ RUN mkdir -p /giovanni-adapter/services/giovanni-adapter
 RUN mkdir -p /tmp/metadata
 
 COPY package.json package-lock.json /giovanni-adapter/services/giovanni-adapter/
-COPY built /giovanni-adapter/
-COPY config/giovanni-datafield.json /giovanni-adapter/services/giovanni-adapter/config
 WORKDIR /giovanni-adapter/services/giovanni-adapter
 RUN npm ci
+
+COPY built /giovanni-adapter/
+COPY config/giovanni-datafield.json /giovanni-adapter/services/giovanni-adapter/config
 WORKDIR /giovanni-adapter
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/giovanni-adapter/node_modules .

diff --git a/services/giovanni-adapter/bin/prebuild b/services/giovanni-adapter/bin/prebuild
@@ -9,4 +9,4 @@ popd
 rimraf built/*
 copyfiles -u 5 "../../packages/util/built/**/*" built/packages/util
 copyfiles ../../packages/util/env-defaults built/packages/util
-copyfiles -u 2 ../harmony/env-defaults built/prebuild/harmony
+copyfiles -u 2 ../harmony/env-defaults built/services/harmony
diff --git a/services/harmony/Dockerfile b/services/harmony/Dockerfile
@@ -1,16 +1,24 @@
 ARG BASE_IMAGE=node:18-buster
 FROM $BASE_IMAGE
+
 RUN apt update && apt-get -y install sqlite3 python3 python3-pip python3-setuptools
 RUN pip3 install --upgrade pip awscli awscli-local
-# Need to downgrade boto3 because there is a bug breaking creating SQS queues
 RUN pip3 install boto3==1.25.5
-RUN mkdir -p /harmony/services/harmony
-COPY built /harmony
-RUN chown node -R /harmony
-USER node
+
+RUN mkdir -p /harmony/services/harmony /harmony/packages/util
+RUN chown -R node:node /harmony
+
+COPY --chown=node:node built/packages/util/package.json built/packages/util/package-lock.json /harmony/packages/util/
 WORKDIR /harmony/packages/util
 RUN env NODE_ENV=production npm ci
+
+COPY --chown=node:node built/services/harmony/package.json built/services/harmony/package-lock.json /harmony/services/harmony/
 WORKDIR /harmony/services/harmony
 RUN env NODE_ENV=production npm ci
-RUN npm install sqlite3 --save
-ENTRYPOINT [ "npm", "run", "start" ]
+RUN npm install sqlite3
+
+COPY --chown=node:node built /harmony
+
+USER node
+
+ENTRYPOINT [ "npm", "run", "start" ]
diff --git a/services/query-cmr/Dockerfile b/services/query-cmr/Dockerfile
@@ -8,10 +8,10 @@ RUN mkdir -p /query-cmr/services/query-cmr
 RUN mkdir -p /tmp/metadata
 
 COPY package.json package-lock.json /query-cmr/services/query-cmr/
-COPY built /query-cmr/
 WORKDIR /query-cmr/services/query-cmr
-RUN npm update
 RUN npm ci
+
+COPY built /query-cmr/
 WORKDIR /query-cmr
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/query-cmr/node_modules .

diff --git a/services/service-runner/Dockerfile b/services/service-runner/Dockerfile
@@ -5,10 +5,13 @@ RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /service-runner/services/service-runner
 RUN mkdir -p /tmp/metadata
-COPY env-defaults package.json package-lock.json /service-runner/services/service-runner/
-COPY built /service-runner/
+
+COPY package.json package-lock.json /service-runner/services/service-runner/
 WORKDIR /service-runner/services/service-runner
 RUN npm ci
+
+COPY env-defaults /service-runner/services/service-runner/
+COPY built /service-runner/
 WORKDIR /service-runner
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/service-runner/node_modules .

diff --git a/services/work-failer/Dockerfile b/services/work-failer/Dockerfile
@@ -4,10 +4,13 @@ RUN apt-get update && apt-get -y install postgresql
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-failer/services/work-failer
-COPY env-defaults package.json package-lock.json /work-failer/services/work-failer/
-COPY built /work-failer/
+
+COPY package.json package-lock.json /work-failer/services/work-failer/
 WORKDIR /work-failer/services/work-failer
 RUN npm ci
+
+COPY env-defaults /work-failer/services/work-failer/
+COPY built /work-failer/
 WORKDIR /work-failer
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/work-failer/node_modules .

diff --git a/services/work-failer/Dockerfile.mac b/services/work-failer/Dockerfile.mac
@@ -5,10 +5,13 @@ RUN apk add bash vim curl git python3 postgresql-client make gcc g++ libc-dev li
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-failer/services/work-failer
-COPY env-defaults package.json package-lock.json /work-failer/services/work-failer/
-COPY built /work-failer/
+
+COPY package.json package-lock.json /work-failer/services/work-failer/
 WORKDIR /work-failer/services/work-failer
 RUN npm ci
+
+COPY env-defaults /work-failer/services/work-failer/
+COPY built /work-failer/
 WORKDIR /work-failer
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/work-failer/node_modules .

diff --git a/services/work-reaper/Dockerfile b/services/work-reaper/Dockerfile
@@ -4,10 +4,13 @@ RUN apt-get update && apt-get -y install postgresql
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-reaper/services/work-reaper
-COPY env-defaults package.json package-lock.json /work-reaper/services/work-reaper/
-COPY built /work-reaper/
+
+COPY package.json package-lock.json /work-reaper/services/work-reaper/
 WORKDIR /work-reaper/services/work-reaper
 RUN npm ci
+
+COPY env-defaults /work-reaper/services/work-reaper/
+COPY built /work-reaper/
 WORKDIR /work-reaper
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/work-reaper/node_modules .

diff --git a/services/work-reaper/Dockerfile.mac b/services/work-reaper/Dockerfile.mac
@@ -5,10 +5,13 @@ RUN apk add bash vim curl git python3 postgresql-client make gcc g++ libc-dev li
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-reaper/services/work-reaper
-COPY env-defaults package.json package-lock.json /work-reaper/services/work-reaper/
-COPY built /work-reaper/
+
+COPY package.json package-lock.json /work-reaper/services/work-reaper/
 WORKDIR /work-reaper/services/work-reaper
 RUN npm ci
+
+COPY env-defaults /work-reaper/services/work-reaper/
+COPY built /work-reaper/
 WORKDIR /work-reaper
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/work-reaper/node_modules .

diff --git a/services/work-scheduler/Dockerfile b/services/work-scheduler/Dockerfile
@@ -4,6 +4,7 @@ RUN apt-get update && apt-get -y install postgresql
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-scheduler/services/work-scheduler
+
 COPY package.json package-lock.json /work-scheduler/services/work-scheduler/
 WORKDIR /work-scheduler/services/work-scheduler
 RUN npm ci

diff --git a/services/work-scheduler/Dockerfile.mac b/services/work-scheduler/Dockerfile.mac
@@ -5,6 +5,7 @@ RUN apk add bash vim curl git python3 postgresql-client make gcc g++ libc-dev li
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-scheduler/services/work-scheduler
+
 COPY package.json package-lock.json /work-scheduler/services/work-scheduler/
 WORKDIR /work-scheduler/services/work-scheduler
 RUN npm ci

diff --git a/services/work-updater/Dockerfile b/services/work-updater/Dockerfile
@@ -4,10 +4,13 @@ RUN apt-get update && apt-get -y install postgresql
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-updater/services/work-updater
-COPY env-defaults package.json package-lock.json /work-updater/services/work-updater/
-COPY built /work-updater/
+
+COPY package.json package-lock.json /work-updater/services/work-updater/
 WORKDIR /work-updater/services/work-updater
 RUN npm ci
+
+COPY env-defaults /work-updater/services/work-updater/
+COPY built /work-updater/
 WORKDIR /work-updater
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/work-updater/node_modules .

diff --git a/services/work-updater/Dockerfile.mac b/services/work-updater/Dockerfile.mac
@@ -5,10 +5,13 @@ RUN apk add bash vim curl git python3 postgresql-client make gcc g++ libc-dev li
 RUN git config --global url."https://".insteadOf ssh://
 
 RUN mkdir -p /work-updater/services/work-updater
-COPY env-defaults package.json package-lock.json /work-updater/services/work-updater/
-COPY built /work-updater/
+
+COPY package.json package-lock.json /work-updater/services/work-updater/
 WORKDIR /work-updater/services/work-updater
 RUN npm ci
+
+COPY env-defaults /work-updater/services/work-updater/
+COPY built /work-updater/
 WORKDIR /work-updater
 # This symlink is needed to make module loading work correctly
 RUN ln -s services/work-updater/node_modules .