Optimize docker image build process

- Run affymetrix/agilent tests in parallel - Set file permissions to 644 while copying - Set directory permissions to 755 - Exclude unnecessary files vie dockerignore
AlexsLemonade · Nov 2, 2023 · b8602a9 · b8602a9
1 parent d95a8e0
commit b8602a9
Show file tree

Hide file tree

Showing 16 changed files with 191 additions and 110 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,15 +1,16 @@
-.git
-*/batch-job-specs
-*/test_volume
-*/volume
+**/.git
+**/__pycache__
+**/*.pyc
+**/batch-job-templates
+**/dockerfiles
+**/environments
+**/test_volume
+**/volume
+**/volumes_postgres
 config/externally_supplied_metadata/metasra/*.tab
 config/externally_supplied_metadata/metasra/metasra_keywords.json
 config/externally_supplied_metadata/metasra/metasra_translated.json
 config/externally_supplied_metadata/metasra/SRAmetadb.sqlite
 dr_env
 env/
-infrastructure/
-terraform/
-test_volume
-volume
-volumes_postgres/
+infrastructure
diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml
@@ -85,7 +85,7 @@ jobs:
     env:
       IMAGES: migrations affymetrix
     needs:
-      - test_affymetrix
+      - test_base
     runs-on: ubuntu-latest-m
     steps:
       - uses: actions/checkout@v3

diff --git a/api/dockerfiles/Dockerfile.api b/api/dockerfiles/Dockerfile.api
@@ -8,8 +8,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN <<EOF
 pip install --ignore-installed --no-cache-dir uwsgi
 chmod +x /home/user/collect_and_run_uwsgi.sh
-mkdir -p /tmp/www/static
+mkdir -m 755 -p /tmp/www/static
 chown user /tmp/www/static
+rm -rf /root/.cache/*
 EOF
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION

diff --git a/api/dockerfiles/Dockerfile.api_base b/api/dockerfiles/Dockerfile.api_base
@@ -15,17 +15,24 @@ EOF
 
 WORKDIR /home/user
 
-COPY api/requirements.txt .
+COPY --chmod=644 api/requirements.txt .
 RUN <<EOF
 pip install --upgrade pip
 pip install --ignore-installed --no-cache-dir -r requirements.txt
 EOF
 
-COPY common/dist/data-refinery-common-* common/
-RUN pip install --ignore-installed common/$(ls common -1 | sort --version-sort | tail -1)
+COPY --chmod=644 common/dist/data-refinery-common-* common/
+RUN <<EOF
+pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+rm -r common
+EOF
+
+COPY --chmod=644 api/ .
+COPY --chmod=644 config/ config/
+COPY --chmod=644 setup.cfg .
 
-COPY api/ .
-COPY config/ config/
-COPY setup.cfg .
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+EOF
 
 ENTRYPOINT []
diff --git a/common/dockerfiles/Dockerfile.base b/common/dockerfiles/Dockerfile.base
@@ -65,9 +65,13 @@ useradd --create-home --home-dir /home/user/ -g user user
 chown -R user /home/user/
 EOF
 
-ENV R_LIBS=/usr/local/lib/R/site-library
+COPY --chmod=644 config/ config/
+COPY --chmod=644 setup.cfg .
 
-COPY config/ config/
-COPY setup.cfg .
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+EOF
+
+ENV R_LIBS=/usr/local/lib/R/site-library
 
 ENTRYPOINT []
diff --git a/common/dockerfiles/Dockerfile.common_tests b/common/dockerfiles/Dockerfile.common_tests
@@ -7,14 +7,19 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /home/user
 
-COPY common/R/dependencies/common_tests/renv.lock .
-COPY common/R/renv_load.R renv_load_common_tests.R
+COPY --chmod=644 common/R/dependencies/common_tests/renv.lock .
+COPY --chmod=644 common/R/renv_load.R renv_load_common_tests.R
 RUN Rscript renv_load_common_tests.R
 
-COPY common/requirements.txt .
+COPY --chmod=644 common/requirements.txt .
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
-COPY common/ .
+COPY --chmod=644 common/ .
+
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+rm -rf /root/.cache/*
+EOF
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 

diff --git a/common/dockerfiles/Dockerfile.migrations b/common/dockerfiles/Dockerfile.migrations
@@ -7,10 +7,15 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /home/user
 
-COPY common/requirements.txt .
+COPY --chmod=644 common/requirements.txt .
 RUN pip install --ignore-installed --no-cache-dir -r requirements.txt
 
-COPY common/ .
+COPY --chmod=644 common/ .
+
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+rm -rf /root/.cache/*
+EOF
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 

diff --git a/foreman/dockerfiles/Dockerfile.foreman b/foreman/dockerfiles/Dockerfile.foreman
@@ -7,21 +7,29 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /home/user
 
-COPY foreman/requirements.txt .
+COPY --chmod=644 foreman/requirements.txt .
 RUN <<EOF
-mkdir -p data/microarray
+mkdir -m 755 -p data/microarray
 wget -q https://gbnci.cancer.gov/geo/GEOmetadb.sqlite.gz -O data/microarray/GEOmetadb.sqlite.gz
 gunzip data/microarray/GEOmetadb.sqlite.gz
 chmod 644 data/microarray/GEOmetadb.sqlite
 pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 EOF
 
 # Get the latest version from the dist directory.
-COPY common/dist/data-refinery-common-* common/
-RUN pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+COPY --chmod=644 common/dist/data-refinery-common-* common/
+RUN <<EOF
+pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+rm -r common
+EOF
+
+COPY --chmod=644 .boto .boto
+COPY --chmod=644 foreman/ .
 
-COPY .boto .boto
-COPY foreman/ .
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+rm -rf /root/.cache/*
+EOF
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 

diff --git a/workers/dockerfiles/Dockerfile.affymetrix b/workers/dockerfiles/Dockerfile.affymetrix
@@ -7,24 +7,30 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /home/user
 
-COPY workers/R/dependencies/affymetrix/renv.lock .
-COPY common/R/renv_load.R renv_load_affymetrix.R
+COPY --chmod=644 workers/R/dependencies/affymetrix/renv.lock .
+COPY --chmod=644 common/R/renv_load.R renv_load_affymetrix.R
 RUN Rscript renv_load_affymetrix.R
 
-COPY workers/R/dependencies/affymetrix/install_ensg_pkgs.R .
+COPY --chmod=644 workers/R/dependencies/affymetrix/install_ensg_pkgs.R .
 RUN Rscript install_ensg_pkgs.R
 
-COPY workers/data_refinery_workers/processors/requirements.txt .
+COPY --chmod=644 workers/data_refinery_workers/processors/requirements.txt .
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
 # Get the latest version from the dist directory.
-COPY common/dist/data-refinery-common-* common/
-RUN pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
-
-COPY .boto .boto
-COPY workers/ .
-
-RUN rm -rf /root/.cache/*
+COPY --chmod=644 common/dist/data-refinery-common-* common/
+RUN <<EOF
+pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+rm -r common
+EOF
+
+COPY --chmod=644 .boto .boto
+COPY --chmod=644 workers/ .
+
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+rm -rf /root/.cache/*
+EOF
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 

diff --git a/workers/dockerfiles/Dockerfile.compendia b/workers/dockerfiles/Dockerfile.compendia
@@ -68,14 +68,14 @@ WORKDIR /home/user
 
 ENV R_LIBS=/usr/local/lib/R/site-library
 
-COPY workers/R/dependencies/compendia/renv.lock .
-COPY common/R/renv_load.R renv_load_compendia.R
+COPY --chmod=644 workers/R/dependencies/compendia/renv.lock .
+COPY --chmod=644 common/R/renv_load.R renv_load_compendia.R
 RUN Rscript renv_load_compendia.R
 
-COPY workers/data_refinery_workers/requirements/compendia.txt requirements.txt
+COPY --chmod=644 workers/data_refinery_workers/requirements/compendia.txt requirements.txt
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
-COPY workers/data_refinery_workers/processors/requirements.txt requirements.txt
+COPY --chmod=644 workers/data_refinery_workers/processors/requirements.txt requirements.txt
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
 # Install phantomjs.
@@ -87,15 +87,21 @@ rm phantomjs-2.1.1-linux-x86_64.tar.bz2
 EOF
 
 # Get the latest version from the dist directory.
-COPY common/dist/data-refinery-common-* common/
-RUN pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+COPY --chmod=644 common/dist/data-refinery-common-* common/
+RUN <<EOF
+pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+rm -r common
+EOF
 
-COPY .boto .boto
-COPY config/ config/
-COPY setup.cfg .
-COPY workers/ .
+COPY --chmod=644 .boto .boto
+COPY --chmod=644 config/ config/
+COPY --chmod=644 setup.cfg .
+COPY --chmod=644 workers/ .
 
-RUN rm -rf /root/.cache/*
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+rm -rf /root/.cache/*
+EOF
 
 ARG SYSTEM_VERSION
 ENV SYSTEM_VERSION=$SYSTEM_VERSION

diff --git a/workers/dockerfiles/Dockerfile.downloaders b/workers/dockerfiles/Dockerfile.downloaders
@@ -7,11 +7,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /home/user
 
-COPY workers/R/dependencies/downloaders/renv.lock .
-COPY common/R/renv_load.R renv_load_downloaders.R
+COPY --chmod=644 workers/R/dependencies/downloaders/renv.lock .
+COPY --chmod=644 common/R/renv_load.R renv_load_downloaders.R
 RUN Rscript renv_load_downloaders.R
 
-COPY workers/data_refinery_workers/downloaders/requirements.txt .
+COPY --chmod=644 workers/data_refinery_workers/downloaders/requirements.txt .
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
 # Install Aspera.
@@ -23,8 +23,10 @@ EOF
 
 USER user
 
+ENV GNUPG_PATH=/home/user/.gnupg
+
 RUN <<EOF
-mkdir -m 700 /home/user/.gnupg
+mkdir -m 700 ${GNUPG_PATH}
 # Disable IPv6 to avoid "Cannot assign requested address" error.
 echo "disable-ipv6" >> /home/user/.gnupg/dirmngr.conf
 . /home/user/rvm/scripts/rvm
@@ -34,13 +36,19 @@ EOF
 USER root
 
 # Get the latest version from the dist directory.
-COPY common/dist/data-refinery-common-* common/
-RUN pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+COPY --chmod=644 common/dist/data-refinery-common-* common/
+RUN <<EOF
+pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+rm -r common
+EOF
 
-COPY .boto .boto
-COPY workers/ .
+COPY --chmod=644 .boto .boto
+COPY --chmod=644 workers/ .
 
-RUN rm -rf /root/.cache/*
+RUN <<EOF
+rm -rf /root/.cache/*
+find /home/user -path ${GNUPG_PATH} -prune -type d ! -perm 755 -exec chmod 755 {} \;
+EOF
 
 ENV PATH="$PATH:/home/user/.aspera/sdk"
 ENV SYSTEM_VERSION=$SYSTEM_VERSION

diff --git a/workers/dockerfiles/Dockerfile.illumina b/workers/dockerfiles/Dockerfile.illumina
@@ -7,23 +7,29 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 WORKDIR /home/user
 
-COPY workers/R/dependencies/illumina/renv.lock .
-COPY common/R/renv_load.R renv_load_illumina.R
+COPY --chmod=644 workers/R/dependencies/illumina/renv.lock .
+COPY --chmod=644 common/R/renv_load.R renv_load_illumina.R
 RUN Rscript renv_load_illumina.R
 
-COPY workers/data_refinery_workers/processors/requirements.txt .
+COPY --chmod=644 workers/data_refinery_workers/processors/requirements.txt .
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
 # Get the latest version from the dist directory.
-COPY common/dist/data-refinery-common-* common/
-RUN pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
-
-COPY .boto .boto
-COPY workers/ .
-COPY workers/data_refinery_workers/processors/detect_database.R .
-COPY workers/illumina_probe_maps/ probe_maps/
-
-RUN rm -rf /root/.cache/*
+COPY --chmod=644 common/dist/data-refinery-common-* common/
+RUN <<EOF
+pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
+rm -r common
+EOF
+
+COPY --chmod=644 .boto .boto
+COPY --chmod=644 workers/ .
+COPY --chmod=644 workers/data_refinery_workers/processors/detect_database.R .
+COPY --chmod=644 workers/illumina_probe_maps/ probe_maps/
+
+RUN <<EOF
+find /home/user -type d ! -perm 755 -exec chmod 755 {} \;
+rm -rf /root/.cache/*
+EOF
 
 ENV SYSTEM_VERSION=$SYSTEM_VERSION