Merge branch 'devel' into JoshuaSBrown-fix-web-server-log-commands

ORNL · Jan 4, 2024 · f7a5edd · f7a5edd
2 parents 28d5d90 + a7529e6
commit f7a5edd
Show file tree

Hide file tree

Showing 16 changed files with 881 additions and 192 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,4 +1,3 @@
-
 stages:
   - ci-infrastructure-check
   - build-infrastructure

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 ## MINOR Feature
 1. [909] - Added Support for Google Analytics
+2. [916] - Reworked Docker build process to be faster and produce simpler and smaller images
 
 ## PATCH Bug fixes/Technical Debt/Documentation
 1. [914] - Improve GitHub template

diff --git a/core/Dockerfile b/core/Dockerfile
@@ -0,0 +1,81 @@
+ARG           BUILD_BASE="ubuntu:focal"
+ARG         DEPENDENCIES="dependencies"
+ARG              RUNTIME="runtime"
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ${DEPENDENCIES} AS core-build
+
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+
+# For communicating with repo server
+EXPOSE 7512
+# For listening to web server
+EXPOSE 7513
+
+COPY ./core/CMakeLists.txt             ${BUILD_DIR}/core/CMakeLists.txt
+COPY ./CMakeLists.txt                  ${BUILD_DIR}
+COPY ./scripts/dependency_versions.sh  ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_datafed.sh     ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_core_config.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_core.sh         ${BUILD_DIR}/scripts/
+COPY ./cmake                           ${BUILD_DIR}/cmake
+COPY ./core/docker/entrypoint.sh       ${BUILD_DIR}/core/docker/
+COPY ./core/server                     ${BUILD_DIR}/core/server
+
+RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
+	cmake -S. -B build						\
+		-DBUILD_REPO_SERVER=False		\
+		-DBUILD_AUTHZ=False					\
+		-DBUILD_CORE_SERVER=True		\
+		-DBUILD_WEB_SERVER=False		\
+		-DBUILD_DOCS=False					\
+		-DBUILD_PYTHON_CLIENT=False	\
+		-DBUILD_FOXX=False
+RUN cmake --build build -j 8
+RUN cmake --build build --target install
+
+FROM ${RUNTIME} AS core
+
+ARG DATAFED_DIR
+ARG DATAFED_INSTALL_PATH
+ARG BUILD_DIR
+ARG LIB_DIR
+
+# The above should also be available at runtime
+ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV          DATAFED_DIR="$DATAFED_DIR"
+ENV            BUILD_DIR="$BUILD_DIR"
+ENV              LIB_DIR="$LIB_DIR"
+
+# copy necessary shared libraries
+COPY --from=core-build /libraries/libprotobuf.so           /libraries/libprotobuf.so
+COPY --from=core-build /libraries/libzmq.so                /libraries/libzmq.so
+COPY --from=core-build /libraries/libsodium.so             /libraries/libsodium.so
+COPY --from=core-build /libraries/libboost_program_options.so /libraries/libboost_program_options.so
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to
+
+RUN ldconfig
+
+USER datafed
+
+COPY --chown=datafed:root ./scripts/generate_datafed.sh     ${DATAFED_DIR}/scripts/generate_datafed.sh
+COPY --chown=datafed:root ./scripts/generate_core_config.sh ${DATAFED_DIR}/scripts/generate_core_config.sh
+COPY --chown=datafed:root ./scripts/install_core.sh         ${DATAFED_DIR}/scripts/install_core.sh
+COPY --chown=datafed:root ./cmake/Version.cmake             ${DATAFED_DIR}/cmake/Version.cmake
+COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh    ${BUILD_DIR}/core/entrypoint.sh
+COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core
+
+ENTRYPOINT ["${BUILD_DIR}/core/entrypoint.sh"]
+CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"]
diff --git a/dockerfiles/dependencies.Dockerfile b/dockerfiles/dependencies.Dockerfile
@@ -0,0 +1,60 @@
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ubuntu:focal
+
+ARG NVM_DIR
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+ARG DEBIAN_FRONTEND=noninteractive
+ARG LIB_DIR
+
+ENV BUILD_DIR="${BUILD_DIR}"
+ENV DATAFED_DIR="${DATAFED_DIR}"
+ENV LIB_DIR="${LIB_DIR}"
+
+RUN mkdir -p ${BUILD_DIR}
+RUN mkdir -p ${BUILD_DIR}/logs
+RUN mkdir -p ${BUILD_DIR}/repository/server
+RUN mkdir -p ${BUILD_DIR}/common/proto
+RUN mkdir -p /libraries
+
+WORKDIR ${BUILD_DIR}
+
+# Copy install scripts
+COPY ./scripts/dependency_install_functions.sh	${BUILD_DIR}/scripts/
+COPY ./scripts/dependency_versions.sh						${BUILD_DIR}/scripts/
+COPY ./scripts/install_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/
+
+RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo
+
+# run build scripts
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh
+
+COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protoc from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem from
+
+RUN mkdir -p ${DATAFED_INSTALL_PATH}
+RUN mkdir -p ${DATAFED_INSTALL_PATH}/keys
+
+WORKDIR ${BUILD_DIR}
+
+COPY ./common ${BUILD_DIR}/common
+
+# RUN cp -R $HOME/.nvm ${NVM_DIR}
diff --git a/dockerfiles/runtime.Dockerfile b/dockerfiles/runtime.Dockerfile
@@ -0,0 +1,37 @@
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ubuntu:focal
+
+ARG DATAFED_DIR
+ARG DATAFED_INSTALL_PATH
+ARG BUILD_DIR
+
+ENV BUILD_DIR="${BUILD_DIR}"
+ENV DATAFED_DIR="${DATAFED_DIR}"
+
+RUN echo $DATAFED_DIR
+
+# Create datafed user, prefer more secure login options than password
+# Recommended to mount ssh public key on run
+RUN adduser --disabled-password --gecos "" datafed
+
+COPY ./scripts/dependency_versions.sh  ${BUILD_DIR}/scripts/
+COPY ./scripts/copy_dependency.sh      ${BUILD_DIR}/scripts/
+RUN mkdir -p ${DATAFED_DIR}
+RUN mkdir -p /opt/datafed
+RUN mkdir -p /var/log/datafed
+RUN chown -R datafed:root /opt/datafed
+RUN chown -R datafed:root /var/log/datafed
+RUN chown -R datafed:root ${DATAFED_DIR}
+WORKDIR ${DATAFED_DIR}
+
+RUN apt update
+RUN apt install -y grep libcurl4
+# ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib:/usr/local/lib"
diff --git a/repository/Dockerfile b/repository/Dockerfile
@@ -0,0 +1,86 @@
+ARG           BUILD_BASE="ubuntu:focal"
+ARG         DEPENDENCIES="dependencies"
+ARG              RUNTIME="runtime"
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ${DEPENDENCIES} AS repo-build
+
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+
+# This port is needed to communicate with the DataFed core server
+EXPOSE 7512
+# Not quite sure what 9000 is doing that 7512 isn't, difference between egress
+# and ingress?
+EXPOSE 9000
+
+COPY ./repository/CMakeLists.txt               ${BUILD_DIR}/repository/CMakeLists.txt
+COPY ./CMakeLists.txt                          ${BUILD_DIR}
+COPY ./scripts/dependency_versions.sh          ${BUILD_DIR}/scripts/
+COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_datafed.sh             ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_repo_config.sh         ${BUILD_DIR}/scripts/
+COPY ./scripts/install_repo.sh                 ${BUILD_DIR}/scripts/
+COPY ./cmake                                   ${BUILD_DIR}/cmake
+COPY ./repository/server                       ${BUILD_DIR}/repository/server
+
+RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
+	cmake -S. -B build						\
+	-DBUILD_REPO_SERVER=True		\
+	-DBUILD_AUTHZ=False					\
+	-DBUILD_CORE_SERVER=False		\
+	-DBUILD_WEB_SERVER=False		\
+	-DBUILD_DOCS=False					\
+	-DBUILD_PYTHON_CLIENT=False	\
+	-DBUILD_FOXX=False
+RUN cmake --build build
+RUN cmake --build build --target install
+
+FROM ${RUNTIME} AS repo
+
+ARG DATAFED_DIR
+ARG DATAFED_INSTALL_PATH
+ARG BUILD_DIR
+ARG LIB_DIR
+
+# The above should also be available at runtime
+ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV          DATAFED_DIR="$DATAFED_DIR"
+ENV            BUILD_DIR="$BUILD_DIR"
+ENV              LIB_DIR="$LIB_DIR"
+
+WORKDIR /datafed
+
+# copy necessary shared libraries
+COPY --from=repo-build /libraries/libprotobuf.so           /libraries/libprotobuf.so
+COPY --from=repo-build /libraries/libzmq.so                /libraries/libzmq.so
+COPY --from=repo-build /libraries/libsodium.so             /libraries/libsodium.so
+COPY --from=repo-build /libraries/libboost_program_options.so /libraries/libboost_program_options.so
+COPY --from=repo-build /libraries/libboost_filesystem.so      /libraries/libboost_filesystem.so
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem to
+
+RUN ldconfig
+
+USER datafed
+
+COPY --chown=datafed:root ./repository/docker/entrypoint_repo.sh ${BUILD_DIR}/repository/entrypoint.sh
+COPY --chown=datafed:root ./scripts/generate_datafed.sh          ${DATAFED_DIR}/scripts/generate_datafed.sh
+COPY --chown=datafed:root ./scripts/generate_repo_config.sh      ${DATAFED_DIR}/scripts/generate_repo_config.sh
+COPY --chown=datafed:root ./scripts/install_repo.sh              ${DATAFED_DIR}/scripts/install_repo.sh
+COPY --chown=datafed:root ./cmake/Version.cmake                  ${DATAFED_DIR}/cmake/Version.cmake
+COPY --from=repo-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/repo/datafed-repo ${DATAFED_INSTALL_PATH}/repo/datafed-repo
+
+ENTRYPOINT ["${BUILD_DIR}/repository/entrypoint.sh"]
+CMD ["${DATAFED_INSTALL_PATH}/repo/datafed-repo","--cfg","${DATAFED_INSTALL_PATH}/repo/datafed-repo.cfg"]
diff --git a/repository/gcs.Dockerfile b/repository/gcs.Dockerfile
@@ -0,0 +1,112 @@
+ARG           BUILD_BASE="ubuntu:focal"
+ARG         DEPENDENCIES="dependencies"
+ARG              RUNTIME="runtime"
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ${DEPENDENCIES} AS dependencies
+
+FROM ${GCS_IMAGE}
+
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+ARG LIB_DIR
+
+ENV GCS_COLLECTION_ROOT_PATH="$DATAFED_DIR/collections/mapped"
+ENV     DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV	             DATAFED_DIR="$DATAFED_DIR"
+ENV                BUILD_DIR="$BUILD_DIR"
+ENV                  LIB_DIR="$LIB_DIR"
+
+RUN mkdir -p ${BUILD_DIR}
+RUN mkdir -p ${BUILD_DIR}/logs
+RUN mkdir -p ${BUILD_DIR}/common/proto
+RUN mkdir -p ${DATAFED_INSTALL_PATH}/authz
+RUN mkdir -p ${DATAFED_DIR}/collections/mapped
+
+# For communicating with public
+EXPOSE 443
+
+# Needed for Globus GridFTP communication
+EXPOSE 50000-51000
+
+WORKDIR ${BUILD_DIR}
+
+RUN adduser --disabled-password --gecos "" datafed
+
+RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo
+
+COPY ./scripts/dependency_versions.sh          ${BUILD_DIR}/scripts/
+COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_authz_dependencies.sh   ${BUILD_DIR}/scripts/
+COPY ./scripts/copy_dependency.sh   ${BUILD_DIR}/scripts/
+
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh unify
+
+
+# All files should be owned by the datafed user
+RUN chown -R datafed:root ${DATAFED_DIR}
+
+COPY --chown=datafed:root ./scripts/dependency_versions.sh        ${BUILD_DIR}/scripts/
+COPY --chown=datafed:root ./scripts/generate_authz_config.sh      ${BUILD_DIR}/scripts/generate_authz_config.sh
+COPY --chown=datafed:root ./scripts/generate_datafed.sh           ${BUILD_DIR}/scripts/generate_datafed.sh
+COPY --chown=datafed:root ./CMakeLists.txt                        ${BUILD_DIR}
+COPY --chown=datafed:root ./cmake                                 ${BUILD_DIR}/cmake
+COPY --chown=datafed:root ./repository/CMakeLists.txt             ${BUILD_DIR}/repository/CMakeLists.txt
+COPY --chown=datafed:root ./repository/gridftp/CMakeLists.txt     ${BUILD_DIR}/repository/gridftp/CMakeLists.txt
+COPY --chown=datafed:root ./scripts/globus/setup_globus.sh        ${BUILD_DIR}/scripts/globus/setup_globus.sh
+COPY --chown=datafed:root ./scripts/globus/generate_repo_form.sh  ${BUILD_DIR}/scripts/globus/generate_repo_form.sh
+COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh
+COPY --chown=datafed:root ./common                                ${BUILD_DIR}/common
+COPY --chown=datafed:root ./repository/gridftp/globus5            ${BUILD_DIR}/repository/gridftp/globus5
+
+# These extra copy statements are necessary as the authz binary is built in the GCS container
+# and not the dependencies container so we must copy the build dependencies as well as the
+# runtime depedencies
+COPY --from=dependencies /usr/local/include/google					/usr/local/include/google
+COPY --from=dependencies /usr/local/include/sodium					/usr/local/include/sodium
+COPY --from=dependencies /usr/local/include/sodium.h				/usr/local/include/sodium.h
+COPY --from=dependencies /usr/local/include/zmq.h						/usr/local/include/zmq.h
+COPY --from=dependencies /usr/local/include/zmq_utils.h			/usr/local/include/zmq_utils.h
+COPY --from=dependencies /usr/include/zmq.hpp								/usr/include/zmq.hpp
+COPY --from=dependencies /usr/local/lib/pkgconfig/libzmq.pc /usr/local/lib/pkgconfig/libzmq.pc
+COPY --from=dependencies /usr/local/lib/cmake								/usr/local/lib/cmake
+COPY --from=dependencies /usr/local/bin/protoc							/usr/local/bin/protoc
+
+COPY --from=dependencies /libraries/libprotobuf.so          /libraries/libprotobuf.so
+COPY --from=dependencies /libraries/libprotoc.so						/libraries/libprotoc.so
+COPY --from=dependencies /libraries/libzmq.so               /libraries/libzmq.so
+COPY --from=dependencies /libraries/libsodium.so            /libraries/libsodium.so
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protoc to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
+
+RUN ${BUILD_DIR}/scripts/generate_datafed.sh
+
+RUN ${BUILD_DIR}/scripts/generate_authz_config.sh &&	\
+	cmake -S. -B build						\
+		-DBUILD_REPO_SERVER=False		\
+		-DBUILD_AUTHZ=True					\
+		-DBUILD_CORE_SERVER=False		\
+		-DBUILD_WEB_SERVER=False		\
+		-DBUILD_DOCS=False					\
+		-DBUILD_PYTHON_CLIENT=False	\
+		-DBUILD_FOXX=False
+RUN	cmake --build build
+RUN cmake --build build --target install
+
+COPY --chown=datafed:root ./scripts/globus/setup_globus.sh        ${BUILD_DIR}/scripts/globus/setup_globus.sh
+COPY --chown=datafed:root ./scripts/globus/generate_repo_form.sh  ${BUILD_DIR}/scripts/globus/generate_repo_form.sh
+COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh
+
+USER root
+
+WORKDIR ${DATAFED_INSTALL_PATH}/authz