Blake nedved rework docker building

dockerfiles/core.Dockerfile

@@ -0,0 +1,86 @@
+ARG           BUILD_BASE="ubuntu:focal"
+ARG         DEPENDENCIES="dependencies"
+ARG              RUNTIME="runtime"
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ${DEPENDENCIES} AS core-build
+
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+
+# For communicating with repo server
+EXPOSE 7512
+# For listening to web server
+EXPOSE 7513
+
+COPY ./core/CMakeLists.txt             ${BUILD_DIR}/core/CMakeLists.txt
+COPY ./CMakeLists.txt                  ${BUILD_DIR}
+COPY ./scripts/dependency_versions.sh  ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_datafed.sh     ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_core_config.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_core.sh         ${BUILD_DIR}/scripts/
+COPY ./cmake                           ${BUILD_DIR}/cmake
+COPY ./core/docker/entrypoint.sh       ${BUILD_DIR}/core/docker/
+COPY ./core/server                     ${BUILD_DIR}/core/server
+
+# All files should be owned by the datafed user
+# RUN chown -R datafed:datafed ${DATAFED_DIR}
+#
+# USER datafed
+
+RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
+	cmake -S. -B build						\
+		-DBUILD_REPO_SERVER=False		\
+		-DBUILD_AUTHZ=False					\
+		-DBUILD_CORE_SERVER=True		\
+		-DBUILD_WEB_SERVER=False		\
+		-DBUILD_DOCS=False					\
+		-DBUILD_PYTHON_CLIENT=False	\
+		-DBUILD_FOXX=False
+RUN cmake --build build -j 8
+RUN cmake --build build --target install
+
+FROM ${RUNTIME} AS core
+
+ARG DATAFED_DIR
+ARG DATAFED_INSTALL_PATH
+ARG BUILD_DIR
+ARG LIB_DIR
+
+# The above should also be available at runtime
+ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV          DATAFED_DIR="$DATAFED_DIR"
+ENV            BUILD_DIR="$BUILD_DIR"
+ENV              LIB_DIR="$LIB_DIR"
+
+# copy necessary shared libraries
+COPY --from=core-build /libraries/libprotobuf.so           /libraries/libprotobuf.so
+COPY --from=core-build /libraries/libzmq.so                /libraries/libzmq.so
+COPY --from=core-build /libraries/libsodium.so             /libraries/libsodium.so
+COPY --from=core-build /libraries/libboost_program_options.so /libraries/libboost_program_options.so
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to
+
+RUN ldconfig
+
+USER datafed
+
+COPY --chown=datafed:root ./scripts/generate_datafed.sh     ${DATAFED_DIR}/scripts/generate_datafed.sh
+COPY --chown=datafed:root ./scripts/generate_core_config.sh ${DATAFED_DIR}/scripts/generate_core_config.sh
+COPY --chown=datafed:root ./scripts/install_core.sh         ${DATAFED_DIR}/scripts/install_core.sh
+COPY --chown=datafed:root ./cmake/Version.cmake             ${DATAFED_DIR}/cmake/Version.cmake
+COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh    ${BUILD_DIR}/core/entrypoint.sh
+COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core
+
+ENTRYPOINT ["${BUILD_DIR}/core/entrypoint.sh"]
+CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"]

dockerfiles/dependencies.Dockerfile

@@ -0,0 +1,60 @@
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ubuntu:focal
+
+ARG NVM_DIR
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+ARG DEBIAN_FRONTEND=noninteractive
+ARG LIB_DIR
+
+ENV BUILD_DIR="${BUILD_DIR}"
+ENV DATAFED_DIR="${DATAFED_DIR}"
+ENV LIB_DIR="${LIB_DIR}"
+
+RUN mkdir -p ${BUILD_DIR}
+RUN mkdir -p ${BUILD_DIR}/logs
+RUN mkdir -p ${BUILD_DIR}/repository/server
+RUN mkdir -p ${BUILD_DIR}/common/proto
+RUN mkdir -p /libraries
+RUN mkdir -p ${NVM_DIR}
+
+WORKDIR ${BUILD_DIR}
+
+# Copy install scripts
+COPY ./scripts/dependency_install_functions.sh	${BUILD_DIR}/scripts/
+COPY ./scripts/dependency_versions.sh						${BUILD_DIR}/scripts/
+COPY ./scripts/install_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/
+
+RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo
+
+# run build scripts
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh
+
+COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem from
+
+RUN mkdir -p ${DATAFED_INSTALL_PATH}
+RUN mkdir -p ${DATAFED_INSTALL_PATH}/keys
+
+WORKDIR ${BUILD_DIR}
+
+COPY ./common ${BUILD_DIR}/common
+
+# RUN cp -R $HOME/.nvm ${NVM_DIR}

dockerfiles/gcs.Dockerfile

@@ -0,0 +1,96 @@
+ARG           BUILD_BASE="ubuntu:focal"
+ARG         DEPENDENCIES="dependencies"
+ARG              RUNTIME="runtime"
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ${GCS_IMAGE}
+
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+
+RUN mkdir -p ${BUILD_DIR}
+RUN mkdir -p ${BUILD_DIR}/logs
+RUN mkdir -p ${BUILD_DIR}/common/proto
+RUN mkdir -p ${DATAFED_INSTALL_PATH}/authz
+RUN mkdir -p ${DATAFED_DIR}/collections/mapped
+
+RUN apt update
+RUN apt install -y vim netcat
+
+# For communicating with public
+EXPOSE 443
+
+# Needed for Globus GridFTP communication
+EXPOSE 50000-51000
+
+WORKDIR ${BUILD_DIR}
+
+RUN adduser --disabled-password --gecos "" datafed
+
+RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo
+
+COPY ./scripts/dependency_versions.sh          ${BUILD_DIR}/scripts/
+COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/install_authz_dependencies.sh   ${BUILD_DIR}/scripts/
+
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh
+
+ARG rebuild=true
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+
+ENV GCS_COLLECTION_ROOT_PATH="$DATAFED_DIR/collections/mapped"
+ENV     DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV	             DATAFED_DIR="$DATAFED_DIR"
+ENV                BUILD_DIR="$BUILD_DIR"
+
+
+# All files should be owned by the datafed user
+RUN chown -R datafed:datafed ${DATAFED_DIR}
+
+COPY --chown=datafed:root ./scripts/dependency_versions.sh        ${BUILD_DIR}/scripts/
+COPY --chown=datafed:root ./scripts/generate_authz_config.sh      ${BUILD_DIR}/scripts/generate_authz_config.sh
+COPY --chown=datafed:root ./scripts/generate_datafed.sh           ${BUILD_DIR}/scripts/generate_datafed.sh
+COPY --chown=datafed:root ./CMakeLists.txt                        ${BUILD_DIR}
+COPY --chown=datafed:root ./cmake                                 ${BUILD_DIR}/cmake
+COPY --chown=datafed:root ./repository/CMakeLists.txt             ${BUILD_DIR}/repository/CMakeLists.txt
+COPY --chown=datafed:root ./repository/gridftp/CMakeLists.txt     ${BUILD_DIR}/repository/gridftp/CMakeLists.txt
+COPY --chown=datafed:root ./scripts/globus/setup_globus.sh        ${BUILD_DIR}/scripts/globus/setup_globus.sh
+COPY --chown=datafed:root ./scripts/globus/generate_repo_form.sh  ${BUILD_DIR}/scripts/globus/generate_repo_form.sh
+COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh
+COPY --chown=datafed:root ./common                                ${BUILD_DIR}/common
+COPY --chown=datafed:root ./repository/gridftp/globus5            ${BUILD_DIR}/repository/gridftp/globus5
+
+# Build as if a non root user
+USER datafed
+
+RUN ${BUILD_DIR}/scripts/generate_datafed.sh
+
+RUN ${BUILD_DIR}/scripts/generate_authz_config.sh &&	\
+	cmake -S. -B build				\
+		-DBUILD_REPO_SERVER=False		\
+		-DBUILD_AUTHZ=True			\
+		-DBUILD_CORE_SERVER=False		\
+		-DBUILD_WEB_SERVER=False		\
+		-DBUILD_DOCS=False			\
+		-DBUILD_PYTHON_CLIENT=False		\
+		-DBUILD_FOXX=False
+RUN cmake --build build
+RUN cmake --build build --target install
+
+COPY ./scripts/globus/setup_globus.sh        ${BUILD_DIR}/scripts/globus/setup_globus.sh
+COPY ./scripts/globus/generate_repo_form.sh  ${BUILD_DIR}/scripts/globus/generate_repo_form.sh
+COPY ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh
+
+USER root
+
+WORKDIR ${DATAFED_INSTALL_PATH}/authz

dockerfiles/repo.Dockerfile

@@ -0,0 +1,86 @@
+ARG           BUILD_BASE="ubuntu:focal"
+ARG         DEPENDENCIES="dependencies"
+ARG              RUNTIME="runtime"
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ${DEPENDENCIES} AS repo-build
+
+ARG DATAFED_DIR
+ARG BUILD_DIR
+ARG DATAFED_INSTALL_PATH
+
+# This port is needed to communicate with the DataFed core server
+EXPOSE 7512
+# Not quite sure what 9000 is doing that 7512 isn't, difference between egress
+# and ingress?
+EXPOSE 9000
+
+COPY ./repository/CMakeLists.txt               ${BUILD_DIR}/repository/CMakeLists.txt
+COPY ./CMakeLists.txt                          ${BUILD_DIR}
+COPY ./scripts/dependency_versions.sh          ${BUILD_DIR}/scripts/
+COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_datafed.sh             ${BUILD_DIR}/scripts/
+COPY ./scripts/generate_repo_config.sh         ${BUILD_DIR}/scripts/
+COPY ./scripts/install_repo.sh                 ${BUILD_DIR}/scripts/
+COPY ./cmake                                   ${BUILD_DIR}/cmake
+COPY ./repository/server                       ${BUILD_DIR}/repository/server
+
+RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
+	cmake -S. -B build						\
+	-DBUILD_REPO_SERVER=True		\
+	-DBUILD_AUTHZ=False					\
+	-DBUILD_CORE_SERVER=False		\
+	-DBUILD_WEB_SERVER=False		\
+	-DBUILD_DOCS=False					\
+	-DBUILD_PYTHON_CLIENT=False	\
+	-DBUILD_FOXX=False
+RUN cmake --build build
+RUN cmake --build build --target install
+
+FROM ${RUNTIME} AS repo
+
+ARG DATAFED_DIR
+ARG DATAFED_INSTALL_PATH
+ARG BUILD_DIR
+ARG LIB_DIR
+
+# The above should also be available at runtime
+ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV          DATAFED_DIR="$DATAFED_DIR"
+ENV            BUILD_DIR="$BUILD_DIR"
+ENV              LIB_DIR="$LIB_DIR"
+
+WORKDIR /datafed
+
+# copy necessary shared libraries
+COPY --from=repo-build /libraries/libprotobuf.so           /libraries/libprotobuf.so
+COPY --from=repo-build /libraries/libzmq.so                /libraries/libzmq.so
+COPY --from=repo-build /libraries/libsodium.so             /libraries/libsodium.so
+COPY --from=repo-build /libraries/libboost_program_options.so /libraries/libboost_program_options.so
+COPY --from=repo-build /libraries/libboost_filesystem.so      /libraries/libboost_filesystem.so
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem to
+
+RUN ldconfig
+
+USER datafed
+
+COPY --chown=datafed:root ./repository/docker/entrypoint_repo.sh ${BUILD_DIR}/repository/entrypoint.sh
+COPY --chown=datafed:root ./scripts/generate_datafed.sh          ${DATAFED_DIR}/scripts/generate_datafed.sh
+COPY --chown=datafed:root ./scripts/generate_repo_config.sh      ${DATAFED_DIR}/scripts/generate_repo_config.sh
+COPY --chown=datafed:root ./scripts/install_repo.sh              ${DATAFED_DIR}/scripts/install_repo.sh
+COPY --chown=datafed:root ./cmake/Version.cmake                  ${DATAFED_DIR}/cmake/Version.cmake
+COPY --from=repo-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/repo/datafed-repo ${DATAFED_INSTALL_PATH}/repo/datafed-repo
+
+ENTRYPOINT ["${BUILD_DIR}/repository/entrypoint.sh"]
+CMD ["${DATAFED_INSTALL_PATH}/repo/datafed-repo","--cfg","${DATAFED_INSTALL_PATH}/repo/datafed-repo.cfg"]

dockerfiles/runtime.Dockerfile

@@ -0,0 +1,37 @@
+ARG          DATAFED_DIR="/datafed"
+ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
+ARG            BUILD_DIR="$DATAFED_DIR/source"
+ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+ARG              LIB_DIR="/usr/local/lib"
+
+FROM ubuntu:focal
+
+ARG DATAFED_DIR
+ARG DATAFED_INSTALL_PATH
+ARG BUILD_DIR
+
+ENV BUILD_DIR="${BUILD_DIR}"
+ENV DATAFED_DIR="${DATAFED_DIR}"
+
+RUN echo $DATAFED_DIR
+
+# Create datafed user, prefer more secure login options than password
+# Recommended to mount ssh public key on run
+RUN adduser --disabled-password --gecos "" datafed
+
+COPY ./scripts/dependency_versions.sh  ${BUILD_DIR}/scripts/
+COPY ./scripts/copy_dependency.sh      ${BUILD_DIR}/scripts/
+RUN mkdir -p ${DATAFED_DIR}
+RUN mkdir -p /opt/datafed
+RUN mkdir -p /var/log/datafed
+RUN chown -R datafed:root /opt/datafed
+RUN chown -R datafed:root /var/log/datafed
+RUN chown -R datafed:root ${DATAFED_DIR}
+WORKDIR ${DATAFED_DIR}
+
+RUN apt update
+RUN apt install -y grep libcurl4
+# ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib:/usr/local/lib"