fixed gcs build by adding missing dependencies

ORNL · Jan 3, 2024 · ab4e9bc · ab4e9bc
1 parent 5d9480b
commit ab4e9bc
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 33 deletions.
diff --git a/dockerfiles/dependencies.Dockerfile b/dockerfiles/dependencies.Dockerfile
@@ -44,6 +44,7 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_depen
 
 COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/
 RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf from
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protoc from
 RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq from
 RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium from
 RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options from

diff --git a/repository/gcs.Dockerfile b/repository/gcs.Dockerfile
@@ -17,6 +17,13 @@ FROM ${GCS_IMAGE}
 ARG DATAFED_DIR
 ARG BUILD_DIR
 ARG DATAFED_INSTALL_PATH
+ARG LIB_DIR
+
+ENV GCS_COLLECTION_ROOT_PATH="$DATAFED_DIR/collections/mapped"
+ENV     DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV	             DATAFED_DIR="$DATAFED_DIR"
+ENV                BUILD_DIR="$BUILD_DIR"
+ENV                  LIB_DIR="$LIB_DIR"
 
 RUN mkdir -p ${BUILD_DIR}
 RUN mkdir -p ${BUILD_DIR}/logs
@@ -43,18 +50,6 @@ COPY ./scripts/copy_dependency.sh   ${BUILD_DIR}/scripts/
 
 RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh unify
 
-ARG rebuild=true
-ARG DATAFED_DIR
-ARG BUILD_DIR
-ARG DATAFED_INSTALL_PATH
-ARG LIB_DIR
-
-ENV GCS_COLLECTION_ROOT_PATH="$DATAFED_DIR/collections/mapped"
-ENV     DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
-ENV	             DATAFED_DIR="$DATAFED_DIR"
-ENV                BUILD_DIR="$BUILD_DIR"
-ENV                  LIB_DIR="$LIB_DIR"
-
 
 # All files should be owned by the datafed user
 RUN chown -R datafed:root ${DATAFED_DIR}
@@ -72,35 +67,40 @@ COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/r
 COPY --chown=datafed:root ./common                                ${BUILD_DIR}/common
 COPY --chown=datafed:root ./repository/gridftp/globus5            ${BUILD_DIR}/repository/gridftp/globus5
 
-COPY --from=dependencies /usr/local/include/google /usr/local/include/google
-COPY --from=dependencies /usr/local/include/sodium /usr/local/include/sodium
-COPY --from=dependencies /usr/local/include/sodium.h /usr/local/include/sodium.h
-COPY --from=dependencies /usr/local/include/zmq.h /usr/local/include/zmq.h
-COPY --from=dependencies /usr/local/include/zmq_utils.h /usr/local/include/zmq_utils.h
+# These extra copy statements are necessary as the authz binary is built in the GCS container
+# and not the dependencies container so we must copy the build dependencies as well as the
+# runtime depedencies
+COPY --from=dependencies /usr/local/include/google					/usr/local/include/google
+COPY --from=dependencies /usr/local/include/sodium					/usr/local/include/sodium
+COPY --from=dependencies /usr/local/include/sodium.h				/usr/local/include/sodium.h
+COPY --from=dependencies /usr/local/include/zmq.h						/usr/local/include/zmq.h
+COPY --from=dependencies /usr/local/include/zmq_utils.h			/usr/local/include/zmq_utils.h
+COPY --from=dependencies /usr/include/zmq.hpp								/usr/include/zmq.hpp
 COPY --from=dependencies /usr/local/lib/pkgconfig/libzmq.pc /usr/local/lib/pkgconfig/libzmq.pc
-RUN rm /usr/include/zmq*
-RUN rm /usr/lib/x86_64-linux-gnu/*zmq*
-COPY --from=dependencies /usr/local/lib/cmake /usr/local/lib/cmake
+COPY --from=dependencies /usr/local/lib/cmake								/usr/local/lib/cmake
+COPY --from=dependencies /usr/local/bin/protoc							/usr/local/bin/protoc
 
-COPY --from=dependencies /libraries/libprotobuf.so           /libraries/libprotobuf.so
-COPY --from=dependencies /libraries/libzmq.so                /libraries/libzmq.so
-COPY --from=dependencies /libraries/libsodium.so             /libraries/libsodium.so
+COPY --from=dependencies /libraries/libprotobuf.so          /libraries/libprotobuf.so
+COPY --from=dependencies /libraries/libprotoc.so						/libraries/libprotoc.so
+COPY --from=dependencies /libraries/libzmq.so               /libraries/libzmq.so
+COPY --from=dependencies /libraries/libsodium.so            /libraries/libsodium.so
 RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
+RUN ${BUILD_DIR}/scripts/copy_dependency.sh protoc to
 RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
 RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
 
 RUN ${BUILD_DIR}/scripts/generate_datafed.sh
 
 RUN ${BUILD_DIR}/scripts/generate_authz_config.sh &&	\
-	cmake -S. -B build				\
+	cmake -S. -B build						\
 		-DBUILD_REPO_SERVER=False		\
-		-DBUILD_AUTHZ=True			\
+		-DBUILD_AUTHZ=True					\
 		-DBUILD_CORE_SERVER=False		\
 		-DBUILD_WEB_SERVER=False		\
-		-DBUILD_DOCS=False			\
-		-DBUILD_PYTHON_CLIENT=False		\
-		-DBUILD_FOXX=False && \
-	cmake --build build
+		-DBUILD_DOCS=False					\
+		-DBUILD_PYTHON_CLIENT=False	\
+		-DBUILD_FOXX=False
+RUN	cmake --build build
 RUN cmake --build build --target install
 
 COPY --chown=datafed:root ./scripts/globus/setup_globus.sh        ${BUILD_DIR}/scripts/globus/setup_globus.sh

diff --git a/scripts/copy_dependency.sh b/scripts/copy_dependency.sh
@@ -5,10 +5,10 @@
 source "$BUILD_DIR/scripts/dependency_versions.sh"
 
 # Define the list of supported libraries and their environment variable names
-supported_libraries=("protobuf" "libsodium" "libzmq" "boost_program_options" "boost_filesystem")
-library_env_variables=("DATAFED_FULL_PROTOBUF_VERSION" "DATAFED_LIB_LIBSODIUM_VERSION" "DATAFED_LIB_LIBZMQ_VERSION" "DATAFED_BOOST" "DATAFED_BOOST")
-library_names=("libprotobuf.so" "libsodium.so" "libzmq.so" "libboost_program_options.so" "libboost_filesystem.so")
-library_locations=("$LIB_DIR" "$LIB_DIR" "$LIB_DIR" "/lib/x86_64-linux-gnu" "/lib/x86_64-linux-gnu")
+supported_libraries=("protobuf" "protoc" "libsodium" "libzmq" "boost_program_options" "boost_filesystem")
+library_env_variables=("DATAFED_FULL_PROTOBUF_VERSION" "DATAFED_FULL_PROTOBUF_VERSION" "DATAFED_LIB_LIBSODIUM_VERSION" "DATAFED_LIB_LIBZMQ_VERSION" "DATAFED_BOOST" "DATAFED_BOOST")
+library_names=("libprotobuf.so" "libprotoc.so" "libsodium.so" "libzmq.so" "libboost_program_options.so" "libboost_filesystem.so")
+library_locations=("$LIB_DIR" "$LIB_DIR" "$LIB_DIR" "$LIB_DIR" "/lib/x86_64-linux-gnu" "/lib/x86_64-linux-gnu")
 
 LIBRARIES_BASE_PATH="/libraries"