Skip to content

Commit

Permalink
added script to copy dependencies with dynamic versions
Browse files Browse the repository at this point in the history
  • Loading branch information
nedvedba committed Nov 8, 2023
1 parent bd860cb commit 3e8546b
Show file tree
Hide file tree
Showing 3 changed files with 162 additions and 77 deletions.
147 changes: 70 additions & 77 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
ARG DATAFED_DIR="/datafed"
ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install"
ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
ARG BUILD_DIR="$DATAFED_DIR/source"
ARG NVM_DIR="$DATAFED_DIR/.nvm"
ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
ARG LIB_DIR="/usr/local/lib"
ARG DATAFED_DIR="/datafed"
ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install"
ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
ARG BUILD_DIR="$DATAFED_DIR/source"
ARG NVM_DIR="$DATAFED_DIR/.nvm"
ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
ARG LIB_DIR="/usr/local/lib"

FROM ubuntu:focal AS base

Expand All @@ -16,13 +16,17 @@ ARG DATAFED_DIR
ARG BUILD_DIR
ARG DATAFED_INSTALL_PATH
ARG DEBIAN_FRONTEND=noninteractive
ARG LIB_DIR

ENV BUILD_DIR="${BUILD_DIR}"
ENV DATAFED_DIR="${DATAFED_DIR}"
ENV LIB_DIR="${LIB_DIR}"

RUN mkdir -p ${BUILD_DIR}
RUN mkdir -p ${BUILD_DIR}/logs
RUN mkdir -p ${BUILD_DIR}/repository/server
RUN mkdir -p ${BUILD_DIR}/common/proto
RUN mkdir -p /libraries
RUN mkdir -p ${NVM_DIR}

WORKDIR ${BUILD_DIR}
Expand All @@ -47,6 +51,13 @@ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo
# RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh

COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/
RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf from
RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq from
RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium from
RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options from
RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem from

# RUN cp -R $HOME/.nvm ${NVM_DIR}

FROM ${GCS_IMAGE} AS gcs-base
Expand Down Expand Up @@ -123,15 +134,15 @@ COPY ./core/server ${BUILD_DIR}/core/server
#
# USER datafed

RUN ${BUILD_DIR}/scripts/generate_datafed.sh
RUN cmake -S. -B build \
-DBUILD_REPO_SERVER=False \
-DBUILD_AUTHZ=False \
-DBUILD_CORE_SERVER=True \
-DBUILD_WEB_SERVER=False \
-DBUILD_DOCS=False \
-DBUILD_PYTHON_CLIENT=False \
-DBUILD_FOXX=False
RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
cmake -S. -B build \
-DBUILD_REPO_SERVER=False \
-DBUILD_AUTHZ=False \
-DBUILD_CORE_SERVER=True \
-DBUILD_WEB_SERVER=False \
-DBUILD_DOCS=False \
-DBUILD_PYTHON_CLIENT=False \
-DBUILD_FOXX=False
RUN cmake --build build -j 8
RUN cmake --build build --target install

Expand All @@ -154,20 +165,18 @@ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/
COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/
COPY ./scripts/generate_repo_config.sh ${BUILD_DIR}/scripts/
COPY ./scripts/install_repo.sh ${BUILD_DIR}/scripts/
COPY ./cmake ${BUILD_DIR}/
COPY ./cmake ${BUILD_DIR}/cmake
COPY ./repository/server ${BUILD_DIR}/repository/server

RUN apt update && apt install -y cmake

RUN ${BUILD_DIR}/scripts/generate_datafed.sh
RUN cmake -S. -B build \
-DBUILD_REPO_SERVER=True \
-DBUILD_AUTHZ=False \
-DBUILD_CORE_SERVER=False \
-DBUILD_WEB_SERVER=False \
-DBUILD_DOCS=False \
-DBUILD_PYTHON_CLIENT=False \
-DBUILD_FOXX=False
RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
cmake -S. -B build \
-DBUILD_REPO_SERVER=True \
-DBUILD_AUTHZ=False \
-DBUILD_CORE_SERVER=False \
-DBUILD_WEB_SERVER=False \
-DBUILD_DOCS=False \
-DBUILD_PYTHON_CLIENT=False \
-DBUILD_FOXX=False
RUN cmake --build build
RUN cmake --build build --target install

Expand All @@ -194,16 +203,16 @@ COPY ./cmake ${BUILD_DIR}/cmake
COPY ./common/proto ${BUILD_DIR}/common/proto
COPY ./web ${BUILD_DIR}/web

RUN ${BUILD_DIR}/scripts/generate_datafed.sh
RUN cmake -S. -B build \
-DBUILD_REPO_SERVER=False \
-DBUILD_AUTHZ=False \
-DBUILD_CORE_SERVER=False \
-DBUILD_WEB_SERVER=True \
-DBUILD_DOCS=False \
-DBUILD_PYTHON_CLIENT=False \
-DBUILD_FOXX=False \
-DBUILD_COMMON=False
RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \
cmake -S. -B build \
-DBUILD_REPO_SERVER=False \
-DBUILD_AUTHZ=False \
-DBUILD_CORE_SERVER=False \
-DBUILD_WEB_SERVER=True \
-DBUILD_DOCS=False \
-DBUILD_PYTHON_CLIENT=False \
-DBUILD_FOXX=False \
-DBUILD_COMMON=False
RUN cmake --build build

ENV NVM_DIR="$NVM_DIR"
Expand All @@ -223,6 +232,8 @@ ARG BUILD_DIR
# Recommended to mount ssh public key on run
RUN adduser --disabled-password --gecos "" datafed

COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/
COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/
RUN mkdir -p ${DATAFED_DIR}
RUN mkdir -p /opt/datafed
RUN mkdir -p /var/log/datafed
Expand All @@ -246,25 +257,17 @@ ARG LIB_DIR
ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
ENV DATAFED_DIR="$DATAFED_DIR"
ENV BUILD_DIR="$BUILD_DIR"
ENV LIB_DIR="$LIB_DIR"

# copy necessary shared libraries
# libprotobuf
COPY --from=dependencies ${LIB_DIR}/libprotobuf.so.3.17.3.0 ${LIB_DIR}/libprotobuf.so.3.17.3.0
RUN ln -s ${LIB_DIR}/libprotobuf.so.3.17.3.0 ${LIB_DIR}/libprotobuf.so

# libzmq
COPY --from=dependencies ${LIB_DIR}/libzmq.so.5.2.4 ${LIB_DIR}/libzmq.so.5.2.4
RUN ln -s ${LIB_DIR}/libzmq.so.5.2.4 ${LIB_DIR}/libzmq.so.5
RUN ln -s ${LIB_DIR}/libzmq.so.5 ${LIB_DIR}/libzmq.so

# libsodium
COPY --from=dependencies ${LIB_DIR}/libsodium.so.23.3.0 ${LIB_DIR}/libsodium.so.23.3.0
RUN ln -s ${LIB_DIR}/libsodium.so.23.3.0 ${LIB_DIR}/libsodium.so.23
RUN ln -s ${LIB_DIR}/libsodium.so.23 ${LIB_DIR}/libsodium.so

# libboost-program-options
COPY --from=dependencies /lib/x86_64-linux-gnu/libboost_program_options.so.1.71.0 ${LIB_DIR}/libboost_program_options.so.1.71.0
RUN ln -s ${LIB_DIR}/libboost_program_options.so.1.71.0 ${LIB_DIR}/libboost_program_options.so
COPY --from=dependencies /libraries/libprotobuf.so /libraries/libprotobuf.so
COPY --from=dependencies /libraries/libzmq.so /libraries/libzmq.so
COPY --from=dependencies /libraries/libsodium.so /libraries/libsodium.so
COPY --from=dependencies /libraries/libboost_program_options.so /libraries/libboost_program_options.so
RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to

RUN ldconfig

Expand All @@ -291,31 +294,21 @@ ARG LIB_DIR
ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
ENV DATAFED_DIR="$DATAFED_DIR"
ENV BUILD_DIR="$BUILD_DIR"
ENV LIB_DIR="$LIB_DIR"

WORKDIR /datafed

# copy necessary shared libraries
# libprotobuf
COPY --from=dependencies ${LIB_DIR}/libprotobuf.so.3.17.3.0 ${LIB_DIR}/libprotobuf.so.3.17.3.0
RUN ln -s ${LIB_DIR}/libprotobuf.so.3.17.3.0 ${LIB_DIR}/libprotobuf.so

# libzmq
COPY --from=dependencies ${LIB_DIR}/libzmq.so.5.2.4 ${LIB_DIR}/libzmq.so.5.2.4
RUN ln -s ${LIB_DIR}/libzmq.so.5.2.4 ${LIB_DIR}/libzmq.so.5
RUN ln -s ${LIB_DIR}/libzmq.so.5 ${LIB_DIR}/libzmq.so

# libsodium
COPY --from=dependencies ${LIB_DIR}/libsodium.so.23.3.0 ${LIB_DIR}/libsodium.so.23.3.0
RUN ln -s ${LIB_DIR}/libsodium.so.23.3.0 ${LIB_DIR}/libsodium.so.23
RUN ln -s ${LIB_DIR}/libsodium.so.23 ${LIB_DIR}/libsodium.so

# libboost-filesystem
COPY --from=dependencies /lib/x86_64-linux-gnu/libboost_filesystem.so.1.71.0 ${LIB_DIR}/libboost_filesystem.so.1.71.0
RUN ln -s ${LIB_DIR}/libboost_filesystem.so.1.71.0 ${LIB_DIR}/libboost_filesystem.so

# libboost-program-options
COPY --from=dependencies /lib/x86_64-linux-gnu/libboost_program_options.so.1.71.0 ${LIB_DIR}/libboost_program_options.so.1.71.0
RUN ln -s ${LIB_DIR}/libboost_program_options.so.1.71.0 ${LIB_DIR}/libboost_program_options.so
COPY --from=dependencies /libraries/libprotobuf.so /libraries/libprotobuf.so
COPY --from=dependencies /libraries/libzmq.so /libraries/libzmq.so
COPY --from=dependencies /libraries/libsodium.so /libraries/libsodium.so
COPY --from=dependencies /libraries/libboost_program_options.so /libraries/libboost_program_options.so
COPY --from=dependencies /libraries/libboost_filesystem.so /libraries/libboost_filesystem.so
RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to
RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to
RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to
RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to
RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem to

RUN ldconfig

Expand Down
89 changes: 89 additions & 0 deletions scripts/copy_dependency.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/bin/bash
# credit to chatgpt for this script

source $BUILD_DIR/scripts/dependency_versions.sh

# Define the list of supported libraries and their environment variable names
supported_libraries=("protobuf" "libsodium" "libzmq" "boost_program_options" "boost_filesystem")
library_env_variables=("DATAFED_FULL_PROTOBUF_VERSION" "DATAFED_LIB_LIBSODIUM_VERSION" "DATAFED_LIB_LIBZMQ_VERSION" "DATAFED_BOOST" "DATAFED_BOOST")
library_names=("libprotobuf.so" "libsodium.so" "libzmq.so" "libboost_program_options.so" "libboost_filesystem.so")
library_locations=("$LIB_DIR" "$LIB_DIR" "$LIB_DIR" "/lib/x86_64-linux-gnu" "/lib/x86_64-linux-gnu")

LIBRARIES_BASE_PATH="/libraries"

# Function to print the list of supported libraries
print_supported_libraries() {
echo "Supported libraries:"
for library in "${supported_libraries[@]}"; do
echo " $library"
done
}

# Check the number of arguments
if [ "$#" -ne 2 ]; then
echo "Usage: $0 library_name (to/from)"
print_supported_libraries
exit 1
fi

library_name="$1"
direction="$2"

# Check if the provided library name is in the list of supported libraries
if [[ ! " ${supported_libraries[@]} " =~ " ${library_name} " ]]; then
echo "Unsupported library: $library_name"
exit 1
fi

# Find the index of the library name in the array
index=0
for ((i = 0; i < ${#supported_libraries[@]}; i++)); do
if [ "${supported_libraries[i]}" == "$library_name" ]; then
index=$i
break
fi
done

# Get the library version from the corresponding environment variable
library_version="${!library_env_variables[index]}"
library_name="${library_names[index]}"
library_location="${library_locations[index]}"

# Define source and destination filenames
source_filename=""
destination_filename=""

if [ "$direction" == "from" ]; then
# Copy from versioned to generic filename
source_filename="$library_location/$library_name.$library_version"
destination_filename="$LIBRARIES_BASE_PATH/$library_name"
elif [ "$direction" == "to" ]; then
# Copy from generic to versioned filename
source_filename="$LIBRARIES_BASE_PATH/$library_name"
destination_filename="$LIB_DIR/$library_name.$library_version"
else
echo "Invalid direction. Use 'to' or 'from'."
exit 1
fi

# Check if the source file exists
if [ ! -e "$source_filename" ]; then
echo "Source file '$source_filename' not found."
exit 1
fi

# Copy the file
cp "$source_filename" "$destination_filename"
if [ $? -eq 0 ]; then
echo "Successfully copied '$source_filename' to '$destination_filename'."
else
echo "Failed to copy '$source_filename' to '$destination_filename'."
fi

if [ "$direction" == "to" ]; then
major_version=$(echo "$library_version" | awk -F'.' '{ print $1 }')
ln -s "$LIB_DIR/$library_name.$library_version" "$LIB_DIR/$library_name.$major_version"
ln -s "$LIB_DIR/$library_name.$library_version" "$LIB_DIR/$library_name"
echo "linking $LIB_DIR/$library_name.$major_version -> $LIB_DIR/$library_name.$library_version"
echo "linking $LIB_DIR/$library_name -> $LIB_DIR/$library_name.$library_version"
fi
3 changes: 3 additions & 0 deletions scripts/dependency_versions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@ DATAFED_GLOBUS_VERSION="6.0.22"
DATAFED_JSON_SCHEMA_VALIDATOR_VERSION="2.1.0"
DATAFED_NLOHMANN_JSON_VERSION="3.10.2"
DATAFED_LIBSODIUM_VERSION="1.0.18"
DATAFED_LIB_LIBSODIUM_VERSION="23.3.0"
DATAFED_LIBZMQ_VERSION="4.3.4"
DATAFED_LIB_LIBZMQ_VERSION="5.2.4"
DATAFED_NODE_VERSION="v14.21.3"
DATAFED_PROTOBUF_VERSION="3.17.3"
DATAFED_FULL_PROTOBUF_VERSION="3.17.3.0"
DATAFED_LIBCURL="7.68.0"
DATAFED_OPENSSL="1.1.1"
DATAFED_BOOST="1.71.0"

0 comments on commit 3e8546b

Please sign in to comment.