diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 532e002db..abde6985a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,3 @@ - stages: - ci-infrastructure-check - build-infrastructure @@ -90,77 +89,49 @@ clear-repo-cache: # # NOTE GitLab is set to remove images after a month, so they will be forced to # rebuild at least once a month. -build-dependencies: +build-ws-base: stage: build-deploy-base variables: - IMAGE_TAG: "dlsw/datafed/dependencies" - CACHE_TAG: "dlsw/datafed/cache:dependencies" + IMAGE_TAG: "dlsw/datafed/ws-base" GIT_STRATEGY: clone tags: - - ci-datafed + - ci-datafed-core - docker script: - - docker buildx build -f dockerfiles/dependencies.Dockerfile --cache-from $CACHE_TAG --cache-to $CACHE_TAG . + - docker system prune -f + - docker build -f web/docker/Dockerfile.web-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - - docker push code.ornl.gov:4567/$IMAGE_TAG + - docker push code.ornl.gov:4567/dlsw/datafed/ws-base -build-runtime: +build-core-base: + needs: ["clear-core-cache"] stage: build-deploy-base variables: - IMAGE_TAG: "dlsw/datafed/runtime" - CACHE_TAG: "dlsw/datafed/cache:runtime" + IMAGE_TAG: "dlsw/datafed/core-base" GIT_STRATEGY: clone tags: - - ci-datafed + - ci-datafed-core - docker script: - - docker buildx build -f dockerfiles/runtime.Dockerfile --cache-from $CACHE_TAG --cache-to $CACHE_TAG . + - docker system prune -f + - docker build -f core/docker/Dockerfile.core-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - - docker push code.ornl.gov:4567/$IMAGE_TAG + - docker push code.ornl.gov:4567/dlsw/datafed/core-base -# build-ws-base: -# stage: build-deploy-base -# variables: -# IMAGE_TAG: "dlsw/datafed/ws-base" -# GIT_STRATEGY: clone -# tags: -# - ci-datafed-core -# - docker -# script: -# - docker system prune -f -# - docker build -f web/docker/Dockerfile.web-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . -# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 -# - docker push code.ornl.gov:4567/dlsw/datafed/ws-base -# -# build-core-base: -# needs: ["clear-core-cache"] -# stage: build-deploy-base -# variables: -# IMAGE_TAG: "dlsw/datafed/core-base" -# GIT_STRATEGY: clone -# tags: -# - ci-datafed-core -# - docker -# script: -# - docker system prune -f -# - docker build -f core/docker/Dockerfile.core-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . -# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 -# - docker push code.ornl.gov:4567/dlsw/datafed/core-base -# -# build-repo-base: -# needs: ["clear-repo-cache"] -# variables: -# IMAGE_TAG: "dlsw/datafed/repo-base" -# GIT_STRATEGY: clone -# stage: build-deploy-base -# tags: -# - ci-datafed-repo -# - docker -# script: -# - docker system prune -f -# - docker build -f repository/docker/Dockerfile.repo-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . -# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 -# - docker push code.ornl.gov:4567/dlsw/datafed/repo-base +build-repo-base: + needs: ["clear-repo-cache"] + variables: + IMAGE_TAG: "dlsw/datafed/repo-base" + GIT_STRATEGY: clone + stage: build-deploy-base + tags: + - ci-datafed-repo + - docker + script: + - docker system prune -f + - docker build -f repository/docker/Dockerfile.repo-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 + - docker push code.ornl.gov:4567/dlsw/datafed/repo-base build-gcs-base: @@ -170,6 +141,8 @@ build-gcs-base: GIT_STRATEGY: clone DISTRO: "ubuntu:focal" GCS_TAG: "code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" + IMAGE_TAG2: "dlsw/datafed/gcs-authz-base" + IMAGE_TAG3: "dlsw/datafed/gcs-authz" stage: build-deploy-base tags: - ci-datafed-globus @@ -182,6 +155,11 @@ build-gcs-base: - docker tag "$LATEST_IMAGE" "$GCS_TAG" - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - docker push "$GCS_TAG" + - cd ../../../ # back too root of project + - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG2} . + - docker push code.ornl.gov:4567/${IMAGE_TAG2} + - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG3} . + - docker push code.ornl.gov:4567/${IMAGE_TAG3} ################################################################################ # STAGE: provision client @@ -204,9 +182,6 @@ provision-client: build-ws: variables: IMAGE_TAG: "dlsw/datafed/ws-" - CACHE_TAG: "dlsw/datafed/cache:ws" - DEPENDENCIES: "dlsw/datafed/dependencies" - RUNTIME: "dlsw/datafed/runtime" GIT_STRATEGY: clone stage: build-unit-test-deploy tags: @@ -228,15 +203,12 @@ build-ws: - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - - docker buildx build -f dockerfiles/ws.Dockerfile -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" --cache-from $CACHE_TAG --cache-to $CACHE_TAG --build-arg DEPENDENCIES=$DEPENDENCIES --build-arg RUNTIME=$RUNTIME . + - docker build -f web/docker/Dockerfile.web.ubuntu -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" build-core: variables: IMAGE_TAG: "dlsw/datafed/core-" - CACHE_TAG: "dlsw/datafed/cache:core" - DEPENDENCIES: "dlsw/datafed/dependencies" - RUNTIME: "dlsw/datafed/runtime" GIT_STRATEGY: clone stage: build-unit-test-deploy tags: @@ -257,7 +229,7 @@ build-core: - ./scripts/generate_datafed.sh - cat ./config/datafed.sh - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - - docker build -f dockerfiles/core.Dockerfile -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" --cache-from $CACHE_TAG --cache-to $CACHE_TAG --build-arg DEPENDENCIES=$DEPENDENCIES --build-arg RUNTIME=$RUNTIME . + - docker build -f core/docker/Dockerfile.core.ubuntu -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" build-foxx: @@ -298,9 +270,6 @@ build-foxx: build-repo: variables: IMAGE_TAG: "dlsw/datafed/repo-" - CACHE_TAG: "dlsw/datafed/cache:repo" - DEPENDENCIES: "dlsw/datafed/dependencies" - RUNTIME: "dlsw/datafed/runtime" GIT_STRATEGY: clone stage: build-unit-test-deploy tags: @@ -319,14 +288,12 @@ build-repo: - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - - docker build -f dockerfiles/repo.Dockerfile -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" --cache-from $CACHE_TAG --cache-to $CACHE_TAG --build-arg DEPENDENCIES=$DEPENDENCIES --build-arg RUNTIME=$RUNTIME . + - docker build -f repository/docker/Dockerfile.repo.ubuntu -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" build-gcs-authz: variables: IMAGE_TAG: "dlsw/datafed/gcs-authz-" - CACHE_TAG: "dlsw/datafed/cache:gcs" - GCS_TAG: "code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" GIT_STRATEGY: clone stage: build-unit-test-deploy tags: @@ -337,7 +304,7 @@ build-gcs-authz: - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - - docker build -f dockerfiles/gcs.Dockerfile -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" --cache-from $CACHE_TAG --cache-to $CACHE_TAG --build-arg GCS_IMAGE=$GCS_TAG . + - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" ################################################################################ diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile deleted file mode 100644 index e8c0950ed..000000000 --- a/dockerfiles/Dockerfile +++ /dev/null @@ -1,365 +0,0 @@ -ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install" -ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" -ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" -ARG LIB_DIR="/usr/local/lib" - -FROM ubuntu:focal AS base - -ARG DATAFED_DIR -ARG DATAFED_INSTALL_PATH -ARG BUILD_DIR -ARG NVM_DIR -ARG NVM_INC -ARG NVM_BIN -ARG LIB_DIR - -FROM base AS dependencies - -ARG NVM_DIR -ARG DATAFED_DIR -ARG BUILD_DIR -ARG DATAFED_INSTALL_PATH -ARG DEBIAN_FRONTEND=noninteractive -ARG LIB_DIR - -ENV BUILD_DIR="${BUILD_DIR}" -ENV DATAFED_DIR="${DATAFED_DIR}" -ENV LIB_DIR="${LIB_DIR}" - -RUN mkdir -p ${BUILD_DIR} -RUN mkdir -p ${BUILD_DIR}/logs -RUN mkdir -p ${BUILD_DIR}/repository/server -RUN mkdir -p ${BUILD_DIR}/common/proto -RUN mkdir -p /libraries -RUN mkdir -p ${NVM_DIR} - -WORKDIR ${BUILD_DIR} - -# Copy install scripts -COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ -# COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/ -# COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/ -# COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ -# COPY ./scripts/install_gcs.sh ${BUILD_DIR}/scripts/ -# COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/install_dependencies.sh ${BUILD_DIR}/scripts/ - -RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo - -# run build scripts -# RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_core_dependencies.sh -# RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_repo_dependencies.sh -# RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_ws_dependencies.sh -n "${DATAFED_DIR}" -# RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_gcs.sh -# RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh - -COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/ -RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf from -RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq from -RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium from -RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options from -RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem from - -# RUN cp -R $HOME/.nvm ${NVM_DIR} - -FROM ${GCS_IMAGE} AS gcs-base - -ARG DATAFED_DIR -ARG BUILD_DIR -ARG DATAFED_INSTALL_PATH - -RUN mkdir -p ${BUILD_DIR} -RUN mkdir -p ${BUILD_DIR}/logs -RUN mkdir -p ${BUILD_DIR}/common/proto -RUN mkdir -p ${DATAFED_INSTALL_PATH}/authz -RUN mkdir -p ${DATAFED_DIR}/collections/mapped - -RUN apt update -RUN apt install -y vim netcat - -# For communicating with public -EXPOSE 443 - -# Needed for Globus GridFTP communication -EXPOSE 50000-51000 - -WORKDIR ${BUILD_DIR} - -RUN adduser --disabled-password --gecos "" datafed - -RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo - -COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ - -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz_dependencies.sh - -FROM dependencies AS build-base - -ARG DATAFED_DIR -ARG BUILD_DIR -ARG DATAFED_INSTALL_PATH - -RUN mkdir -p ${DATAFED_INSTALL_PATH} -RUN mkdir -p ${DATAFED_INSTALL_PATH}/keys - -WORKDIR ${BUILD_DIR} - -COPY ./common ${BUILD_DIR}/common - - -FROM build-base AS repo-build - -ARG DATAFED_DIR -ARG BUILD_DIR -ARG DATAFED_INSTALL_PATH - -# This port is needed to communicate with the DataFed core server -EXPOSE 7512 -# Not quite sure what 9000 is doing that 7512 isn't, difference between egress -# and ingress? -EXPOSE 9000 - -COPY ./repository/CMakeLists.txt ${BUILD_DIR}/repository/CMakeLists.txt -COPY ./CMakeLists.txt ${BUILD_DIR} -COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/generate_repo_config.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/install_repo.sh ${BUILD_DIR}/scripts/ -COPY ./cmake ${BUILD_DIR}/cmake -COPY ./repository/server ${BUILD_DIR}/repository/server - -RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \ - cmake -S. -B build \ - -DBUILD_REPO_SERVER=True \ - -DBUILD_AUTHZ=False \ - -DBUILD_CORE_SERVER=False \ - -DBUILD_WEB_SERVER=False \ - -DBUILD_DOCS=False \ - -DBUILD_PYTHON_CLIENT=False \ - -DBUILD_FOXX=False -RUN cmake --build build -RUN cmake --build build --target install - -FROM build-base AS ws-build - -ARG DATAFED_DIR -ARG BUILD_DIR -ARG DATAFED_INSTALL_PATH -ARG NVM_DIR -ARG NVM_INC -ARG NVM_BIN - -# This port is needed to communicate with the DataFed core server -EXPOSE 7513 -# For communication with the public -EXPOSE 443 - -COPY ./CMakeLists.txt ${BUILD_DIR} -COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/generate_ws_config.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/install_ws.sh ${BUILD_DIR}/scripts/ -COPY ./cmake ${BUILD_DIR}/cmake -COPY ./common/proto ${BUILD_DIR}/common/proto -COPY ./web ${BUILD_DIR}/web - -RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \ - cmake -S. -B build \ - -DBUILD_REPO_SERVER=False \ - -DBUILD_AUTHZ=False \ - -DBUILD_CORE_SERVER=False \ - -DBUILD_WEB_SERVER=True \ - -DBUILD_DOCS=False \ - -DBUILD_PYTHON_CLIENT=False \ - -DBUILD_FOXX=False \ - -DBUILD_COMMON=False -RUN cmake --build build - -ENV NVM_DIR="$NVM_DIR" -ENV NVM_INC="$NVM_INC" -ENV NVM_BIN="$NVM_BIN" -ENV PATH="$NVM_BIN:$PATH" - -RUN cmake --build build --target install - -FROM base AS runtime - -ARG DATAFED_DIR -ARG DATAFED_INSTALL_PATH -ARG BUILD_DIR - -ENV BUILD_DIR="${BUILD_DIR}" -ENV DATAFED_DIR="${DATAFED_DIR}" - -RUN echo $DATAFED_DIR - -# Create datafed user, prefer more secure login options than password -# Recommended to mount ssh public key on run -RUN adduser --disabled-password --gecos "" datafed - -COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/ -RUN mkdir -p ${DATAFED_DIR} -RUN mkdir -p /opt/datafed -RUN mkdir -p /var/log/datafed -RUN chown -R datafed:datafed /opt/datafed -RUN chown -R datafed:datafed /var/log/datafed -RUN chown -R datafed:datafed ${DATAFED_DIR} -WORKDIR ${DATAFED_DIR} - -RUN apt update -RUN apt install -y grep libcurl4 -# ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib:/usr/local/lib" - - -FROM runtime AS repo - -ARG DATAFED_DIR -ARG DATAFED_INSTALL_PATH -ARG BUILD_DIR -ARG LIB_DIR - -# The above should also be available at runtime -ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -ENV DATAFED_DIR="$DATAFED_DIR" -ENV BUILD_DIR="$BUILD_DIR" -ENV LIB_DIR="$LIB_DIR" - -WORKDIR /datafed - -# copy necessary shared libraries -COPY --from=dependencies /libraries/libprotobuf.so /libraries/libprotobuf.so -COPY --from=dependencies /libraries/libzmq.so /libraries/libzmq.so -COPY --from=dependencies /libraries/libsodium.so /libraries/libsodium.so -COPY --from=dependencies /libraries/libboost_program_options.so /libraries/libboost_program_options.so -COPY --from=dependencies /libraries/libboost_filesystem.so /libraries/libboost_filesystem.so -RUN ${BUILD_DIR}/scripts/copy_dependency.sh protobuf to -RUN ${BUILD_DIR}/scripts/copy_dependency.sh libzmq to -RUN ${BUILD_DIR}/scripts/copy_dependency.sh libsodium to -RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to -RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_filesystem to - -RUN ldconfig - -USER datafed - -COPY --chown=datafed:datafed ./repository/docker/entrypoint_repo.sh ${BUILD_DIR}/repository/entrypoint.sh -COPY --chown=datafed:datafed ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh -COPY --chown=datafed:datafed ./scripts/generate_repo_config.sh ${DATAFED_DIR}/scripts/generate_repo_config.sh -COPY --chown=datafed:datafed ./scripts/install_repo.sh ${DATAFED_DIR}/scripts/install_repo.sh -COPY --chown=datafed:datafed ./cmake/Version.cmake ${DATAFED_DIR}/cmake/Version.cmake -COPY --from=repo-build --chown=datafed:datafed ${DATAFED_INSTALL_PATH}/repo/datafed-repo ${DATAFED_INSTALL_PATH}/repo/datafed-repo - -# ENTRYPOINT ["/app/entrypoint.sh"] -# CMD ["/app/datafed-core","--cfg","/app/datafed-core.cfg"] - -FROM runtime AS ws - -ARG DATAFED_NODE_VERSION="" -ARG DATAFED_DIR -ARG DATAFED_INSTALL_PATH -ARG BUILD_DIR -ARG NVM_DIR -ARG NVM_INC -ARG NVM_BIN - -# The above should also be available at runtime -ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -ENV DATAFED_DIR="$DATAFED_DIR" -ENV BUILD_DIR="$BUILD_DIR" -ENV NVM_DIR="$NVM_DIR" -ENV NVM_INC="$NVM_INC" -ENV NVM_BIN="$NVM_BIN" -ENV PATH="$NVM_BIN:$PATH" - -RUN apt install -y python3 make g++ - -WORKDIR /datafed - -RUN mkdir -p /home/cades -RUN chown -R datafed:datafed /home/cades - -COPY --from=dependencies --chown=datafed:datafed "$NVM_DIR" "$NVM_DIR" -RUN ln -s ${DATAFED_INSTALL_PATH}/web ${DATAFED_DIR}/web -RUN ln -s "$NVM_DIR" /home/cades/.nvm - -USER datafed - -COPY --chown=datafed:datafed ./web/docker/entrypoint.sh ${BUILD_DIR}/web/entrypoint.sh -COPY --chown=datafed:datafed ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh -COPY --chown=datafed:datafed ./scripts/dependency_versions.sh ${DATAFED_DIR}/scripts/dependency_versions.sh -COPY --chown=datafed:datafed ./scripts/generate_ws_config.sh ${DATAFED_DIR}/scripts/generate_ws_config.sh -COPY --chown=datafed:datafed ./scripts/install_ws.sh ${DATAFED_DIR}/scripts/install_ws.sh -COPY --chown=datafed:datafed ./cmake/Version.cmake ${DATAFED_DIR}/cmake/Version.cmake - -COPY --from=ws-build --chown=datafed:datafed ${BUILD_DIR}/web/package.json ${DATAFED_INSTALL_PATH}/web/package.json -RUN . ${DATAFED_DIR}/scripts/dependency_versions.sh && \ - . ${DATAFED_DIR}/.nvm/nvm.sh && \ - npm --allow-root --unsafe-perm --prefix ${DATAFED_INSTALL_PATH}/web install - -COPY --from=ws-build --chown=datafed:datafed ${BUILD_DIR}/web ${DATAFED_INSTALL_PATH}/web - -WORKDIR ${DATAFED_INSTALL_PATH}/web - -FROM gcs-base AS gcs-authz - -ARG rebuild=true -ARG DATAFED_DIR -ARG BUILD_DIR -ARG DATAFED_INSTALL_PATH - -ENV GCS_COLLECTION_ROOT_PATH="$DATAFED_DIR/collections/mapped" -ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -ENV DATAFED_DIR="$DATAFED_DIR" -ENV BUILD_DIR="$BUILD_DIR" - - -# All files should be owned by the datafed user -RUN chown -R datafed:datafed ${DATAFED_DIR} - -COPY --chown=datafed:datafed ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ -COPY --chown=datafed:datafed ./scripts/generate_authz_config.sh ${BUILD_DIR}/scripts/generate_authz_config.sh -COPY --chown=datafed:datafed ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh -COPY --chown=datafed:datafed ./CMakeLists.txt ${BUILD_DIR} -COPY --chown=datafed:datafed ./cmake ${BUILD_DIR}/cmake -COPY --chown=datafed:datafed ./repository/CMakeLists.txt ${BUILD_DIR}/repository/CMakeLists.txt -COPY --chown=datafed:datafed ./repository/gridftp/CMakeLists.txt ${BUILD_DIR}/repository/gridftp/CMakeLists.txt -COPY --chown=datafed:datafed ./scripts/globus/setup_globus.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh -COPY --chown=datafed:datafed ./scripts/globus/generate_repo_form.sh ${BUILD_DIR}/scripts/globus/generate_repo_form.sh -COPY --chown=datafed:datafed ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh -COPY --chown=datafed:datafed ./common ${BUILD_DIR}/common -COPY --chown=datafed:datafed ./repository/gridftp/globus5 ${BUILD_DIR}/repository/gridftp/globus5 - -# Build as if a non root user -USER datafed - -RUN ${BUILD_DIR}/scripts/generate_datafed.sh - -RUN ${BUILD_DIR}/scripts/generate_authz_config.sh && \ - cmake -S. -B build \ - -DBUILD_REPO_SERVER=False \ - -DBUILD_AUTHZ=True \ - -DBUILD_CORE_SERVER=False \ - -DBUILD_WEB_SERVER=False \ - -DBUILD_DOCS=False \ - -DBUILD_PYTHON_CLIENT=False \ - -DBUILD_FOXX=False -RUN cmake --build build -RUN cmake --build build --target install - -COPY ./scripts/globus/setup_globus.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh -COPY ./scripts/globus/generate_repo_form.sh ${BUILD_DIR}/scripts/globus/generate_repo_form.sh -COPY ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh - -USER root - -WORKDIR ${DATAFED_INSTALL_PATH}/authz diff --git a/dockerfiles/core.Dockerfile b/dockerfiles/core.Dockerfile index 8e1146d97..2557330fe 100644 --- a/dockerfiles/core.Dockerfile +++ b/dockerfiles/core.Dockerfile @@ -2,7 +2,7 @@ ARG BUILD_BASE="ubuntu:focal" ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" ARG NVM_DIR="$DATAFED_DIR/.nvm" @@ -82,5 +82,5 @@ COPY --chown=datafed:root ./cmake/Version.cmake ${DATAFED_DIR}/cmake COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh ${BUILD_DIR}/core/entrypoint.sh COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core -ENTRYPOINT ["/app/entrypoint.sh"] -CMD ["/app/datafed-core","--cfg","/app/datafed-core.cfg"] +ENTRYPOINT ["${BUILD_DIR}/core/entrypoint.sh"] +CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"] diff --git a/dockerfiles/dependencies.Dockerfile b/dockerfiles/dependencies.Dockerfile index 238270ad2..46f5afb45 100644 --- a/dockerfiles/dependencies.Dockerfile +++ b/dockerfiles/dependencies.Dockerfile @@ -1,5 +1,5 @@ ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" ARG NVM_DIR="$DATAFED_DIR/.nvm" diff --git a/dockerfiles/gcs.Dockerfile b/dockerfiles/gcs.Dockerfile index 51881ec5c..dc109809d 100644 --- a/dockerfiles/gcs.Dockerfile +++ b/dockerfiles/gcs.Dockerfile @@ -2,7 +2,7 @@ ARG BUILD_BASE="ubuntu:focal" ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" ARG NVM_DIR="$DATAFED_DIR/.nvm" diff --git a/dockerfiles/repo.Dockerfile b/dockerfiles/repo.Dockerfile index 3f21335c5..49a286045 100644 --- a/dockerfiles/repo.Dockerfile +++ b/dockerfiles/repo.Dockerfile @@ -2,7 +2,7 @@ ARG BUILD_BASE="ubuntu:focal" ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" ARG NVM_DIR="$DATAFED_DIR/.nvm" @@ -82,5 +82,5 @@ COPY --chown=datafed:root ./scripts/install_repo.sh ${DATAFED_DIR}/ COPY --chown=datafed:root ./cmake/Version.cmake ${DATAFED_DIR}/cmake/Version.cmake COPY --from=repo-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/repo/datafed-repo ${DATAFED_INSTALL_PATH}/repo/datafed-repo -ENTRYPOINT ["/app/entrypoint.sh"] -CMD ["/app/datafed-core","--cfg","/app/datafed-core.cfg"] +ENTRYPOINT ["${BUILD_DIR}/repository/entrypoint.sh"] +CMD ["${DATAFED_INSTALL_PATH}/repo/datafed-repo","--cfg","${DATAFED_INSTALL_PATH}/repo/datafed-repo.cfg"] diff --git a/dockerfiles/runtime.Dockerfile b/dockerfiles/runtime.Dockerfile index 7ba7d4ab7..a372c683d 100644 --- a/dockerfiles/runtime.Dockerfile +++ b/dockerfiles/runtime.Dockerfile @@ -1,5 +1,5 @@ ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" ARG NVM_DIR="$DATAFED_DIR/.nvm" @@ -27,9 +27,9 @@ COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/ RUN mkdir -p ${DATAFED_DIR} RUN mkdir -p /opt/datafed RUN mkdir -p /var/log/datafed -RUN chown -R datafed:datafed /opt/datafed -RUN chown -R datafed:datafed /var/log/datafed -RUN chown -R datafed:datafed ${DATAFED_DIR} +RUN chown -R datafed:root /opt/datafed +RUN chown -R datafed:root /var/log/datafed +RUN chown -R datafed:root ${DATAFED_DIR} WORKDIR ${DATAFED_DIR} RUN apt update diff --git a/dockerfiles/ws.Dockerfile b/dockerfiles/ws.Dockerfile index 1836de1a4..5659b4c62 100644 --- a/dockerfiles/ws.Dockerfile +++ b/dockerfiles/ws.Dockerfile @@ -2,7 +2,7 @@ ARG BUILD_BASE="ubuntu:focal" ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="$DATAFED_DIR/install" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" ARG NVM_DIR="$DATAFED_DIR/.nvm" diff --git a/scripts/copy_dependency.sh b/scripts/copy_dependency.sh index 79fcccef2..9f11f2dd8 100755 --- a/scripts/copy_dependency.sh +++ b/scripts/copy_dependency.sh @@ -1,4 +1,5 @@ #!/bin/bash +# This script is used to rename the dependency libraries that DataFed uses to make them easier to copy between containers by reading the dependency_versions.sh script and renaming them accordingly. # credit to chatgpt for this script source $BUILD_DIR/scripts/dependency_versions.sh diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index ae19f75bc..66c7a666c 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -4,100 +4,160 @@ SOURCE=$(dirname "$SCRIPT") source "${SOURCE}/dependency_versions.sh" install_cmake() { - wget https://github.com/Kitware/CMake/releases/download/v${DATAFED_CMAKE_VERSION}/cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz - tar -xzvf cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz - cp -r cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/bin /usr/local - cp -r cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/share /usr/local + if [ ! -e ".cmake_installed" ]; then + wget https://github.com/Kitware/CMake/releases/download/v${DATAFED_CMAKE_VERSION}/cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz + tar -xzvf cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz + cp -r cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/bin /usr/local + cp -r cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64/share /usr/local - # Cleanup - rm -rf cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64 - rm -rf cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz + # Cleanup + rm -rf cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64 + rm -rf cmake-${DATAFED_CMAKE_VERSION}-Linux-x86_64.tar.gz + + # Mark cmake as installed + touch .cmake_installed + fi } install_protobuf() { - if [ -d protobuf ] - then - # sudo required because of egg file - sudo rm -rf protobuf + if [ ! -e ".protobuf_installed" ]; then + if [ -d protobuf ] + then + # sudo required because of egg file + sudo rm -rf protobuf + fi + git clone https://github.com/google/protobuf.git + cd protobuf + git checkout v${DATAFED_PROTOBUF_VERSION} + git submodule update --init --recursive + cmake -S cmake/ -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON + cmake --build build -j 8 + sudo cmake --build build --target install + cd python + python3 setup.py build + python3 setup.py test + sudo python3 setup.py install + cd ../../ + + # Mark protobuf as installed + touch .protobuf_installed fi - git clone https://github.com/google/protobuf.git - cd protobuf - git checkout v${DATAFED_PROTOBUF_VERSION} - git submodule update --init --recursive - cmake -S cmake/ -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON - cmake --build build -j 8 - sudo cmake --build build --target install - cd python - python3 setup.py build - python3 setup.py test - sudo python3 setup.py install - cd ../../ } install_libsodium() { - if [ -d libsodium ] - then - rm -rf libsodium + if [ ! -e ".libsodium_installed" ]; then + if [ -d libsodium ] + then + rm -rf libsodium + fi + git clone https://github.com/jedisct1/libsodium.git + cd libsodium + git checkout "$DATAFED_LIBSODIUM_VERSION" + ./autogen.sh + ./configure + make check + sudo make install + sudo ldconfig + cd ../ + + # Mark libsodium as installed + touch .libsodium_installed fi - git clone https://github.com/jedisct1/libsodium.git - cd libsodium - git checkout "$DATAFED_LIBSODIUM_VERSION" - ./autogen.sh - ./configure - make check - sudo make install - sudo ldconfig - cd ../ } install_libzmq() { - if [ -d libzmq ] - then - rm -rf libzmq + if [ ! -e ".libzmq_installed" ]; then + if [ -d libzmq ] + then + rm -rf libzmq + fi + git clone https://github.com/zeromq/libzmq.git + cd libzmq + git checkout v${DATAFED_LIBZMQ_VERSION} + cmake -S. -B build -DBUILD_STATIC=ON -DBUILD_SHARED=ON + cmake --build build -j 8 + sudo cmake --build build --target install + + # Mark libzmq as installed + touch .libzmq_installed fi - git clone https://github.com/zeromq/libzmq.git - cd libzmq - git checkout v${DATAFED_LIBZMQ_VERSION} - cmake -S. -B build -DBUILD_STATIC=ON -DBUILD_SHARED=ON - cmake --build build -j 8 - sudo cmake --build build --target install } install_nlohmann_json() { - if [ -d json ] - then - rm -rf json + if [ ! -e ".nlohmann_json_installed" ]; then + if [ -d json ] + then + rm -rf json + fi + git clone https://github.com/nlohmann/json.git + cd json + git checkout v${DATAFED_NLOHMANN_JSON_VERSION} + echo "FILE STRUCTURE $(ls)" + cmake -S . -B build + cmake --build build -j 8 + sudo cmake --build build --target install + cd ../ + + # Mark nlohmann_json as installed + touch .nlohmann_json_installed fi - git clone https://github.com/nlohmann/json.git - cd json - git checkout v${DATAFED_NLOHMANN_JSON_VERSION} - cmake -S . -B build - cmake --build build -j 8 - sudo cmake --build build --target install - cd ../ } install_json_schema_validator() { - if [ -d json-schema-validator ] - then - rm -rf json-schema-validator + if [ ! -e ".json_schema_validator_installed" ]; then + if [ -d json-schema-validator ] + then + rm -rf json-schema-validator + fi + git clone https://github.com/pboettch/json-schema-validator + cd json-schema-validator + git checkout ${DATAFED_JSON_SCHEMA_VALIDATOR_VERSION} + cmake -S . -B build + cmake --build build -j 8 + sudo cmake --build build --target install + cd ../ + + # Mark json-schema-validator as installed + touch .json_schema_validator_installed fi - git clone https://github.com/pboettch/json-schema-validator - cd json-schema-validator - git checkout ${DATAFED_JSON_SCHEMA_VALIDATOR_VERSION} - cmake -S . -B build - cmake --build build -j 8 - sudo cmake --build build --target install - cd ../ } install_gcs() { - sudo apt update - sudo apt install -y curl git gnupg - curl -LOs https://downloads.globus.org/globus-connect-server/stable/installers/repo/deb/globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb - sudo dpkg -i globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb - sudo apt-key add /usr/share/globus-repo/RPM-GPG-KEY-Globus - # Need a second update command after adding the globus GPG key - sudo apt update - sudo apt-get install globus-connect-server54 -y + if [ ! -e ".gcs_installed" ]; then + sudo apt update + sudo apt install -y curl git gnupg + curl -LOs https://downloads.globus.org/globus-connect-server/stable/installers/repo/deb/globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb + sudo dpkg -i globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb + sudo apt-key add /usr/share/globus-repo/RPM-GPG-KEY-Globus + # Need a second update command after adding the globus GPG key + sudo apt update + sudo apt-get install globus-connect-server54 -y + + # Mark gcs as installed + touch .gcs_installed + fi +} + +install_dep_by_name() { + case "$1" in + "protobuf") + install_protobuf + ;; + "nlohmann_json") + install_nlohmann_json + ;; + "json_schema_validator") + install_json_schema_validator + ;; + "gcs") + install_gcs + ;; + "libsodium") + install_libsodium + ;; + "libzmq") + install_libzmq + ;; + esac + cd ~ }