From ac0a4507007cac8d629787de3fa1e3e9ce54037d Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 11:25:59 -0400 Subject: [PATCH 01/69] First steps for setting up docker compose file --- compose/build_images_for_compose.sh | 15 +++++++++++++++ compose/compose.yml | 12 ++++++++++++ compose/generate_env.sh | 11 +++++++++++ 3 files changed, 38 insertions(+) create mode 100755 compose/build_images_for_compose.sh create mode 100644 compose/compose.yml create mode 100755 compose/generate_env.sh diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh new file mode 100755 index 000000000..5e19a2540 --- /dev/null +++ b/compose/build_images_for_compose.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euf -o pipefail + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath ${SOURCE}/../) + +REPO_LIST=("repository" "web" "core") + +for REPO in "${REPO_LIST[@]}" +do + CONTAINER=${REPO} + docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile ${PROJECT_ROOT}/${REPO}/ -t dataafed/${CONTAINER}:latest +done diff --git a/compose/compose.yml b/compose/compose.yml new file mode 100644 index 000000000..80137924f --- /dev/null +++ b/compose/compose.yml @@ -0,0 +1,12 @@ +version: '3.9' + + +services: + datafed-web: + image: datafed/web:latest + + datafed-core: + image: datafed/core:latest + + arango: + diff --git a/compose/generate_env.sh b/compose/generate_env.sh new file mode 100755 index 000000000..5186c1f93 --- /dev/null +++ b/compose/generate_env.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ -f ".env" ] +then + echo ".env already exist! Will not overwrite!" + exit 1 +fi + + +cat << EOF > ".env" +EOF From da66d18494502c71e7635bb5cf35c7e1f226251b Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 15:33:08 -0400 Subject: [PATCH 02/69] Initial commit for compose build --- compose/README.md | 4 ++++ compose/build_images_for_compose.sh | 19 +++++++++++++++++-- compose/compose.yml | 13 ++++++++++++- compose/generate_env.sh | 4 ++++ 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 compose/README.md diff --git a/compose/README.md b/compose/README.md new file mode 100644 index 000000000..94dd7da94 --- /dev/null +++ b/compose/README.md @@ -0,0 +1,4 @@ +# Compose Dev environment + +The files in this folder are incomplete but are the start for setting up a full +docker compose instance of datafed. diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index 5e19a2540..c64343c4f 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -6,10 +6,25 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../) -REPO_LIST=("repository" "web" "core") +# Repository server and authz library +docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo-base.ubuntu ${PROJECT_ROOT}/repository/ -t datafed/repo-base:latest +# Retag +docker tag datafed/repo-base:latest code.ornl.gov:4567/dlsw/datafed/repo-base:latest +docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo.ubuntu ${PROJECT_ROOT}/repository/ -t datafed/repo:latest +# Core and Web servers +REPO_LIST=("web" "core") for REPO in "${REPO_LIST[@]}" do CONTAINER=${REPO} - docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile ${PROJECT_ROOT}/${REPO}/ -t dataafed/${CONTAINER}:latest + docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}-base.ubuntu ${PROJECT_ROOT}/${REPO}/ -t datafed/${CONTAINER}-base:latest + # Retag + docker tag datafed/${CONTAINER}-base:latest code.ornl.gov:4567/dlsw/datafed/core-base:latest + docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}.ubuntu ${PROJECT_ROOT}/${REPO}/ -t datafed/${CONTAINER}:latest done + +# Python client +docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client-base.ubuntu ${PROJECT_ROOT}/python/ -t datafed/python-client-base:latest +# Retag +docker tag datafed/python-client-base:latest code.ornl.gov:4567/dlsw/datafed/python-client:latest +docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client.ubuntu ${PROJECT_ROOT}/python/ -t datafed/python-client:latest diff --git a/compose/compose.yml b/compose/compose.yml index 80137924f..4811c24be 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -4,9 +4,20 @@ version: '3.9' services: datafed-web: image: datafed/web:latest + ports: + - 8080:443 datafed-core: image: datafed/core:latest + environment: + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + ports: + - 7513 # Communication web server + - 7512 # Secure core server communication arango: - + image: arangodb + environment: + ARANGO_ROOT_PASSWORD: "${ARANGO_ROOT_PASSWORD}" + ports: + - 8529:8529 # Arangodb web UI diff --git a/compose/generate_env.sh b/compose/generate_env.sh index 5186c1f93..99ea67b98 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -8,4 +8,8 @@ fi cat << EOF > ".env" +DATAFED_DOMAIN="" # The domain of the metadata core web server +ARANGO_ROOT_PASSWORD="" +DATAFED_USER89_PASSWORD="" # For End to end testing +DATAFED_REPO_FORM_PATH="" # Where the repo form is located also needed for testing EOF From a19a9f9c80e1d26de8350ad25e8b553c1a2ba96b Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 16:38:23 -0400 Subject: [PATCH 03/69] Create build for python client --- .gitlab-ci.yml | 108 ++++++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76ac2ae32..06938d39e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,7 +72,6 @@ build-repo-base: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - docker push code.ornl.gov:4567/dlsw/datafed/repo-base - build-gcs-base: variables: GIT_SUBMODULE_STRATEGY: recursive @@ -95,26 +94,19 @@ build-gcs-base: - docker push code.ornl.gov:4567/${IMAGE_TAG2} - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG3} . - docker push code.ornl.gov:4567/${IMAGE_TAG3} -# This build runs two builds, one to build the gcs image provided by Globus -# The second to build on top of the gcs image and add the authz dependencies -#build-gcs-authz-base: -# variables: -# IMAGE_TAG: "dlsw/datafed/gcs-base" -# IMAGE_TAG2: "dlsw/datafed/gcs-authz-base" -# IMAGE_TAG3: "dlsw/datafed/gcs-authz" -# GIT_STRATEGY: clone -# stage: build-deploy-base -# tags: -# - ci-datafed-globus -# - docker -# script: -# - docker build -f repository/docker/Dockerfile.gcs-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . -# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 -# - docker push code.ornl.gov:4567/${IMAGE_TAG} -# - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG2} . -# - docker push code.ornl.gov:4567/${IMAGE_TAG2} -# - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG3} . -# - docker push code.ornl.gov:4567/${IMAGE_TAG3} + +build-python-client-base: + variables: + IMAGE_TAG: "dlsw/datafed/python-client-base" + GIT_STRATEGY: clone + stage: build-deploy-base + tags: + - docker + script: + - docker build -f python/docker/Dockerfile.python-client-base.ubuntu -t code.ornl.gov:4567/${IMAGE_TAG} . + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 + - docker push code.ornl.gov:4567/dlsw/datafed/python-client-base + ################################################################################ # STAGE: build @@ -249,6 +241,30 @@ build-gcs-authz: - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" +build-python-client: + variables: + IMAGE_TAG: "dlsw/datafed/python-client-" + GIT_STRATEGY: clone + stage: build-unit-test-deploy + tags: + - docker + rules: + - changes: + - python/* + - scripts/* + - common/* + - .gitlab-ci.yml + when: always + script: + - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') + - echo "$BRANCH_LOWER" + - ./scripts/generate_datafed.sh + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 + - docker build -f python/docker/Dockerfile.python-client.ubuntu -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" . + - docker push "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" + + + ################################################################################ # STAGE: Integration ################################################################################ @@ -397,28 +413,28 @@ end-to-end-arango-setup: - ./scripts/run_arango_service.sh -end-to-end-client-test: - variables: - GIT_STRATEGY: clone - stage: end-to-end-test - tags: - - ci-datafed-client - script: - - ./scripts/generate_datafed.sh - - env - - cat ./config/datafed.sh - - > - cmake -S. -B build - -DENABLE_FOXX_TESTS=OFF - -DBUILD_CORE_SERVER=OFF - -DBUILD_COMMON=OFF - -DBUILD_WEB_SERVER=OFF - -DBUILD_DOCS=OFF - -DBUILD_PYTHON_CLIENT=ON - -DBUILD_TESTS=ON - -DENABLE_END_TO_END_TESTS=ON - -DINSTALL_FOXX=OFF - - cmake --build build - - cmake --build build --target install - - +# end-to-end-client-test: +# variables: +# GIT_STRATEGY: clone +# stage: end-to-end-test +# tags: +# - ci-datafed-client +# script: +# - ./scripts/generate_datafed.sh +# - env +# - cat ./config/datafed.sh +# - > +# cmake -S. -B build +# -DENABLE_FOXX_TESTS=OFF +# -DBUILD_CORE_SERVER=OFF +# -DBUILD_COMMON=OFF +# -DBUILD_WEB_SERVER=OFF +# -DBUILD_DOCS=OFF +# -DBUILD_PYTHON_CLIENT=ON +# -DBUILD_TESTS=ON +# -DENABLE_END_TO_END_TESTS=ON +# -DINSTALL_FOXX=OFF +# - cmake --build build +# - cmake --build build --target install +# +# From 73c3a9855e01530d81a92ea607a2843d26ab9cb9 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 16:38:54 -0400 Subject: [PATCH 04/69] Add default to DATAFED_SERVER_PORT --- scripts/generate_datafed.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index ccb3138ef..02fe7f21b 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -82,7 +82,7 @@ fi local_DATAFED_SERVER_PORT="" if [ -z "${DATAFED_SERVER_PORT}" ] then - local_DATAFED_SERVER_PORT="" + local_DATAFED_SERVER_PORT="7512" else local_DATAFED_SERVER_PORT=$(printenv DATAFED_SERVER_PORT) fi From 6b70ceb97d891f3f91789feb2eeaf8d6c66ce6ef Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 16:39:43 -0400 Subject: [PATCH 05/69] Add script for installing dependencies for building the python client --- scripts/install_python_client_dependencies.sh | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 scripts/install_python_client_dependencies.sh diff --git a/scripts/install_python_client_dependencies.sh b/scripts/install_python_client_dependencies.sh new file mode 100755 index 000000000..14bbc9eeb --- /dev/null +++ b/scripts/install_python_client_dependencies.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Exit on error +set -e + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath ${SOURCE}/..) + +source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" + +# This script will install all of the dependencies needed by DataFed 1.0 +sudo apt-get update +sudo dpkg --configure -a +sudo apt-get install -y libtool build-essential g++ gcc make libboost-all-dev \ +pkg-config autoconf automake unzip libcurl4-openssl-dev wget \ +rapidjson-dev libkrb5-dev git python3-pkg-resources python3-pip libssl-dev + +cd ~ +install_cmake +cd ~ + +# Install cmake 3.17 + +python3 -m pip install --upgrade pip +python3 -m pip install setuptools + +install_protobuf +cd ~ + + From 4131860646e7b59ed9188eea46061297ffea0cdb Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 16:40:12 -0400 Subject: [PATCH 06/69] Add docker files for building python client --- .../Dockerfile.python-client-base.ubuntu | 15 +++++ python/docker/Dockerfile.python-client.ubuntu | 59 +++++++++++++++++++ python/docker/entrypoint.sh | 40 +++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 python/docker/Dockerfile.python-client-base.ubuntu create mode 100644 python/docker/Dockerfile.python-client.ubuntu create mode 100755 python/docker/entrypoint.sh diff --git a/python/docker/Dockerfile.python-client-base.ubuntu b/python/docker/Dockerfile.python-client-base.ubuntu new file mode 100644 index 000000000..3395b072a --- /dev/null +++ b/python/docker/Dockerfile.python-client-base.ubuntu @@ -0,0 +1,15 @@ +FROM ubuntu:focal as build + +ARG DATAFED_DIR="/datafed" +ARG BUILD_DIR="/datafed/source" + +RUN mkdir -p ${BUILD_DIR} + +WORKDIR ${BUILD_DIR} + +COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/install_python_client_dependencies.sh ${BUILD_DIR}/scripts/ + +RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_python_client_dependencies.sh diff --git a/python/docker/Dockerfile.python-client.ubuntu b/python/docker/Dockerfile.python-client.ubuntu new file mode 100644 index 000000000..b22eddc59 --- /dev/null +++ b/python/docker/Dockerfile.python-client.ubuntu @@ -0,0 +1,59 @@ +FROM code.ornl.gov:4567/dlsw/datafed/python-client-base:latest as build + +ARG DATAFED_DIR="/datafed" +ARG BUILD_DIR="/datafed/source" +ARG DATAFED_INSTALL_PATH="/datafed/install" + +# The above should also be available at runtime +ENV DATAFED_DIR="$DATAFED_DIR" +ENV BUILD_DIR="$BUILD_DIR" +ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH/python-client" + +RUN mkdir -p ${BUILD_DIR}/logs +RUN mkdir -p ${DATAFED_INSTALL_PATH} +RUN mkdir -p ${DATAFED_INSTALL_PATH}/keys + +# For communicating with repo server +# EXPOSE 7512 +# For listening to web server +# EXPOSE 7513 +# ArangoDB port +# EXPOSE 8529 + +# Create datafed user, prefer more secure login options than password +# Recommended to mount ssh public key on run +RUN adduser --disabled-password --gecos "" datafed + +WORKDIR ${BUILD_DIR} + +COPY ./common ${BUILD_DIR}/common +COPY ./CMakeLists.txt ${BUILD_DIR} +COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ +COPY ./cmake ${BUILD_DIR}/cmake +COPY ./python ${BUILD_DIR}/python + +# All files should be owned by the datafed user +RUN chown -R datafed:datafed ${DATAFED_DIR} + +USER datafed + +RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ + cmake -S. -B build -DBUILD_REPO_SERVER=False -DBUILD_AUTHZ=False \ + -DBUILD_CORE_SERVER=False -DBUILD_WEB_SERVER=False \ + -DBUILD_DOCS=False -DBUILD_PYTHON_CLIENT=True \ + -DBUILD_FOXX=False -DENABLE_UNIT_TESTS=False \ + -DBUILD_TESTS=False -DBUILD_COMMON=False &&\ + cmake --build build -j 8 +RUN cmake --build build --target pydatafed + +WORKDIR ${BUILD_DIR}/python/datafed_pkg + +# Install datafed client +RUN python3 -m pip install . + +WORKDIR /home/datafed + +#ENTRYPOINT ["/bin/bash"] +ENTRYPOINT ["/datafed/source/python/docker/entrypoint.sh"] + diff --git a/python/docker/entrypoint.sh b/python/docker/entrypoint.sh new file mode 100755 index 000000000..f9573344f --- /dev/null +++ b/python/docker/entrypoint.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euf -o pipefail + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") + +# Entry point file expects that the directory where the DataFed source file +# is passed in as the first argument + +echo "SOURCE BUILD DIR $BUILD_DIR" + +# Here we will rerun datafed configuration script to create a new set of +# default variables that are useful for setting up the DataFed Python client +# ini file + +"${BUILD_DIR}/scripts/generate_datafed.sh" +source "${BUILD_DIR}/config/datafed.sh" + +mkdir -p "/home/datafed/.datafed" + +# At this point we will create an ini file +cat << EOF > "/home/datafed/.datafed/datafed-client.ini" +[server] +host = ${DATAFED_DOMAIN} +port = ${DATAFED_SERVER_PORT} +config_dir = /home/datafed/.datafed + +[client] +config_dir = /home/datafed/.datafed + +EOF + +if [ "$#" -eq 0 ]; then + echo "No arguments were passed, running bash" + exec "/home/datafed/.local/bin/datafed --cfg /home/datafed/.datafed/datafed-client.ini" +fi + +"$@" + From 57f1396e971fabea3198f7473509ca5aaaf7d7fe Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 16:42:32 -0400 Subject: [PATCH 07/69] Applied formatting --- python/datafed_pkg/datafed/CLI.py | 4 +++- python/datafed_pkg/datafed/CommandLib.py | 4 ---- python/datafed_pkg/datafed/Config.py | 2 +- python/datafed_pkg/datafed/Connection.py | 2 +- python/datafed_pkg/datafed/MessageLib.py | 2 -- python/datafed_pkg/test/Test_EndToEnd.py | 1 - tests/end-to-end/test_api_alloc.py | 1 - tests/end-to-end/test_api_collection.py | 1 - tests/end-to-end/test_api_context.py | 2 +- tests/end-to-end/test_api_endpoint.py | 1 - tests/end-to-end/test_api_repo.py | 3 +-- 11 files changed, 7 insertions(+), 16 deletions(-) diff --git a/python/datafed_pkg/datafed/CLI.py b/python/datafed_pkg/datafed/CLI.py index 7b8a676c6..f7cb787bd 100644 --- a/python/datafed_pkg/datafed/CLI.py +++ b/python/datafed_pkg/datafed/CLI.py @@ -346,6 +346,7 @@ def command(command): # @cond + # Aliases click commands class _AliasedGroup(click.Group): # Allows command matching by unique suffix @@ -442,6 +443,7 @@ def _global_context_options(func): # click.option("-T", "--text", is_flag=True, help="Set _cli output format to human-friendly text.") ] + # Decorator to add output options to click commands def _global_output_options(func): for option in reversed(__global_output_options): @@ -2285,6 +2287,7 @@ def _more(count): # ------------------------------------------------------------- Print Functions # ============================================================================= + # Interactive and verbosity-aware print def _print_msg(level, message, err=False): global _verbosity @@ -2997,7 +3000,6 @@ def _bar_adaptive_human_readable(current, total, width=80): # render output = "" for field in selected: - if field == "percent": # fixed size width for percentage output += ("%s%%" % (100 * current // total)).rjust(min_width["percent"]) diff --git a/python/datafed_pkg/datafed/CommandLib.py b/python/datafed_pkg/datafed/CommandLib.py index 0b31b6650..25b6b0ed3 100644 --- a/python/datafed_pkg/datafed/CommandLib.py +++ b/python/datafed_pkg/datafed/CommandLib.py @@ -70,7 +70,6 @@ class methods and replies are (currently) returned as Google Protobuf message ob ) def __init__(self, opts={}): - if not isinstance(opts, dict): raise Exception("CommandLib API options parameter must be a dictionary.") @@ -280,7 +279,6 @@ def repoDelete(self, repo_id): return self._mapi.sendRecv(msg) def repoAllocationCreate(self, repo_id, subject, data_limit, rec_limit): - if not repo_id.startswith("repo/"): repo_id = "repo/" + repo_id @@ -1391,7 +1389,6 @@ def queryCreate( sort=None, sort_rev=None, ): - """ Create a new saved query @@ -1656,7 +1653,6 @@ def _buildSearchRequest( offset=0, count=20, ): - if coll_mode and (schema != None or meta != None or meta_err == True): raise Exception( "Cannot specify metadata terms when searching for collection." diff --git a/python/datafed_pkg/datafed/Config.py b/python/datafed_pkg/datafed/Config.py index a39348422..736f71e47 100644 --- a/python/datafed_pkg/datafed/Config.py +++ b/python/datafed_pkg/datafed/Config.py @@ -126,6 +126,7 @@ ], } + ## # @class API # @brief A client configuration helper class. @@ -163,7 +164,6 @@ # 5. set by environment variable # class API: - ## # @brief Class initialization method. # diff --git a/python/datafed_pkg/datafed/Connection.py b/python/datafed_pkg/datafed/Connection.py index 9d9d09faf..592c8572b 100644 --- a/python/datafed_pkg/datafed/Connection.py +++ b/python/datafed_pkg/datafed/Connection.py @@ -24,6 +24,7 @@ import sys import uuid + ## # @class Connection # @brief Provides low-level message-oriented communication @@ -57,7 +58,6 @@ def __init__( zmq_ctxt=None, log_level=logging.INFO, ): - self._log_level = log_level self._format = "%(asctime)s datafed-cli %(levelname)s %(message)" logging.Formatter(self._format) diff --git a/python/datafed_pkg/datafed/MessageLib.py b/python/datafed_pkg/datafed/MessageLib.py index c4f1755aa..2667a558c 100644 --- a/python/datafed_pkg/datafed/MessageLib.py +++ b/python/datafed_pkg/datafed/MessageLib.py @@ -49,7 +49,6 @@ def get_latest_version(package_name): # and both synchronous ans asynchronous message send/recv methods. # class API: - ## # @brief MessageLib.API class initialization method. # @param server_host The DataFed core server hostname or IP address. @@ -84,7 +83,6 @@ def __init__( manual_auth=None, **kwargs, ): - self._ctxt = 0 self._auth = False self._nack_except = True diff --git a/python/datafed_pkg/test/Test_EndToEnd.py b/python/datafed_pkg/test/Test_EndToEnd.py index 386a46782..d03175191 100644 --- a/python/datafed_pkg/test/Test_EndToEnd.py +++ b/python/datafed_pkg/test/Test_EndToEnd.py @@ -445,7 +445,6 @@ def forbidden_char(num): @classmethod def generate(cls, fits_requirements=True): - if fits_requirements: dr = DataRecord( "d/data_id", diff --git a/tests/end-to-end/test_api_alloc.py b/tests/end-to-end/test_api_alloc.py index 4844f35bd..c581d6b0f 100755 --- a/tests/end-to-end/test_api_alloc.py +++ b/tests/end-to-end/test_api_alloc.py @@ -96,7 +96,6 @@ def setUp(self): print(result) def test_repo_alloc_list_create_delete(self): - repo_id = self._repo_form["id"] if not repo_id.startswith("repo/"): repo_id = "repo/" + repo_id diff --git a/tests/end-to-end/test_api_collection.py b/tests/end-to-end/test_api_collection.py index 7a87df498..9481440de 100755 --- a/tests/end-to-end/test_api_collection.py +++ b/tests/end-to-end/test_api_collection.py @@ -126,7 +126,6 @@ def setUp(self): count = count + 1 def test_collection_create_delete(self): - # collectionItemsList in "root" of context list_response = self._df_api.collectionItemsList("root") self.assertEqual(list_response[0].total, 0) diff --git a/tests/end-to-end/test_api_context.py b/tests/end-to-end/test_api_context.py index be40603b5..2af9909d1 100755 --- a/tests/end-to-end/test_api_context.py +++ b/tests/end-to-end/test_api_context.py @@ -5,6 +5,7 @@ import sys import unittest + # Depends on the provided tests first passing # user_login class TestDataFedPythonAPIContext(unittest.TestCase): @@ -51,7 +52,6 @@ def setUp(self): assert count < 3 def test_context(self): - context = self._df_api.getContext() self.assertEqual(self._df_api.getContext(), f"u/{self._username}") diff --git a/tests/end-to-end/test_api_endpoint.py b/tests/end-to-end/test_api_endpoint.py index a772254f7..7e1aa688a 100755 --- a/tests/end-to-end/test_api_endpoint.py +++ b/tests/end-to-end/test_api_endpoint.py @@ -52,7 +52,6 @@ def setUp(self): assert count < 3 def test_endpoint_set_and_default(self): - endpoint = os.environ.get("DATAFED_USER89_GLOBUS_UUID") if endpoint is None: self.fail( diff --git a/tests/end-to-end/test_api_repo.py b/tests/end-to-end/test_api_repo.py index 3cdbae17c..b22fe54e5 100755 --- a/tests/end-to-end/test_api_repo.py +++ b/tests/end-to-end/test_api_repo.py @@ -5,6 +5,7 @@ import sys import unittest + # Should only run after api login password test has been run class TestDataFedPythonAPIRepo(unittest.TestCase): def setUp(self): @@ -66,12 +67,10 @@ def setUp(self): self._repo_form = json.load(json_file) def test_repo_list(self): - result = self._df_api.repoList(list_all=True) self.assertEqual(len(result[0].repo), 0) def test_repo_create_delete(self): - result = self._df_api.repoCreate( repo_id=self._repo_form["id"], title=self._repo_form["title"], From 2d35350c46152b99552f02d7bd58310fe8584aea Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 18 Oct 2023 16:47:41 -0400 Subject: [PATCH 08/69] Add changelog command --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94a282930..71f1bd592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ + +## MINOR Feature +1. [912] - Adds initial compose file and docker files for python-client + # v2023.8.21.10.40 ## MAJOR Breaking changes From 17538088ba9e7037d7d5f7ecb27b370ee33c4903 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 4 Dec 2023 08:38:23 -0500 Subject: [PATCH 09/69] fix build images script --- compose/build_images_for_compose.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index c64343c4f..5db00adbb 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -7,24 +7,24 @@ SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../) # Repository server and authz library -docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo-base.ubuntu ${PROJECT_ROOT}/repository/ -t datafed/repo-base:latest +docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo-base.ubuntu ${PROJECT_ROOT} -t datafed/repo-base:latest # Retag docker tag datafed/repo-base:latest code.ornl.gov:4567/dlsw/datafed/repo-base:latest -docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo.ubuntu ${PROJECT_ROOT}/repository/ -t datafed/repo:latest +docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo.ubuntu ${PROJECT_ROOT} -t datafed/repo:latest # Core and Web servers REPO_LIST=("web" "core") for REPO in "${REPO_LIST[@]}" do CONTAINER=${REPO} - docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}-base.ubuntu ${PROJECT_ROOT}/${REPO}/ -t datafed/${CONTAINER}-base:latest + docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}-base.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}-base:latest # Retag docker tag datafed/${CONTAINER}-base:latest code.ornl.gov:4567/dlsw/datafed/core-base:latest - docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}.ubuntu ${PROJECT_ROOT}/${REPO}/ -t datafed/${CONTAINER}:latest + docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}:latest done # Python client -docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client-base.ubuntu ${PROJECT_ROOT}/python/ -t datafed/python-client-base:latest +docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client-base.ubuntu ${PROJECT_ROOT} -t datafed/python-client-base:latest # Retag docker tag datafed/python-client-base:latest code.ornl.gov:4567/dlsw/datafed/python-client:latest -docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client.ubuntu ${PROJECT_ROOT}/python/ -t datafed/python-client:latest +docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client.ubuntu ${PROJECT_ROOT} -t datafed/python-client:latest From cd5feeb3b09cf91ded1e88627def14aa1a21e0d4 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 6 Feb 2024 10:53:00 -0500 Subject: [PATCH 10/69] Fix dockerfiles --- compose/build_images_for_compose.sh | 56 ++++++++++++------- {dockerfiles => docker}/Dockerfile.base | 0 {dockerfiles => docker}/Dockerfile.core | 0 .../Dockerfile.datafed-centos7 | 0 .../Dockerfile.dependencies | 0 .../Dockerfile.kickstart-centos7 | 0 .../Dockerfile.runtime | 0 {dockerfiles => docker}/Dockerfile.web | 0 {dockerfiles => docker}/README.md | 0 9 files changed, 36 insertions(+), 20 deletions(-) rename {dockerfiles => docker}/Dockerfile.base (100%) rename {dockerfiles => docker}/Dockerfile.core (100%) rename {dockerfiles => docker}/Dockerfile.datafed-centos7 (100%) rename dockerfiles/dependencies.Dockerfile => docker/Dockerfile.dependencies (100%) rename {dockerfiles => docker}/Dockerfile.kickstart-centos7 (100%) rename dockerfiles/runtime.Dockerfile => docker/Dockerfile.runtime (100%) rename {dockerfiles => docker}/Dockerfile.web (100%) rename {dockerfiles => docker}/README.md (100%) diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index 5db00adbb..91f83c614 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -6,25 +6,41 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../) -# Repository server and authz library -docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo-base.ubuntu ${PROJECT_ROOT} -t datafed/repo-base:latest -# Retag -docker tag datafed/repo-base:latest code.ornl.gov:4567/dlsw/datafed/repo-base:latest -docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo.ubuntu ${PROJECT_ROOT} -t datafed/repo:latest +docker build \ + -f "${PROJECT_ROOT}/dockerfiles/Dockerfile.dependencies" \ + "${PROJECT_ROOT}" \ + -t datafed-dependencies:latest +docker build \ + -f "${PROJECT_ROOT}/dockerfiles/Dockerfile.runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-runtime:latest +docker build -f \ + "${PROJECT_ROOT}/core/dockerfiles/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-core:latest -# Core and Web servers -REPO_LIST=("web" "core") -for REPO in "${REPO_LIST[@]}" -do - CONTAINER=${REPO} - docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}-base.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}-base:latest - # Retag - docker tag datafed/${CONTAINER}-base:latest code.ornl.gov:4567/dlsw/datafed/core-base:latest - docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}:latest -done -# Python client -docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client-base.ubuntu ${PROJECT_ROOT} -t datafed/python-client-base:latest -# Retag -docker tag datafed/python-client-base:latest code.ornl.gov:4567/dlsw/datafed/python-client:latest -docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client.ubuntu ${PROJECT_ROOT} -t datafed/python-client:latest +## Repository server and authz library +#docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo-base.ubuntu ${PROJECT_ROOT} -t datafed/repo-base:latest +## Retag +#docker tag datafed/repo-base:latest code.ornl.gov:4567/dlsw/datafed/repo-base:latest +#docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo.ubuntu ${PROJECT_ROOT} -t datafed/repo:latest +# +## Core and Web servers +#REPO_LIST=("web" "core") +#for REPO in "${REPO_LIST[@]}" +#do +# CONTAINER=${REPO} +# docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}-base.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}-base:latest +# # Retag +# docker tag datafed/${CONTAINER}-base:latest code.ornl.gov:4567/dlsw/datafed/core-base:latest +# docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}:latest +#done +# +## Python client +#docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client-base.ubuntu ${PROJECT_ROOT} -t datafed/python-client-base:latest +## Retag +#docker tag datafed/python-client-base:latest code.ornl.gov:4567/dlsw/datafed/python-client:latest +#docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client.ubuntu ${PROJECT_ROOT} -t datafed/python-client:latest diff --git a/dockerfiles/Dockerfile.base b/docker/Dockerfile.base similarity index 100% rename from dockerfiles/Dockerfile.base rename to docker/Dockerfile.base diff --git a/dockerfiles/Dockerfile.core b/docker/Dockerfile.core similarity index 100% rename from dockerfiles/Dockerfile.core rename to docker/Dockerfile.core diff --git a/dockerfiles/Dockerfile.datafed-centos7 b/docker/Dockerfile.datafed-centos7 similarity index 100% rename from dockerfiles/Dockerfile.datafed-centos7 rename to docker/Dockerfile.datafed-centos7 diff --git a/dockerfiles/dependencies.Dockerfile b/docker/Dockerfile.dependencies similarity index 100% rename from dockerfiles/dependencies.Dockerfile rename to docker/Dockerfile.dependencies diff --git a/dockerfiles/Dockerfile.kickstart-centos7 b/docker/Dockerfile.kickstart-centos7 similarity index 100% rename from dockerfiles/Dockerfile.kickstart-centos7 rename to docker/Dockerfile.kickstart-centos7 diff --git a/dockerfiles/runtime.Dockerfile b/docker/Dockerfile.runtime similarity index 100% rename from dockerfiles/runtime.Dockerfile rename to docker/Dockerfile.runtime diff --git a/dockerfiles/Dockerfile.web b/docker/Dockerfile.web similarity index 100% rename from dockerfiles/Dockerfile.web rename to docker/Dockerfile.web diff --git a/dockerfiles/README.md b/docker/README.md similarity index 100% rename from dockerfiles/README.md rename to docker/README.md From af6c6653036e7cd7066b0151b41e37f7d0fab061 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 7 Feb 2024 13:22:13 -0500 Subject: [PATCH 11/69] addressing differences --- compose/build_images_for_compose.sh | 13 ++++++++++--- compose/compose.yml | 4 ++-- core/docker/Dockerfile | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index 91f83c614..05e407ae2 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -7,19 +7,26 @@ SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../) docker build \ - -f "${PROJECT_ROOT}/dockerfiles/Dockerfile.dependencies" \ + -f "${PROJECT_ROOT}/docker/Dockerfile.dependencies" \ "${PROJECT_ROOT}" \ -t datafed-dependencies:latest docker build \ - -f "${PROJECT_ROOT}/dockerfiles/Dockerfile.runtime" \ + -f "${PROJECT_ROOT}/docker/Dockerfile.runtime" \ "${PROJECT_ROOT}" \ -t datafed-runtime:latest docker build -f \ - "${PROJECT_ROOT}/core/dockerfiles/Dockerfile" \ + "${PROJECT_ROOT}/core/docker/Dockerfile" \ --build-arg DEPENDENCIES="datafed-dependencies" \ --build-arg RUNTIME="datafed-runtime" \ "${PROJECT_ROOT}" \ -t datafed-core:latest +docker build -f \ + "${PROJECT_ROOT}/web/docker/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-web:latest + ## Repository server and authz library diff --git a/compose/compose.yml b/compose/compose.yml index 4811c24be..f6e36aa97 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -3,12 +3,12 @@ version: '3.9' services: datafed-web: - image: datafed/web:latest + image: datafed-web:latest ports: - 8080:443 datafed-core: - image: datafed/core:latest + image: datafed-core:latest environment: DATAFED_DOMAIN: "${DATAFED_DOMAIN}" ports: diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index 79b0120f9..ca856822e 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -81,5 +81,5 @@ COPY --chown=datafed:root ./cmake/Version.cmake ${DATAFED_DIR}/cmake COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh ${BUILD_DIR}/core/entrypoint.sh COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core -ENTRYPOINT ["${BUILD_DIR}/core/entrypoint.sh"] -CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"] +ENTRYPOINT ${BUILD_DIR}/core/entrypoint.sh +#CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"] From 21d604bae412214abc13fd00086b5e96bf3ba066 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 9 Feb 2024 15:04:21 -0500 Subject: [PATCH 12/69] Add CHangelog comment --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17ebf287a..13e3ceb36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,10 @@ ## MAJOR Breaking changes ## MINOR Feature +1. [912] - Adds initial compose file and docker files for python-client 1. [909] - Added Support for Google Analytics 2. [916] - Reworked Docker build process to be faster and produce simpler and smaller images - -## MINOR Feature -1. [912] - Adds initial compose file and docker files for python-client +2. [912] - Adding working compose instance for core metadata services. ## PATCH Bug fixes/Technical Debt/Documentation 1. [914] - Improve GitHub template From ae89d739ac074ced4fd680d4b71330bb01eb4f8d Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 9 Feb 2024 15:19:48 -0500 Subject: [PATCH 13/69] Add missing env files --- compose/compose.yml | 16 ++++++++++++++++ compose/generate_env.sh | 11 +++++++++++ 2 files changed, 27 insertions(+) diff --git a/compose/compose.yml b/compose/compose.yml index f6e36aa97..71d82e896 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -2,6 +2,15 @@ version: '3.9' services: + environment: + DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" + DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" + DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" + DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" + DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_DEFAULT_LOG_PATH}" datafed-web: image: datafed-web:latest ports: @@ -10,7 +19,14 @@ services: datafed-core: image: datafed-core:latest environment: + DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" + DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" + DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" + DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" + DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_DEFAULT_LOG_PATH}" ports: - 7513 # Communication web server - 7512 # Secure core server communication diff --git a/compose/generate_env.sh b/compose/generate_env.sh index 99ea67b98..368139deb 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -12,4 +12,15 @@ DATAFED_DOMAIN="" # The domain of the metadata core web server ARANGO_ROOT_PASSWORD="" DATAFED_USER89_PASSWORD="" # For End to end testing DATAFED_REPO_FORM_PATH="" # Where the repo form is located also needed for testing +GLOBUS_DATAFED_APP_SECRET="" +GLOBUS_DATAFED_APP_ID="" +DATAFED_ZEROMQ_SESSION_SECRET="" +DATAFED_ZEROMQ_SYSTEM_SECRET="" +DATAFED_DOMAIN="" +DATAFED_WEB_CERT_PATH="" +DATAFED_WEB_KEY_PATH="" +CONTAINER_LOG_FILE_PATH="" +DATAFED_DATABASE_PASSWORD="" +DATAFED_DATABASE_IP_ADDRESS_PORT="" + EOF From a933cf7d5276e5c3d96a1050688fe5df38e112a4 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sat, 10 Feb 2024 16:11:34 -0500 Subject: [PATCH 14/69] Debug --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57a3dc820..168b108ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -356,7 +356,7 @@ end-to-end-ws-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run -d + docker run --name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="GLOBUS_DATAFED_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$GLOBUS_DATAFED_APP_ID" @@ -397,7 +397,7 @@ end-to-end-core-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run -d + docker run --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" @@ -436,7 +436,7 @@ end-to-end-repo-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run -d + docker run --name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" @@ -470,7 +470,7 @@ end-to-end-gcs-authz-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run -d + docker run --name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" From 6df918464b634c89ce52aade8c7971888856a5ce Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sat, 10 Feb 2024 22:53:50 -0500 Subject: [PATCH 15/69] Make image customizable --- repository/docker/Dockerfile.repo.ubuntu | 12 ++++++++---- web/docker/Dockerfile.web.ubuntu | 8 ++++---- web/docker/entrypoint.sh | 6 +++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/repository/docker/Dockerfile.repo.ubuntu b/repository/docker/Dockerfile.repo.ubuntu index 4e663c37d..892e3004f 100644 --- a/repository/docker/Dockerfile.repo.ubuntu +++ b/repository/docker/Dockerfile.repo.ubuntu @@ -1,4 +1,6 @@ -FROM camden.ornl.gov/datafed/repo-base:latest as build +ARG REPO_BASE_IMAGE="camden.ornl.gov/datafed/repo-base:latest" + +FROM "$REPO_BASE_IMAGE" as build ARG DATAFED_DIR="/datafed" ARG BUILD_DIR="/datafed/source" @@ -9,9 +11,9 @@ ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -RUN mkdir -p ${BUILD_DIR} -RUN mkdir -p ${BUILD_DIR}/repository/server -RUN mkdir -p ${DATAFED_INSTALL_PATH}/keys +RUN mkdir -p "${BUILD_DIR}" +RUN mkdir -p "${BUILD_DIR}/repository/server" +RUN mkdir -p "${DATAFED_INSTALL_PATH}/keys" # This port is needed to communicate with the DataFed core server EXPOSE 7512 @@ -48,6 +50,8 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ cmake --build build RUN cmake --build build --target install +ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" + ENTRYPOINT ["/datafed/source/repository/docker/entrypoint_repo.sh"] CMD ["/datafed/install/repo/datafed-repo","--cfg","/datafed/install/repo/datafed-repo.cfg"] diff --git a/web/docker/Dockerfile.web.ubuntu b/web/docker/Dockerfile.web.ubuntu index ff4c5c0f5..3234a6167 100644 --- a/web/docker/Dockerfile.web.ubuntu +++ b/web/docker/Dockerfile.web.ubuntu @@ -50,10 +50,10 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ -DBUILD_FOXX=False -DBUILD_COMMON=False &&\ cmake --build build -ENV NVM_DIR="$DATAFED_DIR/.nvm" -ENV NVM_INC=$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node -ENV PATH=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ENV NVM_BIN=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin +#ENV NVM_DIR="$DATAFED_DIR/.nvm" +#ENV NVM_INC=$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node +#ENV PATH=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +#ENV NVM_BIN=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin # Note that building the web server this way assumes that a temporary core key # has been passed in, this will need to be overwritten when the web server diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index e7a33bf6a..006b0191d 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -8,10 +8,14 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) -source $NVM_DIR/nvm.sh ${PROJECT_ROOT}/scripts/generate_datafed.sh ${PROJECT_ROOT}/scripts/generate_ws_config.sh ${PROJECT_ROOT}/scripts/install_ws.sh +${PROJECT_ROOT}/scripts/dependency_versions.sh + +export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" +export NODE_VERSION="$DATAFED_NODE_VERSION" +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm # Send output to file as well as print to terminal log_path=$(grep "log-path" /datafed/install/web/datafed-ws.cfg | cut -d "=" -f 2 | tr -d ' ') From f494f54e5037af85e454e9f3fe7f1d81a2dfb9bb Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sat, 10 Feb 2024 23:23:09 -0500 Subject: [PATCH 16/69] Fix web build and env variables --- scripts/install_ws.sh | 56 +++------------------------ web/docker/Dockerfile.web-base.ubuntu | 2 + web/docker/Dockerfile.web.ubuntu | 5 ++- 3 files changed, 12 insertions(+), 51 deletions(-) diff --git a/scripts/install_ws.sh b/scripts/install_ws.sh index 61d1a35e6..e2678f20d 100755 --- a/scripts/install_ws.sh +++ b/scripts/install_ws.sh @@ -7,11 +7,12 @@ SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) source ${PROJECT_ROOT}/config/datafed.sh source ${SOURCE}/dependency_versions.sh +source ${SOURCE}/dependency_install_functions.sh #NVM_DIR=/home/cades/.nvm #[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" -export NVM_DIR="/home/cades/.nvm" -source ${NVM_DIR}/nvm.sh +#export NVM_DIR="/home/cades/.nvm" +#source ${NVM_DIR}/nvm.sh # Make sure paths exist mkdir -p ${DATAFED_INSTALL_PATH}/web mkdir -p ${DATAFED_INSTALL_PATH}/keys @@ -20,54 +21,9 @@ mkdir -p ${DATAFED_DEFAULT_LOG_PATH} # Install web node modules cp "$PROJECT_ROOT/web/package.json" ${DATAFED_INSTALL_PATH}/web/ -nvm use $DATAFED_NODE_VERSION -export npm_config_cache=${DATAFED_INSTALL_PATH}/web -# Check if npm exists -{ - npm_path=$(which npm) -} || { - echo "npm_path not found." -} -if [ -z "$npm_path" ] -then - - { - # Will return a result if nvm can be found and returns nothing otherwise - # which does not work with nvm - nvm_command_exists=$(command -v nvm) - } || { - echo "nvm_command_exists not found." - } - - if [ -z "$nvm_path" ] - then - # Check for nvm in default location when installed with web dependencies - # script - if [ -d "$NVM_DIR" ] - then - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm - elif [ -d "$HOME/.nvm" ] - then - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm - { - nvm_command_exists=$(command -v nvm) - } || { - echo "nvm_command_exists not found after sourcing." - } - if [ -z "$nvm_path" ] - then - echo "ERROR Unable to locate npm or nvm." - exit 1 - fi - else - echo "ERROR Unable to locate npm or nvm." - exit 1 - fi - - fi - nvm use $DATAFED_NODE_VERSION -fi +install_nvm +install_node +install_foxx_cli { npm --allow-root --unsafe-perm --prefix ${DATAFED_INSTALL_PATH}/web install } || { diff --git a/web/docker/Dockerfile.web-base.ubuntu b/web/docker/Dockerfile.web-base.ubuntu index b504c455f..60e7e2e96 100644 --- a/web/docker/Dockerfile.web-base.ubuntu +++ b/web/docker/Dockerfile.web-base.ubuntu @@ -13,6 +13,8 @@ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DIR}/dependenies" + # Docker dependency RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo RUN ${BUILD_DIR}/scripts/generate_datafed.sh diff --git a/web/docker/Dockerfile.web.ubuntu b/web/docker/Dockerfile.web.ubuntu index 3234a6167..aa187b175 100644 --- a/web/docker/Dockerfile.web.ubuntu +++ b/web/docker/Dockerfile.web.ubuntu @@ -1,4 +1,6 @@ -FROM camden.ornl.gov/datafed/ws-base:latest as build +ARG BASE_WEB_IMAGE="camden.ornl.gov/datafed/ws-base:latest" + +FROM "$BASE_WEB_IMAGE" as build ARG DATAFED_DIR="/datafed" ARG BUILD_DIR="/datafed/source" @@ -8,6 +10,7 @@ ARG DATAFED_INSTALL_PATH="/datafed/install" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DIR}/dependenies" RUN mkdir -p ${BUILD_DIR}/logs RUN mkdir -p ${BUILD_DIR}/common/proto From 81c0a2788e2059813ec98719492c26187956bc49 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sat, 10 Feb 2024 23:29:18 -0500 Subject: [PATCH 17/69] Make core build customizable --- core/docker/Dockerfile.core.ubuntu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/docker/Dockerfile.core.ubuntu b/core/docker/Dockerfile.core.ubuntu index 60a8891b5..fecd2ccf3 100644 --- a/core/docker/Dockerfile.core.ubuntu +++ b/core/docker/Dockerfile.core.ubuntu @@ -1,4 +1,6 @@ -FROM camden.ornl.gov/datafed/core-base:latest as build +ARG BASE_CORE_IMAGE="camden.ornl.gov/datafed/core-base:latest" + +FROM "$BASE_CORE_IMAGE" as build ARG DATAFED_DIR="/datafed" ARG BUILD_DIR="/datafed/source" From c60c64f6ff1aa897df49eb01da17ec004a0da500 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sat, 10 Feb 2024 23:32:33 -0500 Subject: [PATCH 18/69] Sleep for 10 seconds before checking if container is still running --- .gitlab-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 168b108ef..125add341 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -373,6 +373,7 @@ end-to-end-ws-setup: -v "${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" # Note here we use network=host to run the docker container this is @@ -413,9 +414,11 @@ end-to-end-core-setup: -v "$CI_DATAFED_CORE_PRIV_KEY":/source/install/keys/datafed-core-key.priv -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - +# Repo server currently will crash on startup if it cannot connect to the core +# server. end-to-end-repo-setup: variables: IMAGE_TAG: "datafed/repo-" @@ -423,6 +426,7 @@ end-to-end-repo-setup: HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs" CONTAINER_LOG_FILE_PATH: "/datafed/logs" stage: end-to-end-setup + needs: ["end-to-end-core-setup"] tags: - ci-datafed-globus - docker @@ -448,6 +452,7 @@ end-to-end-repo-setup: -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" end-to-end-gcs-authz-setup: @@ -482,6 +487,7 @@ end-to-end-gcs-authz-setup: -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" end-to-end-arango-setup: From 8882c11221c85b694aaa9c59f605c551a65e8e86 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sat, 10 Feb 2024 23:33:38 -0500 Subject: [PATCH 19/69] Run as deamon --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 125add341..196562df0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -356,7 +356,7 @@ end-to-end-ws-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run + docker run -d --name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="GLOBUS_DATAFED_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$GLOBUS_DATAFED_APP_ID" @@ -398,7 +398,7 @@ end-to-end-core-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run + docker run -d --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" @@ -440,7 +440,7 @@ end-to-end-repo-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run + docker run -d --name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" @@ -475,7 +475,7 @@ end-to-end-gcs-authz-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - docker run + docker run -d --name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" From 8448c901b470f732eb1ca618946a16b5ce7f21f5 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 12 Feb 2024 14:57:52 -0500 Subject: [PATCH 20/69] Save run script --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 196562df0..ea73247f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -356,6 +356,8 @@ end-to-end-ws-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > + cat < run_web.sh + #!/bin/bash docker run -d --name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="GLOBUS_DATAFED_APP_SECRET" @@ -373,6 +375,9 @@ end-to-end-ws-setup: -v "${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + EOF + - chmod +x run_web.sh + - ./run_web.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" @@ -398,6 +403,7 @@ end-to-end-core-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > + cat < run_web.sh docker run -d --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" @@ -414,6 +420,8 @@ end-to-end-core-setup: -v "$CI_DATAFED_CORE_PRIV_KEY":/source/install/keys/datafed-core-key.priv -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + - chmod +x run_core.sh + - ./run_core.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" From db16621ea738d1d25170d0c1f74c50bdc1a93e6e Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 12 Feb 2024 20:51:14 +0000 Subject: [PATCH 21/69] Remove comments --- .gitignore | 1 + compose/build_images_for_compose.sh | 24 ------------------------ compose/generate_env.sh | 1 - 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 1ee074785..8a0b30651 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.idea/ build common/proto/common/Version.proto +compose/.env config/datafed.sh config/datafed-authz.cfg config/datafed-core.cfg diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index 05e407ae2..239ba8f2d 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -27,27 +27,3 @@ docker build -f \ "${PROJECT_ROOT}" \ -t datafed-web:latest - - -## Repository server and authz library -#docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo-base.ubuntu ${PROJECT_ROOT} -t datafed/repo-base:latest -## Retag -#docker tag datafed/repo-base:latest code.ornl.gov:4567/dlsw/datafed/repo-base:latest -#docker build -f ${PROJECT_ROOT}/repository/docker/Dockerfile.repo.ubuntu ${PROJECT_ROOT} -t datafed/repo:latest -# -## Core and Web servers -#REPO_LIST=("web" "core") -#for REPO in "${REPO_LIST[@]}" -#do -# CONTAINER=${REPO} -# docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}-base.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}-base:latest -# # Retag -# docker tag datafed/${CONTAINER}-base:latest code.ornl.gov:4567/dlsw/datafed/core-base:latest -# docker build -f ${PROJECT_ROOT}/${REPO}/docker/Dockerfile.${REPO}.ubuntu ${PROJECT_ROOT} -t datafed/${CONTAINER}:latest -#done -# -## Python client -#docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client-base.ubuntu ${PROJECT_ROOT} -t datafed/python-client-base:latest -## Retag -#docker tag datafed/python-client-base:latest code.ornl.gov:4567/dlsw/datafed/python-client:latest -#docker build -f ${PROJECT_ROOT}/python/docker/Dockerfile.python-client.ubuntu ${PROJECT_ROOT} -t datafed/python-client:latest diff --git a/compose/generate_env.sh b/compose/generate_env.sh index 368139deb..a13a4d11b 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -22,5 +22,4 @@ DATAFED_WEB_KEY_PATH="" CONTAINER_LOG_FILE_PATH="" DATAFED_DATABASE_PASSWORD="" DATAFED_DATABASE_IP_ADDRESS_PORT="" - EOF From 68c59199a92dbaba0e32122e8db7911a87513d4a Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 12 Feb 2024 20:52:04 +0000 Subject: [PATCH 22/69] Add missing EOF --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea73247f2..2696b31df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -420,6 +420,7 @@ end-to-end-core-setup: -v "$CI_DATAFED_CORE_PRIV_KEY":/source/install/keys/datafed-core-key.priv -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + EOF - chmod +x run_core.sh - ./run_core.sh - sleep 10 From a6eb53f635de08b2e162195e1cba0dcae1ee3807 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 12 Feb 2024 16:41:47 -0500 Subject: [PATCH 23/69] minor fixes --- compose/compose.yml | 4 ++-- compose/generate_env.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compose/compose.yml b/compose/compose.yml index 71d82e896..1485b4c81 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -10,7 +10,7 @@ services: DATAFED_DOMAIN: "${DATAFED_DOMAIN}" DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" - DATAFED_DEFAULT_LOG_PATH: "${DATAFED_DEFAULT_LOG_PATH}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" datafed-web: image: datafed-web:latest ports: @@ -26,7 +26,7 @@ services: DATAFED_DOMAIN: "${DATAFED_DOMAIN}" DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" - DATAFED_DEFAULT_LOG_PATH: "${DATAFED_DEFAULT_LOG_PATH}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" ports: - 7513 # Communication web server - 7512 # Secure core server communication diff --git a/compose/generate_env.sh b/compose/generate_env.sh index a13a4d11b..1aa9d3ba2 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -12,14 +12,14 @@ DATAFED_DOMAIN="" # The domain of the metadata core web server ARANGO_ROOT_PASSWORD="" DATAFED_USER89_PASSWORD="" # For End to end testing DATAFED_REPO_FORM_PATH="" # Where the repo form is located also needed for testing -GLOBUS_DATAFED_APP_SECRET="" -GLOBUS_DATAFED_APP_ID="" +DATAFED_GLOBUS_APP_SECRET="" +DATAFED_GLOBUS_APP_ID="" DATAFED_ZEROMQ_SESSION_SECRET="" DATAFED_ZEROMQ_SYSTEM_SECRET="" DATAFED_DOMAIN="" DATAFED_WEB_CERT_PATH="" DATAFED_WEB_KEY_PATH="" -CONTAINER_LOG_FILE_PATH="" +DATAFED_CONTAINER_LOG_FILE_PATH="" DATAFED_DATABASE_PASSWORD="" DATAFED_DATABASE_IP_ADDRESS_PORT="" EOF From 7c9372f2e9c899ee4deb53450c136453896f462a Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 12 Feb 2024 16:45:09 -0500 Subject: [PATCH 24/69] Add missing here doc --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2696b31df..d5a82399a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,6 +404,7 @@ end-to-end-core-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > cat < run_web.sh + #!/bin/bash docker run -d --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" @@ -449,6 +450,8 @@ end-to-end-repo-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > + cat < run_repo.sh + #!/bin/bash docker run -d --name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" @@ -461,6 +464,9 @@ end-to-end-repo-setup: -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + EOF + - chmod +x run_repo.sh + - ./run_repo.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" @@ -484,6 +490,8 @@ end-to-end-gcs-authz-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > + cat < run_globus.sh + #!/bin/bash docker run -d --name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" @@ -496,6 +504,9 @@ end-to-end-gcs-authz-setup: -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" /dev/null & + EOF + - chmod +x run_globus.sh + - ./run_globus.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" From ab7f4cec12525f127bde2001f3ccf8dcb424cad2 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 10:09:58 -0500 Subject: [PATCH 25/69] Try to limit the number of times a build happens --- .gitlab-ci.yml | 60 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5a82399a..20fd88eb9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ stages: - signal - clear-docker-cache - build-deploy-base + - build-deploy-base_2 - provision-client - build - end-to-end-setup @@ -149,7 +150,6 @@ build-repo-base: - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - docker push "${REGISTRY}/${IMAGE_TAG}" - build-gcs-base: needs: ["clear-repo-cache"] variables: @@ -159,23 +159,68 @@ build-gcs-base: GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal" IMAGE_TAG2: "datafed/gcs-authz-base" IMAGE_TAG3: "datafed/gcs-authz" + GCS_SUBMODULE_VERSION: "v2.6.0" stage: build-deploy-base tags: - ci-datafed-globus - docker script: - docker system prune -f - - cd external/globus-connect-server-deploy/docker - - ./docker-build-gcs.sh "${DISTRO}" - - LATEST_IMAGE=$(docker images | grep ubuntu-focal | head -n1 | awk '{print $3}') - - docker tag "$LATEST_IMAGE" "$GCS_TAG" - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - - docker push "$GCS_TAG" - - cd ../../../ # back too root of project + - if ! docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}"; then + cd external/globus-connect-server-deploy/docker + git checkout "${GCS_SUBMODULE_VERSION}" + ./docker-build-gcs.sh "${DISTRO}" + LATEST_IMAGE=$(docker images | grep ubuntu-focal | head -n1 | awk '{print $3}') + docker tag "$LATEST_IMAGE" "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + docker push "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" + docker push "$GCS_TAG" + cd ../../../ # back too root of project + else + echo "Docker image already exists in the registry. Skipping build." + fi + +build-gcs-authz-base: + variables: + GIT_SUBMODULE_STRATEGY: recursive + GIT_STRATEGY: clone + DISTRO: "ubuntu:focal" + GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal" + IMAGE_TAG2: "datafed/gcs-authz-base" + IMAGE_TAG3: "datafed/gcs-authz" + GCS_SUBMODULE_VERSION: "v2.6.0" + stage: build-deploy-base_2 + tags: + - ci-datafed-globus + - docker + script: + - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" + - docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + - docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG2}" . - docker push "${REGISTRY}/${IMAGE_TAG2}" - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG3}" . - docker push "${REGISTRY}/${IMAGE_TAG3}" + rules: + - changes: + - scripts/generate_datafed.sh + - scripts/dependency_versions.sh + - scripts/dependency_install_functions.sh + - scripts/install_authz_dependencies.sh + - scripts/generate_authz_config.sh + - scripts/generate_datafed.sh + - scripts/globus/setup_globus.sh + - scripts/globus/generate_repo_form.sh + - repository/docker/entrypoint_authz.sh + - CMakeLists.txt + - cmake/* + - repository/CMakeLists.txt + - repository/gridftp/CMakeLists.txt + - repository/gridftp/globus5/**/* + - common/**/* + - .gitlab-ci.ym + ################################################################################ # STAGE: provision client @@ -314,7 +359,6 @@ build-repo: - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" build-gcs-authz: - needs: ["build-gcs-base"] stage: build variables: IMAGE_TAG: "datafed/gcs-authz-" From fa378afc85a578f2c9ce396a4e7c01a590cbd9af Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 10:11:15 -0500 Subject: [PATCH 26/69] Remove tabs --- .gitlab-ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20fd88eb9..a31ce086e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,14 +168,14 @@ build-gcs-base: - docker system prune -f - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - if ! docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}"; then - cd external/globus-connect-server-deploy/docker - git checkout "${GCS_SUBMODULE_VERSION}" - ./docker-build-gcs.sh "${DISTRO}" - LATEST_IMAGE=$(docker images | grep ubuntu-focal | head -n1 | awk '{print $3}') - docker tag "$LATEST_IMAGE" "$GCS_TAG:${GCS_SUBMODULE_VERSION}" - docker push "$GCS_TAG:${GCS_SUBMODULE_VERSION}" - docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" - docker push "$GCS_TAG" + cd external/globus-connect-server-deploy/docker + git checkout "${GCS_SUBMODULE_VERSION}" + ./docker-build-gcs.sh "${DISTRO}" + LATEST_IMAGE=$(docker images | grep ubuntu-focal | head -n1 | awk '{print $3}') + docker tag "$LATEST_IMAGE" "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + docker push "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" + docker push "$GCS_TAG" cd ../../../ # back too root of project else echo "Docker image already exists in the registry. Skipping build." @@ -197,7 +197,7 @@ build-gcs-authz-base: script: - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}" - - docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" + - docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG2}" . - docker push "${REGISTRY}/${IMAGE_TAG2}" - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG3}" . From dce32eac30309679e99db055fb1f300661bc0fc3 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 10:13:23 -0500 Subject: [PATCH 27/69] Multiline yml command --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a31ce086e..570f72145 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,7 +167,8 @@ build-gcs-base: script: - docker system prune -f - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - - if ! docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}"; then + - > + if ! docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}"; then cd external/globus-connect-server-deploy/docker git checkout "${GCS_SUBMODULE_VERSION}" ./docker-build-gcs.sh "${DISTRO}" From 1dd933f81fbec8a32c87eb813edb1ce4aea7d7f5 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 10:14:53 -0500 Subject: [PATCH 28/69] Fix missing stage --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 570f72145..ec6d085e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - signal - clear-docker-cache - build-deploy-base - - build-deploy-base_2 + - build-deploy-base-2 - provision-client - build - end-to-end-setup @@ -191,7 +191,7 @@ build-gcs-authz-base: IMAGE_TAG2: "datafed/gcs-authz-base" IMAGE_TAG3: "datafed/gcs-authz" GCS_SUBMODULE_VERSION: "v2.6.0" - stage: build-deploy-base_2 + stage: build-deploy-base-2 tags: - ci-datafed-globus - docker From 377d98a96fa30f19c9f1f6a60914bddc336d0d46 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 14:48:48 -0500 Subject: [PATCH 29/69] Address new globus connect server tag --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ec6d085e9..1d16ab0cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,7 +155,7 @@ build-gcs-base: variables: GIT_SUBMODULE_STRATEGY: recursive GIT_STRATEGY: clone - DISTRO: "ubuntu:focal" + DISTRO: "ubuntu-20.04" GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal" IMAGE_TAG2: "datafed/gcs-authz-base" IMAGE_TAG3: "datafed/gcs-authz" @@ -168,7 +168,7 @@ build-gcs-base: - docker system prune -f - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - > - if ! docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}"; then + if ! docker pull "${GCS_TAG}:${GCS_SUBMODULE_VERSION}"; then cd external/globus-connect-server-deploy/docker git checkout "${GCS_SUBMODULE_VERSION}" ./docker-build-gcs.sh "${DISTRO}" From 0c3c55d866dc2ff48a5269cfde0993dec947e767 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 15:25:12 -0500 Subject: [PATCH 30/69] Remove heredoc in favor of echo --- .gitlab-ci.yml | 128 +++++++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 67 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d16ab0cf..5ee8fbb42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -402,24 +402,24 @@ end-to-end-ws-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > cat < run_web.sh - #!/bin/bash - docker run -d - --name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="GLOBUS_DATAFED_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$GLOBUS_DATAFED_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -e DATAFED_WEB_CERT_PATH="${CI_DATAFED_WEB_CERT_PATH}" - -e DATAFED_WEB_KEY_PATH="${CI_DATAFED_WEB_KEY_PATH}" - -e DATAFED_DEFAULT_LOG_PATH="$CONTAINER_LOG_FILE_PATH" - -p 443:443 - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub" - -v "${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}" - -v "${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}" - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & + echo "#!/bin/bash" > run_web.sh + echo "docker run -d \\" >> run_web.sh + echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh + echo "-e DATAFED_GLOBUS_APP_SECRET=\"GLOBUS_DATAFED_APP_SECRET\" \\" >> run_web.sh + echo "-e DATAFED_GLOBUS_APP_ID=\"$GLOBUS_DATAFED_APP_ID\" \\" >> run_web.sh + echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh + echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh + echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh + echo "-e DATAFED_WEB_CERT_PATH=\"${CI_DATAFED_WEB_CERT_PATH}\" \\" >> run_web.sh + echo "-e DATAFED_WEB_KEY_PATH=\"${CI_DATAFED_WEB_KEY_PATH}\" \\" >> run_web.sh + echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_web.sh + echo "-p 443:443 \\" >> run_web.sh + echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh + echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub\" \\" >> run_web.sh + echo "-v \"${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}\" \\" >> run_web.sh + echo "-v \"${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}\" \\" >> run_web.sh + echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_web.sh + echo " /dev/null &" >> run_web.sh EOF - chmod +x run_web.sh - ./run_web.sh @@ -448,25 +448,23 @@ end-to-end-core-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - cat < run_web.sh - #!/bin/bash - docker run -d - --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -e DATAFED_DATABASE_PASSWORD="$CI_DATAFED_DATABASE_PASSWORD" - -e DATAFED_DATABASE_IP_ADDRESS_PORT="$CI_DATAFED_DATABASE_IP_ADDRESS_PORT" - -e DATAFED_DEFAULT_LOG_PATH="$CONTAINER_LOG_FILE_PATH" - --network host - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "$CI_DATAFED_CORE_PUB_KEY":/source/install/keys/datafed-core-key.pub - -v "$CI_DATAFED_CORE_PRIV_KEY":/source/install/keys/datafed-core-key.priv - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & - EOF + echo "#!/bin/bash" > run_core.sh + echo "docker run -d \\" >> run_core.sh + echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh + echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh + echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh + echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh + echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_core.sh + echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_core.sh + echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> run_core.sh + echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> run_core.sh + echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_core.sh + echo "--network host \\" >> run_core.sh + echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh + echo "-v \"$CI_DATAFED_CORE_PUB_KEY\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh + echo "-v \"$CI_DATAFED_CORE_PRIV_KEY\":/source/install/keys/datafed-core-key.priv \\" >> run_core.sh + echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_core.sh + echo " /dev/null &" >> run_core.sh - chmod +x run_core.sh - ./run_core.sh - sleep 10 @@ -495,21 +493,19 @@ end-to-end-repo-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - cat < run_repo.sh - #!/bin/bash - docker run -d - --name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub - -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & - EOF + echo "#!/bin/bash" > run_repo.sh + echo "docker run -d \\" >> run_repo.sh + echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_repo.sh + echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_repo.sh + echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_repo.sh + echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh + echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_repo.sh + echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_repo.sh + echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_repo.sh + echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_repo.sh + echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_repo.sh + echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_repo.sh + echo " /dev/null &" >> run_repo.sh - chmod +x run_repo.sh - ./run_repo.sh - sleep 10 @@ -535,21 +531,19 @@ end-to-end-gcs-authz-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - > - cat < run_globus.sh - #!/bin/bash - docker run -d - --name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub - -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & - EOF + echo "#!/bin/bash" > run_globus.sh + echo "docker run -d \\" >> run_globus.sh + echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh + echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh + echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh + echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh + echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh + echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh + echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh + echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_globus.sh + echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_globus.sh + echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_globus.sh + echo " /dev/null &" >> run_globus.sh - chmod +x run_globus.sh - ./run_globus.sh - sleep 10 From 5d61f2879cb3482795d6717de7fa8ec3ca489d26 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 15:36:35 -0500 Subject: [PATCH 31/69] Fix gitlab --- .gitlab-ci.yml | 130 +++++++++++++++++++++++-------------------------- 1 file changed, 62 insertions(+), 68 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5ee8fbb42..f3f5a4b81 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -160,7 +160,7 @@ build-gcs-base: IMAGE_TAG2: "datafed/gcs-authz-base" IMAGE_TAG3: "datafed/gcs-authz" GCS_SUBMODULE_VERSION: "v2.6.0" - stage: build-deploy-base + stage: pre-build-deploy-base tags: - ci-datafed-globus - docker @@ -400,27 +400,24 @@ end-to-end-ws-setup: - KEY_FILE_NAME=$(basename "${CI_DATAFED_WEB_KEY_PATH}") - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - cat < run_web.sh - echo "#!/bin/bash" > run_web.sh - echo "docker run -d \\" >> run_web.sh - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"GLOBUS_DATAFED_APP_SECRET\" \\" >> run_web.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$GLOBUS_DATAFED_APP_ID\" \\" >> run_web.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh - echo "-e DATAFED_WEB_CERT_PATH=\"${CI_DATAFED_WEB_CERT_PATH}\" \\" >> run_web.sh - echo "-e DATAFED_WEB_KEY_PATH=\"${CI_DATAFED_WEB_KEY_PATH}\" \\" >> run_web.sh - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_web.sh - echo "-p 443:443 \\" >> run_web.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}\" \\" >> run_web.sh - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_web.sh - echo " /dev/null &" >> run_web.sh - EOF + - echo "#!/bin/bash" > run_web.sh + - echo "docker run -d \\" >> run_web.sh + - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"GLOBUS_DATAFED_APP_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$GLOBUS_DATAFED_APP_ID\" \\" >> run_web.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh + - echo "-e DATAFED_WEB_CERT_PATH=\"${CI_DATAFED_WEB_CERT_PATH}\" \\" >> run_web.sh + - echo "-e DATAFED_WEB_KEY_PATH=\"${CI_DATAFED_WEB_KEY_PATH}\" \\" >> run_web.sh + - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_web.sh + - echo "-p 443:443 \\" >> run_web.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh + - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub\" \\" >> run_web.sh + - echo "-v \"${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}\" \\" >> run_web.sh + - echo "-v \"${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}\" \\" >> run_web.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_web.sh + - echo " /dev/null &" >> run_web.sh - chmod +x run_web.sh - ./run_web.sh - sleep 10 @@ -447,24 +444,23 @@ end-to-end-core-setup: - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - echo "#!/bin/bash" > run_core.sh - echo "docker run -d \\" >> run_core.sh - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_core.sh - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_core.sh - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> run_core.sh - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> run_core.sh - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_core.sh - echo "--network host \\" >> run_core.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh - echo "-v \"$CI_DATAFED_CORE_PUB_KEY\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh - echo "-v \"$CI_DATAFED_CORE_PRIV_KEY\":/source/install/keys/datafed-core-key.priv \\" >> run_core.sh - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_core.sh - echo " /dev/null &" >> run_core.sh + - echo "#!/bin/bash" > run_core.sh + - echo "docker run -d \\" >> run_core.sh + - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_core.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_core.sh + - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> run_core.sh + - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> run_core.sh + - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_core.sh + - echo "--network host \\" >> run_core.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh + - echo "-v \"$CI_DATAFED_CORE_PUB_KEY\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh + - echo "-v \"$CI_DATAFED_CORE_PRIV_KEY\":/source/install/keys/datafed-core-key.priv \\" >> run_core.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_core.sh + - echo " /dev/null &" >> run_core.sh - chmod +x run_core.sh - ./run_core.sh - sleep 10 @@ -492,20 +488,19 @@ end-to-end-repo-setup: - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - echo "#!/bin/bash" > run_repo.sh - echo "docker run -d \\" >> run_repo.sh - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_repo.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_repo.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_repo.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_repo.sh - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_repo.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_repo.sh - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_repo.sh - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_repo.sh - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_repo.sh - echo " /dev/null &" >> run_repo.sh + - echo "#!/bin/bash" > run_repo.sh + - echo "docker run -d \\" >> run_repo.sh + - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_repo.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_repo.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_repo.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_repo.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_repo.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_repo.sh + - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_repo.sh + - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_repo.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_repo.sh + - echo " /dev/null &" >> run_repo.sh - chmod +x run_repo.sh - ./run_repo.sh - sleep 10 @@ -530,20 +525,19 @@ end-to-end-gcs-authz-setup: - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - echo "#!/bin/bash" > run_globus.sh - echo "docker run -d \\" >> run_globus.sh - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_globus.sh - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_globus.sh - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_globus.sh - echo " /dev/null &" >> run_globus.sh + - echo "#!/bin/bash" > run_globus.sh + - echo "docker run -d \\" >> run_globus.sh + - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh + - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_globus.sh + - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_globus.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_globus.sh + - echo " /dev/null &" >> run_globus.sh - chmod +x run_globus.sh - ./run_globus.sh - sleep 10 From 811c1adae275599ec04900dae7bd6584dbcddb04 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 15:37:48 -0500 Subject: [PATCH 32/69] Temporarily turn off build --- .gitlab-ci.yml | 78 +++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3f5a4b81..fa9c3566d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -182,45 +182,45 @@ build-gcs-base: echo "Docker image already exists in the registry. Skipping build." fi -build-gcs-authz-base: - variables: - GIT_SUBMODULE_STRATEGY: recursive - GIT_STRATEGY: clone - DISTRO: "ubuntu:focal" - GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal" - IMAGE_TAG2: "datafed/gcs-authz-base" - IMAGE_TAG3: "datafed/gcs-authz" - GCS_SUBMODULE_VERSION: "v2.6.0" - stage: build-deploy-base-2 - tags: - - ci-datafed-globus - - docker - script: - - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - - docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}" - - docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" - - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG2}" . - - docker push "${REGISTRY}/${IMAGE_TAG2}" - - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG3}" . - - docker push "${REGISTRY}/${IMAGE_TAG3}" - rules: - - changes: - - scripts/generate_datafed.sh - - scripts/dependency_versions.sh - - scripts/dependency_install_functions.sh - - scripts/install_authz_dependencies.sh - - scripts/generate_authz_config.sh - - scripts/generate_datafed.sh - - scripts/globus/setup_globus.sh - - scripts/globus/generate_repo_form.sh - - repository/docker/entrypoint_authz.sh - - CMakeLists.txt - - cmake/* - - repository/CMakeLists.txt - - repository/gridftp/CMakeLists.txt - - repository/gridftp/globus5/**/* - - common/**/* - - .gitlab-ci.ym + #build-gcs-authz-base: + # variables: + # GIT_SUBMODULE_STRATEGY: recursive + # GIT_STRATEGY: clone + # DISTRO: "ubuntu:focal" + # GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal" + # IMAGE_TAG2: "datafed/gcs-authz-base" + # IMAGE_TAG3: "datafed/gcs-authz" + # GCS_SUBMODULE_VERSION: "v2.6.0" + # stage: build-deploy-base-2 + # tags: + # - ci-datafed-globus + # - docker + # script: + # - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" + # - docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + # - docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" + # - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG2}" . + # - docker push "${REGISTRY}/${IMAGE_TAG2}" + # - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG3}" . + # - docker push "${REGISTRY}/${IMAGE_TAG3}" + # rules: + # - changes: + # - scripts/generate_datafed.sh + # - scripts/dependency_versions.sh + # - scripts/dependency_install_functions.sh + # - scripts/install_authz_dependencies.sh + # - scripts/generate_authz_config.sh + # - scripts/generate_datafed.sh + # - scripts/globus/setup_globus.sh + # - scripts/globus/generate_repo_form.sh + # - repository/docker/entrypoint_authz.sh + # - CMakeLists.txt + # - cmake/* + # - repository/CMakeLists.txt + # - repository/gridftp/CMakeLists.txt + # - repository/gridftp/globus5/**/* + # - common/**/* + # - .gitlab-ci.ym ################################################################################ From b0d2a2ccc318314dbdf65ca22da28183d86efb37 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 15:39:22 -0500 Subject: [PATCH 33/69] comment out stage --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa9c3566d..8f85451f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: - signal - clear-docker-cache - build-deploy-base - - build-deploy-base-2 + # - build-deploy-base-2 - provision-client - build - end-to-end-setup From dfb854e4ba001c89a10a6d68756327656cb4d6ff Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 15:40:24 -0500 Subject: [PATCH 34/69] Fix stage --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f85451f8..c8d6248fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -160,7 +160,7 @@ build-gcs-base: IMAGE_TAG2: "datafed/gcs-authz-base" IMAGE_TAG3: "datafed/gcs-authz" GCS_SUBMODULE_VERSION: "v2.6.0" - stage: pre-build-deploy-base + stage: build-deploy-base tags: - ci-datafed-globus - docker From 483f5cec686197b15dde7308d47ca60565eb6cf0 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 15:58:00 -0500 Subject: [PATCH 35/69] Hopefully fix log file path --- .gitlab-ci.yml | 8 +++++++- core/docker/entrypoint.sh | 5 +++++ repository/docker/entrypoint_authz.sh | 7 +++++++ repository/docker/entrypoint_repo.sh | 7 +++++++ web/docker/entrypoint.sh | 5 +++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8d6248fa..b1be4ace9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -238,7 +238,13 @@ provision-client: script: - ./scripts/generate_datafed.sh - ./scripts/install_client_dependencies.sh - + rules: + - changes: + - scripts/generate_datafed.sh + - scripts/install_client_dependencies.sh + - scripts/dependency_install_functions.sh + - scripts/dependency_versions.sh + ################################################################################ # STAGE: build ################################################################################ diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 0b45898f2..5bbdc3ec5 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -12,6 +12,11 @@ ${PROJECT_ROOT}/scripts/install_core.sh log_path="$DATAFED_DEFAULT_LOG_PATH" +if [ ! -d "${log_path}" ] +then + mkdir -p "${log_path}" +fi + if [ "$#" -eq 0 ]; then echo "No arguments were passed, running bash" exec "bash" diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index 3a072d107..b5bd7ff66 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -12,4 +12,11 @@ GLOBUS_CLIENT_ID="$DATAFED_GLOBUS_APP_ID" ${PROJECT_ROOT}/scripts/generate_datafed.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh +log_path="$DATAFED_DEFAULT_LOG_PATH" + +if [ ! -d "${log_path}" ] +then + mkdir -p "${log_path}" +fi + exec "$@" diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index 4eebc6225..57813d81d 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -10,4 +10,11 @@ ${PROJECT_ROOT}/scripts/generate_datafed.sh ${PROJECT_ROOT}/scripts/generate_repo_config.sh ${PROJECT_ROOT}/scripts/install_repo.sh +log_path="$DATAFED_DEFAULT_LOG_PATH" + +if [ ! -d "${log_path}" ] +then + mkdir -p "${log_path}" +fi + exec "$@" diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 006b0191d..297ffa2c8 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -20,6 +20,11 @@ export NODE_VERSION="$DATAFED_NODE_VERSION" # Send output to file as well as print to terminal log_path=$(grep "log-path" /datafed/install/web/datafed-ws.cfg | cut -d "=" -f 2 | tr -d ' ') +if [ ! -d "${log_path}" ] +then + mkdir -p "${log_path}" +fi + if [ "$#" -eq 0 ]; then echo "No arguments were passed, running bash" exec "bash" From 58daf0065294bea5e09edd90816eb2e000037d8e Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 16:22:19 -0500 Subject: [PATCH 36/69] Fix permissions --- .gitlab-ci.yml | 4 ++++ core/docker/Dockerfile.core.ubuntu | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1be4ace9..3c2fa37c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -399,6 +399,7 @@ end-to-end-ws-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" + - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" @@ -444,6 +445,7 @@ end-to-end-core-setup: script: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - mkdir -p "$HOST_LOG_FILE_PATH" + - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh @@ -489,6 +491,7 @@ end-to-end-repo-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" + - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" @@ -526,6 +529,7 @@ end-to-end-gcs-authz-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" + - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" diff --git a/core/docker/Dockerfile.core.ubuntu b/core/docker/Dockerfile.core.ubuntu index fecd2ccf3..7c752ab11 100644 --- a/core/docker/Dockerfile.core.ubuntu +++ b/core/docker/Dockerfile.core.ubuntu @@ -52,7 +52,7 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ cmake --build build -j 8 RUN cmake --build build --target install -WORKDIR ${DATAFED_INSTALL_PATHl}/core +WORKDIR ${DATAFED_INSTALL_PATH}/core ENTRYPOINT ["/datafed/source/core/docker/entrypoint.sh"] From 8626e7e7b28c826576b39244c2c2bf0e875e5770 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 03:07:34 +0000 Subject: [PATCH 37/69] Correct install path hopefully --- core/docker/Dockerfile.core.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/docker/Dockerfile.core.ubuntu b/core/docker/Dockerfile.core.ubuntu index 7c752ab11..88ba3e797 100644 --- a/core/docker/Dockerfile.core.ubuntu +++ b/core/docker/Dockerfile.core.ubuntu @@ -9,7 +9,7 @@ ARG DATAFED_INSTALL_PATH="/datafed/install" # The above should also be available at runtime ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" -ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH/core" +ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" RUN mkdir -p ${BUILD_DIR}/logs RUN mkdir -p ${DATAFED_INSTALL_PATH} From bafd4fd1bae4d799a85cb2eb6d0309e9251026eb Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Tue, 13 Feb 2024 22:14:35 -0500 Subject: [PATCH 38/69] Fix ci variables --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c2fa37c4..72d5dbc44 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -410,10 +410,10 @@ end-to-end-ws-setup: - echo "#!/bin/bash" > run_web.sh - echo "docker run -d \\" >> run_web.sh - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh - - echo "-e DATAFED_GLOBUS_APP_SECRET=\"GLOBUS_DATAFED_APP_SECRET\" \\" >> run_web.sh - - echo "-e DATAFED_GLOBUS_APP_ID=\"$GLOBUS_DATAFED_APP_ID\" \\" >> run_web.sh - - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh - - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh - echo "-e DATAFED_WEB_CERT_PATH=\"${CI_DATAFED_WEB_CERT_PATH}\" \\" >> run_web.sh - echo "-e DATAFED_WEB_KEY_PATH=\"${CI_DATAFED_WEB_KEY_PATH}\" \\" >> run_web.sh From ae33e194c01895e20213851bd02bcdbc462fbb97 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 03:28:18 +0000 Subject: [PATCH 39/69] Fix permissions --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72d5dbc44..2d5275faa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -405,11 +405,14 @@ end-to-end-ws-setup: - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - CERT_FILE_NAME=$(basename "${CI_DATAFED_WEB_CERT_PATH}") - KEY_FILE_NAME=$(basename "${CI_DATAFED_WEB_KEY_PATH}") + - USER_ID=$(id -u) + - GROUP_ID=$(id -g) - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - echo "#!/bin/bash" > run_web.sh - echo "docker run -d \\" >> run_web.sh - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh + - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_web.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh @@ -455,6 +458,7 @@ end-to-end-core-setup: - echo "#!/bin/bash" > run_core.sh - echo "docker run -d \\" >> run_core.sh - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh + - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh @@ -500,6 +504,7 @@ end-to-end-repo-setup: - echo "#!/bin/bash" > run_repo.sh - echo "docker run -d \\" >> run_repo.sh - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_repo.sh + - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_repo.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_repo.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_repo.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh @@ -538,6 +543,7 @@ end-to-end-gcs-authz-setup: - echo "#!/bin/bash" > run_globus.sh - echo "docker run -d \\" >> run_globus.sh - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh + - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_globus.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh From 37475e8aef142feddf8619e80e58f590e7aa52ee Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 04:06:36 +0000 Subject: [PATCH 40/69] Debugging volume mounts --- .gitlab-ci.yml | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d5275faa..8701fa5bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -410,9 +410,8 @@ end-to-end-ws-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - echo "#!/bin/bash" > run_web.sh - - echo "docker run -d \\" >> run_web.sh + - echo "docker run \\" >> run_web.sh - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh - - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_web.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh @@ -426,13 +425,14 @@ end-to-end-ws-setup: - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}\" \\" >> run_web.sh - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_web.sh - - echo " /dev/null &" >> run_web.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" " >> run_web.sh - chmod +x run_web.sh - ./run_web.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" +#- echo " /dev/null &" >> run_web.sh +#- echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_web.sh # Note here we use network=host to run the docker container this is # because we do not need it to communicate with an external port end-to-end-core-setup: @@ -456,9 +456,8 @@ end-to-end-core-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - echo "#!/bin/bash" > run_core.sh - - echo "docker run -d \\" >> run_core.sh + - echo "docker run \\" >> run_core.sh - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh - - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh @@ -471,13 +470,13 @@ end-to-end-core-setup: - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh - echo "-v \"$CI_DATAFED_CORE_PUB_KEY\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh - echo "-v \"$CI_DATAFED_CORE_PRIV_KEY\":/source/install/keys/datafed-core-key.priv \\" >> run_core.sh - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_core.sh - - echo " /dev/null &" >> run_core.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_core.sh - chmod +x run_core.sh - ./run_core.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" +#- echo " /dev/null &" >> run_core.sh # Repo server currently will crash on startup if it cannot connect to the core # server. end-to-end-repo-setup: @@ -502,9 +501,8 @@ end-to-end-repo-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - echo "#!/bin/bash" > run_repo.sh - - echo "docker run -d \\" >> run_repo.sh + - echo "docker run \\" >> run_repo.sh - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_repo.sh - - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_repo.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_repo.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_repo.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh @@ -513,13 +511,13 @@ end-to-end-repo-setup: - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_repo.sh - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_repo.sh - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_repo.sh - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_repo.sh - - echo " /dev/null &" >> run_repo.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_repo.sh - chmod +x run_repo.sh - ./run_repo.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" +# - echo " /dev/null &" >> run_repo.sh end-to-end-gcs-authz-setup: variables: IMAGE_TAG: "datafed/gcs-authz-" @@ -541,9 +539,8 @@ end-to-end-gcs-authz-setup: - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - echo "#!/bin/bash" > run_globus.sh - - echo "docker run -d \\" >> run_globus.sh + - echo "docker run \\" >> run_globus.sh - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh - - echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_globus.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh @@ -552,13 +549,13 @@ end-to-end-gcs-authz-setup: - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_globus.sh - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_globus.sh - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" \\" >> run_globus.sh - - echo " /dev/null &" >> run_globus.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_globus.sh - chmod +x run_globus.sh - ./run_globus.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" +# - echo " /dev/null &" >> run_globus.sh end-to-end-arango-setup: variables: GIT_STRATEGY: clone From 9e059fe080ca84b9c2c89f2baf9d24540a4f58ff Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 04:57:26 +0000 Subject: [PATCH 41/69] Try to fix user id troubles with volume mounts --- .gitlab-ci.yml | 7 +++++++ core/docker/Dockerfile.core.ubuntu | 2 ++ core/docker/entrypoint.sh | 5 +++++ repository/docker/Dockerfile.repo.ubuntu | 2 ++ repository/docker/entrypoint_authz.sh | 6 ++++++ repository/docker/entrypoint_repo.sh | 5 +++++ web/docker/Dockerfile.web.ubuntu | 2 ++ web/docker/entrypoint.sh | 7 ++++++- 8 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8701fa5bb..e8ccd0846 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -420,6 +420,7 @@ end-to-end-ws-setup: - echo "-e DATAFED_WEB_CERT_PATH=\"${CI_DATAFED_WEB_CERT_PATH}\" \\" >> run_web.sh - echo "-e DATAFED_WEB_KEY_PATH=\"${CI_DATAFED_WEB_KEY_PATH}\" \\" >> run_web.sh - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_web.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_web.sh - echo "-p 443:443 \\" >> run_web.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub\" \\" >> run_web.sh @@ -449,6 +450,7 @@ end-to-end-core-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - mkdir -p "$HOST_LOG_FILE_PATH" - chmod o+w "${HOST_LOG_FILE_PATH}" + - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh @@ -466,6 +468,7 @@ end-to-end-core-setup: - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> run_core.sh - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> run_core.sh - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_core.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_core.sh - echo "--network host \\" >> run_core.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh - echo "-v \"$CI_DATAFED_CORE_PUB_KEY\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh @@ -495,6 +498,7 @@ end-to-end-repo-setup: - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" - chmod o+w "${HOST_LOG_FILE_PATH}" + - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" @@ -508,6 +512,7 @@ end-to-end-repo-setup: - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_repo.sh - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_repo.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_repo.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_repo.sh - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_repo.sh - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_repo.sh @@ -532,6 +537,7 @@ end-to-end-gcs-authz-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" + - USER_ID=$(id -u) - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh @@ -546,6 +552,7 @@ end-to-end-gcs-authz-setup: - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_globus.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_globus.sh - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_globus.sh diff --git a/core/docker/Dockerfile.core.ubuntu b/core/docker/Dockerfile.core.ubuntu index 88ba3e797..8f8d6ae98 100644 --- a/core/docker/Dockerfile.core.ubuntu +++ b/core/docker/Dockerfile.core.ubuntu @@ -54,6 +54,8 @@ RUN cmake --build build --target install WORKDIR ${DATAFED_INSTALL_PATH}/core +USER root + ENTRYPOINT ["/datafed/source/core/docker/entrypoint.sh"] CMD ["/datafed/install/core/datafed-core","--cfg","/datafed/install/core/datafed-core.cfg"] diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 5bbdc3ec5..d84e96917 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -2,6 +2,11 @@ set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed + su datafed +fi + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) diff --git a/repository/docker/Dockerfile.repo.ubuntu b/repository/docker/Dockerfile.repo.ubuntu index 892e3004f..e3b7abedc 100644 --- a/repository/docker/Dockerfile.repo.ubuntu +++ b/repository/docker/Dockerfile.repo.ubuntu @@ -52,6 +52,8 @@ RUN cmake --build build --target install ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" +USER root + ENTRYPOINT ["/datafed/source/repository/docker/entrypoint_repo.sh"] CMD ["/datafed/install/repo/datafed-repo","--cfg","/datafed/install/repo/datafed-repo.cfg"] diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index b5bd7ff66..c26e1606f 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -4,6 +4,12 @@ # To run it just pass in /entrypoint.sh as an argument set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed + su datafed +fi + + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index 57813d81d..5edc6a28c 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -2,6 +2,11 @@ set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed + su datafed +fi + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) diff --git a/web/docker/Dockerfile.web.ubuntu b/web/docker/Dockerfile.web.ubuntu index aa187b175..9e9929678 100644 --- a/web/docker/Dockerfile.web.ubuntu +++ b/web/docker/Dockerfile.web.ubuntu @@ -65,5 +65,7 @@ RUN cmake --build build --target install WORKDIR ${DATAFED_INSTALL_PATH}/web +USER root + ENTRYPOINT ["/datafed/source/web/docker/entrypoint.sh"] CMD ["/datafed/install/web/datafed-ws.js","/datafed/install/web/datafed-ws.cfg"] diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 297ffa2c8..6d191da22 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -1,9 +1,14 @@ #!/bin/bash - # NVM_DIR must be defined set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed + su datafed +fi + + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) From f5817b362cc424f7638632e654fad9efcdfd0135 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 10:11:18 -0500 Subject: [PATCH 42/69] Changes to how entrypoint handles arguments --- core/docker/entrypoint.sh | 8 ++++---- repository/docker/Dockerfile.repo.ubuntu | 2 ++ repository/docker/entrypoint_authz.sh | 5 ++--- repository/docker/entrypoint_repo.sh | 5 ++--- web/docker/entrypoint.sh | 7 +++---- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index d84e96917..2222d5c74 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -4,7 +4,6 @@ set -euf -o pipefail if [ -n "$UID" ]; then usermod -u $UID datafed - su datafed fi SCRIPT=$(realpath "$0") @@ -19,7 +18,7 @@ log_path="$DATAFED_DEFAULT_LOG_PATH" if [ ! -d "${log_path}" ] then - mkdir -p "${log_path}" + su -c "mkdir -p ${log_path}" datafed fi if [ "$#" -eq 0 ]; then @@ -32,9 +31,10 @@ datafed_core_exec=$(basename "$1") if [ "${datafed_core_exec}" = "datafed-core" ] then # Send output to log file - "$@" 2>&1 | tee "$log_path/datafed-core.log" + # For this to work all commands must be passed in as a single string + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" else echo "Not sending output to datafed-core.log" # If not do not by default send to log file - exec "$@" + su datafed -c '"$@"' -- argv0 "$@" fi diff --git a/repository/docker/Dockerfile.repo.ubuntu b/repository/docker/Dockerfile.repo.ubuntu index e3b7abedc..a980a41bd 100644 --- a/repository/docker/Dockerfile.repo.ubuntu +++ b/repository/docker/Dockerfile.repo.ubuntu @@ -42,6 +42,8 @@ COPY ./repository/docker/entrypoint_repo.sh ${BUILD_DIR}/repository/docker/ # All files should be owned by the datafed user RUN chown -R datafed:datafed ${DATAFED_DIR} +USER datafed + RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ cmake -S. -B build -DBUILD_REPO_SERVER=True -DBUILD_AUTHZ=False \ -DBUILD_CORE_SERVER=False -DBUILD_WEB_SERVER=False \ diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index c26e1606f..3be1b602a 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -6,7 +6,6 @@ set -euf -o pipefail if [ -n "$UID" ]; then usermod -u $UID datafed - su datafed fi @@ -22,7 +21,7 @@ log_path="$DATAFED_DEFAULT_LOG_PATH" if [ ! -d "${log_path}" ] then - mkdir -p "${log_path}" + su -c "mkdir -p ${log_path}" datafed fi -exec "$@" +su datafed -c '"$@"' -- argv0 "$@" diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index 5edc6a28c..997da30f2 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -4,7 +4,6 @@ set -euf -o pipefail if [ -n "$UID" ]; then usermod -u $UID datafed - su datafed fi SCRIPT=$(realpath "$0") @@ -19,7 +18,7 @@ log_path="$DATAFED_DEFAULT_LOG_PATH" if [ ! -d "${log_path}" ] then - mkdir -p "${log_path}" + su -c "mkdir -p ${log_path}" datafed fi -exec "$@" +su datafed -c '"$@"' -- argv0 "$@" diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 6d191da22..98f2ee2b7 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -5,7 +5,6 @@ set -euf -o pipefail if [ -n "$UID" ]; then usermod -u $UID datafed - su datafed fi @@ -27,7 +26,7 @@ log_path=$(grep "log-path" /datafed/install/web/datafed-ws.cfg | cut -d "=" -f 2 if [ ! -d "${log_path}" ] then - mkdir -p "${log_path}" + su -c "mkdir -p ${log_path}" datafed fi if [ "$#" -eq 0 ]; then @@ -40,9 +39,9 @@ datafed_ws_exec=$(basename "$1") if [ "${datafed_ws_exec}" = "datafed-ws.js" ] then # Send output to log file - "$@" 2>&1 | tee "$log_path/datafed-ws.log" + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-ws.log" else echo "Not sending output to datafed-ws.log" # If not do not by default send to log file - exec "$@" + su datafed -c '"$@"' -- argv0 "$@" fi From 1c0cffa21b37e09dbccd7d9c13c494e63ae187ae Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 15:26:06 -0500 Subject: [PATCH 43/69] End image build as root user --- core/docker/Dockerfile | 2 +- web/docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index ca856822e..396c41489 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -72,7 +72,7 @@ RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to RUN ldconfig -USER datafed +USER root COPY --chown=datafed:root ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh COPY --chown=datafed:root ./scripts/generate_core_config.sh ${DATAFED_DIR}/scripts/generate_core_config.sh diff --git a/web/docker/Dockerfile b/web/docker/Dockerfile index a8c3f1609..e3895a76e 100644 --- a/web/docker/Dockerfile +++ b/web/docker/Dockerfile @@ -81,7 +81,7 @@ WORKDIR ${DATAFED_DIR} COPY --from=ws-build --chown=datafed:root "$NVM_DIR" "$NVM_DIR" RUN ln -s ${DATAFED_INSTALL_PATH}/web ${DATAFED_DIR}/web -USER datafed +USER root COPY --chown=datafed:root ./web/docker/entrypoint.sh ${BUILD_DIR}/web/entrypoint.sh COPY --chown=datafed:root ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh From 5e352a7454ae9d7899726c249c523e3d55edd321 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 15:26:59 -0500 Subject: [PATCH 44/69] Change user permissions --- core/docker/Dockerfile | 4 +++- web/docker/Dockerfile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index 396c41489..cedbb1f90 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -72,7 +72,7 @@ RUN ${BUILD_DIR}/scripts/copy_dependency.sh boost_program_options to RUN ldconfig -USER root +USER datafed COPY --chown=datafed:root ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh COPY --chown=datafed:root ./scripts/generate_core_config.sh ${DATAFED_DIR}/scripts/generate_core_config.sh @@ -81,5 +81,7 @@ COPY --chown=datafed:root ./cmake/Version.cmake ${DATAFED_DIR}/cmake COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh ${BUILD_DIR}/core/entrypoint.sh COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core +USER root + ENTRYPOINT ${BUILD_DIR}/core/entrypoint.sh #CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"] diff --git a/web/docker/Dockerfile b/web/docker/Dockerfile index e3895a76e..decbb0ff9 100644 --- a/web/docker/Dockerfile +++ b/web/docker/Dockerfile @@ -81,7 +81,7 @@ WORKDIR ${DATAFED_DIR} COPY --from=ws-build --chown=datafed:root "$NVM_DIR" "$NVM_DIR" RUN ln -s ${DATAFED_INSTALL_PATH}/web ${DATAFED_DIR}/web -USER root +USER datafed COPY --chown=datafed:root ./web/docker/entrypoint.sh ${BUILD_DIR}/web/entrypoint.sh COPY --chown=datafed:root ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh @@ -97,4 +97,6 @@ RUN . ${DATAFED_DIR}/scripts/dependency_versions.sh && \ COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${DATAFED_INSTALL_PATH}/web +USER root + WORKDIR ${DATAFED_INSTALL_PATH}/web From 428caa345cc95828798d88c97d1c6d3f175f068c Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Wed, 14 Feb 2024 16:08:45 -0500 Subject: [PATCH 45/69] Moving towards working compose file --- compose/compose.yml | 17 +++++++-- compose/generate_env.sh | 5 +-- core/docker/entrypoint.sh | 14 ++++++-- docker/Dockerfile.foxx | 46 ++++++++++++++++++++++++ docker/entrypoint_foxx.sh | 50 +++++++++++++++++++++++++++ repository/docker/entrypoint_authz.sh | 14 +++++++- repository/docker/entrypoint_repo.sh | 13 ++++++- web/docker/entrypoint.sh | 21 ++++++++--- 8 files changed, 167 insertions(+), 13 deletions(-) create mode 100644 docker/Dockerfile.foxx create mode 100755 docker/entrypoint_foxx.sh diff --git a/compose/compose.yml b/compose/compose.yml index 1485b4c81..652a2509c 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -2,7 +2,8 @@ version: '3.9' services: - environment: + datafed-web: + environment: DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" @@ -11,22 +12,26 @@ services: DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" - datafed-web: + UID: "${DATAFED_UID}" image: datafed-web:latest ports: - 8080:443 datafed-core: image: datafed-core:latest + depends_on: ["arango"] environment: DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" + DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" + UID: "${DATAFED_UID}" DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" - DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" ports: - 7513 # Communication web server - 7512 # Secure core server communication @@ -37,3 +42,9 @@ services: ARANGO_ROOT_PASSWORD: "${ARANGO_ROOT_PASSWORD}" ports: - 8529:8529 # Arangodb web UI + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8529/_admin/cluster/health"] + interval: 30s + timeout: 10s + retries: 5 + diff --git a/compose/generate_env.sh b/compose/generate_env.sh index 1aa9d3ba2..9cf202edb 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -19,7 +19,8 @@ DATAFED_ZEROMQ_SYSTEM_SECRET="" DATAFED_DOMAIN="" DATAFED_WEB_CERT_PATH="" DATAFED_WEB_KEY_PATH="" -DATAFED_CONTAINER_LOG_FILE_PATH="" +DATAFED_CONTAINER_LOG_PATH="" DATAFED_DATABASE_PASSWORD="" -DATAFED_DATABASE_IP_ADDRESS_PORT="" +DATAFED_DATABASE_IP_ADDRESS_PORT="http://arango:8529" +UID="$(id -u)" EOF diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 0b45898f2..2222d5c74 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -2,6 +2,10 @@ set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) @@ -12,6 +16,11 @@ ${PROJECT_ROOT}/scripts/install_core.sh log_path="$DATAFED_DEFAULT_LOG_PATH" +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + if [ "$#" -eq 0 ]; then echo "No arguments were passed, running bash" exec "bash" @@ -22,9 +31,10 @@ datafed_core_exec=$(basename "$1") if [ "${datafed_core_exec}" = "datafed-core" ] then # Send output to log file - "$@" 2>&1 | tee "$log_path/datafed-core.log" + # For this to work all commands must be passed in as a single string + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" else echo "Not sending output to datafed-core.log" # If not do not by default send to log file - exec "$@" + su datafed -c '"$@"' -- argv0 "$@" fi diff --git a/docker/Dockerfile.foxx b/docker/Dockerfile.foxx new file mode 100644 index 000000000..086ef0dec --- /dev/null +++ b/docker/Dockerfile.foxx @@ -0,0 +1,46 @@ +# NOTE this image must be built with respect to the base of the project i.e. +# cd ${PROJECT_ROOT} or cd DataFed +# docker build -f core/docker/Dockerfile . + +ARG BUILD_BASE="ubuntu:focal" +ARG DEPENDENCIES="dependencies" +ARG RUNTIME="runtime" +ARG DATAFED_DIR="/datafed" +ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" +ARG BUILD_DIR="$DATAFED_DIR/source" +ARG NVM_DIR="$DATAFED_DIR/.nvm" +ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +ARG LIB_DIR="/usr/local/lib" + +FROM ${DEPENDENCIES} AS foxx-build + +ARG DATAFED_DIR +ARG BUILD_DIR +ARG DATAFED_INSTALL_PATH + +COPY ./core/CMakeLists.txt ${BUILD_DIR}/core/CMakeLists.txt +COPY ./CMakeLists.txt ${BUILD_DIR} +COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ +COPY ./cmake ${BUILD_DIR}/cmake +COPY ./docker/entrypoint_foxx.sh ${BUILD_DIR}/docker +COPY ./core/database ${BUILD_DIR}/core/database +COPY ./scripts/install_foxx.sh ${BUILD_DIR}/scripts/ + +ENV BUILD_DIR="${BUILD_DIR}" +ENV DATAFED_DIR="${DATAFED_DIR}" + +# Create datafed user, prefer more secure login options than password +# Recommended to mount ssh public key on run +RUN adduser --disabled-password --gecos "" datafed + +RUN mkdir -p ${DATAFED_DIR} +RUN mkdir -p /opt/datafed +RUN mkdir -p /var/log/datafed +RUN chown -R datafed:root /opt/datafed +RUN chown -R datafed:root /var/log/datafed +RUN chown -R datafed:root ${DATAFED_DIR} + +WORKDIR ${DATAFED_DIR} diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh new file mode 100755 index 000000000..d3710e4a7 --- /dev/null +++ b/docker/entrypoint_foxx.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euf -o pipefail + +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath ${SOURCE}/../..) + +${PROJECT_ROOT}/scripts/generate_datafed.sh + + +cmake -S. -B build \ + -DBUILD_REPO_SERVER=False \ + -DBUILD_AUTHZ=False \ + -DBUILD_CORE_SERVER=False \ + -DBUILD_WEB_SERVER=False \ + -DBUILD_DOCS=False \ + -DBUILD_PYTHON_CLIENT=False \ + -DBUILD_FOXX=True \ + -DINSTALL_FOXX=True + + +log_path="$DATAFED_DEFAULT_LOG_PATH" + +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + +if [ "$#" -eq 0 ]; then + echo "No arguments were passed, running bash" + exec "bash" + exit 0 +fi + +datafed_core_exec=$(basename "$1") +if [ "${datafed_core_exec}" = "datafed-core" ] +then + # Send output to log file + # For this to work all commands must be passed in as a single string + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" +else + echo "Not sending output to datafed-core.log" + # If not do not by default send to log file + su datafed -c '"$@"' -- argv0 "$@" +fi diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index 3a072d107..3be1b602a 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -4,6 +4,11 @@ # To run it just pass in /entrypoint.sh as an argument set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) @@ -12,4 +17,11 @@ GLOBUS_CLIENT_ID="$DATAFED_GLOBUS_APP_ID" ${PROJECT_ROOT}/scripts/generate_datafed.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh -exec "$@" +log_path="$DATAFED_DEFAULT_LOG_PATH" + +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + +su datafed -c '"$@"' -- argv0 "$@" diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index 4eebc6225..997da30f2 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -2,6 +2,10 @@ set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) @@ -10,4 +14,11 @@ ${PROJECT_ROOT}/scripts/generate_datafed.sh ${PROJECT_ROOT}/scripts/generate_repo_config.sh ${PROJECT_ROOT}/scripts/install_repo.sh -exec "$@" +log_path="$DATAFED_DEFAULT_LOG_PATH" + +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + +su datafed -c '"$@"' -- argv0 "$@" diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index e7a33bf6a..98f2ee2b7 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -1,21 +1,34 @@ #!/bin/bash - # NVM_DIR must be defined set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) -source $NVM_DIR/nvm.sh ${PROJECT_ROOT}/scripts/generate_datafed.sh ${PROJECT_ROOT}/scripts/generate_ws_config.sh ${PROJECT_ROOT}/scripts/install_ws.sh +${PROJECT_ROOT}/scripts/dependency_versions.sh + +export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" +export NODE_VERSION="$DATAFED_NODE_VERSION" +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm # Send output to file as well as print to terminal log_path=$(grep "log-path" /datafed/install/web/datafed-ws.cfg | cut -d "=" -f 2 | tr -d ' ') +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + if [ "$#" -eq 0 ]; then echo "No arguments were passed, running bash" exec "bash" @@ -26,9 +39,9 @@ datafed_ws_exec=$(basename "$1") if [ "${datafed_ws_exec}" = "datafed-ws.js" ] then # Send output to log file - "$@" 2>&1 | tee "$log_path/datafed-ws.log" + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-ws.log" else echo "Not sending output to datafed-ws.log" # If not do not by default send to log file - exec "$@" + su datafed -c '"$@"' -- argv0 "$@" fi From edb34361ec5c678b99476061e88bef75908a9c59 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 16 Feb 2024 15:25:23 -0500 Subject: [PATCH 46/69] Working dev version of web server --- compose/README.md | 6 ++ compose/build_images_for_compose.sh | 6 ++ compose/compose.yml | 43 +++++++++-- core/docker/Dockerfile | 22 ++++-- core/docker/entrypoint.sh | 7 ++ docker/Dockerfile.dependencies | 12 ++- docker/Dockerfile.foxx | 19 ++++- docker/Dockerfile.runtime | 8 +- docker/entrypoint_foxx.sh | 76 +++++++++++-------- repository/docker/Dockerfile | 10 ++- scripts/dependency_install_functions.sh | 48 +++++++----- scripts/export_dependency_version.sh | 18 +++++ scripts/generate_ws_config.sh | 10 +++ scripts/install_authz_dependencies.sh | 9 ++- scripts/install_client_dependencies.sh | 6 +- scripts/install_core_dependencies.sh | 9 ++- scripts/install_dependencies.sh | 22 ++++-- scripts/install_docs_dependencies.sh | 8 +- scripts/install_foxx.sh | 10 +-- scripts/install_lego_and_certificates.sh | 15 ++-- scripts/install_python_client_dependencies.sh | 9 ++- scripts/install_repo_dependencies.sh | 9 ++- scripts/install_ws.sh | 2 +- scripts/install_ws_dependencies.sh | 9 ++- scripts/utils.sh | 65 ++++++++++++++++ web/docker/Dockerfile | 73 ++++++++++-------- web/docker/entrypoint.sh | 14 +++- 27 files changed, 393 insertions(+), 152 deletions(-) create mode 100755 scripts/export_dependency_version.sh create mode 100755 scripts/utils.sh diff --git a/compose/README.md b/compose/README.md index 94dd7da94..d653ba098 100644 --- a/compose/README.md +++ b/compose/README.md @@ -2,3 +2,9 @@ The files in this folder are incomplete but are the start for setting up a full docker compose instance of datafed. + +## Generate self signed web key + +openssl genrsa -out cert.key 2048 +openssl req -new -key cert.key -out cert.csr +openssl x509 -req -days 3650 -in cert.csr -signkey cert.key -out cert.crt diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index 239ba8f2d..1c8aa08cf 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -26,4 +26,10 @@ docker build -f \ --build-arg RUNTIME="datafed-runtime" \ "${PROJECT_ROOT}" \ -t datafed-web:latest +docker build -f \ + "${PROJECT_ROOT}/docker/Dockerfile.foxx" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-foxx:latest diff --git a/compose/compose.yml b/compose/compose.yml index 652a2509c..82a7993fa 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -3,6 +3,7 @@ version: '3.9' services: datafed-web: + depends_on: ["datafed-core"] environment: DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" @@ -12,14 +13,20 @@ services: DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" + DATAFED_CORE_ADDRESS_PORT_INTERNAL: "datafed-core:7513" UID: "${DATAFED_UID}" image: datafed-web:latest ports: - 8080:443 + volumes: + - keys:/opt/datafed/keys + - ./web_keys:/opt/datafed/web_keys datafed-core: image: datafed-core:latest - depends_on: ["arango"] + depends_on: + datafed-foxx: + condition: service_healthy environment: DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" @@ -32,19 +39,41 @@ services: UID: "${DATAFED_UID}" DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" + DATAFED_DATABASE_IP_ADDRESS_PORT: "${DATAFED_DATABASE_IP_ADDRESS_PORT}" ports: - 7513 # Communication web server - 7512 # Secure core server communication + volumes: + - keys:/opt/datafed/keys + + datafed-foxx: + image: datafed-foxx:latest + depends_on: ["arango"] + environment: + DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" + UID: "${DATAFED_UID}" + DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" + DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" + DATAFED_DATABASE_HOST: "arango" + healthcheck: + test: ["CMD", "/bin/bash", "-c", "[ -f /tmp/.foxx_is_installed ]"] + interval: 10s + timeout: 5s + retries: 10 arango: image: arangodb environment: - ARANGO_ROOT_PASSWORD: "${ARANGO_ROOT_PASSWORD}" + ARANGO_ROOT_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" ports: - 8529:8529 # Arangodb web UI - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8529/_admin/cluster/health"] - interval: 30s - timeout: 10s - retries: 5 + # healthcheck: + # test: ["CMD", "curl", "-f", "http://localhost:8529/_admin/cluster/health"] + # interval: 30s + # timeout: 10s + # retries: 5 +volumes: + keys: diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index cedbb1f90..f11d17edd 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -7,15 +7,18 @@ ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" +#ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +#ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS core-build +SHELL ["/bin/bash", "-c"] + ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH @@ -49,6 +52,8 @@ RUN cmake --build build --target install FROM ${RUNTIME} AS core +SHELL ["/bin/bash", "-c"] + ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH ARG BUILD_DIR @@ -56,6 +61,7 @@ ARG LIB_DIR # The above should also be available at runtime ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" @@ -83,5 +89,11 @@ COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed USER root -ENTRYPOINT ${BUILD_DIR}/core/entrypoint.sh -#CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"] +RUN chown -R datafed:root /datafed +RUN chown -R datafed:root /opt + +#SHELL ["/bin/bash"] +# NOTE docker does not recognize curly braces around variables in entrypoint +# and CMD +ENTRYPOINT [ "/datafed/source/core/entrypoint.sh", "/opt/datafed/core/datafed-core","--cfg", "/opt/datafed/core/datafed-core.cfg"] +#CMD ["$DATAFED_INSTALL_PATH/core/datafed-core","--cfg","$DATAFED_INSTALL_PATH/core/datafed-core.cfg"] diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 2222d5c74..4039d27a7 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -10,6 +10,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) +env ${PROJECT_ROOT}/scripts/generate_datafed.sh ${PROJECT_ROOT}/scripts/generate_core_config.sh ${PROJECT_ROOT}/scripts/install_core.sh @@ -21,6 +22,9 @@ then su -c "mkdir -p ${log_path}" datafed fi +echo "Number of arguments is $#" +echo "arguments are $@" + if [ "$#" -eq 0 ]; then echo "No arguments were passed, running bash" exec "bash" @@ -38,3 +42,6 @@ else # If not do not by default send to log file su datafed -c '"$@"' -- argv0 "$@" fi + +echo "Give a few minutes to debug the problem" +sleep 10000 diff --git a/docker/Dockerfile.dependencies b/docker/Dockerfile.dependencies index 602ebd524..6db15262b 100644 --- a/docker/Dockerfile.dependencies +++ b/docker/Dockerfile.dependencies @@ -1,15 +1,15 @@ ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DEPENDENCIES_INSTALL_PATH/nvm" ARG LIB_DIR="/usr/local/lib" FROM ubuntu:focal -ARG NVM_DIR +SHELL ["/bin/bash", "-c"] +#ARG NVM_DIR ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH @@ -18,6 +18,7 @@ ARG LIB_DIR ENV BUILD_DIR="${BUILD_DIR}" ENV DATAFED_DIR="${DATAFED_DIR}" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV LIB_DIR="${LIB_DIR}" RUN mkdir -p ${BUILD_DIR} @@ -32,14 +33,17 @@ WORKDIR ${BUILD_DIR} COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo # run build scripts +RUN ${BUILD_DIR}/scripts/generate_datafed.sh RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/ diff --git a/docker/Dockerfile.foxx b/docker/Dockerfile.foxx index 086ef0dec..38d5c15c7 100644 --- a/docker/Dockerfile.foxx +++ b/docker/Dockerfile.foxx @@ -20,18 +20,18 @@ ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH +ENV BUILD_DIR="${BUILD_DIR}" +ENV DATAFED_DIR="${DATAFED_DIR}" + COPY ./core/CMakeLists.txt ${BUILD_DIR}/core/CMakeLists.txt COPY ./CMakeLists.txt ${BUILD_DIR} COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./cmake ${BUILD_DIR}/cmake -COPY ./docker/entrypoint_foxx.sh ${BUILD_DIR}/docker +COPY ./docker/entrypoint_foxx.sh ${BUILD_DIR}/docker/entrypoint_foxx.sh COPY ./core/database ${BUILD_DIR}/core/database COPY ./scripts/install_foxx.sh ${BUILD_DIR}/scripts/ -ENV BUILD_DIR="${BUILD_DIR}" -ENV DATAFED_DIR="${DATAFED_DIR}" - # Create datafed user, prefer more secure login options than password # Recommended to mount ssh public key on run RUN adduser --disabled-password --gecos "" datafed @@ -43,4 +43,15 @@ RUN chown -R datafed:root /opt/datafed RUN chown -R datafed:root /var/log/datafed RUN chown -R datafed:root ${DATAFED_DIR} +RUN curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key; apt-key add - < Release.key +#RUN wget -q https://www.arangodb.com/repositories/arangodb38/DEBIAN/Release.key -O- | apt-key add - +RUN echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list +RUN apt update +RUN apt install arangodb3-client -y +RUN arangosh --version + +USER root + WORKDIR ${DATAFED_DIR} + +ENTRYPOINT ${BUILD_DIR}/docker/entrypoint_foxx.sh diff --git a/docker/Dockerfile.runtime b/docker/Dockerfile.runtime index a372c683d..86e142731 100644 --- a/docker/Dockerfile.runtime +++ b/docker/Dockerfile.runtime @@ -2,15 +2,17 @@ ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" +#ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +#ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" ARG LIB_DIR="/usr/local/lib" FROM ubuntu:focal +SHELL ["/bin/bash", "-c"] ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH +ARG DATAFED_DEPENDENCIES_INSTALL_PATH ARG BUILD_DIR ENV BUILD_DIR="${BUILD_DIR}" diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh index d3710e4a7..8e1f189ef 100755 --- a/docker/entrypoint_foxx.sh +++ b/docker/entrypoint_foxx.sh @@ -8,43 +8,57 @@ fi SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../..) - -${PROJECT_ROOT}/scripts/generate_datafed.sh - - -cmake -S. -B build \ - -DBUILD_REPO_SERVER=False \ - -DBUILD_AUTHZ=False \ - -DBUILD_CORE_SERVER=False \ - -DBUILD_WEB_SERVER=False \ - -DBUILD_DOCS=False \ - -DBUILD_PYTHON_CLIENT=False \ - -DBUILD_FOXX=True \ - -DINSTALL_FOXX=True - +PROJECT_ROOT=$(realpath ${SOURCE}/../) log_path="$DATAFED_DEFAULT_LOG_PATH" - if [ ! -d "${log_path}" ] then su -c "mkdir -p ${log_path}" datafed fi -if [ "$#" -eq 0 ]; then - echo "No arguments were passed, running bash" - exec "bash" - exit 0 -fi - -datafed_core_exec=$(basename "$1") -if [ "${datafed_core_exec}" = "datafed-core" ] +cd "${PROJECT_ROOT}" +# Check to see if foxx has previously been installed +install_flag="/tmp/.foxx_is_installed" +if [ ! -f "$install_flag" ] then - # Send output to log file - # For this to work all commands must be passed in as a single string - su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" -else - echo "Not sending output to datafed-core.log" - # If not do not by default send to log file - su datafed -c '"$@"' -- argv0 "$@" + su datafed -c "${PROJECT_ROOT}/scripts/generate_datafed.sh" + + su datafed -c "cmake -S. -B build \ + -DBUILD_REPO_SERVER=False \ + -DBUILD_COMMON=False \ + -DBUILD_AUTHZ=False \ + -DBUILD_CORE_SERVER=False \ + -DBUILD_WEB_SERVER=False \ + -DBUILD_DOCS=False \ + -DBUILD_PYTHON_CLIENT=False \ + -DBUILD_FOXX=True \ + -DINSTALL_FOXX=True" + + + su datafed -c "cmake --build build" + + # Give arango container a minute to initialize + # should be replaced with health check at some point + sleep 10 + su datafed -c "cmake --build build --target install" + + #if [ "$#" -eq 0 ]; then + # echo "No arguments were passed, running bash" + # exec "bash" + # exit 0 + #fi + touch "$install_flag" fi + +sleep 1000 +#datafed_core_exec=$(basename "$1") +#if [ "${datafed_core_exec}" = "datafed-core" ] +#then +# # Send output to log file +# # For this to work all commands must be passed in as a single string +# su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" +#else +# echo "Not sending output to datafed-core.log" +# # If not do not by default send to log file +# su datafed -c '"$@"' -- argv0 "$@" +#fi diff --git a/repository/docker/Dockerfile b/repository/docker/Dockerfile index 73a73ff47..dbadfeb3f 100644 --- a/repository/docker/Dockerfile +++ b/repository/docker/Dockerfile @@ -7,15 +7,17 @@ ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" +#ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +#ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS repo-build +SHELL ["/bin/bash", "-c"] ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH @@ -50,6 +52,7 @@ RUN cmake --build build --target install FROM ${RUNTIME} AS repo +SHELL ["/bin/bash", "-c"] ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH ARG BUILD_DIR @@ -57,6 +60,7 @@ ARG LIB_DIR # The above should also be available at runtime ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index 1699548f5..26d87f1dc 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -2,6 +2,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") source "${SOURCE}/dependency_versions.sh" +source "${SOURCE}/utils.sh" # these are the dependencies to be installed by apt apt_file_path="/tmp/apt_deps" @@ -41,7 +42,7 @@ install_protobuf() { if [ -d protobuf ] then # sudo required because of egg file - sudo rm -rf protobuf + "$SUDO_CMD" rm -rf protobuf fi git clone https://github.com/google/protobuf.git cd protobuf @@ -49,7 +50,7 @@ install_protobuf() { git submodule update --init --recursive cmake -S cmake/ -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install cd python python3 setup.py build python3 setup.py test @@ -58,7 +59,7 @@ install_protobuf() { # Cleanup build file with root ownership if [ -f build/install_manifest.txt ] then - sudo rm build/install_manifest.txt + "$SUDO_CMD" rm build/install_manifest.txt fi cd ../ @@ -79,8 +80,8 @@ install_libsodium() { ./autogen.sh ./configure make check - sudo make install - sudo ldconfig + "$SUDO_CMD" make install + "$SUDO_CMD" ldconfig cd ../ # Mark libsodium as installed @@ -99,7 +100,7 @@ install_libzmq() { git checkout v${DATAFED_LIBZMQ_VERSION} cmake -S. -B build -DBUILD_STATIC=ON -DBUILD_SHARED=ON cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install # Mark libzmq as installed touch ".libzmq_installed-${DATAFED_LIBZMQ_VERSION}" @@ -118,7 +119,7 @@ install_nlohmann_json() { echo "FILE STRUCTURE $(ls)" cmake -S . -B build cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install cd ../ # Mark nlohmann_json as installed @@ -137,7 +138,7 @@ install_json_schema_validator() { git checkout ${DATAFED_JSON_SCHEMA_VALIDATOR_VERSION} cmake -S . -B build cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install cd ../ # Mark json-schema-validator as installed @@ -147,14 +148,14 @@ install_json_schema_validator() { install_gcs() { if [ ! -e ".gcs_installed-${DATAFED_GLOBUS_VERSION}" ]; then - sudo apt update - sudo apt install -y curl git gnupg + "$SUDO_CMD" apt update + "$SUDO_CMD" apt install -y curl git gnupg curl -LOs https://downloads.globus.org/globus-connect-server/stable/installers/repo/deb/globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb - sudo dpkg -i globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb - sudo apt-key add /usr/share/globus-repo/RPM-GPG-KEY-Globus + "$SUDO_CMD" dpkg -i globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb + "$SUDO_CMD" apt-key add /usr/share/globus-repo/RPM-GPG-KEY-Globus # Need a second update command after adding the globus GPG key - sudo apt update - sudo apt-get install globus-connect-server54 -y + "$SUDO_CMD" apt update + "$SUDO_CMD" apt-get install globus-connect-server54 -y # Mark gcs as installed touch ".gcs_installed-${DATAFED_GLOBUS_VERSION}" @@ -171,6 +172,8 @@ install_nvm() { curl -o- "https://raw.githubusercontent.com/nvm-sh/nvm/${DATAFED_NVM_VERSION}/install.sh" | bash # Mark nvm as installed touch ".nvm_installed-${DATAFED_NVM_VERSION}" + else + export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" fi } @@ -188,6 +191,9 @@ install_node() { nvm install "$DATAFED_NODE_VERSION" # Mark node as installed touch ".node_installed-${DATAFED_NODE_VERSION}" + else + export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm fi } @@ -208,16 +214,20 @@ install_foxx_cli() { "$NVM_DIR/nvm-exec" npm install --global foxx-cli --prefix "${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm" # Mark foxx_cli as installed touch ".foxx_cli_installed" + else + export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm + export NODE_VERSION="$DATAFED_NODE_VERSION" fi } install_arangodb() { curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key - sudo apt-key add - < Release.key - echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | sudo tee /etc/apt/sources.list.d/arangodb.list - sudo apt-get install apt-transport-https - sudo apt-get update - sudo apt-get install arangodb3 + "$SUDO_CMD" apt-key add - < Release.key + echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | "$SUDO_CMD" tee /etc/apt/sources.list.d/arangodb.list + "$SUDO_CMD" apt-get install apt-transport-https + "$SUDO_CMD" apt-get update + "$SUDO_CMD" apt-get install arangodb3 } install_dep_by_name() { diff --git a/scripts/export_dependency_version.sh b/scripts/export_dependency_version.sh new file mode 100755 index 000000000..a965144af --- /dev/null +++ b/scripts/export_dependency_version.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SCRIPT=$(realpath "$0") +# Assumes sourcing not running +SCRIPT=$( realpath "${BASH_SOURCE[0]}" ) +SCRIPT_DIR=$( dirname "${SCRIPT}" ) +PROJECT_ROOT=$(realpath ${SCRIPT_DIR}/..) + +echo "SCRIPT $SCRIPT" +echo "SCRIPT_DIR $SCRIPT_DIR" +echo "PROJECT_ROOT $PROJECT_ROOT" +#"SOURCE ${SOURCEl}" +. "${PROJECT_ROOT}/scripts/utils.sh" +# +#echo "PROJECT ROOT $PROJECT_ROOT" +## WARNING +## For this script to work it must be called with source +## source export_dependency_version_numbers +export_dependency_version_numbers diff --git a/scripts/generate_ws_config.sh b/scripts/generate_ws_config.sh index f14a00e4c..a8fd76ea6 100755 --- a/scripts/generate_ws_config.sh +++ b/scripts/generate_ws_config.sh @@ -47,6 +47,16 @@ local_DATAFED_WEB_KEY_PATH="${DATAFED_INSTALL_PATH}/keys/datafed-server-test.orn local_DATAFED_HTTPS_SERVER_PORT="443" +if [ ! -z "${DATAFED_WEB_KEY_PATH}" ] +then + local_DATAFED_WEB_KEY_PATH=$(printenv DATAFED_WEB_KEY_PATH) +fi + +if [ ! -z "${DATAFED_WEB_CERT_PATH}" ] +then + local_DATAFED_WEB_CERT_PATH=$(printenv DATAFED_WEB_CERT_PATH) +fi + if [ -z "${DATAFED_GLOBUS_APP_ID}" ] then local_DATAFED_SERVER_DOMAIN_NAME="datafed.ornl.gov" diff --git a/scripts/install_authz_dependencies.sh b/scripts/install_authz_dependencies.sh index 92ee5bd9c..e3806ffc8 100755 --- a/scripts/install_authz_dependencies.sh +++ b/scripts/install_authz_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "build-essential" "g++" "gcc" "libboost-all-dev" "autoconf" "automake" "make" "git" "python3-pkg-resources" "python3-pip" "pkg-config" "libglobus-common-dev" "wget" "libssl-dev" "libzmq3-dev") @@ -33,9 +34,11 @@ if [ $# -eq 1 ]; then esac fi -sudo apt-get update -sudo dpkg --configure -a -sudo apt-get install -y "${packages[@]}" +sudo_command + +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a +"$SUDO_CMD" apt-get install -y "${packages[@]}" cd ~ install_cmake diff --git a/scripts/install_client_dependencies.sh b/scripts/install_client_dependencies.sh index 574e0457c..cea8afb38 100755 --- a/scripts/install_client_dependencies.sh +++ b/scripts/install_client_dependencies.sh @@ -7,11 +7,13 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" +sudo_command # This script will install all of the dependencies needed by DataFed 1.0 -sudo apt-get update -sudo dpkg --configure -a +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a python3 -m pip install -r "${PROJECT_ROOT}/python/datafed_pkg/requirements.txt" diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index a3baa4c95..70b78b7e7 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "libcurl4-openssl-dev" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip" "libssl-dev" "libzmq3-dev") @@ -33,10 +34,12 @@ if [ $# -eq 1 ]; then esac fi +sudo_command + if [[ $local_UNIFY = false ]]; then - sudo apt-get update - sudo dpkg --configure -a - sudo apt-get install -y "${packages[@]}" + "$SUDO_CMD" apt-get update + "$SUDO_CMD" dpkg --configure -a + "$SUDO_CMD" apt-get install -y "${packages[@]}" python3 -m pip install --upgrade pip python3 -m pip install setuptools diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh index 5c4e0cb0c..ecf5aa244 100755 --- a/scripts/install_dependencies.sh +++ b/scripts/install_dependencies.sh @@ -7,23 +7,29 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" source "${SOURCE}/dependency_versions.sh" touch "$apt_file_path" touch "$ext_file_path" -sudo apt-get update -sudo apt install -y wget git curl +# Defines SUDO_CMD which is empty if root +# sudo path if exists +# throws error otherwise +sudo_command + +"$SUDO_CMD" apt-get update +"$SUDO_CMD" apt install -y wget git curl install_cmake # This script will install all of the dependencies needed by DataFed 1.0 -sudo dpkg --configure -a +"$SUDO_CMD" dpkg --configure -a -sudo "$SOURCE/install_core_dependencies.sh" unify -sudo "$SOURCE/install_repo_dependencies.sh" unify -sudo "$SOURCE/install_ws_dependencies.sh" unify -sudo "$SOURCE/install_authz_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_core_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_repo_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_ws_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_authz_dependencies.sh" unify all_packages=$(cat $apt_file_path) IFS=' ' read -r -a all_packages_array <<< "$all_packages" @@ -32,7 +38,7 @@ deduplicated_packages_array=($(printf "%s\n" "${all_packages_array[@]}" | sort - all_externals=$(cat $ext_file_path) IFS=' ' read -r -a all_externals_array <<< "$all_externals" -sudo apt-get install -y "${deduplicated_packages_array[@]}" +"$SUDO_CMD" apt-get install -y "${deduplicated_packages_array[@]}" echo "DEPENDENCIES (${deduplicated_externals_array[@]})" diff --git a/scripts/install_docs_dependencies.sh b/scripts/install_docs_dependencies.sh index d68d1e18e..927104890 100755 --- a/scripts/install_docs_dependencies.sh +++ b/scripts/install_docs_dependencies.sh @@ -7,12 +7,14 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" +sudo_command # This script will install all of the dependencies needed by DataFed 1.0 -sudo apt-get update -sudo dpkg --configure -a -sudo apt-get install -y +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a +"$SUDO_CMD" apt-get install -y install_cmake cd ~ diff --git a/scripts/install_foxx.sh b/scripts/install_foxx.sh index 09ad12519..9f918cdb0 100755 --- a/scripts/install_foxx.sh +++ b/scripts/install_foxx.sh @@ -254,12 +254,10 @@ install_foxx_cli #fi FOXX_PREFIX="" -{ - # Determine if exists globally first - which foxx -} || { - FOXX_PREFIX="${NPM_CONFIG_PREFIX}/bin/" -} +if ! command -v foxx > /dev/null 2>&1; then + FOXX_PREFIX="${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm/bin/" +fi + PATH_TO_PASSWD_FILE="${SOURCE}/database_temp.password" diff --git a/scripts/install_lego_and_certificates.sh b/scripts/install_lego_and_certificates.sh index b49edc834..e098e790f 100755 --- a/scripts/install_lego_and_certificates.sh +++ b/scripts/install_lego_and_certificates.sh @@ -6,6 +6,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) source ${PROJECT_ROOT}/config/datafed.sh +source ${PROJECT_ROOT}/scripts/utils.sh Help() { @@ -78,9 +79,11 @@ then exit 1 fi -sudo add-apt-repository --yes ppa:longsleep/golang-backports -sudo apt-get update -sudo apt-get install golang-go +sudo_commands + +"$SUDO_CMD" add-apt-repository --yes ppa:longsleep/golang-backports +"$SUDO_CMD" apt-get update +"$SUDO_CMD" apt-get install golang-go #This was verified for go 1.17 export GO111MODULE=on @@ -89,13 +92,13 @@ GOBIN=/usr/local/bin/ go install github.com/go-acme/lego/v4/cmd/lego@latest # Create the folder if [ ! -d "${DATAFED_INSTALL_PATH}/keys" ] then - sudo mkdir -p "${DATAFED_INSTALL_PATH}/keys" + "$SUDO_CMD" mkdir -p "${DATAFED_INSTALL_PATH}/keys" fi # Check if the datafed-ws server is already running, will need to stop it if we want # to use port 443 to start the domain name datafed_ws_service=$(systemctl list-unit-files --type service | grep datafed-ws | awk '{print $1}') -[[ "$datafed_ws_service" == 'datafed-ws.service' ]] && sudo systemctl stop datafed-ws.service +[[ "$datafed_ws_service" == 'datafed-ws.service' ]] && "$SUDO_CMD" systemctl stop datafed-ws.service # This should create a folder in ~/.lego/certificates, that contains the # certificate files you need, we are going to copy them over to the @@ -109,7 +112,7 @@ cert_file="datafed-server-test.ornl.gov.crt" key_file="datafed-server-test.ornl.gov.key" if [ ! -f "${DATAFED_INSTALL_PATH}/keys/$cert_file" ] || [ ! -f "${DATAFED_INSTALL_PATH}/keys/$key_file" ] then - sudo lego --accept-tos --email="$DATAFED_LEGO_EMAIL" --domains="$local_DATAFED_DOMAIN" --path "${DATAFED_INSTALL_PATH}/keys/" --tls run + "$SUDO_CMD" lego --accept-tos --email="$DATAFED_LEGO_EMAIL" --domains="$local_DATAFED_DOMAIN" --path "${DATAFED_INSTALL_PATH}/keys/" --tls run mv ${DATAFED_INSTALL_PATH}/keys/certificates/$cert_file ${DATAFED_INSTALL_PATH}/keys/ mv ${DATAFED_INSTALL_PATH}/keys/certificates/$key_file ${DATAFED_INSTALL_PATH}/keys/ rm -rf ${DATAFED_INSTALL_PATH}/keys/certificates diff --git a/scripts/install_python_client_dependencies.sh b/scripts/install_python_client_dependencies.sh index 14bbc9eeb..7d48ba721 100755 --- a/scripts/install_python_client_dependencies.sh +++ b/scripts/install_python_client_dependencies.sh @@ -7,12 +7,15 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" +sudo_command + # This script will install all of the dependencies needed by DataFed 1.0 -sudo apt-get update -sudo dpkg --configure -a -sudo apt-get install -y libtool build-essential g++ gcc make libboost-all-dev \ +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a +"$SUDO_CMD" apt-get install -y libtool build-essential g++ gcc make libboost-all-dev \ pkg-config autoconf automake unzip libcurl4-openssl-dev wget \ rapidjson-dev libkrb5-dev git python3-pkg-resources python3-pip libssl-dev diff --git a/scripts/install_repo_dependencies.sh b/scripts/install_repo_dependencies.sh index 6580db90c..06ee55fb1 100755 --- a/scripts/install_repo_dependencies.sh +++ b/scripts/install_repo_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "wget" "build-essential" "g++" "gcc" "libboost-all-dev" "pkg-config" "autoconf" "automake" "make" "unzip" "git" "python3-pkg-resources" "libssl-dev" "libzmq3-dev" "python3-pip") @@ -34,10 +35,12 @@ if [ $# -eq 1 ]; then esac fi +sudo_command + if [[ $local_UNIFY = false ]]; then - sudo apt-get update - sudo dpkg --configure -a - sudo apt-get install -y "${packages[@]}" + "$SUDO_CMD" apt-get update + "$SUDO_CMD" dpkg --configure -a + "$SUDO_CMD" apt-get install -y "${packages[@]}" python3 -m pip install --upgrade pip python3 -m pip install setuptools diff --git a/scripts/install_ws.sh b/scripts/install_ws.sh index e2678f20d..425b4104e 100755 --- a/scripts/install_ws.sh +++ b/scripts/install_ws.sh @@ -45,4 +45,4 @@ then exit 1 fi -cp ${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub "$PROJECT_ROOT/web/static/" +cp ${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub "$DATAFED_INSTALL_PATH/web/static/" diff --git a/scripts/install_ws_dependencies.sh b/scripts/install_ws_dependencies.sh index 1d1d425a1..1de114446 100755 --- a/scripts/install_ws_dependencies.sh +++ b/scripts/install_ws_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${SOURCE}/dependency_versions.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" @@ -62,10 +63,12 @@ while [ : ]; do esac done +sudo_command + if [[ $local_UNIFY = false ]]; then - sudo apt-get update - sudo dpkg --configure -a - sudo apt-get install -y "${packages[@]}" + "$SUDO_CMD" apt-get update + "$SUDO_CMD" dpkg --configure -a + "$SUDO_CMD" apt-get install -y "${packages[@]}" for ext in "${externals[@]}"; do install_dep_by_name "$ext" diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100755 index 000000000..f88e8c488 --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,65 @@ +#!/bin/bash +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") + +if [ -z "${PROJECT_ROOT}" ] +then + PROJECT_ROOT=$(realpath ${SOURCE}/..) +fi + +echo "PROJECT ROOT $PROJECT_ROOT" + +export_dependency_version_numbers() { + # Get the content of the function and remove comments + variables=$(cat "${PROJECT_ROOT}/scripts/dependency_versions.sh") + + local content="$(echo "${variables}" | sed '/^$/d;/^#/d')" + + # Extract variable assignments from the content + local assignments=$(echo "$content" | grep -Eo '\b[a-zA-Z_][a-zA-Z_0-9]*="[^\"]*"') + + echo "Variables are $variables" + echo "Content is $content" + echo "Assignments is $assignments" + # Loop through each assignment, export the variable + # Note: This may override existing variables + for assignment in $assignments; do + echo "export $assignment" + export "$assignment" + done +} + +empty_command() { + "$@" +} + +# The purpose of this function is to detect the sudo command +# if it exists use it, if we are running as root set SUDO_CMD to empty_command +# empty_command is needed so that I can do this where sudo doesn't exist +# +# "$SUDO_CMD" apt install curl +# +# If running as root this will expand to +# +# empty_command apt install curl +# +# which expands to +# +# apt install curl +# +# If I left SUDO_CMD blank i.e. "" apt install curl bash would complain +sudo_command() { + if [ "$(id -u)" -eq 0 ]; then + export SUDO_CMD="empty_command" # Ignore sudo running as root + else + # Check if sudo is available + if command -v sudo &> /dev/null; then + export SUDO_CMD=$(command -v sudo) + else + echo "Error: This script requires sudo but sudo is not installed." >&2 + echo "You are not running as root!" >&2 + exit 1 + fi + exit $? # Exit with the same status as the sudo command + fi +} diff --git a/web/docker/Dockerfile b/web/docker/Dockerfile index decbb0ff9..ed7adb715 100644 --- a/web/docker/Dockerfile +++ b/web/docker/Dockerfile @@ -6,11 +6,10 @@ ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS ws-build @@ -18,9 +17,7 @@ FROM ${DEPENDENCIES} AS ws-build ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH -ARG NVM_DIR -ARG NVM_INC -ARG NVM_BIN +ARG DATAFED_DEPENDENCIES_INSTALL_PATH # This port is needed to communicate with the DataFed core server EXPOSE 7513 @@ -32,6 +29,7 @@ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/generate_ws_config.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/export_dependency_version.sh ${BUILD_DIR}/scripts/ COPY ./cmake ${BUILD_DIR}/cmake COPY ./common/proto ${BUILD_DIR}/common/proto COPY ./web ${BUILD_DIR}/web @@ -47,56 +45,65 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \ -DBUILD_FOXX=False \ -DBUILD_COMMON=False RUN cmake --build build - -ENV NVM_DIR="$NVM_DIR" -ENV NVM_INC="$NVM_INC" -ENV NVM_BIN="$NVM_BIN" -ENV PATH="$NVM_BIN:$PATH" - RUN cmake --build build --target install +# Only part of the final install can be done here +#RUN ${BUILD_DIR}/scripts/install_ws.sh FROM ${RUNTIME} AS ws -ARG DATAFED_NODE_VERSION="" +SHELL ["/bin/bash", "-c"] +#ARG DATAFED_NODE_VERSION="" ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH ARG BUILD_DIR -ARG NVM_DIR -ARG NVM_INC -ARG NVM_BIN +ARG DATAFED_DEPENDENCIES_INSTALL_PATH # The above should also be available at runtime ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="$DATAFED_DEPENDENCIES_INSTALL_PATH" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" -ENV NVM_DIR="$NVM_DIR" -ENV NVM_INC="$NVM_INC" -ENV NVM_BIN="$NVM_BIN" -ENV PATH="$NVM_BIN:$PATH" RUN apt install -y python3 make g++ WORKDIR ${DATAFED_DIR} -COPY --from=ws-build --chown=datafed:root "$NVM_DIR" "$NVM_DIR" -RUN ln -s ${DATAFED_INSTALL_PATH}/web ${DATAFED_DIR}/web +#COPY --from=ws-build --chown=datafed:root "$NVM_DIR" "$NVM_DIR" +#RUN ln -s ${DATAFED_INSTALL_PATH}/web ${BUILD_DIR}/web USER datafed -COPY --chown=datafed:root ./web/docker/entrypoint.sh ${BUILD_DIR}/web/entrypoint.sh -COPY --chown=datafed:root ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh -COPY --chown=datafed:root ./scripts/dependency_versions.sh ${DATAFED_DIR}/scripts/dependency_versions.sh -COPY --chown=datafed:root ./scripts/generate_ws_config.sh ${DATAFED_DIR}/scripts/generate_ws_config.sh -COPY --chown=datafed:root ./scripts/install_ws.sh ${DATAFED_DIR}/scripts/install_ws.sh -COPY --chown=datafed:root ./cmake/Version.cmake ${DATAFED_DIR}/cmake/Version.cmake +#COPY --chown=datafed:root ./web/docker/entrypoint.sh ${BUILD_DIR}/web/docker/entrypoint.sh +COPY --chown=datafed:root ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh +COPY --chown=datafed:root ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/dependency_versions.sh +COPY --chown=datafed:root ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/dependency_install_functions.sh +COPY --chown=datafed:root ./scripts/generate_ws_config.sh ${BUILD_DIR}/scripts/generate_ws_config.sh +COPY --chown=datafed:root ./scripts/install_ws.sh ${BUILD_DIR}/scripts/install_ws.sh +COPY --chown=datafed:root ./cmake/Version.cmake ${BUILD_DIR}/cmake/Version.cmake +COPY --chown=datafed:root ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh +COPY --chown=datafed:root ./scripts/export_dependency_version.sh ${BUILD_DIR}/scripts/export_dependency_version.sh +##COPY --chown=datafed:root ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} +# +COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${BUILD_DIR}/web +COPY --from=ws-build --chown=datafed:root ${DATAFED_INSTALL_PATH} ${DATAFED_INSTALL_PATH} +#COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web/package.json ${DATAFED_INSTALL_PATH}/web/package.json +COPY --from=ws-build --chown=datafed:root ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} +COPY --from=ws-build --chown=datafed:root /usr/bin/curl /usr/bin/curl + +RUN chown -R datafed:root ${DATAFED_INSTALL_PATH}/web +RUN chown -R datafed:root ${BUILD_DIR} + +WORKDIR ${DATAFED_DIR} -COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web/package.json ${DATAFED_INSTALL_PATH}/web/package.json -RUN . ${DATAFED_DIR}/scripts/dependency_versions.sh && \ - . ${DATAFED_DIR}/.nvm/nvm.sh && \ +RUN echo "${DATAFED_DIR}" +RUN . ${BUILD_DIR}/scripts/export_dependency_version.sh && \ + . ${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm/nvm.sh && \ npm --allow-root --unsafe-perm --prefix ${DATAFED_INSTALL_PATH}/web install -COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${DATAFED_INSTALL_PATH}/web +#COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${DATAFED_INSTALL_PATH}/web USER root -WORKDIR ${DATAFED_INSTALL_PATH}/web +#ENTRYPOINT ["sleep", "100000" ] +ENTRYPOINT ["/datafed/source/web/docker/entrypoint.sh", "/opt/datafed/web/datafed-ws.js", "/datafed/source/config/datafed-ws.cfg" ] +#WORKDIR ${DATAFED_INSTALL_PATH}/web diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 98f2ee2b7..d5b1eb597 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -12,17 +12,25 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) + ${PROJECT_ROOT}/scripts/generate_datafed.sh ${PROJECT_ROOT}/scripts/generate_ws_config.sh ${PROJECT_ROOT}/scripts/install_ws.sh -${PROJECT_ROOT}/scripts/dependency_versions.sh +. ${PROJECT_ROOT}/scripts/export_dependency_version.sh export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" export NODE_VERSION="$DATAFED_NODE_VERSION" [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm +echo "NODE VERSION $NODE_VERSION" +echo "Which node" +NODE_PATH=$(su datafed -c ". ${NVM_DIR}/nvm.sh; nvm which node") +# Do this as root +if [ ! -L "/usr/local/bin/node" ]; then + ln -s "$NODE_PATH" /usr/local/bin/node +fi # Send output to file as well as print to terminal -log_path=$(grep "log-path" /datafed/install/web/datafed-ws.cfg | cut -d "=" -f 2 | tr -d ' ') +log_path=$(grep "log-path" ${BUILD_DIR}/config/datafed-ws.cfg | cut -d "=" -f 2 | tr -d ' ') if [ ! -d "${log_path}" ] then @@ -35,6 +43,7 @@ if [ "$#" -eq 0 ]; then exit 0 fi +cd "$DATAFED_INSTALL_PATH/web" datafed_ws_exec=$(basename "$1") if [ "${datafed_ws_exec}" = "datafed-ws.js" ] then @@ -45,3 +54,4 @@ else # If not do not by default send to log file su datafed -c '"$@"' -- argv0 "$@" fi + From a13e386482e0d2b7159804288990d108d2464d80 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 16 Feb 2024 17:38:36 -0500 Subject: [PATCH 47/69] Fix stage --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 281811263..75937c462 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -385,7 +385,7 @@ build-python-client: variables: IMAGE_TAG: "dlsw/datafed/python-client-" GIT_STRATEGY: clone - stage: build-unit-test-deploy + stage: build tags: - docker rules: From e19f1729f8d54ffb6b40408aa35963452894449a Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 16 Feb 2024 18:23:09 -0500 Subject: [PATCH 48/69] Fix ci file --- .gitlab-ci.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75937c462..f8109bf13 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -399,11 +399,9 @@ build-python-client: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN code.ornl.gov:4567 - - docker build -f python/docker/Dockerfile.python-client.ubuntu -t "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" . - - docker push "code.ornl.gov:4567/${IMAGE_TAG}${BRANCH_LOWER}" - - + - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" + - docker build -f repository/docker/Dockerfile.python-client.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . + - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" ################################################################################ # STAGE: Integration From 78d9f2fef7e1ac8cadd46d0b47d38bf86f879da8 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 16 Feb 2024 18:26:12 -0500 Subject: [PATCH 49/69] Fix path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8109bf13..1f0225732 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -400,7 +400,7 @@ build-python-client: - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - - docker build -f repository/docker/Dockerfile.python-client.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . + - docker build -f python/docker/Dockerfile.python-client.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" ################################################################################ From e12d40a881fa5a83e011e09436b4d5e7dbd53e2b Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 16 Feb 2024 18:42:33 -0500 Subject: [PATCH 50/69] Comment out python client --- .gitlab-ci.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1f0225732..abc7893e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -381,27 +381,27 @@ build-gcs-authz: - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" -build-python-client: - variables: - IMAGE_TAG: "dlsw/datafed/python-client-" - GIT_STRATEGY: clone - stage: build - tags: - - docker - rules: - - changes: - - python/* - - scripts/* - - common/* - - .gitlab-ci.yml - when: always - script: - - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - - echo "$BRANCH_LOWER" - - ./scripts/generate_datafed.sh - - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - - docker build -f python/docker/Dockerfile.python-client.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . - - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" + #build-python-client: + # variables: + # IMAGE_TAG: "dlsw/datafed/python-client-" + # GIT_STRATEGY: clone + # stage: build + # tags: + # - docker + # rules: + # - changes: + # - python/* + # - scripts/* + # - common/* + # - .gitlab-ci.yml + # when: always + # script: + # - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') + # - echo "$BRANCH_LOWER" + # - ./scripts/generate_datafed.sh + # - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" + # - docker build -f python/docker/Dockerfile.python-client.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . + # - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" ################################################################################ # STAGE: Integration From b9c2a5901f7c897c3355b6a0bc351e53679c131a Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Fri, 16 Feb 2024 18:47:50 -0500 Subject: [PATCH 51/69] Update images --- core/docker/Dockerfile.core-base.ubuntu | 1 + repository/docker/Dockerfile.gcs-authz-base.ubuntu | 1 + repository/docker/Dockerfile.gcs-authz.ubuntu | 1 + repository/docker/Dockerfile.repo-base.ubuntu | 1 + web/docker/Dockerfile.web-base.ubuntu | 1 + 5 files changed, 5 insertions(+) diff --git a/core/docker/Dockerfile.core-base.ubuntu b/core/docker/Dockerfile.core-base.ubuntu index 9f3eed307..65fb32106 100644 --- a/core/docker/Dockerfile.core-base.ubuntu +++ b/core/docker/Dockerfile.core-base.ubuntu @@ -11,6 +11,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo RUN ${BUILD_DIR}/scripts/generate_datafed.sh diff --git a/repository/docker/Dockerfile.gcs-authz-base.ubuntu b/repository/docker/Dockerfile.gcs-authz-base.ubuntu index fd62494e4..79b4660f6 100644 --- a/repository/docker/Dockerfile.gcs-authz-base.ubuntu +++ b/repository/docker/Dockerfile.gcs-authz-base.ubuntu @@ -11,6 +11,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo RUN ${BUILD_DIR}/scripts/generate_datafed.sh diff --git a/repository/docker/Dockerfile.gcs-authz.ubuntu b/repository/docker/Dockerfile.gcs-authz.ubuntu index 56ec69e2d..e71bb01c7 100644 --- a/repository/docker/Dockerfile.gcs-authz.ubuntu +++ b/repository/docker/Dockerfile.gcs-authz.ubuntu @@ -33,6 +33,7 @@ WORKDIR ${BUILD_DIR} COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/generate_authz_config.sh ${BUILD_DIR}/scripts/generate_authz_config.sh COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ COPY ./CMakeLists.txt ${BUILD_DIR} COPY ./cmake ${BUILD_DIR}/cmake COPY ./repository/CMakeLists.txt ${BUILD_DIR}/repository/CMakeLists.txt diff --git a/repository/docker/Dockerfile.repo-base.ubuntu b/repository/docker/Dockerfile.repo-base.ubuntu index 4c6b40d21..b7a84e1a5 100644 --- a/repository/docker/Dockerfile.repo-base.ubuntu +++ b/repository/docker/Dockerfile.repo-base.ubuntu @@ -11,6 +11,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo RUN ${BUILD_DIR}/scripts/generate_datafed.sh diff --git a/web/docker/Dockerfile.web-base.ubuntu b/web/docker/Dockerfile.web-base.ubuntu index 60e7e2e96..7a231f483 100644 --- a/web/docker/Dockerfile.web-base.ubuntu +++ b/web/docker/Dockerfile.web-base.ubuntu @@ -12,6 +12,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DIR}/dependenies" From 6e6bb50d5e8b3a9921067237a4ca5e1cb57d472e Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sun, 18 Feb 2024 21:53:33 -0500 Subject: [PATCH 52/69] Address codacy issues --- CHANGELOG.md | 6 +++--- compose/build_images_for_compose.sh | 2 +- compose/compose.yml | 9 +++++++++ docker/entrypoint_foxx.sh | 2 +- scripts/export_dependency_version.sh | 2 +- scripts/install_lego_and_certificates.sh | 6 +++--- scripts/install_ws.sh | 25 ++++++++++++------------ scripts/utils.sh | 2 +- web/docker/entrypoint.sh | 14 ++++++------- 9 files changed, 38 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a633525cb..f8b11c8ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ ## MINOR Feature 1. [912] - Adds initial compose file and docker files for python-client -1. [909] - Added Support for Google Analytics -2. [916] - Reworked Docker build process to be faster and produce simpler and smaller images -2. [912] - Adding working compose instance for core metadata services. +2. [909] - Added Support for Google Analytics +3. [916] - Reworked Docker build process to be faster and produce simpler and smaller images +4. [912] - Adding working compose instance for core metadata services. ## PATCH Bug fixes/Technical Debt/Documentation 1. [914] - Improve GitHub template diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index 1c8aa08cf..7b155465f 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -4,7 +4,7 @@ set -euf -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../) +PROJECT_ROOT=$(realpath "${SOURCE}/../") docker build \ -f "${PROJECT_ROOT}/docker/Dockerfile.dependencies" \ diff --git a/compose/compose.yml b/compose/compose.yml index 82a7993fa..cf7467e4e 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -2,6 +2,15 @@ version: '3.9' services: + + keycloak: + image: jboss/keycloak:latest + environment: + KEYCLOAK_USER: admin + KEYCLOAK_PASSWORD: admin + ports: + - "8888:8080" + datafed-web: depends_on: ["datafed-core"] environment: diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh index 8e1f189ef..f63516839 100755 --- a/docker/entrypoint_foxx.sh +++ b/docker/entrypoint_foxx.sh @@ -8,7 +8,7 @@ fi SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../) +PROJECT_ROOT=$(realpath "${SOURCE}/../") log_path="$DATAFED_DEFAULT_LOG_PATH" if [ ! -d "${log_path}" ] diff --git a/scripts/export_dependency_version.sh b/scripts/export_dependency_version.sh index a965144af..8fd5302f4 100755 --- a/scripts/export_dependency_version.sh +++ b/scripts/export_dependency_version.sh @@ -3,7 +3,7 @@ # Assumes sourcing not running SCRIPT=$( realpath "${BASH_SOURCE[0]}" ) SCRIPT_DIR=$( dirname "${SCRIPT}" ) -PROJECT_ROOT=$(realpath ${SCRIPT_DIR}/..) +PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/..") echo "SCRIPT $SCRIPT" echo "SCRIPT_DIR $SCRIPT_DIR" diff --git a/scripts/install_lego_and_certificates.sh b/scripts/install_lego_and_certificates.sh index e098e790f..6f19b0c3d 100755 --- a/scripts/install_lego_and_certificates.sh +++ b/scripts/install_lego_and_certificates.sh @@ -4,9 +4,9 @@ set -euf -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/..) -source ${PROJECT_ROOT}/config/datafed.sh -source ${PROJECT_ROOT}/scripts/utils.sh +PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/config/datafed.sh" +source "${PROJECT_ROOT}/scripts/utils.sh" Help() { diff --git a/scripts/install_ws.sh b/scripts/install_ws.sh index 425b4104e..18b248cac 100755 --- a/scripts/install_ws.sh +++ b/scripts/install_ws.sh @@ -4,28 +4,27 @@ set -uf -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/..) -source ${PROJECT_ROOT}/config/datafed.sh -source ${SOURCE}/dependency_versions.sh -source ${SOURCE}/dependency_install_functions.sh +PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/config/datafed.sh" +source "${SOURCE}/dependency_versions.sh" +source "${SOURCE}/dependency_install_functions.sh" #NVM_DIR=/home/cades/.nvm #[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" #export NVM_DIR="/home/cades/.nvm" #source ${NVM_DIR}/nvm.sh # Make sure paths exist -mkdir -p ${DATAFED_INSTALL_PATH}/web -mkdir -p ${DATAFED_INSTALL_PATH}/keys -mkdir -p ${DATAFED_DEFAULT_LOG_PATH} +mkdir -p "${DATAFED_INSTALL_PATH}/web" +mkdir -p "${DATAFED_INSTALL_PATH}/keys" +mkdir -p "${DATAFED_DEFAULT_LOG_PATH}" # Install web node modules -cp "$PROJECT_ROOT/web/package.json" ${DATAFED_INSTALL_PATH}/web/ - +cp "$PROJECT_ROOT/web/package.json" "${DATAFED_INSTALL_PATH}/web/" install_nvm install_node install_foxx_cli { - npm --allow-root --unsafe-perm --prefix ${DATAFED_INSTALL_PATH}/web install + npm --allow-root --unsafe-perm --prefix "${DATAFED_INSTALL_PATH}/web" install } || { echo "ERROR npm command failed!" exit 1 @@ -33,10 +32,10 @@ install_foxx_cli # Install javascript web server repo and core server were # already installed by CMake -cp "$PROJECT_ROOT/web/datafed-ws.js" ${DATAFED_INSTALL_PATH}/web +cp "$PROJECT_ROOT/web/datafed-ws.js" "${DATAFED_INSTALL_PATH}/web" # Copy configuration files -cp "$PROJECT_ROOT/config/datafed-ws.cfg" ${DATAFED_INSTALL_PATH}/web +cp "$PROJECT_ROOT/config/datafed-ws.cfg" "${DATAFED_INSTALL_PATH}/web" # Make sure that the datafed-core-pub.key exists in the web/static folder if [[ -z "${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub" ]] @@ -45,4 +44,4 @@ then exit 1 fi -cp ${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub "$DATAFED_INSTALL_PATH/web/static/" +cp "${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub" "$DATAFED_INSTALL_PATH/web/static/" diff --git a/scripts/utils.sh b/scripts/utils.sh index f88e8c488..c0ee7455a 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -4,7 +4,7 @@ SOURCE=$(dirname "$SCRIPT") if [ -z "${PROJECT_ROOT}" ] then - PROJECT_ROOT=$(realpath ${SOURCE}/..) + PROJECT_ROOT=$(realpath "${SOURCE}/..") fi echo "PROJECT ROOT $PROJECT_ROOT" diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index d5b1eb597..945c7cdc1 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -4,19 +4,19 @@ set -euf -o pipefail if [ -n "$UID" ]; then - usermod -u $UID datafed + usermod -u "$UID" datafed fi SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../..) +PROJECT_ROOT=$(realpath "${SOURCE}/../..") -${PROJECT_ROOT}/scripts/generate_datafed.sh -${PROJECT_ROOT}/scripts/generate_ws_config.sh -${PROJECT_ROOT}/scripts/install_ws.sh -. ${PROJECT_ROOT}/scripts/export_dependency_version.sh +"${PROJECT_ROOT}/scripts/generate_datafed.sh" +"${PROJECT_ROOT}/scripts/generate_ws_config.sh" +"${PROJECT_ROOT}/scripts/install_ws.sh" +. "${PROJECT_ROOT}/scripts/export_dependency_version.sh" export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" export NODE_VERSION="$DATAFED_NODE_VERSION" @@ -30,7 +30,7 @@ if [ ! -L "/usr/local/bin/node" ]; then ln -s "$NODE_PATH" /usr/local/bin/node fi # Send output to file as well as print to terminal -log_path=$(grep "log-path" ${BUILD_DIR}/config/datafed-ws.cfg | cut -d "=" -f 2 | tr -d ' ') +log_path=$(grep "log-path" "${BUILD_DIR}/config/datafed-ws.cfg" | cut -d "=" -f 2 | tr -d ' ') if [ ! -d "${log_path}" ] then From 73dac5b6b3bc629985daaec445cce14758d0f5fd Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sun, 18 Feb 2024 23:46:38 -0500 Subject: [PATCH 53/69] A few small fixes --- compose/compose.yml | 8 -------- scripts/generate_ws_config.sh | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/compose/compose.yml b/compose/compose.yml index cf7467e4e..9aa747981 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -3,14 +3,6 @@ version: '3.9' services: - keycloak: - image: jboss/keycloak:latest - environment: - KEYCLOAK_USER: admin - KEYCLOAK_PASSWORD: admin - ports: - - "8888:8080" - datafed-web: depends_on: ["datafed-core"] environment: diff --git a/scripts/generate_ws_config.sh b/scripts/generate_ws_config.sh index a8fd76ea6..0d6e19773 100755 --- a/scripts/generate_ws_config.sh +++ b/scripts/generate_ws_config.sh @@ -57,7 +57,7 @@ then local_DATAFED_WEB_CERT_PATH=$(printenv DATAFED_WEB_CERT_PATH) fi -if [ -z "${DATAFED_GLOBUS_APP_ID}" ] +if [ -z "${DATAFED_DOMAIN}" ] then local_DATAFED_SERVER_DOMAIN_NAME="datafed.ornl.gov" else From 2484b463637bbd20f0579cfa945e9ec606b608dd Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 00:57:48 -0500 Subject: [PATCH 54/69] Small fixes to get Globus working with DataFed compose --- compose/compose.yml | 3 ++- compose/generate_env.sh | 36 ++++++++++++++++++++++++++++++----- scripts/generate_ws_config.sh | 9 +++++++-- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/compose/compose.yml b/compose/compose.yml index 9aa747981..d28a6fe56 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -11,6 +11,7 @@ services: DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_HTTPS_SERVER_PORT: "${DATAFED_HTTPS_SERVER_PORT}" DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" @@ -18,7 +19,7 @@ services: UID: "${DATAFED_UID}" image: datafed-web:latest ports: - - 8080:443 + - 443:443 # This must be the same port that is mapped to the host for redirects to work volumes: - keys:/opt/datafed/keys - ./web_keys:/opt/datafed/web_keys diff --git a/compose/generate_env.sh b/compose/generate_env.sh index 9cf202edb..631728e65 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -1,4 +1,7 @@ #!/bin/bash +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath ${SOURCE}/..) if [ -f ".env" ] then @@ -6,10 +9,32 @@ then exit 1 fi +local_DATAFED_WEB_KEY_DIR="${PROJECT_ROOT}/compose/web_keys" +if [ ! -d "$local_DATAFED_WEB_KEY_DIR" ] +then + mkdir -p "$local_DATAFED_WEB_KEY_DIR" +fi + +local_DATAFED_WEB_CERT_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.crt" +local_DATAFED_WEB_CSR_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.csr" +local_DATAFED_WEB_KEY_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.key" +if [ ! -e "$local_DATAFED_WEB_CERT_PATH" ] || [ ! -e "$local_DATAFED_WEB_KEY_PATH" ] +then + rm "${local_DATAFED_WEB_CERT_PATH}" + rm "${local_DATAFED_WEB_KEY_PATH}" + rm "${local_DATAFED_WEB_CSR_PATH}" + + openssl genrsa -out "$local_DATAFED_WEB_KEY_PATH" 2048 + openssl req -new -key "$local_DATAFED_WEB_KEY_PATH" \ + -out "${local_DATAFED_WEB_CSR_PATH}" + openssl x509 -req -days 3650 \ + -in "${local_DATAFED_WEB_CSR_PATH}" \ + -signkey "$local_DATAFED_WEB_KEY_PATH" \ + -out "$local_DATAFED_WEB_CERT_PATH" +fi cat << EOF > ".env" -DATAFED_DOMAIN="" # The domain of the metadata core web server -ARANGO_ROOT_PASSWORD="" +DATAFED_DOMAIN="localhost" # The domain of the metadata web server DATAFED_USER89_PASSWORD="" # For End to end testing DATAFED_REPO_FORM_PATH="" # Where the repo form is located also needed for testing DATAFED_GLOBUS_APP_SECRET="" @@ -17,9 +42,10 @@ DATAFED_GLOBUS_APP_ID="" DATAFED_ZEROMQ_SESSION_SECRET="" DATAFED_ZEROMQ_SYSTEM_SECRET="" DATAFED_DOMAIN="" -DATAFED_WEB_CERT_PATH="" -DATAFED_WEB_KEY_PATH="" -DATAFED_CONTAINER_LOG_PATH="" +DATAFED_HTTPS_SERVER_PORT="443" +DATAFED_WEB_CERT_PATH="${local_DATAFED_WEB_CERT_PATH}" +DATAFED_WEB_KEY_PATH="${local_DATAFED_WEB_KEY_PATH}" +DATAFED_CONTAINER_LOG_PATH="/opt/datafed/logs" DATAFED_DATABASE_PASSWORD="" DATAFED_DATABASE_IP_ADDRESS_PORT="http://arango:8529" UID="$(id -u)" diff --git a/scripts/generate_ws_config.sh b/scripts/generate_ws_config.sh index 0d6e19773..33211d675 100755 --- a/scripts/generate_ws_config.sh +++ b/scripts/generate_ws_config.sh @@ -45,8 +45,6 @@ Help() local_DATAFED_WEB_CERT_PATH="${DATAFED_INSTALL_PATH}/keys/datafed-server-test.ornl.gov.crt" local_DATAFED_WEB_KEY_PATH="${DATAFED_INSTALL_PATH}/keys/datafed-server-test.ornl.gov.key" -local_DATAFED_HTTPS_SERVER_PORT="443" - if [ ! -z "${DATAFED_WEB_KEY_PATH}" ] then local_DATAFED_WEB_KEY_PATH=$(printenv DATAFED_WEB_KEY_PATH) @@ -64,6 +62,13 @@ else local_DATAFED_SERVER_DOMAIN_NAME=$(printenv DATAFED_DOMAIN) fi +if [ -z "${DATAFED_HTTPS_SERVER_PORT}" ] +then + local_DATAFED_HTTPS_SERVER_PORT="443" +else + local_DATAFED_HTTPS_SERVER_PORT=$(printenv DATAFED_HTTPS_SERVER_PORT) +fi + if [ -z "${DATAFED_GLOBUS_APP_ID}" ] then local_DATAFED_GLOBUS_APP_ID="" From 2c2480dc503b794325a061db4c012984bebf5723 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 10:37:50 -0500 Subject: [PATCH 55/69] Cleanup a little --- compose/README.md | 31 ++++- compose/build_images_for_compose.sh | 6 + compose/{compose.yml => compose_core.yml} | 28 +++-- compose/generate_env.sh | 137 ++++++++++++++++++---- core/docker/entrypoint.sh | 8 +- scripts/install_foxx.sh | 6 +- 6 files changed, 172 insertions(+), 44 deletions(-) rename compose/{compose.yml => compose_core.yml} (87%) diff --git a/compose/README.md b/compose/README.md index d653ba098..d40570f5d 100644 --- a/compose/README.md +++ b/compose/README.md @@ -3,8 +3,31 @@ The files in this folder are incomplete but are the start for setting up a full docker compose instance of datafed. -## Generate self signed web key +```bash +./build_images_for_compose.sh +``` -openssl genrsa -out cert.key 2048 -openssl req -new -key cert.key -out cert.csr -openssl x509 -req -days 3650 -in cert.csr -signkey cert.key -out cert.crt +Create the .env file fill in the missing components that are required. +```bash +./generate_env.sh +``` + +Stand up the core services. +```bash +docker compose -f ./compose_core.yml up +``` + +At this point you should be able to navigate in your browser to +https://localhost + +NOTE we are using a self signed certificate so you will have to force your +browser to allow you to see the page. + +Standing up the repo services has been separated because of Globus. You will +need a machine with firewall exceptions to use it. + +## Cleaning up + +```bash +docker compose -f ./compose_core.yml down +``` diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh index 7b155465f..6b472dcb2 100755 --- a/compose/build_images_for_compose.sh +++ b/compose/build_images_for_compose.sh @@ -26,6 +26,12 @@ docker build -f \ --build-arg RUNTIME="datafed-runtime" \ "${PROJECT_ROOT}" \ -t datafed-web:latest +docker build -f \ + "${PROJECT_ROOT}/repository/docker/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-repo:latest docker build -f \ "${PROJECT_ROOT}/docker/Dockerfile.foxx" \ --build-arg DEPENDENCIES="datafed-dependencies" \ diff --git a/compose/compose.yml b/compose/compose_core.yml similarity index 87% rename from compose/compose.yml rename to compose/compose_core.yml index d28a6fe56..18d9b6c62 100644 --- a/compose/compose.yml +++ b/compose/compose_core.yml @@ -21,8 +21,9 @@ services: ports: - 443:443 # This must be the same port that is mapped to the host for redirects to work volumes: - - keys:/opt/datafed/keys - - ./web_keys:/opt/datafed/web_keys + - ./keys:/opt/datafed/keys + networks: + - datafed-internal datafed-core: image: datafed-core:latest @@ -41,12 +42,15 @@ services: UID: "${DATAFED_UID}" DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" - DATAFED_DATABASE_IP_ADDRESS_PORT: "${DATAFED_DATABASE_IP_ADDRESS_PORT}" + DATAFED_DATABASE_IP_ADDRESS_PORT: "${DATAFED_DATABASE_IP_ADDRESS}:${DATAFED_DATABASE_PORT}" ports: - 7513 # Communication web server - 7512 # Secure core server communication volumes: - - keys:/opt/datafed/keys + - ./keys:/opt/datafed/keys + networks: + - datafed-core-secure-api + - datafed-internal datafed-foxx: image: datafed-foxx:latest @@ -64,6 +68,8 @@ services: interval: 10s timeout: 5s retries: 10 + networks: + - datafed-internal arango: image: arangodb @@ -71,11 +77,15 @@ services: ARANGO_ROOT_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" ports: - 8529:8529 # Arangodb web UI - # healthcheck: - # test: ["CMD", "curl", "-f", "http://localhost:8529/_admin/cluster/health"] - # interval: 30s - # timeout: 10s - # retries: 5 + networks: + - datafed-internal volumes: keys: + +networks: + datafed-core-secure-api: + driver: bridge + datafed-internal: + driver: bridge + diff --git a/compose/generate_env.sh b/compose/generate_env.sh index 631728e65..b09f76da1 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -1,7 +1,7 @@ #!/bin/bash SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/..) +PROJECT_ROOT=$(realpath "${SOURCE}/..") if [ -f ".env" ] then @@ -9,44 +9,133 @@ then exit 1 fi -local_DATAFED_WEB_KEY_DIR="${PROJECT_ROOT}/compose/web_keys" +local_DATAFED_WEB_KEY_DIR="${PROJECT_ROOT}/compose/keys" if [ ! -d "$local_DATAFED_WEB_KEY_DIR" ] then mkdir -p "$local_DATAFED_WEB_KEY_DIR" fi -local_DATAFED_WEB_CERT_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.crt" +if [ -z "${DATAFED_COMPOSE_DOMAIN}" ] +then + local_DATAFED_COMPOSE_DOMAIN="localhost" +else + local_DATAFED_COMPOSE_DOMAIN=$(printenv DATAFED_COMPOSE_DOMAIN) +fi + +local_DATAFED_WEB_CERT_NAME="cert.crt" +local_DATAFED_WEB_KEY_NAME="cert.key" + +local_DATAFED_WEB_CERT_PATH="${local_DATAFED_WEB_KEY_DIR}/${local_DATAFED_WEB_CERT_NAME}" local_DATAFED_WEB_CSR_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.csr" -local_DATAFED_WEB_KEY_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.key" +local_DATAFED_WEB_KEY_PATH="${local_DATAFED_WEB_KEY_DIR}/${local_DATAFED_WEB_KEY_NAME}" + if [ ! -e "$local_DATAFED_WEB_CERT_PATH" ] || [ ! -e "$local_DATAFED_WEB_KEY_PATH" ] then - rm "${local_DATAFED_WEB_CERT_PATH}" - rm "${local_DATAFED_WEB_KEY_PATH}" - rm "${local_DATAFED_WEB_CSR_PATH}" - + if [ -e "$local_DATAFED_WEB_CERT_PATH" ] + then + rm "${local_DATAFED_WEB_CERT_PATH}" + fi + if [ -e "$local_DATAFED_WEB_KEY_PATH" ] + then + rm "${local_DATAFED_WEB_KEY_PATH}" + fi + if [ -e "$local_DATAFED_WEB_CSR_PATH" ] + then + rm "${local_DATAFED_WEB_CSR_PATH}" + fi openssl genrsa -out "$local_DATAFED_WEB_KEY_PATH" 2048 openssl req -new -key "$local_DATAFED_WEB_KEY_PATH" \ - -out "${local_DATAFED_WEB_CSR_PATH}" + -out "${local_DATAFED_WEB_CSR_PATH}" \ + -subj "/C=US/ST=TN/L=Oak Ridge/O=ORNL/OU=DLT/CN=${local_DATAFED_COMPOSE_DOMAIN}" openssl x509 -req -days 3650 \ -in "${local_DATAFED_WEB_CSR_PATH}" \ -signkey "$local_DATAFED_WEB_KEY_PATH" \ -out "$local_DATAFED_WEB_CERT_PATH" fi +if [ -z "${DATAFED_COMPOSE_USER89_PASSWORD}" ] +then + local_DATAFED_COMPOSE_USER89_PASSWORD="" # For End to end testing +else + local_DATAFED_COMPOSE_USER89_PASSWORD=$(printenv DATAFED_COMPOSE_USER89_PASSWORD) +fi +if [ -z "${DATAFED_COMPOSE_REPO_FORM_PATH}" ] +then + local_DATAFED_COMPOSE_REPO_FORM_PATH="" # Where the repo form is located also needed for testing +else + local_DATAFED_COMPOSE_REPO_FORM_PATH=$(printenv DATAFED_COMPOSE_REPO_FORM_PATH) +fi +if [ -z "${DATAFED_COMPOSE_GLOBUS_APP_SECRET}" ] +then + local_DATAFED_COMPOSE_GLOBUS_APP_SECRET="" +else + local_DATAFED_COMPOSE_GLOBUS_APP_SECRET=$(printenv DATAFED_COMPOSE_GLOBUS_APP_SECRET) +fi +if [ -z "${DATAFED_COMPOSE_GLOBUS_APP_ID}" ] +then + local_DATAFED_COMPOSE_GLOBUS_APP_ID="" +else + local_DATAFED_COMPOSE_GLOBUS_APP_ID=$(printenv DATAFED_COMPOSE_GLOBUS_APP_ID) +fi +if [ -z "${DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET}" ] +then + local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET="" +else + local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET=$(printenv DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET) +fi +if [ -z "${DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET}" ] +then + local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET="" +else + local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET=$(printenv DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET) +fi +if [ -z "${DATAFED_COMPOSE_HTTPS_SERVER_PORT}" ] +then + local_DATAFED_COMPOSE_HTTPS_SERVER_PORT="443" +else + local_DATAFED_COMPOSE_HTTPS_SERVER_PORT=$(printenv DATAFED_COMPOSE_HTTPS_SERVER_PORT) +fi +if [ -z "${DATAFED_COMPOSE_CONTAINER_LOG_PATH}" ] +then + local_DATAFED_COMPOSE_CONTAINER_LOG_PATH="/opt/datafed/logs" +else + local_DATAFED_COMPOSE_CONTAINER_LOG_PATH=$(printenv DATAFED_COMPOSE_CONTAINER_LOG_PATH) +fi +if [ -z "${DATAFED_COMPOSE_DATABASE_PASSWORD}" ] +then + local_DATAFED_COMPOSE_DATABASE_PASSWORD="butterscotch" +else + local_DATAFED_COMPOSE_DATABASE_PASSWORD=$(printenv DATAFED_COMPOSE_DATABASE_PASSWORD) +fi + +if [ -z "${DATAFED_COMPOSE_DATABASE_IP_ADDRESS}" ] +then + local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS="http://arango" +else + local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS=$(printenv DATAFED_COMPOSE_DATABASE_IP_ADDRESS) +fi + +if [ -z "${DATAFED_COMPOSE_DATABASE_PORT}" ] +then + local_DATAFED_COMPOSE_DATABASE_PORT="8529" +else + local_DATAFED_COMPOSE_DATABASE_PORT=$(printenv DATAFED_COMPOSE_DATABASE_PORT) +fi + cat << EOF > ".env" -DATAFED_DOMAIN="localhost" # The domain of the metadata web server -DATAFED_USER89_PASSWORD="" # For End to end testing -DATAFED_REPO_FORM_PATH="" # Where the repo form is located also needed for testing -DATAFED_GLOBUS_APP_SECRET="" -DATAFED_GLOBUS_APP_ID="" -DATAFED_ZEROMQ_SESSION_SECRET="" -DATAFED_ZEROMQ_SYSTEM_SECRET="" -DATAFED_DOMAIN="" -DATAFED_HTTPS_SERVER_PORT="443" -DATAFED_WEB_CERT_PATH="${local_DATAFED_WEB_CERT_PATH}" -DATAFED_WEB_KEY_PATH="${local_DATAFED_WEB_KEY_PATH}" -DATAFED_CONTAINER_LOG_PATH="/opt/datafed/logs" -DATAFED_DATABASE_PASSWORD="" -DATAFED_DATABASE_IP_ADDRESS_PORT="http://arango:8529" -UID="$(id -u)" +DATAFED_USER89_PASSWORD="${local_DATAFED_COMPOSE_USER89_PASSWORD}" # For End to end testing +DATAFED_REPO_FORM_PATH="${local_DATAFED_COMPOSE_REPO_FORM_PATH}" # Where the repo form is located also needed for testing +DATAFED_GLOBUS_APP_SECRET="${local_DATAFED_COMPOSE_GLOBUS_APP_SECRET}" +DATAFED_GLOBUS_APP_ID="${local_DATAFED_COMPOSE_GLOBUS_APP_ID}" +DATAFED_ZEROMQ_SESSION_SECRET="${local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET}" +DATAFED_ZEROMQ_SYSTEM_SECRET="${local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET}" +DATAFED_DOMAIN="${local_DATAFED_COMPOSE_DOMAIN}" +DATAFED_HTTPS_SERVER_PORT="${local_DATAFED_COMPOSE_HTTPS_SERVER_PORT}" +DATAFED_WEB_CERT_PATH="/opt/datafed/keys/${local_DATAFED_WEB_CERT_NAME}" +DATAFED_WEB_KEY_PATH="/opt/datafed/keys/${local_DATAFED_WEB_KEY_NAME}" +DATAFED_CONTAINER_LOG_PATH="${local_DATAFED_COMPOSE_CONTAINER_LOG_PATH}" +DATAFED_DATABASE_PASSWORD="${local_DATAFED_COMPOSE_DATABASE_PASSWORD}" +DATAFED_DATABASE_IP_ADDRESS="${local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS}" +DATAFED_DATABASE_PORT="${local_DATAFED_COMPOSE_DATABASE_PORT}" +DATAFED_UID="$(id -u)" EOF diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 4039d27a7..6f04fb1de 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -8,12 +8,12 @@ fi SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../..) +PROJECT_ROOT=$(realpath "${SOURCE}/../..") env -${PROJECT_ROOT}/scripts/generate_datafed.sh -${PROJECT_ROOT}/scripts/generate_core_config.sh -${PROJECT_ROOT}/scripts/install_core.sh +"${PROJECT_ROOT}/scripts/generate_datafed.sh" +"${PROJECT_ROOT}/scripts/generate_core_config.sh" +"${PROJECT_ROOT}/scripts/install_core.sh" log_path="$DATAFED_DEFAULT_LOG_PATH" diff --git a/scripts/install_foxx.sh b/scripts/install_foxx.sh index 9f918cdb0..6664e68f7 100755 --- a/scripts/install_foxx.sh +++ b/scripts/install_foxx.sh @@ -237,9 +237,9 @@ fi # # The web deployment requires manual interaction, and I could not figure out the # syntax for the REST http endpoints with curl so we are going to try the node module -actual_version=$(node --version) -semantic_version_compatible "$actual_version" "$DATAFED_NODE_VERSION" -compatible=$? +#actual_version=$(node --version) +#semantic_version_compatible "$actual_version" "$DATAFED_NODE_VERSION" +#compatible=$? #if [ "$compatible" -eq "0" ] #then From 959bcb9f7d6c26a70f06a72c3db3127c5cb99890 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 10:38:26 -0500 Subject: [PATCH 56/69] Small changes --- docker/entrypoint_foxx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh index f63516839..128f0a9d1 100755 --- a/docker/entrypoint_foxx.sh +++ b/docker/entrypoint_foxx.sh @@ -39,7 +39,7 @@ then # Give arango container a minute to initialize # should be replaced with health check at some point - sleep 10 + sleep 5 su datafed -c "cmake --build build --target install" #if [ "$#" -eq 0 ]; then From 005b70715e25e521e84e3f62ef761eb0f8946063 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 10:51:32 -0500 Subject: [PATCH 57/69] add comment --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8ccd0846..9a347a308 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -521,8 +521,10 @@ end-to-end-repo-setup: - ./run_repo.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - # - echo " /dev/null &" >> run_repo.sh + +# Requires setting up Globus Connect Server, requires firewall exceptions on +# the machine running this. end-to-end-gcs-authz-setup: variables: IMAGE_TAG: "datafed/gcs-authz-" From cf2daf6a3bbafe0aff31d0bae1fba8deeb00d413 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 17:38:27 +0000 Subject: [PATCH 58/69] Add check for database connection --- .gitlab-ci.yml | 2 + scripts/ci_database_health_check.sh | 72 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100755 scripts/ci_database_health_check.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a347a308..e14be7d74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -442,6 +442,7 @@ end-to-end-core-setup: GIT_STRATEGY: clone HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs" CONTAINER_LOG_FILE_PATH: "/datafed/logs" + DATAFED_DATABASE_HOST: "$CI_DATAFED_DATABASE_HOST" stage: end-to-end-setup tags: - ci-datafed-core @@ -456,6 +457,7 @@ end-to-end-core-setup: - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" + - ./scripts/ci_database_health_check.sh - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - echo "#!/bin/bash" > run_core.sh - echo "docker run \\" >> run_core.sh diff --git a/scripts/ci_database_health_check.sh b/scripts/ci_database_health_check.sh new file mode 100755 index 000000000..6fe58c8d1 --- /dev/null +++ b/scripts/ci_database_health_check.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +DATABASE_PORT="8529" +DATAFED_DATABASE="sdms" +RESPONSE="" + +if [ -z "${DATAFED_DATABASE_HOST}" ] +then + local_DATAFED_DATABASE_HOST="" +else + local_DATAFED_DATABASE_HOST=$(printenv DATAFED_DATABASE_HOST) +fi + +establish_connection() { + + local URL="http://${local_DATAFED_DATABASE_HOST}:${DATABASE_PORT}/_db/${DATAFED_DATABASE}/api/0/version" + local CONNECTION="FALSE" + local count=0 + local max_count=20 + while [ "$CONNECTION" == "FALSE" ] + do + echo "Attempt $count - Checking Database Connection at: $URL" + RESPONSE=$(curl -s "$URL") + local error_code="$?" + ((count++)) + if [ "$error_code" = "0" ] + then + echo "Connection made" + CONNECTION="TRUE" + else + sleep 5 + fi + + if [ "$count" -eq "$max_count" ] + then + echo "Max attempts made exiting!" + exit 1 + fi + done +} + +foxx_provisioned() { + + local URL="http://${local_DATAFED_DATABASE_HOST}:${DATABASE_PORT}/_db/${DATAFED_DATABASE}/api/0/version" + local FOXX_PROVISIONED="FALSE" + local count=0 + local max_count=20 + while [ "$FOXX_PROVISIONED" == "FALSE" ] + do + echo "Attempt $count - Checking foxx endpoint exists: $URL" + RESPONSE=$(curl -s "$URL") + local error=$(echo "$RESPONSE" | jq -r .error) + ((count++)) + if [ "$error" == "false" ] + then + echo "Endpoint exists!" + FOXX_PROVISIONED="TRUE" + else + sleep 5 + fi + + if [ "$count" -eq "$max_count" ] + then + echo "Max attempts made exiting!" + exit 1 + fi + done +} + +establish_connection +foxx_provisioned +exit 0 From f3063058823d92c003b00a336b06423323f2b9b5 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 18:57:40 +0000 Subject: [PATCH 59/69] Fix CI pipeline health check script --- scripts/ci_database_health_check.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ci_database_health_check.sh b/scripts/ci_database_health_check.sh index 6fe58c8d1..24adb8afb 100755 --- a/scripts/ci_database_health_check.sh +++ b/scripts/ci_database_health_check.sh @@ -16,7 +16,7 @@ establish_connection() { local URL="http://${local_DATAFED_DATABASE_HOST}:${DATABASE_PORT}/_db/${DATAFED_DATABASE}/api/0/version" local CONNECTION="FALSE" local count=0 - local max_count=20 + local max_count=40 while [ "$CONNECTION" == "FALSE" ] do echo "Attempt $count - Checking Database Connection at: $URL" @@ -51,12 +51,12 @@ foxx_provisioned() { RESPONSE=$(curl -s "$URL") local error=$(echo "$RESPONSE" | jq -r .error) ((count++)) - if [ "$error" == "false" ] + if [ "$error" == "true" ] then + sleep 5 + else echo "Endpoint exists!" FOXX_PROVISIONED="TRUE" - else - sleep 5 fi if [ "$count" -eq "$max_count" ] From 29b626dfdfb18f425148b377eee54ad2eb216879 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 18:59:55 +0000 Subject: [PATCH 60/69] Fix order --- .gitlab-ci.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e14be7d74..9f28179ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,7 @@ stages: # - build-deploy-base-2 - provision-client - build + - end-to-end-setup-arango - end-to-end-setup - end-to-end-test - end-to-end-teardown @@ -385,6 +386,17 @@ build-gcs-authz: # STAGE: Integration ################################################################################ +# - echo " /dev/null &" >> run_globus.sh +end-to-end-arango-setup: + variables: + GIT_STRATEGY: clone + stage: end-to-end-setup-arango + tags: + - ci-datafed-arango + script: + - arangod --version + - ./scripts/run_arango_service.sh + end-to-end-ws-setup: variables: IMAGE_TAG: "datafed/ws-" @@ -444,6 +456,7 @@ end-to-end-core-setup: CONTAINER_LOG_FILE_PATH: "/datafed/logs" DATAFED_DATABASE_HOST: "$CI_DATAFED_DATABASE_HOST" stage: end-to-end-setup + needs: ["end-to-end-arango-setup"] tags: - ci-datafed-core - docker @@ -566,17 +579,6 @@ end-to-end-gcs-authz-setup: - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" -# - echo " /dev/null &" >> run_globus.sh -end-to-end-arango-setup: - variables: - GIT_STRATEGY: clone - stage: end-to-end-setup - tags: - - ci-datafed-arango - script: - - arangod --version - - ./scripts/run_arango_service.sh - ################################################################################ # Actual Testing ################################################################################ From 86684d34b4d6a3eddc9313450103f20150ea3408 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 19:34:16 +0000 Subject: [PATCH 61/69] Fix core --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f28179ff..b9da03b7c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -404,6 +404,7 @@ end-to-end-ws-setup: HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs" CONTAINER_LOG_FILE_PATH: "/datafed/logs" stage: end-to-end-setup + needs: ["end-to-end-core-setup"] tags: - ci-datafed-core - docker @@ -473,7 +474,7 @@ end-to-end-core-setup: - ./scripts/ci_database_health_check.sh - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - echo "#!/bin/bash" > run_core.sh - - echo "docker run \\" >> run_core.sh + - echo "docker run -d \\" >> run_core.sh - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh From 57d490f0d7cd8e30e9981a554abb197e4fc9d4b2 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 19:53:10 +0000 Subject: [PATCH 62/69] Fix ci script --- scripts/container_stop.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/container_stop.sh b/scripts/container_stop.sh index 96ee1b18d..28f03e2ba 100755 --- a/scripts/container_stop.sh +++ b/scripts/container_stop.sh @@ -74,7 +74,7 @@ then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do IMAGE="${CONTAINER_IMAGES[$i]}" - if [ "$IMAGE" = "local_CONTAINER_TAG" ] + if [ "$IMAGE" = "$local_CONTAINER_TAG" ] then echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE" docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}" @@ -86,7 +86,7 @@ then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do NAME="${CONTAINER_NAMES[$i]}" - if [ "$NAME" = "local_CONTAINER_NAME" ] + if [ "$NAME" = "$local_CONTAINER_NAME" ] then echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}" docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}" From 25c82363973cca9db0730fd5d0762eb0af5c7f2b Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 19:54:28 +0000 Subject: [PATCH 63/69] Switch to double == --- scripts/container_stop.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/container_stop.sh b/scripts/container_stop.sh index 28f03e2ba..6d0506de8 100755 --- a/scripts/container_stop.sh +++ b/scripts/container_stop.sh @@ -81,12 +81,12 @@ then fi done -elif [ "$local_NAME_FLAG_DETECTED" = "1" ] +elif [ "$local_NAME_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do NAME="${CONTAINER_NAMES[$i]}" - if [ "$NAME" = "$local_CONTAINER_NAME" ] + if [ "$NAME" == "$local_CONTAINER_NAME" ] then echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}" docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}" @@ -102,7 +102,7 @@ else fi -if [ ! "${#CONTAINER_IDS[@]}" = "0" ] +if [ ! "${#CONTAINER_IDS[@]}" == "0" ] then sleep "$SLEEP_TIME" fi @@ -112,24 +112,24 @@ CONTAINER_IDS=($(docker container ls --format "{{.ID}}")) CONTAINER_IMAGES=($(docker container ls --format "{{.Image}}")) CONTAINER_NAMES=($(docker container ls --format "{{.Names}}")) -if [ "$local_TAG_FLAG_DETECTED" = "1" ] +if [ "$local_TAG_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do IMAGE="${CONTAINER_IMAGES[$i]}" - if [ "$IMAGE" = "local_CONTAINER_TAG" ] + if [ "$IMAGE" == "local_CONTAINER_TAG" ] then echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE" exit 1 fi done -elif [ "$local_NAME_FLAG_DETECTED" = "1" ] +elif [ "$local_NAME_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do NAME="${CONTAINER_NAMES[$i]}" - if [ "$NAME" = "local_CONTAINER_NAME" ] + if [ "$NAME" == "local_CONTAINER_NAME" ] then echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}" exit 1 @@ -138,7 +138,7 @@ then else - if [ ! "${#CONTAINER_IDS[@]}" = "0" ] + if [ ! "${#CONTAINER_IDS[@]}" == "0" ] then docker container ls exit 1 From 1f300b489e3d92d89370da4cbe9a31aaf3aae285 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 14:59:29 -0500 Subject: [PATCH 64/69] Change piped tee command owner --- core/docker/entrypoint.sh | 2 +- web/docker/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 6f04fb1de..56c200de7 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -36,7 +36,7 @@ if [ "${datafed_core_exec}" = "datafed-core" ] then # Send output to log file # For this to work all commands must be passed in as a single string - su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | su datafed -c "tee $log_path/datafed-core.log" else echo "Not sending output to datafed-core.log" # If not do not by default send to log file diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 945c7cdc1..6e352c5bb 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -48,7 +48,7 @@ datafed_ws_exec=$(basename "$1") if [ "${datafed_ws_exec}" = "datafed-ws.js" ] then # Send output to log file - su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-ws.log" + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | su datafed -c "tee $log_path/datafed-ws.log" else echo "Not sending output to datafed-ws.log" # If not do not by default send to log file From 34695d53261552dc8c8f8edc78ebb2b0fc9d2fc9 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 20:01:29 +0000 Subject: [PATCH 65/69] Fix output tee command ownership --- core/docker/entrypoint.sh | 2 +- web/docker/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 2222d5c74..e8c54047c 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -32,7 +32,7 @@ if [ "${datafed_core_exec}" = "datafed-core" ] then # Send output to log file # For this to work all commands must be passed in as a single string - su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | su datafed -c "tee $log_path/datafed-core.log" else echo "Not sending output to datafed-core.log" # If not do not by default send to log file diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 98f2ee2b7..24bd0eb9e 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -39,7 +39,7 @@ datafed_ws_exec=$(basename "$1") if [ "${datafed_ws_exec}" = "datafed-ws.js" ] then # Send output to log file - su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-ws.log" + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | su datafed -c "tee $log_path/datafed-ws.log" else echo "Not sending output to datafed-ws.log" # If not do not by default send to log file From f5546a10003d2a8b707b571a29fb1c660f619ddb Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 20:03:19 +0000 Subject: [PATCH 66/69] Fix == and variables that were missing $ --- scripts/container_stop.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/container_stop.sh b/scripts/container_stop.sh index 96ee1b18d..101881b8a 100755 --- a/scripts/container_stop.sh +++ b/scripts/container_stop.sh @@ -58,7 +58,7 @@ while [ : ]; do esac done -if [ "$local_TAG_FLAG_DETECTED" = "1" ] && [ "$local_NAME_FLAG_DETECTED" = "1" ] +if [ "$local_TAG_FLAG_DETECTED" == "1" ] && [ "$local_NAME_FLAG_DETECTED" == "1" ] then echo "ERROR can only specify -t or -n cannot specify both" exit 1 @@ -69,24 +69,24 @@ CONTAINER_IDS=($(docker container ls --format "{{.ID}}")) CONTAINER_IMAGES=($(docker container ls --format "{{.Image}}")) CONTAINER_NAMES=($(docker container ls --format "{{.Names}}")) -if [ "$local_TAG_FLAG_DETECTED" = "1" ] +if [ "$local_TAG_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do IMAGE="${CONTAINER_IMAGES[$i]}" - if [ "$IMAGE" = "local_CONTAINER_TAG" ] + if [ "$IMAGE" == "$local_CONTAINER_TAG" ] then echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE" docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}" fi done -elif [ "$local_NAME_FLAG_DETECTED" = "1" ] +elif [ "$local_NAME_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do NAME="${CONTAINER_NAMES[$i]}" - if [ "$NAME" = "local_CONTAINER_NAME" ] + if [ "$NAME" == "$local_CONTAINER_NAME" ] then echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}" docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}" @@ -102,7 +102,7 @@ else fi -if [ ! "${#CONTAINER_IDS[@]}" = "0" ] +if [ ! "${#CONTAINER_IDS[@]}" == "0" ] then sleep "$SLEEP_TIME" fi @@ -112,24 +112,24 @@ CONTAINER_IDS=($(docker container ls --format "{{.ID}}")) CONTAINER_IMAGES=($(docker container ls --format "{{.Image}}")) CONTAINER_NAMES=($(docker container ls --format "{{.Names}}")) -if [ "$local_TAG_FLAG_DETECTED" = "1" ] +if [ "$local_TAG_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do IMAGE="${CONTAINER_IMAGES[$i]}" - if [ "$IMAGE" = "local_CONTAINER_TAG" ] + if [ "$IMAGE" == "$local_CONTAINER_TAG" ] then echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE" exit 1 fi done -elif [ "$local_NAME_FLAG_DETECTED" = "1" ] +elif [ "$local_NAME_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do NAME="${CONTAINER_NAMES[$i]}" - if [ "$NAME" = "local_CONTAINER_NAME" ] + if [ "$NAME" == "$local_CONTAINER_NAME" ] then echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}" exit 1 @@ -138,7 +138,7 @@ then else - if [ ! "${#CONTAINER_IDS[@]}" = "0" ] + if [ ! "${#CONTAINER_IDS[@]}" == "0" ] then docker container ls exit 1 From 4a2ae489167abebcdf42dcf85183c74106649a0a Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 15:18:53 -0500 Subject: [PATCH 67/69] Make sure web server gets core key --- .gitlab-ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9da03b7c..bac3eaecd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -405,6 +405,8 @@ end-to-end-ws-setup: CONTAINER_LOG_FILE_PATH: "/datafed/logs" stage: end-to-end-setup needs: ["end-to-end-core-setup"] + dependencies: + - end-to-end-core-setup tags: - ci-datafed-core - docker @@ -487,13 +489,16 @@ end-to-end-core-setup: - echo "-e UID=\"$USER_ID\" \\" >> run_core.sh - echo "--network host \\" >> run_core.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh - - echo "-v \"$CI_DATAFED_CORE_PUB_KEY\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh - - echo "-v \"$CI_DATAFED_CORE_PRIV_KEY\":/source/install/keys/datafed-core-key.priv \\" >> run_core.sh + - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh + - echo "-v \"${CI_DATAFED_CORE_PRIV_KEY}\":/source/install/keys/datafed-core-key.priv \\" >> run_core.sh - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_core.sh - chmod +x run_core.sh - ./run_core.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" + artifacts: + paths: + - "${CI_DATAFED_CORE_PUB_KEY}" #- echo " /dev/null &" >> run_core.sh # Repo server currently will crash on startup if it cannot connect to the core From 37e637c3913d49bcead14f96ff634254fb190b57 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 15:21:06 -0500 Subject: [PATCH 68/69] Fix web path --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bac3eaecd..1d56bcc61 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -405,8 +405,8 @@ end-to-end-ws-setup: CONTAINER_LOG_FILE_PATH: "/datafed/logs" stage: end-to-end-setup needs: ["end-to-end-core-setup"] - dependencies: - - end-to-end-core-setup + # dependencies: + # - end-to-end-core-setup tags: - ci-datafed-core - docker @@ -438,7 +438,7 @@ end-to-end-ws-setup: - echo "-e UID=\"$USER_ID\" \\" >> run_web.sh - echo "-p 443:443 \\" >> run_web.sh - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh - - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub\" \\" >> run_web.sh + - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/datafed/install/keys/datafed-core-key.pub\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}\" \\" >> run_web.sh - echo "-v \"${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}\" \\" >> run_web.sh - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" " >> run_web.sh From 6161576bd1d25a40f701332695c455c376a5be81 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 19 Feb 2024 15:47:22 -0500 Subject: [PATCH 69/69] A few small fixes --- .gitlab-ci.yml | 3 --- core/docker/entrypoint.sh | 8 ++++---- web/docker/entrypoint.sh | 8 ++++---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d56bcc61..0e3476df2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -496,9 +496,6 @@ end-to-end-core-setup: - ./run_core.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - artifacts: - paths: - - "${CI_DATAFED_CORE_PUB_KEY}" #- echo " /dev/null &" >> run_core.sh # Repo server currently will crash on startup if it cannot connect to the core diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index e8c54047c..75efe097e 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -8,11 +8,11 @@ fi SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../..) +PROJECT_ROOT=$(realpath "${SOURCE}/../..") -${PROJECT_ROOT}/scripts/generate_datafed.sh -${PROJECT_ROOT}/scripts/generate_core_config.sh -${PROJECT_ROOT}/scripts/install_core.sh +"${PROJECT_ROOT}/scripts/generate_datafed.sh" +"${PROJECT_ROOT}/scripts/generate_core_config.sh" +"${PROJECT_ROOT}/scripts/install_core.sh" log_path="$DATAFED_DEFAULT_LOG_PATH" diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 24bd0eb9e..3c84b5367 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -12,10 +12,10 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) -${PROJECT_ROOT}/scripts/generate_datafed.sh -${PROJECT_ROOT}/scripts/generate_ws_config.sh -${PROJECT_ROOT}/scripts/install_ws.sh -${PROJECT_ROOT}/scripts/dependency_versions.sh +"${PROJECT_ROOT}/scripts/generate_datafed.sh" +"${PROJECT_ROOT}/scripts/generate_ws_config.sh" +"${PROJECT_ROOT}/scripts/install_ws.sh" +. "${PROJECT_ROOT}/scripts/dependency_versions.sh" export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" export NODE_VERSION="$DATAFED_NODE_VERSION"