diff --git a/.gitignore b/.gitignore index 1ee074785..8a0b30651 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.idea/ build common/proto/common/Version.proto +compose/.env config/datafed.sh config/datafed-authz.cfg config/datafed-core.cfg diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57a3dc820..ed564b5a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,8 +4,10 @@ stages: - signal - clear-docker-cache - build-deploy-base + # - build-deploy-base-2 - provision-client - build + - end-to-end-setup-arango - end-to-end-setup - end-to-end-test - end-to-end-teardown @@ -149,33 +151,78 @@ build-repo-base: - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - docker push "${REGISTRY}/${IMAGE_TAG}" - build-gcs-base: needs: ["clear-repo-cache"] variables: GIT_SUBMODULE_STRATEGY: recursive GIT_STRATEGY: clone - DISTRO: "ubuntu:focal" + DISTRO: "ubuntu-20.04" GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal" IMAGE_TAG2: "datafed/gcs-authz-base" IMAGE_TAG3: "datafed/gcs-authz" + GCS_SUBMODULE_VERSION: "v2.6.0" stage: build-deploy-base tags: - ci-datafed-globus - docker script: - docker system prune -f - - cd external/globus-connect-server-deploy/docker - - ./docker-build-gcs.sh "${DISTRO}" - - LATEST_IMAGE=$(docker images | grep ubuntu-focal | head -n1 | awk '{print $3}') - - docker tag "$LATEST_IMAGE" "$GCS_TAG" - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - - docker push "$GCS_TAG" - - cd ../../../ # back too root of project - - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG2}" . - - docker push "${REGISTRY}/${IMAGE_TAG2}" - - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG3}" . - - docker push "${REGISTRY}/${IMAGE_TAG3}" + - > + if ! docker pull "${GCS_TAG}:${GCS_SUBMODULE_VERSION}"; then + cd external/globus-connect-server-deploy/docker + git checkout "${GCS_SUBMODULE_VERSION}" + ./docker-build-gcs.sh "${DISTRO}" + LATEST_IMAGE=$(docker images | grep ubuntu-focal | head -n1 | awk '{print $3}') + docker tag "$LATEST_IMAGE" "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + docker push "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" + docker push "$GCS_TAG" + cd ../../../ # back too root of project + else + echo "Docker image already exists in the registry. Skipping build." + fi + + #build-gcs-authz-base: + # variables: + # GIT_SUBMODULE_STRATEGY: recursive + # GIT_STRATEGY: clone + # DISTRO: "ubuntu:focal" + # GCS_TAG: "${REGISTRY}/datafed/gcs-ubuntu-focal" + # IMAGE_TAG2: "datafed/gcs-authz-base" + # IMAGE_TAG3: "datafed/gcs-authz" + # GCS_SUBMODULE_VERSION: "v2.6.0" + # stage: build-deploy-base-2 + # tags: + # - ci-datafed-globus + # - docker + # script: + # - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" + # - docker pull "$GCS_TAG:${GCS_SUBMODULE_VERSION}" + # - docker tag "$GCS_TAG:${GCS_SUBMODULE_VERSION}" "$GCS_TAG" + # - docker build -f repository/docker/Dockerfile.gcs-authz-base.ubuntu -t "${REGISTRY}/${IMAGE_TAG2}" . + # - docker push "${REGISTRY}/${IMAGE_TAG2}" + # - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG3}" . + # - docker push "${REGISTRY}/${IMAGE_TAG3}" + # rules: + # - changes: + # - scripts/generate_datafed.sh + # - scripts/dependency_versions.sh + # - scripts/dependency_install_functions.sh + # - scripts/install_authz_dependencies.sh + # - scripts/generate_authz_config.sh + # - scripts/generate_datafed.sh + # - scripts/globus/setup_globus.sh + # - scripts/globus/generate_repo_form.sh + # - repository/docker/entrypoint_authz.sh + # - CMakeLists.txt + # - cmake/* + # - repository/CMakeLists.txt + # - repository/gridftp/CMakeLists.txt + # - repository/gridftp/globus5/**/* + # - common/**/* + # - .gitlab-ci.ym + ################################################################################ # STAGE: provision client @@ -192,7 +239,13 @@ provision-client: script: - ./scripts/generate_datafed.sh - ./scripts/install_client_dependencies.sh - + rules: + - changes: + - scripts/generate_datafed.sh + - scripts/install_client_dependencies.sh + - scripts/dependency_install_functions.sh + - scripts/dependency_versions.sh + ################################################################################ # STAGE: build ################################################################################ @@ -314,7 +367,6 @@ build-repo: - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" build-gcs-authz: - needs: ["build-gcs-base"] stage: build variables: IMAGE_TAG: "datafed/gcs-authz-" @@ -330,10 +382,43 @@ build-gcs-authz: - docker build -f repository/docker/Dockerfile.gcs-authz.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" + #build-python-client: + # variables: + # IMAGE_TAG: "dlsw/datafed/python-client-" + # GIT_STRATEGY: clone + # stage: build + # tags: + # - docker + # rules: + # - changes: + # - python/* + # - scripts/* + # - common/* + # - .gitlab-ci.yml + # when: always + # script: + # - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') + # - echo "$BRANCH_LOWER" + # - ./scripts/generate_datafed.sh + # - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" + # - docker build -f python/docker/Dockerfile.python-client.ubuntu -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" . + # - docker push "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" + ################################################################################ # STAGE: Integration ################################################################################ +# - echo " /dev/null &" >> run_globus.sh +end-to-end-arango-setup: + variables: + GIT_STRATEGY: clone + stage: end-to-end-setup-arango + tags: + - ci-datafed-arango + script: + - arangod --version + - ./scripts/run_arango_service.sh + end-to-end-ws-setup: variables: IMAGE_TAG: "datafed/ws-" @@ -341,6 +426,9 @@ end-to-end-ws-setup: HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs" CONTAINER_LOG_FILE_PATH: "/datafed/logs" stage: end-to-end-setup + needs: ["end-to-end-core-setup"] + # dependencies: + # - end-to-end-core-setup tags: - ci-datafed-core - docker @@ -348,33 +436,41 @@ end-to-end-ws-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" + - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - CERT_FILE_NAME=$(basename "${CI_DATAFED_WEB_CERT_PATH}") - KEY_FILE_NAME=$(basename "${CI_DATAFED_WEB_KEY_PATH}") + - USER_ID=$(id -u) + - GROUP_ID=$(id -g) - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - docker run -d - --name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="GLOBUS_DATAFED_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$GLOBUS_DATAFED_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -e DATAFED_WEB_CERT_PATH="${CI_DATAFED_WEB_CERT_PATH}" - -e DATAFED_WEB_KEY_PATH="${CI_DATAFED_WEB_KEY_PATH}" - -e DATAFED_DEFAULT_LOG_PATH="$CONTAINER_LOG_FILE_PATH" - -p 443:443 - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "${CI_DATAFED_CORE_PUB_KEY}:/source/install/keys/datafed-core-key.pub" - -v "${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}" - -v "${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}" - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & + - echo "#!/bin/bash" > run_web.sh + - echo "docker run \\" >> run_web.sh + - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh + - echo "-e DATAFED_WEB_CERT_PATH=\"${CI_DATAFED_WEB_CERT_PATH}\" \\" >> run_web.sh + - echo "-e DATAFED_WEB_KEY_PATH=\"${CI_DATAFED_WEB_KEY_PATH}\" \\" >> run_web.sh + - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_web.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_web.sh + - echo "-p 443:443 \\" >> run_web.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh + - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}:/datafed/install/keys/datafed-core-key.pub\" \\" >> run_web.sh + - echo "-v \"${CI_DATAFED_WEB_CERT_PATH}:/datafed/install/keys/${CERT_FILE_NAME}\" \\" >> run_web.sh + - echo "-v \"${CI_DATAFED_WEB_KEY_PATH}:/datafed/install/keys/${KEY_FILE_NAME}\" \\" >> run_web.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\" " >> run_web.sh + - chmod +x run_web.sh + - ./run_web.sh + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" +#- echo " /dev/null &" >> run_web.sh +#- echo "-u ${USER_ID}:${GROUP_ID} \\" >> run_web.sh # Note here we use network=host to run the docker container this is # because we do not need it to communicate with an external port end-to-end-core-setup: @@ -383,39 +479,49 @@ end-to-end-core-setup: GIT_STRATEGY: clone HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs" CONTAINER_LOG_FILE_PATH: "/datafed/logs" + DATAFED_DATABASE_HOST: "$CI_DATAFED_DATABASE_HOST" stage: end-to-end-setup + needs: ["end-to-end-arango-setup"] tags: - ci-datafed-core - docker script: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - mkdir -p "$HOST_LOG_FILE_PATH" + - chmod o+w "${HOST_LOG_FILE_PATH}" + - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - echo "$BRANCH_LOWER" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" + - ./scripts/ci_database_health_check.sh - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - docker run -d - --name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -e DATAFED_DATABASE_PASSWORD="$CI_DATAFED_DATABASE_PASSWORD" - -e DATAFED_DATABASE_IP_ADDRESS_PORT="$CI_DATAFED_DATABASE_IP_ADDRESS_PORT" - -e DATAFED_DEFAULT_LOG_PATH="$CONTAINER_LOG_FILE_PATH" - --network host - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "$CI_DATAFED_CORE_PUB_KEY":/source/install/keys/datafed-core-key.pub - -v "$CI_DATAFED_CORE_PRIV_KEY":/source/install/keys/datafed-core-key.priv - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & + - echo "#!/bin/bash" > run_core.sh + - echo "docker run -d \\" >> run_core.sh + - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_core.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_core.sh + - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> run_core.sh + - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> run_core.sh + - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_core.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_core.sh + - echo "--network host \\" >> run_core.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh + - echo "-v \"${CI_DATAFED_CORE_PUB_KEY}\":/source/install/keys/datafed-core-key.pub \\" >> run_core.sh + - echo "-v \"${CI_DATAFED_CORE_PRIV_KEY}\":/source/install/keys/datafed-core-key.priv \\" >> run_core.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_core.sh + - chmod +x run_core.sh + - ./run_core.sh + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - +#- echo " /dev/null &" >> run_core.sh +# Repo server currently will crash on startup if it cannot connect to the core +# server. end-to-end-repo-setup: variables: IMAGE_TAG: "datafed/repo-" @@ -423,6 +529,7 @@ end-to-end-repo-setup: HOST_LOG_FILE_PATH: "$CI_PROJECT_DIR/logs" CONTAINER_LOG_FILE_PATH: "/datafed/logs" stage: end-to-end-setup + needs: ["end-to-end-core-setup"] tags: - ci-datafed-globus - docker @@ -430,26 +537,34 @@ end-to-end-repo-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" + - chmod o+w "${HOST_LOG_FILE_PATH}" + - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - docker run -d - --name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub - -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & + - echo "#!/bin/bash" > run_repo.sh + - echo "docker run \\" >> run_repo.sh + - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_repo.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_repo.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_repo.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_repo.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_repo.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_repo.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_repo.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_repo.sh + - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_repo.sh + - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_repo.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_repo.sh + - chmod +x run_repo.sh + - ./run_repo.sh + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" +# - echo " /dev/null &" >> run_repo.sh +# Requires setting up Globus Connect Server, requires firewall exceptions on +# the machine running this. end-to-end-gcs-authz-setup: variables: IMAGE_TAG: "datafed/gcs-authz-" @@ -464,36 +579,31 @@ end-to-end-gcs-authz-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" + - USER_ID=$(id -u) + - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY}" - ./scripts/container_stop.sh -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - > - docker run -d - --name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - -e DATAFED_GLOBUS_APP_SECRET="$CI_DATAFED_GLOBUS_APP_SECRET" - -e DATAFED_GLOBUS_APP_ID="$CI_DATAFED_GLOBUS_APP_ID" - -e DATAFED_ZEROMQ_SESSION_SECRET="$CI_DATAFED_ZEROMQ_SESSION_SECRET" - -e DATAFED_ZEROMQ_SYSTEM_SECRET="$CI_DATAFED_ZEROMQ_SYSTEM_SECRET" - -e DATAFED_DOMAIN="$CI_DATAFED_DOMAIN" - -v "${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}" - -v "$CI_DATAFED_REPO_PUB_KEY":/source/install/keys/datafed-repo-key.pub - -v "$CI_DATAFED_REPO_PRIV_KEY":/source/install/keys/datafed-repo-key.priv - -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" - /dev/null & + - echo "#!/bin/bash" > run_globus.sh + - echo "docker run \\" >> run_globus.sh + - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh + - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh + - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh + - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh + - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh + - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh + - echo "-e UID=\"$USER_ID\" \\" >> run_globus.sh + - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh + - echo "-v \"$CI_DATAFED_REPO_PUB_KEY\":/source/install/keys/datafed-repo-key.pub \\" >> run_globus.sh + - echo "-v \"$CI_DATAFED_REPO_PRIV_KEY\":/source/install/keys/datafed-repo-key.priv \\" >> run_globus.sh + - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}\"" >> run_globus.sh + - chmod +x run_globus.sh + - ./run_globus.sh + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" -end-to-end-arango-setup: - variables: - GIT_STRATEGY: clone - stage: end-to-end-setup - tags: - - ci-datafed-arango - script: - - arangod --version - - ./scripts/run_arango_service.sh - ################################################################################ # Actual Testing ################################################################################ diff --git a/CHANGELOG.md b/CHANGELOG.md index a6e9c6c1c..f8b11c8ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,10 @@ ## MAJOR Breaking changes ## MINOR Feature -1. [909] - Added Support for Google Analytics -2. [916] - Reworked Docker build process to be faster and produce simpler and smaller images +1. [912] - Adds initial compose file and docker files for python-client +2. [909] - Added Support for Google Analytics +3. [916] - Reworked Docker build process to be faster and produce simpler and smaller images +4. [912] - Adding working compose instance for core metadata services. ## PATCH Bug fixes/Technical Debt/Documentation 1. [914] - Improve GitHub template diff --git a/compose/README.md b/compose/README.md new file mode 100644 index 000000000..d40570f5d --- /dev/null +++ b/compose/README.md @@ -0,0 +1,33 @@ +# Compose Dev environment + +The files in this folder are incomplete but are the start for setting up a full +docker compose instance of datafed. + +```bash +./build_images_for_compose.sh +``` + +Create the .env file fill in the missing components that are required. +```bash +./generate_env.sh +``` + +Stand up the core services. +```bash +docker compose -f ./compose_core.yml up +``` + +At this point you should be able to navigate in your browser to +https://localhost + +NOTE we are using a self signed certificate so you will have to force your +browser to allow you to see the page. + +Standing up the repo services has been separated because of Globus. You will +need a machine with firewall exceptions to use it. + +## Cleaning up + +```bash +docker compose -f ./compose_core.yml down +``` diff --git a/compose/build_images_for_compose.sh b/compose/build_images_for_compose.sh new file mode 100755 index 000000000..6b472dcb2 --- /dev/null +++ b/compose/build_images_for_compose.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -euf -o pipefail + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath "${SOURCE}/../") + +docker build \ + -f "${PROJECT_ROOT}/docker/Dockerfile.dependencies" \ + "${PROJECT_ROOT}" \ + -t datafed-dependencies:latest +docker build \ + -f "${PROJECT_ROOT}/docker/Dockerfile.runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-runtime:latest +docker build -f \ + "${PROJECT_ROOT}/core/docker/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-core:latest +docker build -f \ + "${PROJECT_ROOT}/web/docker/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-web:latest +docker build -f \ + "${PROJECT_ROOT}/repository/docker/Dockerfile" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-repo:latest +docker build -f \ + "${PROJECT_ROOT}/docker/Dockerfile.foxx" \ + --build-arg DEPENDENCIES="datafed-dependencies" \ + --build-arg RUNTIME="datafed-runtime" \ + "${PROJECT_ROOT}" \ + -t datafed-foxx:latest + diff --git a/compose/compose_core.yml b/compose/compose_core.yml new file mode 100644 index 000000000..18d9b6c62 --- /dev/null +++ b/compose/compose_core.yml @@ -0,0 +1,91 @@ +version: '3.9' + + +services: + + datafed-web: + depends_on: ["datafed-core"] + environment: + DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" + DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" + DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" + DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_HTTPS_SERVER_PORT: "${DATAFED_HTTPS_SERVER_PORT}" + DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" + DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" + DATAFED_CORE_ADDRESS_PORT_INTERNAL: "datafed-core:7513" + UID: "${DATAFED_UID}" + image: datafed-web:latest + ports: + - 443:443 # This must be the same port that is mapped to the host for redirects to work + volumes: + - ./keys:/opt/datafed/keys + networks: + - datafed-internal + + datafed-core: + image: datafed-core:latest + depends_on: + datafed-foxx: + condition: service_healthy + environment: + DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}" + DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}" + DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" + DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" + DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" + UID: "${DATAFED_UID}" + DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" + DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" + DATAFED_DATABASE_IP_ADDRESS_PORT: "${DATAFED_DATABASE_IP_ADDRESS}:${DATAFED_DATABASE_PORT}" + ports: + - 7513 # Communication web server + - 7512 # Secure core server communication + volumes: + - ./keys:/opt/datafed/keys + networks: + - datafed-core-secure-api + - datafed-internal + + datafed-foxx: + image: datafed-foxx:latest + depends_on: ["arango"] + environment: + DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" + DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" + UID: "${DATAFED_UID}" + DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" + DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}" + DATAFED_DATABASE_HOST: "arango" + healthcheck: + test: ["CMD", "/bin/bash", "-c", "[ -f /tmp/.foxx_is_installed ]"] + interval: 10s + timeout: 5s + retries: 10 + networks: + - datafed-internal + + arango: + image: arangodb + environment: + ARANGO_ROOT_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" + ports: + - 8529:8529 # Arangodb web UI + networks: + - datafed-internal + +volumes: + keys: + +networks: + datafed-core-secure-api: + driver: bridge + datafed-internal: + driver: bridge + diff --git a/compose/generate_env.sh b/compose/generate_env.sh new file mode 100755 index 000000000..b09f76da1 --- /dev/null +++ b/compose/generate_env.sh @@ -0,0 +1,141 @@ +#!/bin/bash +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath "${SOURCE}/..") + +if [ -f ".env" ] +then + echo ".env already exist! Will not overwrite!" + exit 1 +fi + +local_DATAFED_WEB_KEY_DIR="${PROJECT_ROOT}/compose/keys" +if [ ! -d "$local_DATAFED_WEB_KEY_DIR" ] +then + mkdir -p "$local_DATAFED_WEB_KEY_DIR" +fi + +if [ -z "${DATAFED_COMPOSE_DOMAIN}" ] +then + local_DATAFED_COMPOSE_DOMAIN="localhost" +else + local_DATAFED_COMPOSE_DOMAIN=$(printenv DATAFED_COMPOSE_DOMAIN) +fi + +local_DATAFED_WEB_CERT_NAME="cert.crt" +local_DATAFED_WEB_KEY_NAME="cert.key" + +local_DATAFED_WEB_CERT_PATH="${local_DATAFED_WEB_KEY_DIR}/${local_DATAFED_WEB_CERT_NAME}" +local_DATAFED_WEB_CSR_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.csr" +local_DATAFED_WEB_KEY_PATH="${local_DATAFED_WEB_KEY_DIR}/${local_DATAFED_WEB_KEY_NAME}" + +if [ ! -e "$local_DATAFED_WEB_CERT_PATH" ] || [ ! -e "$local_DATAFED_WEB_KEY_PATH" ] +then + if [ -e "$local_DATAFED_WEB_CERT_PATH" ] + then + rm "${local_DATAFED_WEB_CERT_PATH}" + fi + if [ -e "$local_DATAFED_WEB_KEY_PATH" ] + then + rm "${local_DATAFED_WEB_KEY_PATH}" + fi + if [ -e "$local_DATAFED_WEB_CSR_PATH" ] + then + rm "${local_DATAFED_WEB_CSR_PATH}" + fi + openssl genrsa -out "$local_DATAFED_WEB_KEY_PATH" 2048 + openssl req -new -key "$local_DATAFED_WEB_KEY_PATH" \ + -out "${local_DATAFED_WEB_CSR_PATH}" \ + -subj "/C=US/ST=TN/L=Oak Ridge/O=ORNL/OU=DLT/CN=${local_DATAFED_COMPOSE_DOMAIN}" + openssl x509 -req -days 3650 \ + -in "${local_DATAFED_WEB_CSR_PATH}" \ + -signkey "$local_DATAFED_WEB_KEY_PATH" \ + -out "$local_DATAFED_WEB_CERT_PATH" +fi + +if [ -z "${DATAFED_COMPOSE_USER89_PASSWORD}" ] +then + local_DATAFED_COMPOSE_USER89_PASSWORD="" # For End to end testing +else + local_DATAFED_COMPOSE_USER89_PASSWORD=$(printenv DATAFED_COMPOSE_USER89_PASSWORD) +fi +if [ -z "${DATAFED_COMPOSE_REPO_FORM_PATH}" ] +then + local_DATAFED_COMPOSE_REPO_FORM_PATH="" # Where the repo form is located also needed for testing +else + local_DATAFED_COMPOSE_REPO_FORM_PATH=$(printenv DATAFED_COMPOSE_REPO_FORM_PATH) +fi +if [ -z "${DATAFED_COMPOSE_GLOBUS_APP_SECRET}" ] +then + local_DATAFED_COMPOSE_GLOBUS_APP_SECRET="" +else + local_DATAFED_COMPOSE_GLOBUS_APP_SECRET=$(printenv DATAFED_COMPOSE_GLOBUS_APP_SECRET) +fi +if [ -z "${DATAFED_COMPOSE_GLOBUS_APP_ID}" ] +then + local_DATAFED_COMPOSE_GLOBUS_APP_ID="" +else + local_DATAFED_COMPOSE_GLOBUS_APP_ID=$(printenv DATAFED_COMPOSE_GLOBUS_APP_ID) +fi +if [ -z "${DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET}" ] +then + local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET="" +else + local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET=$(printenv DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET) +fi +if [ -z "${DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET}" ] +then + local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET="" +else + local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET=$(printenv DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET) +fi +if [ -z "${DATAFED_COMPOSE_HTTPS_SERVER_PORT}" ] +then + local_DATAFED_COMPOSE_HTTPS_SERVER_PORT="443" +else + local_DATAFED_COMPOSE_HTTPS_SERVER_PORT=$(printenv DATAFED_COMPOSE_HTTPS_SERVER_PORT) +fi +if [ -z "${DATAFED_COMPOSE_CONTAINER_LOG_PATH}" ] +then + local_DATAFED_COMPOSE_CONTAINER_LOG_PATH="/opt/datafed/logs" +else + local_DATAFED_COMPOSE_CONTAINER_LOG_PATH=$(printenv DATAFED_COMPOSE_CONTAINER_LOG_PATH) +fi +if [ -z "${DATAFED_COMPOSE_DATABASE_PASSWORD}" ] +then + local_DATAFED_COMPOSE_DATABASE_PASSWORD="butterscotch" +else + local_DATAFED_COMPOSE_DATABASE_PASSWORD=$(printenv DATAFED_COMPOSE_DATABASE_PASSWORD) +fi + +if [ -z "${DATAFED_COMPOSE_DATABASE_IP_ADDRESS}" ] +then + local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS="http://arango" +else + local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS=$(printenv DATAFED_COMPOSE_DATABASE_IP_ADDRESS) +fi + +if [ -z "${DATAFED_COMPOSE_DATABASE_PORT}" ] +then + local_DATAFED_COMPOSE_DATABASE_PORT="8529" +else + local_DATAFED_COMPOSE_DATABASE_PORT=$(printenv DATAFED_COMPOSE_DATABASE_PORT) +fi + +cat << EOF > ".env" +DATAFED_USER89_PASSWORD="${local_DATAFED_COMPOSE_USER89_PASSWORD}" # For End to end testing +DATAFED_REPO_FORM_PATH="${local_DATAFED_COMPOSE_REPO_FORM_PATH}" # Where the repo form is located also needed for testing +DATAFED_GLOBUS_APP_SECRET="${local_DATAFED_COMPOSE_GLOBUS_APP_SECRET}" +DATAFED_GLOBUS_APP_ID="${local_DATAFED_COMPOSE_GLOBUS_APP_ID}" +DATAFED_ZEROMQ_SESSION_SECRET="${local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET}" +DATAFED_ZEROMQ_SYSTEM_SECRET="${local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET}" +DATAFED_DOMAIN="${local_DATAFED_COMPOSE_DOMAIN}" +DATAFED_HTTPS_SERVER_PORT="${local_DATAFED_COMPOSE_HTTPS_SERVER_PORT}" +DATAFED_WEB_CERT_PATH="/opt/datafed/keys/${local_DATAFED_WEB_CERT_NAME}" +DATAFED_WEB_KEY_PATH="/opt/datafed/keys/${local_DATAFED_WEB_KEY_NAME}" +DATAFED_CONTAINER_LOG_PATH="${local_DATAFED_COMPOSE_CONTAINER_LOG_PATH}" +DATAFED_DATABASE_PASSWORD="${local_DATAFED_COMPOSE_DATABASE_PASSWORD}" +DATAFED_DATABASE_IP_ADDRESS="${local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS}" +DATAFED_DATABASE_PORT="${local_DATAFED_COMPOSE_DATABASE_PORT}" +DATAFED_UID="$(id -u)" +EOF diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index 79b0120f9..f11d17edd 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -7,15 +7,18 @@ ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" +#ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +#ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS core-build +SHELL ["/bin/bash", "-c"] + ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH @@ -49,6 +52,8 @@ RUN cmake --build build --target install FROM ${RUNTIME} AS core +SHELL ["/bin/bash", "-c"] + ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH ARG BUILD_DIR @@ -56,6 +61,7 @@ ARG LIB_DIR # The above should also be available at runtime ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" @@ -81,5 +87,13 @@ COPY --chown=datafed:root ./cmake/Version.cmake ${DATAFED_DIR}/cmake COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh ${BUILD_DIR}/core/entrypoint.sh COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core -ENTRYPOINT ["${BUILD_DIR}/core/entrypoint.sh"] -CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"] +USER root + +RUN chown -R datafed:root /datafed +RUN chown -R datafed:root /opt + +#SHELL ["/bin/bash"] +# NOTE docker does not recognize curly braces around variables in entrypoint +# and CMD +ENTRYPOINT [ "/datafed/source/core/entrypoint.sh", "/opt/datafed/core/datafed-core","--cfg", "/opt/datafed/core/datafed-core.cfg"] +#CMD ["$DATAFED_INSTALL_PATH/core/datafed-core","--cfg","$DATAFED_INSTALL_PATH/core/datafed-core.cfg"] diff --git a/core/docker/Dockerfile.core-base.ubuntu b/core/docker/Dockerfile.core-base.ubuntu index 9f3eed307..65fb32106 100644 --- a/core/docker/Dockerfile.core-base.ubuntu +++ b/core/docker/Dockerfile.core-base.ubuntu @@ -11,6 +11,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo RUN ${BUILD_DIR}/scripts/generate_datafed.sh diff --git a/core/docker/Dockerfile.core.ubuntu b/core/docker/Dockerfile.core.ubuntu index 60a8891b5..8f8d6ae98 100644 --- a/core/docker/Dockerfile.core.ubuntu +++ b/core/docker/Dockerfile.core.ubuntu @@ -1,4 +1,6 @@ -FROM camden.ornl.gov/datafed/core-base:latest as build +ARG BASE_CORE_IMAGE="camden.ornl.gov/datafed/core-base:latest" + +FROM "$BASE_CORE_IMAGE" as build ARG DATAFED_DIR="/datafed" ARG BUILD_DIR="/datafed/source" @@ -7,7 +9,7 @@ ARG DATAFED_INSTALL_PATH="/datafed/install" # The above should also be available at runtime ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" -ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH/core" +ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" RUN mkdir -p ${BUILD_DIR}/logs RUN mkdir -p ${DATAFED_INSTALL_PATH} @@ -50,7 +52,9 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ cmake --build build -j 8 RUN cmake --build build --target install -WORKDIR ${DATAFED_INSTALL_PATHl}/core +WORKDIR ${DATAFED_INSTALL_PATH}/core + +USER root ENTRYPOINT ["/datafed/source/core/docker/entrypoint.sh"] diff --git a/core/docker/entrypoint.sh b/core/docker/entrypoint.sh index 0b45898f2..efdd65c2c 100755 --- a/core/docker/entrypoint.sh +++ b/core/docker/entrypoint.sh @@ -2,16 +2,30 @@ set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../..) +PROJECT_ROOT=$(realpath "${SOURCE}/../..") -${PROJECT_ROOT}/scripts/generate_datafed.sh -${PROJECT_ROOT}/scripts/generate_core_config.sh -${PROJECT_ROOT}/scripts/install_core.sh +env + +"${PROJECT_ROOT}/scripts/generate_datafed.sh" +"${PROJECT_ROOT}/scripts/generate_core_config.sh" +"${PROJECT_ROOT}/scripts/install_core.sh" log_path="$DATAFED_DEFAULT_LOG_PATH" +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + +echo "Number of arguments is $#" +echo "arguments are $@" + if [ "$#" -eq 0 ]; then echo "No arguments were passed, running bash" exec "bash" @@ -22,9 +36,13 @@ datafed_core_exec=$(basename "$1") if [ "${datafed_core_exec}" = "datafed-core" ] then # Send output to log file - "$@" 2>&1 | tee "$log_path/datafed-core.log" + # For this to work all commands must be passed in as a single string + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | su datafed -c "tee $log_path/datafed-core.log" else echo "Not sending output to datafed-core.log" # If not do not by default send to log file - exec "$@" + su datafed -c '"$@"' -- argv0 "$@" fi + +echo "Give a few minutes to debug the problem" +sleep 10000 diff --git a/dockerfiles/Dockerfile.base b/docker/Dockerfile.base similarity index 100% rename from dockerfiles/Dockerfile.base rename to docker/Dockerfile.base diff --git a/dockerfiles/Dockerfile.core b/docker/Dockerfile.core similarity index 100% rename from dockerfiles/Dockerfile.core rename to docker/Dockerfile.core diff --git a/dockerfiles/Dockerfile.datafed-centos7 b/docker/Dockerfile.datafed-centos7 similarity index 100% rename from dockerfiles/Dockerfile.datafed-centos7 rename to docker/Dockerfile.datafed-centos7 diff --git a/dockerfiles/dependencies.Dockerfile b/docker/Dockerfile.dependencies similarity index 81% rename from dockerfiles/dependencies.Dockerfile rename to docker/Dockerfile.dependencies index 602ebd524..6db15262b 100644 --- a/dockerfiles/dependencies.Dockerfile +++ b/docker/Dockerfile.dependencies @@ -1,15 +1,15 @@ ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DEPENDENCIES_INSTALL_PATH/nvm" ARG LIB_DIR="/usr/local/lib" FROM ubuntu:focal -ARG NVM_DIR +SHELL ["/bin/bash", "-c"] +#ARG NVM_DIR ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH @@ -18,6 +18,7 @@ ARG LIB_DIR ENV BUILD_DIR="${BUILD_DIR}" ENV DATAFED_DIR="${DATAFED_DIR}" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV LIB_DIR="${LIB_DIR}" RUN mkdir -p ${BUILD_DIR} @@ -32,14 +33,17 @@ WORKDIR ${BUILD_DIR} COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo # run build scripts +RUN ${BUILD_DIR}/scripts/generate_datafed.sh RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/ diff --git a/docker/Dockerfile.foxx b/docker/Dockerfile.foxx new file mode 100644 index 000000000..38d5c15c7 --- /dev/null +++ b/docker/Dockerfile.foxx @@ -0,0 +1,57 @@ +# NOTE this image must be built with respect to the base of the project i.e. +# cd ${PROJECT_ROOT} or cd DataFed +# docker build -f core/docker/Dockerfile . + +ARG BUILD_BASE="ubuntu:focal" +ARG DEPENDENCIES="dependencies" +ARG RUNTIME="runtime" +ARG DATAFED_DIR="/datafed" +ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" +ARG BUILD_DIR="$DATAFED_DIR/source" +ARG NVM_DIR="$DATAFED_DIR/.nvm" +ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +ARG LIB_DIR="/usr/local/lib" + +FROM ${DEPENDENCIES} AS foxx-build + +ARG DATAFED_DIR +ARG BUILD_DIR +ARG DATAFED_INSTALL_PATH + +ENV BUILD_DIR="${BUILD_DIR}" +ENV DATAFED_DIR="${DATAFED_DIR}" + +COPY ./core/CMakeLists.txt ${BUILD_DIR}/core/CMakeLists.txt +COPY ./CMakeLists.txt ${BUILD_DIR} +COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ +COPY ./cmake ${BUILD_DIR}/cmake +COPY ./docker/entrypoint_foxx.sh ${BUILD_DIR}/docker/entrypoint_foxx.sh +COPY ./core/database ${BUILD_DIR}/core/database +COPY ./scripts/install_foxx.sh ${BUILD_DIR}/scripts/ + +# Create datafed user, prefer more secure login options than password +# Recommended to mount ssh public key on run +RUN adduser --disabled-password --gecos "" datafed + +RUN mkdir -p ${DATAFED_DIR} +RUN mkdir -p /opt/datafed +RUN mkdir -p /var/log/datafed +RUN chown -R datafed:root /opt/datafed +RUN chown -R datafed:root /var/log/datafed +RUN chown -R datafed:root ${DATAFED_DIR} + +RUN curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key; apt-key add - < Release.key +#RUN wget -q https://www.arangodb.com/repositories/arangodb38/DEBIAN/Release.key -O- | apt-key add - +RUN echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list +RUN apt update +RUN apt install arangodb3-client -y +RUN arangosh --version + +USER root + +WORKDIR ${DATAFED_DIR} + +ENTRYPOINT ${BUILD_DIR}/docker/entrypoint_foxx.sh diff --git a/dockerfiles/Dockerfile.kickstart-centos7 b/docker/Dockerfile.kickstart-centos7 similarity index 100% rename from dockerfiles/Dockerfile.kickstart-centos7 rename to docker/Dockerfile.kickstart-centos7 diff --git a/dockerfiles/runtime.Dockerfile b/docker/Dockerfile.runtime similarity index 79% rename from dockerfiles/runtime.Dockerfile rename to docker/Dockerfile.runtime index a372c683d..86e142731 100644 --- a/dockerfiles/runtime.Dockerfile +++ b/docker/Dockerfile.runtime @@ -2,15 +2,17 @@ ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" +#ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +#ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" ARG LIB_DIR="/usr/local/lib" FROM ubuntu:focal +SHELL ["/bin/bash", "-c"] ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH +ARG DATAFED_DEPENDENCIES_INSTALL_PATH ARG BUILD_DIR ENV BUILD_DIR="${BUILD_DIR}" diff --git a/dockerfiles/Dockerfile.web b/docker/Dockerfile.web similarity index 100% rename from dockerfiles/Dockerfile.web rename to docker/Dockerfile.web diff --git a/dockerfiles/README.md b/docker/README.md similarity index 100% rename from dockerfiles/README.md rename to docker/README.md diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh new file mode 100755 index 000000000..128f0a9d1 --- /dev/null +++ b/docker/entrypoint_foxx.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -euf -o pipefail + +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath "${SOURCE}/../") + +log_path="$DATAFED_DEFAULT_LOG_PATH" +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + +cd "${PROJECT_ROOT}" +# Check to see if foxx has previously been installed +install_flag="/tmp/.foxx_is_installed" +if [ ! -f "$install_flag" ] +then + su datafed -c "${PROJECT_ROOT}/scripts/generate_datafed.sh" + + su datafed -c "cmake -S. -B build \ + -DBUILD_REPO_SERVER=False \ + -DBUILD_COMMON=False \ + -DBUILD_AUTHZ=False \ + -DBUILD_CORE_SERVER=False \ + -DBUILD_WEB_SERVER=False \ + -DBUILD_DOCS=False \ + -DBUILD_PYTHON_CLIENT=False \ + -DBUILD_FOXX=True \ + -DINSTALL_FOXX=True" + + + su datafed -c "cmake --build build" + + # Give arango container a minute to initialize + # should be replaced with health check at some point + sleep 5 + su datafed -c "cmake --build build --target install" + + #if [ "$#" -eq 0 ]; then + # echo "No arguments were passed, running bash" + # exec "bash" + # exit 0 + #fi + touch "$install_flag" +fi + +sleep 1000 +#datafed_core_exec=$(basename "$1") +#if [ "${datafed_core_exec}" = "datafed-core" ] +#then +# # Send output to log file +# # For this to work all commands must be passed in as a single string +# su datafed -c '"$@"' -- argv0 "$@" 2>&1 | tee "$log_path/datafed-core.log" +#else +# echo "Not sending output to datafed-core.log" +# # If not do not by default send to log file +# su datafed -c '"$@"' -- argv0 "$@" +#fi diff --git a/python/datafed_pkg/datafed/CommandLib.py b/python/datafed_pkg/datafed/CommandLib.py index 3ee98befe..a4016e52b 100644 --- a/python/datafed_pkg/datafed/CommandLib.py +++ b/python/datafed_pkg/datafed/CommandLib.py @@ -1660,6 +1660,7 @@ def _buildSearchRequest( offset=0, count=20, ): + if coll_mode and (schema is not None or meta is not None or meta_err): raise Exception( "Cannot specify metadata terms when searching for collection." diff --git a/python/docker/Dockerfile.python-client-base.ubuntu b/python/docker/Dockerfile.python-client-base.ubuntu new file mode 100644 index 000000000..3395b072a --- /dev/null +++ b/python/docker/Dockerfile.python-client-base.ubuntu @@ -0,0 +1,15 @@ +FROM ubuntu:focal as build + +ARG DATAFED_DIR="/datafed" +ARG BUILD_DIR="/datafed/source" + +RUN mkdir -p ${BUILD_DIR} + +WORKDIR ${BUILD_DIR} + +COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/install_python_client_dependencies.sh ${BUILD_DIR}/scripts/ + +RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_python_client_dependencies.sh diff --git a/python/docker/Dockerfile.python-client.ubuntu b/python/docker/Dockerfile.python-client.ubuntu new file mode 100644 index 000000000..b22eddc59 --- /dev/null +++ b/python/docker/Dockerfile.python-client.ubuntu @@ -0,0 +1,59 @@ +FROM code.ornl.gov:4567/dlsw/datafed/python-client-base:latest as build + +ARG DATAFED_DIR="/datafed" +ARG BUILD_DIR="/datafed/source" +ARG DATAFED_INSTALL_PATH="/datafed/install" + +# The above should also be available at runtime +ENV DATAFED_DIR="$DATAFED_DIR" +ENV BUILD_DIR="$BUILD_DIR" +ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH/python-client" + +RUN mkdir -p ${BUILD_DIR}/logs +RUN mkdir -p ${DATAFED_INSTALL_PATH} +RUN mkdir -p ${DATAFED_INSTALL_PATH}/keys + +# For communicating with repo server +# EXPOSE 7512 +# For listening to web server +# EXPOSE 7513 +# ArangoDB port +# EXPOSE 8529 + +# Create datafed user, prefer more secure login options than password +# Recommended to mount ssh public key on run +RUN adduser --disabled-password --gecos "" datafed + +WORKDIR ${BUILD_DIR} + +COPY ./common ${BUILD_DIR}/common +COPY ./CMakeLists.txt ${BUILD_DIR} +COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ +COPY ./cmake ${BUILD_DIR}/cmake +COPY ./python ${BUILD_DIR}/python + +# All files should be owned by the datafed user +RUN chown -R datafed:datafed ${DATAFED_DIR} + +USER datafed + +RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ + cmake -S. -B build -DBUILD_REPO_SERVER=False -DBUILD_AUTHZ=False \ + -DBUILD_CORE_SERVER=False -DBUILD_WEB_SERVER=False \ + -DBUILD_DOCS=False -DBUILD_PYTHON_CLIENT=True \ + -DBUILD_FOXX=False -DENABLE_UNIT_TESTS=False \ + -DBUILD_TESTS=False -DBUILD_COMMON=False &&\ + cmake --build build -j 8 +RUN cmake --build build --target pydatafed + +WORKDIR ${BUILD_DIR}/python/datafed_pkg + +# Install datafed client +RUN python3 -m pip install . + +WORKDIR /home/datafed + +#ENTRYPOINT ["/bin/bash"] +ENTRYPOINT ["/datafed/source/python/docker/entrypoint.sh"] + diff --git a/python/docker/entrypoint.sh b/python/docker/entrypoint.sh new file mode 100755 index 000000000..f9573344f --- /dev/null +++ b/python/docker/entrypoint.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euf -o pipefail + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") + +# Entry point file expects that the directory where the DataFed source file +# is passed in as the first argument + +echo "SOURCE BUILD DIR $BUILD_DIR" + +# Here we will rerun datafed configuration script to create a new set of +# default variables that are useful for setting up the DataFed Python client +# ini file + +"${BUILD_DIR}/scripts/generate_datafed.sh" +source "${BUILD_DIR}/config/datafed.sh" + +mkdir -p "/home/datafed/.datafed" + +# At this point we will create an ini file +cat << EOF > "/home/datafed/.datafed/datafed-client.ini" +[server] +host = ${DATAFED_DOMAIN} +port = ${DATAFED_SERVER_PORT} +config_dir = /home/datafed/.datafed + +[client] +config_dir = /home/datafed/.datafed + +EOF + +if [ "$#" -eq 0 ]; then + echo "No arguments were passed, running bash" + exec "/home/datafed/.local/bin/datafed --cfg /home/datafed/.datafed/datafed-client.ini" +fi + +"$@" + diff --git a/repository/docker/Dockerfile b/repository/docker/Dockerfile index 73a73ff47..dbadfeb3f 100644 --- a/repository/docker/Dockerfile +++ b/repository/docker/Dockerfile @@ -7,15 +7,17 @@ ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" +#ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +#ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS repo-build +SHELL ["/bin/bash", "-c"] ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH @@ -50,6 +52,7 @@ RUN cmake --build build --target install FROM ${RUNTIME} AS repo +SHELL ["/bin/bash", "-c"] ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH ARG BUILD_DIR @@ -57,6 +60,7 @@ ARG LIB_DIR # The above should also be available at runtime ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" diff --git a/repository/docker/Dockerfile.gcs-authz-base.ubuntu b/repository/docker/Dockerfile.gcs-authz-base.ubuntu index fd62494e4..79b4660f6 100644 --- a/repository/docker/Dockerfile.gcs-authz-base.ubuntu +++ b/repository/docker/Dockerfile.gcs-authz-base.ubuntu @@ -11,6 +11,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo RUN ${BUILD_DIR}/scripts/generate_datafed.sh diff --git a/repository/docker/Dockerfile.gcs-authz.ubuntu b/repository/docker/Dockerfile.gcs-authz.ubuntu index 56ec69e2d..e71bb01c7 100644 --- a/repository/docker/Dockerfile.gcs-authz.ubuntu +++ b/repository/docker/Dockerfile.gcs-authz.ubuntu @@ -33,6 +33,7 @@ WORKDIR ${BUILD_DIR} COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/generate_authz_config.sh ${BUILD_DIR}/scripts/generate_authz_config.sh COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ COPY ./CMakeLists.txt ${BUILD_DIR} COPY ./cmake ${BUILD_DIR}/cmake COPY ./repository/CMakeLists.txt ${BUILD_DIR}/repository/CMakeLists.txt diff --git a/repository/docker/Dockerfile.repo-base.ubuntu b/repository/docker/Dockerfile.repo-base.ubuntu index 4c6b40d21..b7a84e1a5 100644 --- a/repository/docker/Dockerfile.repo-base.ubuntu +++ b/repository/docker/Dockerfile.repo-base.ubuntu @@ -11,6 +11,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo RUN ${BUILD_DIR}/scripts/generate_datafed.sh diff --git a/repository/docker/Dockerfile.repo.ubuntu b/repository/docker/Dockerfile.repo.ubuntu index 4e663c37d..a980a41bd 100644 --- a/repository/docker/Dockerfile.repo.ubuntu +++ b/repository/docker/Dockerfile.repo.ubuntu @@ -1,4 +1,6 @@ -FROM camden.ornl.gov/datafed/repo-base:latest as build +ARG REPO_BASE_IMAGE="camden.ornl.gov/datafed/repo-base:latest" + +FROM "$REPO_BASE_IMAGE" as build ARG DATAFED_DIR="/datafed" ARG BUILD_DIR="/datafed/source" @@ -9,9 +11,9 @@ ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -RUN mkdir -p ${BUILD_DIR} -RUN mkdir -p ${BUILD_DIR}/repository/server -RUN mkdir -p ${DATAFED_INSTALL_PATH}/keys +RUN mkdir -p "${BUILD_DIR}" +RUN mkdir -p "${BUILD_DIR}/repository/server" +RUN mkdir -p "${DATAFED_INSTALL_PATH}/keys" # This port is needed to communicate with the DataFed core server EXPOSE 7512 @@ -40,6 +42,8 @@ COPY ./repository/docker/entrypoint_repo.sh ${BUILD_DIR}/repository/docker/ # All files should be owned by the datafed user RUN chown -R datafed:datafed ${DATAFED_DIR} +USER datafed + RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ cmake -S. -B build -DBUILD_REPO_SERVER=True -DBUILD_AUTHZ=False \ -DBUILD_CORE_SERVER=False -DBUILD_WEB_SERVER=False \ @@ -48,6 +52,10 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ cmake --build build RUN cmake --build build --target install +ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" + +USER root + ENTRYPOINT ["/datafed/source/repository/docker/entrypoint_repo.sh"] CMD ["/datafed/install/repo/datafed-repo","--cfg","/datafed/install/repo/datafed-repo.cfg"] diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index 3a072d107..3be1b602a 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -4,6 +4,11 @@ # To run it just pass in /entrypoint.sh as an argument set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) @@ -12,4 +17,11 @@ GLOBUS_CLIENT_ID="$DATAFED_GLOBUS_APP_ID" ${PROJECT_ROOT}/scripts/generate_datafed.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh -exec "$@" +log_path="$DATAFED_DEFAULT_LOG_PATH" + +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + +su datafed -c '"$@"' -- argv0 "$@" diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index 4eebc6225..997da30f2 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -2,6 +2,10 @@ set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u $UID datafed +fi + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) @@ -10,4 +14,11 @@ ${PROJECT_ROOT}/scripts/generate_datafed.sh ${PROJECT_ROOT}/scripts/generate_repo_config.sh ${PROJECT_ROOT}/scripts/install_repo.sh -exec "$@" +log_path="$DATAFED_DEFAULT_LOG_PATH" + +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi + +su datafed -c '"$@"' -- argv0 "$@" diff --git a/scripts/ci_database_health_check.sh b/scripts/ci_database_health_check.sh new file mode 100755 index 000000000..24adb8afb --- /dev/null +++ b/scripts/ci_database_health_check.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +DATABASE_PORT="8529" +DATAFED_DATABASE="sdms" +RESPONSE="" + +if [ -z "${DATAFED_DATABASE_HOST}" ] +then + local_DATAFED_DATABASE_HOST="" +else + local_DATAFED_DATABASE_HOST=$(printenv DATAFED_DATABASE_HOST) +fi + +establish_connection() { + + local URL="http://${local_DATAFED_DATABASE_HOST}:${DATABASE_PORT}/_db/${DATAFED_DATABASE}/api/0/version" + local CONNECTION="FALSE" + local count=0 + local max_count=40 + while [ "$CONNECTION" == "FALSE" ] + do + echo "Attempt $count - Checking Database Connection at: $URL" + RESPONSE=$(curl -s "$URL") + local error_code="$?" + ((count++)) + if [ "$error_code" = "0" ] + then + echo "Connection made" + CONNECTION="TRUE" + else + sleep 5 + fi + + if [ "$count" -eq "$max_count" ] + then + echo "Max attempts made exiting!" + exit 1 + fi + done +} + +foxx_provisioned() { + + local URL="http://${local_DATAFED_DATABASE_HOST}:${DATABASE_PORT}/_db/${DATAFED_DATABASE}/api/0/version" + local FOXX_PROVISIONED="FALSE" + local count=0 + local max_count=20 + while [ "$FOXX_PROVISIONED" == "FALSE" ] + do + echo "Attempt $count - Checking foxx endpoint exists: $URL" + RESPONSE=$(curl -s "$URL") + local error=$(echo "$RESPONSE" | jq -r .error) + ((count++)) + if [ "$error" == "true" ] + then + sleep 5 + else + echo "Endpoint exists!" + FOXX_PROVISIONED="TRUE" + fi + + if [ "$count" -eq "$max_count" ] + then + echo "Max attempts made exiting!" + exit 1 + fi + done +} + +establish_connection +foxx_provisioned +exit 0 diff --git a/scripts/container_stop.sh b/scripts/container_stop.sh index 96ee1b18d..101881b8a 100755 --- a/scripts/container_stop.sh +++ b/scripts/container_stop.sh @@ -58,7 +58,7 @@ while [ : ]; do esac done -if [ "$local_TAG_FLAG_DETECTED" = "1" ] && [ "$local_NAME_FLAG_DETECTED" = "1" ] +if [ "$local_TAG_FLAG_DETECTED" == "1" ] && [ "$local_NAME_FLAG_DETECTED" == "1" ] then echo "ERROR can only specify -t or -n cannot specify both" exit 1 @@ -69,24 +69,24 @@ CONTAINER_IDS=($(docker container ls --format "{{.ID}}")) CONTAINER_IMAGES=($(docker container ls --format "{{.Image}}")) CONTAINER_NAMES=($(docker container ls --format "{{.Names}}")) -if [ "$local_TAG_FLAG_DETECTED" = "1" ] +if [ "$local_TAG_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do IMAGE="${CONTAINER_IMAGES[$i]}" - if [ "$IMAGE" = "local_CONTAINER_TAG" ] + if [ "$IMAGE" == "$local_CONTAINER_TAG" ] then echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE" docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}" fi done -elif [ "$local_NAME_FLAG_DETECTED" = "1" ] +elif [ "$local_NAME_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do NAME="${CONTAINER_NAMES[$i]}" - if [ "$NAME" = "local_CONTAINER_NAME" ] + if [ "$NAME" == "$local_CONTAINER_NAME" ] then echo "Stopping ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}" docker container stop --time "$SLEEP_TIME" "${CONTAINER_IDS[$i]}" @@ -102,7 +102,7 @@ else fi -if [ ! "${#CONTAINER_IDS[@]}" = "0" ] +if [ ! "${#CONTAINER_IDS[@]}" == "0" ] then sleep "$SLEEP_TIME" fi @@ -112,24 +112,24 @@ CONTAINER_IDS=($(docker container ls --format "{{.ID}}")) CONTAINER_IMAGES=($(docker container ls --format "{{.Image}}")) CONTAINER_NAMES=($(docker container ls --format "{{.Names}}")) -if [ "$local_TAG_FLAG_DETECTED" = "1" ] +if [ "$local_TAG_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do IMAGE="${CONTAINER_IMAGES[$i]}" - if [ "$IMAGE" = "local_CONTAINER_TAG" ] + if [ "$IMAGE" == "$local_CONTAINER_TAG" ] then echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} $IMAGE" exit 1 fi done -elif [ "$local_NAME_FLAG_DETECTED" = "1" ] +elif [ "$local_NAME_FLAG_DETECTED" == "1" ] then for (( i=0; i<${#CONTAINER_IDS[@]}; i++)); do NAME="${CONTAINER_NAMES[$i]}" - if [ "$NAME" = "local_CONTAINER_NAME" ] + if [ "$NAME" == "$local_CONTAINER_NAME" ] then echo "ERROR still running ${CONTAINER_IDS[$i]} ${CONTAINER_NAMES[$i]} ${CONTAINER_IMAGES[$i]}" exit 1 @@ -138,7 +138,7 @@ then else - if [ ! "${#CONTAINER_IDS[@]}" = "0" ] + if [ ! "${#CONTAINER_IDS[@]}" == "0" ] then docker container ls exit 1 diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index 1699548f5..26d87f1dc 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -2,6 +2,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") source "${SOURCE}/dependency_versions.sh" +source "${SOURCE}/utils.sh" # these are the dependencies to be installed by apt apt_file_path="/tmp/apt_deps" @@ -41,7 +42,7 @@ install_protobuf() { if [ -d protobuf ] then # sudo required because of egg file - sudo rm -rf protobuf + "$SUDO_CMD" rm -rf protobuf fi git clone https://github.com/google/protobuf.git cd protobuf @@ -49,7 +50,7 @@ install_protobuf() { git submodule update --init --recursive cmake -S cmake/ -B build -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install cd python python3 setup.py build python3 setup.py test @@ -58,7 +59,7 @@ install_protobuf() { # Cleanup build file with root ownership if [ -f build/install_manifest.txt ] then - sudo rm build/install_manifest.txt + "$SUDO_CMD" rm build/install_manifest.txt fi cd ../ @@ -79,8 +80,8 @@ install_libsodium() { ./autogen.sh ./configure make check - sudo make install - sudo ldconfig + "$SUDO_CMD" make install + "$SUDO_CMD" ldconfig cd ../ # Mark libsodium as installed @@ -99,7 +100,7 @@ install_libzmq() { git checkout v${DATAFED_LIBZMQ_VERSION} cmake -S. -B build -DBUILD_STATIC=ON -DBUILD_SHARED=ON cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install # Mark libzmq as installed touch ".libzmq_installed-${DATAFED_LIBZMQ_VERSION}" @@ -118,7 +119,7 @@ install_nlohmann_json() { echo "FILE STRUCTURE $(ls)" cmake -S . -B build cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install cd ../ # Mark nlohmann_json as installed @@ -137,7 +138,7 @@ install_json_schema_validator() { git checkout ${DATAFED_JSON_SCHEMA_VALIDATOR_VERSION} cmake -S . -B build cmake --build build -j 8 - sudo cmake --build build --target install + "$SUDO_CMD" cmake --build build --target install cd ../ # Mark json-schema-validator as installed @@ -147,14 +148,14 @@ install_json_schema_validator() { install_gcs() { if [ ! -e ".gcs_installed-${DATAFED_GLOBUS_VERSION}" ]; then - sudo apt update - sudo apt install -y curl git gnupg + "$SUDO_CMD" apt update + "$SUDO_CMD" apt install -y curl git gnupg curl -LOs https://downloads.globus.org/globus-connect-server/stable/installers/repo/deb/globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb - sudo dpkg -i globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb - sudo apt-key add /usr/share/globus-repo/RPM-GPG-KEY-Globus + "$SUDO_CMD" dpkg -i globus-repo_${DATAFED_GLOBUS_VERSION}_all.deb + "$SUDO_CMD" apt-key add /usr/share/globus-repo/RPM-GPG-KEY-Globus # Need a second update command after adding the globus GPG key - sudo apt update - sudo apt-get install globus-connect-server54 -y + "$SUDO_CMD" apt update + "$SUDO_CMD" apt-get install globus-connect-server54 -y # Mark gcs as installed touch ".gcs_installed-${DATAFED_GLOBUS_VERSION}" @@ -171,6 +172,8 @@ install_nvm() { curl -o- "https://raw.githubusercontent.com/nvm-sh/nvm/${DATAFED_NVM_VERSION}/install.sh" | bash # Mark nvm as installed touch ".nvm_installed-${DATAFED_NVM_VERSION}" + else + export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" fi } @@ -188,6 +191,9 @@ install_node() { nvm install "$DATAFED_NODE_VERSION" # Mark node as installed touch ".node_installed-${DATAFED_NODE_VERSION}" + else + export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm fi } @@ -208,16 +214,20 @@ install_foxx_cli() { "$NVM_DIR/nvm-exec" npm install --global foxx-cli --prefix "${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm" # Mark foxx_cli as installed touch ".foxx_cli_installed" + else + export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm + export NODE_VERSION="$DATAFED_NODE_VERSION" fi } install_arangodb() { curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key - sudo apt-key add - < Release.key - echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | sudo tee /etc/apt/sources.list.d/arangodb.list - sudo apt-get install apt-transport-https - sudo apt-get update - sudo apt-get install arangodb3 + "$SUDO_CMD" apt-key add - < Release.key + echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | "$SUDO_CMD" tee /etc/apt/sources.list.d/arangodb.list + "$SUDO_CMD" apt-get install apt-transport-https + "$SUDO_CMD" apt-get update + "$SUDO_CMD" apt-get install arangodb3 } install_dep_by_name() { diff --git a/scripts/export_dependency_version.sh b/scripts/export_dependency_version.sh new file mode 100755 index 000000000..8fd5302f4 --- /dev/null +++ b/scripts/export_dependency_version.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#SCRIPT=$(realpath "$0") +# Assumes sourcing not running +SCRIPT=$( realpath "${BASH_SOURCE[0]}" ) +SCRIPT_DIR=$( dirname "${SCRIPT}" ) +PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/..") + +echo "SCRIPT $SCRIPT" +echo "SCRIPT_DIR $SCRIPT_DIR" +echo "PROJECT_ROOT $PROJECT_ROOT" +#"SOURCE ${SOURCEl}" +. "${PROJECT_ROOT}/scripts/utils.sh" +# +#echo "PROJECT ROOT $PROJECT_ROOT" +## WARNING +## For this script to work it must be called with source +## source export_dependency_version_numbers +export_dependency_version_numbers diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index 1e53c704a..519af8b9a 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -91,7 +91,7 @@ fi local_DATAFED_SERVER_PORT="" if [ -z "${DATAFED_SERVER_PORT}" ] then - local_DATAFED_SERVER_PORT="" + local_DATAFED_SERVER_PORT="7512" else local_DATAFED_SERVER_PORT=$(printenv DATAFED_SERVER_PORT) fi diff --git a/scripts/generate_ws_config.sh b/scripts/generate_ws_config.sh index f14a00e4c..33211d675 100755 --- a/scripts/generate_ws_config.sh +++ b/scripts/generate_ws_config.sh @@ -45,15 +45,30 @@ Help() local_DATAFED_WEB_CERT_PATH="${DATAFED_INSTALL_PATH}/keys/datafed-server-test.ornl.gov.crt" local_DATAFED_WEB_KEY_PATH="${DATAFED_INSTALL_PATH}/keys/datafed-server-test.ornl.gov.key" -local_DATAFED_HTTPS_SERVER_PORT="443" +if [ ! -z "${DATAFED_WEB_KEY_PATH}" ] +then + local_DATAFED_WEB_KEY_PATH=$(printenv DATAFED_WEB_KEY_PATH) +fi -if [ -z "${DATAFED_GLOBUS_APP_ID}" ] +if [ ! -z "${DATAFED_WEB_CERT_PATH}" ] +then + local_DATAFED_WEB_CERT_PATH=$(printenv DATAFED_WEB_CERT_PATH) +fi + +if [ -z "${DATAFED_DOMAIN}" ] then local_DATAFED_SERVER_DOMAIN_NAME="datafed.ornl.gov" else local_DATAFED_SERVER_DOMAIN_NAME=$(printenv DATAFED_DOMAIN) fi +if [ -z "${DATAFED_HTTPS_SERVER_PORT}" ] +then + local_DATAFED_HTTPS_SERVER_PORT="443" +else + local_DATAFED_HTTPS_SERVER_PORT=$(printenv DATAFED_HTTPS_SERVER_PORT) +fi + if [ -z "${DATAFED_GLOBUS_APP_ID}" ] then local_DATAFED_GLOBUS_APP_ID="" diff --git a/scripts/install_authz_dependencies.sh b/scripts/install_authz_dependencies.sh index 92ee5bd9c..e3806ffc8 100755 --- a/scripts/install_authz_dependencies.sh +++ b/scripts/install_authz_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "build-essential" "g++" "gcc" "libboost-all-dev" "autoconf" "automake" "make" "git" "python3-pkg-resources" "python3-pip" "pkg-config" "libglobus-common-dev" "wget" "libssl-dev" "libzmq3-dev") @@ -33,9 +34,11 @@ if [ $# -eq 1 ]; then esac fi -sudo apt-get update -sudo dpkg --configure -a -sudo apt-get install -y "${packages[@]}" +sudo_command + +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a +"$SUDO_CMD" apt-get install -y "${packages[@]}" cd ~ install_cmake diff --git a/scripts/install_client_dependencies.sh b/scripts/install_client_dependencies.sh index 574e0457c..cea8afb38 100755 --- a/scripts/install_client_dependencies.sh +++ b/scripts/install_client_dependencies.sh @@ -7,11 +7,13 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" +sudo_command # This script will install all of the dependencies needed by DataFed 1.0 -sudo apt-get update -sudo dpkg --configure -a +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a python3 -m pip install -r "${PROJECT_ROOT}/python/datafed_pkg/requirements.txt" diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index a3baa4c95..70b78b7e7 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "libcurl4-openssl-dev" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip" "libssl-dev" "libzmq3-dev") @@ -33,10 +34,12 @@ if [ $# -eq 1 ]; then esac fi +sudo_command + if [[ $local_UNIFY = false ]]; then - sudo apt-get update - sudo dpkg --configure -a - sudo apt-get install -y "${packages[@]}" + "$SUDO_CMD" apt-get update + "$SUDO_CMD" dpkg --configure -a + "$SUDO_CMD" apt-get install -y "${packages[@]}" python3 -m pip install --upgrade pip python3 -m pip install setuptools diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh index 5c4e0cb0c..ecf5aa244 100755 --- a/scripts/install_dependencies.sh +++ b/scripts/install_dependencies.sh @@ -7,23 +7,29 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" source "${SOURCE}/dependency_versions.sh" touch "$apt_file_path" touch "$ext_file_path" -sudo apt-get update -sudo apt install -y wget git curl +# Defines SUDO_CMD which is empty if root +# sudo path if exists +# throws error otherwise +sudo_command + +"$SUDO_CMD" apt-get update +"$SUDO_CMD" apt install -y wget git curl install_cmake # This script will install all of the dependencies needed by DataFed 1.0 -sudo dpkg --configure -a +"$SUDO_CMD" dpkg --configure -a -sudo "$SOURCE/install_core_dependencies.sh" unify -sudo "$SOURCE/install_repo_dependencies.sh" unify -sudo "$SOURCE/install_ws_dependencies.sh" unify -sudo "$SOURCE/install_authz_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_core_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_repo_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_ws_dependencies.sh" unify +"$SUDO_CMD" "$SOURCE/install_authz_dependencies.sh" unify all_packages=$(cat $apt_file_path) IFS=' ' read -r -a all_packages_array <<< "$all_packages" @@ -32,7 +38,7 @@ deduplicated_packages_array=($(printf "%s\n" "${all_packages_array[@]}" | sort - all_externals=$(cat $ext_file_path) IFS=' ' read -r -a all_externals_array <<< "$all_externals" -sudo apt-get install -y "${deduplicated_packages_array[@]}" +"$SUDO_CMD" apt-get install -y "${deduplicated_packages_array[@]}" echo "DEPENDENCIES (${deduplicated_externals_array[@]})" diff --git a/scripts/install_docs_dependencies.sh b/scripts/install_docs_dependencies.sh index d68d1e18e..927104890 100755 --- a/scripts/install_docs_dependencies.sh +++ b/scripts/install_docs_dependencies.sh @@ -7,12 +7,14 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" +sudo_command # This script will install all of the dependencies needed by DataFed 1.0 -sudo apt-get update -sudo dpkg --configure -a -sudo apt-get install -y +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a +"$SUDO_CMD" apt-get install -y install_cmake cd ~ diff --git a/scripts/install_foxx.sh b/scripts/install_foxx.sh index 09ad12519..6664e68f7 100755 --- a/scripts/install_foxx.sh +++ b/scripts/install_foxx.sh @@ -237,9 +237,9 @@ fi # # The web deployment requires manual interaction, and I could not figure out the # syntax for the REST http endpoints with curl so we are going to try the node module -actual_version=$(node --version) -semantic_version_compatible "$actual_version" "$DATAFED_NODE_VERSION" -compatible=$? +#actual_version=$(node --version) +#semantic_version_compatible "$actual_version" "$DATAFED_NODE_VERSION" +#compatible=$? #if [ "$compatible" -eq "0" ] #then @@ -254,12 +254,10 @@ install_foxx_cli #fi FOXX_PREFIX="" -{ - # Determine if exists globally first - which foxx -} || { - FOXX_PREFIX="${NPM_CONFIG_PREFIX}/bin/" -} +if ! command -v foxx > /dev/null 2>&1; then + FOXX_PREFIX="${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm/bin/" +fi + PATH_TO_PASSWD_FILE="${SOURCE}/database_temp.password" diff --git a/scripts/install_lego_and_certificates.sh b/scripts/install_lego_and_certificates.sh index b49edc834..6f19b0c3d 100755 --- a/scripts/install_lego_and_certificates.sh +++ b/scripts/install_lego_and_certificates.sh @@ -4,8 +4,9 @@ set -euf -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/..) -source ${PROJECT_ROOT}/config/datafed.sh +PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/config/datafed.sh" +source "${PROJECT_ROOT}/scripts/utils.sh" Help() { @@ -78,9 +79,11 @@ then exit 1 fi -sudo add-apt-repository --yes ppa:longsleep/golang-backports -sudo apt-get update -sudo apt-get install golang-go +sudo_commands + +"$SUDO_CMD" add-apt-repository --yes ppa:longsleep/golang-backports +"$SUDO_CMD" apt-get update +"$SUDO_CMD" apt-get install golang-go #This was verified for go 1.17 export GO111MODULE=on @@ -89,13 +92,13 @@ GOBIN=/usr/local/bin/ go install github.com/go-acme/lego/v4/cmd/lego@latest # Create the folder if [ ! -d "${DATAFED_INSTALL_PATH}/keys" ] then - sudo mkdir -p "${DATAFED_INSTALL_PATH}/keys" + "$SUDO_CMD" mkdir -p "${DATAFED_INSTALL_PATH}/keys" fi # Check if the datafed-ws server is already running, will need to stop it if we want # to use port 443 to start the domain name datafed_ws_service=$(systemctl list-unit-files --type service | grep datafed-ws | awk '{print $1}') -[[ "$datafed_ws_service" == 'datafed-ws.service' ]] && sudo systemctl stop datafed-ws.service +[[ "$datafed_ws_service" == 'datafed-ws.service' ]] && "$SUDO_CMD" systemctl stop datafed-ws.service # This should create a folder in ~/.lego/certificates, that contains the # certificate files you need, we are going to copy them over to the @@ -109,7 +112,7 @@ cert_file="datafed-server-test.ornl.gov.crt" key_file="datafed-server-test.ornl.gov.key" if [ ! -f "${DATAFED_INSTALL_PATH}/keys/$cert_file" ] || [ ! -f "${DATAFED_INSTALL_PATH}/keys/$key_file" ] then - sudo lego --accept-tos --email="$DATAFED_LEGO_EMAIL" --domains="$local_DATAFED_DOMAIN" --path "${DATAFED_INSTALL_PATH}/keys/" --tls run + "$SUDO_CMD" lego --accept-tos --email="$DATAFED_LEGO_EMAIL" --domains="$local_DATAFED_DOMAIN" --path "${DATAFED_INSTALL_PATH}/keys/" --tls run mv ${DATAFED_INSTALL_PATH}/keys/certificates/$cert_file ${DATAFED_INSTALL_PATH}/keys/ mv ${DATAFED_INSTALL_PATH}/keys/certificates/$key_file ${DATAFED_INSTALL_PATH}/keys/ rm -rf ${DATAFED_INSTALL_PATH}/keys/certificates diff --git a/scripts/install_python_client_dependencies.sh b/scripts/install_python_client_dependencies.sh new file mode 100755 index 000000000..7d48ba721 --- /dev/null +++ b/scripts/install_python_client_dependencies.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Exit on error +set -e + +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath ${SOURCE}/..) + +source "${PROJECT_ROOT}/scripts/utils.sh" +source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" + +sudo_command + +# This script will install all of the dependencies needed by DataFed 1.0 +"$SUDO_CMD" apt-get update +"$SUDO_CMD" dpkg --configure -a +"$SUDO_CMD" apt-get install -y libtool build-essential g++ gcc make libboost-all-dev \ +pkg-config autoconf automake unzip libcurl4-openssl-dev wget \ +rapidjson-dev libkrb5-dev git python3-pkg-resources python3-pip libssl-dev + +cd ~ +install_cmake +cd ~ + +# Install cmake 3.17 + +python3 -m pip install --upgrade pip +python3 -m pip install setuptools + +install_protobuf +cd ~ + + diff --git a/scripts/install_repo_dependencies.sh b/scripts/install_repo_dependencies.sh index 6580db90c..06ee55fb1 100755 --- a/scripts/install_repo_dependencies.sh +++ b/scripts/install_repo_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "wget" "build-essential" "g++" "gcc" "libboost-all-dev" "pkg-config" "autoconf" "automake" "make" "unzip" "git" "python3-pkg-resources" "libssl-dev" "libzmq3-dev" "python3-pip") @@ -34,10 +35,12 @@ if [ $# -eq 1 ]; then esac fi +sudo_command + if [[ $local_UNIFY = false ]]; then - sudo apt-get update - sudo dpkg --configure -a - sudo apt-get install -y "${packages[@]}" + "$SUDO_CMD" apt-get update + "$SUDO_CMD" dpkg --configure -a + "$SUDO_CMD" apt-get install -y "${packages[@]}" python3 -m pip install --upgrade pip python3 -m pip install setuptools diff --git a/scripts/install_ws.sh b/scripts/install_ws.sh index 61d1a35e6..18b248cac 100755 --- a/scripts/install_ws.sh +++ b/scripts/install_ws.sh @@ -4,72 +4,27 @@ set -uf -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/..) -source ${PROJECT_ROOT}/config/datafed.sh -source ${SOURCE}/dependency_versions.sh +PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/config/datafed.sh" +source "${SOURCE}/dependency_versions.sh" +source "${SOURCE}/dependency_install_functions.sh" #NVM_DIR=/home/cades/.nvm #[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" -export NVM_DIR="/home/cades/.nvm" -source ${NVM_DIR}/nvm.sh +#export NVM_DIR="/home/cades/.nvm" +#source ${NVM_DIR}/nvm.sh # Make sure paths exist -mkdir -p ${DATAFED_INSTALL_PATH}/web -mkdir -p ${DATAFED_INSTALL_PATH}/keys -mkdir -p ${DATAFED_DEFAULT_LOG_PATH} +mkdir -p "${DATAFED_INSTALL_PATH}/web" +mkdir -p "${DATAFED_INSTALL_PATH}/keys" +mkdir -p "${DATAFED_DEFAULT_LOG_PATH}" # Install web node modules -cp "$PROJECT_ROOT/web/package.json" ${DATAFED_INSTALL_PATH}/web/ - -nvm use $DATAFED_NODE_VERSION -export npm_config_cache=${DATAFED_INSTALL_PATH}/web -# Check if npm exists -{ - npm_path=$(which npm) -} || { - echo "npm_path not found." -} -if [ -z "$npm_path" ] -then - - { - # Will return a result if nvm can be found and returns nothing otherwise - # which does not work with nvm - nvm_command_exists=$(command -v nvm) - } || { - echo "nvm_command_exists not found." - } - - if [ -z "$nvm_path" ] - then - # Check for nvm in default location when installed with web dependencies - # script - if [ -d "$NVM_DIR" ] - then - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm - elif [ -d "$HOME/.nvm" ] - then - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm - { - nvm_command_exists=$(command -v nvm) - } || { - echo "nvm_command_exists not found after sourcing." - } - if [ -z "$nvm_path" ] - then - echo "ERROR Unable to locate npm or nvm." - exit 1 - fi - else - echo "ERROR Unable to locate npm or nvm." - exit 1 - fi - - fi - nvm use $DATAFED_NODE_VERSION -fi +cp "$PROJECT_ROOT/web/package.json" "${DATAFED_INSTALL_PATH}/web/" +install_nvm +install_node +install_foxx_cli { - npm --allow-root --unsafe-perm --prefix ${DATAFED_INSTALL_PATH}/web install + npm --allow-root --unsafe-perm --prefix "${DATAFED_INSTALL_PATH}/web" install } || { echo "ERROR npm command failed!" exit 1 @@ -77,10 +32,10 @@ fi # Install javascript web server repo and core server were # already installed by CMake -cp "$PROJECT_ROOT/web/datafed-ws.js" ${DATAFED_INSTALL_PATH}/web +cp "$PROJECT_ROOT/web/datafed-ws.js" "${DATAFED_INSTALL_PATH}/web" # Copy configuration files -cp "$PROJECT_ROOT/config/datafed-ws.cfg" ${DATAFED_INSTALL_PATH}/web +cp "$PROJECT_ROOT/config/datafed-ws.cfg" "${DATAFED_INSTALL_PATH}/web" # Make sure that the datafed-core-pub.key exists in the web/static folder if [[ -z "${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub" ]] @@ -89,4 +44,4 @@ then exit 1 fi -cp ${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub "$PROJECT_ROOT/web/static/" +cp "${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub" "$DATAFED_INSTALL_PATH/web/static/" diff --git a/scripts/install_ws_dependencies.sh b/scripts/install_ws_dependencies.sh index 1d1d425a1..1de114446 100755 --- a/scripts/install_ws_dependencies.sh +++ b/scripts/install_ws_dependencies.sh @@ -7,6 +7,7 @@ SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/..) +source "${PROJECT_ROOT}/scripts/utils.sh" source "${SOURCE}/dependency_versions.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" @@ -62,10 +63,12 @@ while [ : ]; do esac done +sudo_command + if [[ $local_UNIFY = false ]]; then - sudo apt-get update - sudo dpkg --configure -a - sudo apt-get install -y "${packages[@]}" + "$SUDO_CMD" apt-get update + "$SUDO_CMD" dpkg --configure -a + "$SUDO_CMD" apt-get install -y "${packages[@]}" for ext in "${externals[@]}"; do install_dep_by_name "$ext" diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100755 index 000000000..c0ee7455a --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,65 @@ +#!/bin/bash +SCRIPT=$(realpath "$0") +SOURCE=$(dirname "$SCRIPT") + +if [ -z "${PROJECT_ROOT}" ] +then + PROJECT_ROOT=$(realpath "${SOURCE}/..") +fi + +echo "PROJECT ROOT $PROJECT_ROOT" + +export_dependency_version_numbers() { + # Get the content of the function and remove comments + variables=$(cat "${PROJECT_ROOT}/scripts/dependency_versions.sh") + + local content="$(echo "${variables}" | sed '/^$/d;/^#/d')" + + # Extract variable assignments from the content + local assignments=$(echo "$content" | grep -Eo '\b[a-zA-Z_][a-zA-Z_0-9]*="[^\"]*"') + + echo "Variables are $variables" + echo "Content is $content" + echo "Assignments is $assignments" + # Loop through each assignment, export the variable + # Note: This may override existing variables + for assignment in $assignments; do + echo "export $assignment" + export "$assignment" + done +} + +empty_command() { + "$@" +} + +# The purpose of this function is to detect the sudo command +# if it exists use it, if we are running as root set SUDO_CMD to empty_command +# empty_command is needed so that I can do this where sudo doesn't exist +# +# "$SUDO_CMD" apt install curl +# +# If running as root this will expand to +# +# empty_command apt install curl +# +# which expands to +# +# apt install curl +# +# If I left SUDO_CMD blank i.e. "" apt install curl bash would complain +sudo_command() { + if [ "$(id -u)" -eq 0 ]; then + export SUDO_CMD="empty_command" # Ignore sudo running as root + else + # Check if sudo is available + if command -v sudo &> /dev/null; then + export SUDO_CMD=$(command -v sudo) + else + echo "Error: This script requires sudo but sudo is not installed." >&2 + echo "You are not running as root!" >&2 + exit 1 + fi + exit $? # Exit with the same status as the sudo command + fi +} diff --git a/web/docker/Dockerfile b/web/docker/Dockerfile index a8c3f1609..ed7adb715 100644 --- a/web/docker/Dockerfile +++ b/web/docker/Dockerfile @@ -6,11 +6,10 @@ ARG DEPENDENCIES="dependencies" ARG RUNTIME="runtime" ARG DATAFED_DIR="/datafed" ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +#ARG NVM_DIR="$DATAFED_DIR/.nvm" ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS ws-build @@ -18,9 +17,7 @@ FROM ${DEPENDENCIES} AS ws-build ARG DATAFED_DIR ARG BUILD_DIR ARG DATAFED_INSTALL_PATH -ARG NVM_DIR -ARG NVM_INC -ARG NVM_BIN +ARG DATAFED_DEPENDENCIES_INSTALL_PATH # This port is needed to communicate with the DataFed core server EXPOSE 7513 @@ -32,6 +29,7 @@ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/generate_ws_config.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/export_dependency_version.sh ${BUILD_DIR}/scripts/ COPY ./cmake ${BUILD_DIR}/cmake COPY ./common/proto ${BUILD_DIR}/common/proto COPY ./web ${BUILD_DIR}/web @@ -47,54 +45,65 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh && \ -DBUILD_FOXX=False \ -DBUILD_COMMON=False RUN cmake --build build - -ENV NVM_DIR="$NVM_DIR" -ENV NVM_INC="$NVM_INC" -ENV NVM_BIN="$NVM_BIN" -ENV PATH="$NVM_BIN:$PATH" - RUN cmake --build build --target install +# Only part of the final install can be done here +#RUN ${BUILD_DIR}/scripts/install_ws.sh FROM ${RUNTIME} AS ws -ARG DATAFED_NODE_VERSION="" +SHELL ["/bin/bash", "-c"] +#ARG DATAFED_NODE_VERSION="" ARG DATAFED_DIR ARG DATAFED_INSTALL_PATH ARG BUILD_DIR -ARG NVM_DIR -ARG NVM_INC -ARG NVM_BIN +ARG DATAFED_DEPENDENCIES_INSTALL_PATH # The above should also be available at runtime ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="$DATAFED_DEPENDENCIES_INSTALL_PATH" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" -ENV NVM_DIR="$NVM_DIR" -ENV NVM_INC="$NVM_INC" -ENV NVM_BIN="$NVM_BIN" -ENV PATH="$NVM_BIN:$PATH" RUN apt install -y python3 make g++ WORKDIR ${DATAFED_DIR} -COPY --from=ws-build --chown=datafed:root "$NVM_DIR" "$NVM_DIR" -RUN ln -s ${DATAFED_INSTALL_PATH}/web ${DATAFED_DIR}/web +#COPY --from=ws-build --chown=datafed:root "$NVM_DIR" "$NVM_DIR" +#RUN ln -s ${DATAFED_INSTALL_PATH}/web ${BUILD_DIR}/web USER datafed -COPY --chown=datafed:root ./web/docker/entrypoint.sh ${BUILD_DIR}/web/entrypoint.sh -COPY --chown=datafed:root ./scripts/generate_datafed.sh ${DATAFED_DIR}/scripts/generate_datafed.sh -COPY --chown=datafed:root ./scripts/dependency_versions.sh ${DATAFED_DIR}/scripts/dependency_versions.sh -COPY --chown=datafed:root ./scripts/generate_ws_config.sh ${DATAFED_DIR}/scripts/generate_ws_config.sh -COPY --chown=datafed:root ./scripts/install_ws.sh ${DATAFED_DIR}/scripts/install_ws.sh -COPY --chown=datafed:root ./cmake/Version.cmake ${DATAFED_DIR}/cmake/Version.cmake +#COPY --chown=datafed:root ./web/docker/entrypoint.sh ${BUILD_DIR}/web/docker/entrypoint.sh +COPY --chown=datafed:root ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh +COPY --chown=datafed:root ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/dependency_versions.sh +COPY --chown=datafed:root ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/dependency_install_functions.sh +COPY --chown=datafed:root ./scripts/generate_ws_config.sh ${BUILD_DIR}/scripts/generate_ws_config.sh +COPY --chown=datafed:root ./scripts/install_ws.sh ${BUILD_DIR}/scripts/install_ws.sh +COPY --chown=datafed:root ./cmake/Version.cmake ${BUILD_DIR}/cmake/Version.cmake +COPY --chown=datafed:root ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh +COPY --chown=datafed:root ./scripts/export_dependency_version.sh ${BUILD_DIR}/scripts/export_dependency_version.sh +##COPY --chown=datafed:root ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} +# +COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${BUILD_DIR}/web +COPY --from=ws-build --chown=datafed:root ${DATAFED_INSTALL_PATH} ${DATAFED_INSTALL_PATH} +#COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web/package.json ${DATAFED_INSTALL_PATH}/web/package.json +COPY --from=ws-build --chown=datafed:root ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} +COPY --from=ws-build --chown=datafed:root /usr/bin/curl /usr/bin/curl + +RUN chown -R datafed:root ${DATAFED_INSTALL_PATH}/web +RUN chown -R datafed:root ${BUILD_DIR} -COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web/package.json ${DATAFED_INSTALL_PATH}/web/package.json -RUN . ${DATAFED_DIR}/scripts/dependency_versions.sh && \ - . ${DATAFED_DIR}/.nvm/nvm.sh && \ +WORKDIR ${DATAFED_DIR} + +RUN echo "${DATAFED_DIR}" +RUN . ${BUILD_DIR}/scripts/export_dependency_version.sh && \ + . ${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm/nvm.sh && \ npm --allow-root --unsafe-perm --prefix ${DATAFED_INSTALL_PATH}/web install -COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${DATAFED_INSTALL_PATH}/web +#COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${DATAFED_INSTALL_PATH}/web + +USER root -WORKDIR ${DATAFED_INSTALL_PATH}/web +#ENTRYPOINT ["sleep", "100000" ] +ENTRYPOINT ["/datafed/source/web/docker/entrypoint.sh", "/opt/datafed/web/datafed-ws.js", "/datafed/source/config/datafed-ws.cfg" ] +#WORKDIR ${DATAFED_INSTALL_PATH}/web diff --git a/web/docker/Dockerfile.web-base.ubuntu b/web/docker/Dockerfile.web-base.ubuntu index b504c455f..7a231f483 100644 --- a/web/docker/Dockerfile.web-base.ubuntu +++ b/web/docker/Dockerfile.web-base.ubuntu @@ -12,6 +12,9 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/ + +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DIR}/dependenies" # Docker dependency RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo diff --git a/web/docker/Dockerfile.web.ubuntu b/web/docker/Dockerfile.web.ubuntu index ff4c5c0f5..9e9929678 100644 --- a/web/docker/Dockerfile.web.ubuntu +++ b/web/docker/Dockerfile.web.ubuntu @@ -1,4 +1,6 @@ -FROM camden.ornl.gov/datafed/ws-base:latest as build +ARG BASE_WEB_IMAGE="camden.ornl.gov/datafed/ws-base:latest" + +FROM "$BASE_WEB_IMAGE" as build ARG DATAFED_DIR="/datafed" ARG BUILD_DIR="/datafed/source" @@ -8,6 +10,7 @@ ARG DATAFED_INSTALL_PATH="/datafed/install" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DIR}/dependenies" RUN mkdir -p ${BUILD_DIR}/logs RUN mkdir -p ${BUILD_DIR}/common/proto @@ -50,10 +53,10 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh &&\ -DBUILD_FOXX=False -DBUILD_COMMON=False &&\ cmake --build build -ENV NVM_DIR="$DATAFED_DIR/.nvm" -ENV NVM_INC=$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node -ENV PATH=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ENV NVM_BIN=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin +#ENV NVM_DIR="$DATAFED_DIR/.nvm" +#ENV NVM_INC=$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node +#ENV PATH=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +#ENV NVM_BIN=$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin # Note that building the web server this way assumes that a temporary core key # has been passed in, this will need to be overwritten when the web server @@ -62,5 +65,7 @@ RUN cmake --build build --target install WORKDIR ${DATAFED_INSTALL_PATH}/web +USER root + ENTRYPOINT ["/datafed/source/web/docker/entrypoint.sh"] CMD ["/datafed/install/web/datafed-ws.js","/datafed/install/web/datafed-ws.cfg"] diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index e7a33bf6a..92482ca4a 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -1,20 +1,40 @@ #!/bin/bash - # NVM_DIR must be defined set -euf -o pipefail +if [ -n "$UID" ]; then + usermod -u "$UID" datafed +fi + + SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../..) +PROJECT_ROOT=$(realpath "${SOURCE}/../..") + +"${PROJECT_ROOT}/scripts/generate_datafed.sh" +"${PROJECT_ROOT}/scripts/generate_ws_config.sh" +"${PROJECT_ROOT}/scripts/install_ws.sh" +. "${PROJECT_ROOT}/scripts/dependency_versions.sh" -source $NVM_DIR/nvm.sh -${PROJECT_ROOT}/scripts/generate_datafed.sh -${PROJECT_ROOT}/scripts/generate_ws_config.sh -${PROJECT_ROOT}/scripts/install_ws.sh +export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm" +export NODE_VERSION="$DATAFED_NODE_VERSION" +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm +echo "NODE VERSION $NODE_VERSION" +echo "Which node" +NODE_PATH=$(su datafed -c ". ${NVM_DIR}/nvm.sh; nvm which node") +# Do this as root +if [ ! -L "/usr/local/bin/node" ]; then + ln -s "$NODE_PATH" /usr/local/bin/node +fi # Send output to file as well as print to terminal -log_path=$(grep "log-path" /datafed/install/web/datafed-ws.cfg | cut -d "=" -f 2 | tr -d ' ') +log_path=$(grep "log-path" "${BUILD_DIR}/config/datafed-ws.cfg" | cut -d "=" -f 2 | tr -d ' ') + +if [ ! -d "${log_path}" ] +then + su -c "mkdir -p ${log_path}" datafed +fi if [ "$#" -eq 0 ]; then echo "No arguments were passed, running bash" @@ -22,13 +42,15 @@ if [ "$#" -eq 0 ]; then exit 0 fi +cd "$DATAFED_INSTALL_PATH/web" datafed_ws_exec=$(basename "$1") if [ "${datafed_ws_exec}" = "datafed-ws.js" ] then # Send output to log file - "$@" 2>&1 | tee "$log_path/datafed-ws.log" + su datafed -c '"$@"' -- argv0 "$@" 2>&1 | su datafed -c "tee $log_path/datafed-ws.log" else echo "Not sending output to datafed-ws.log" # If not do not by default send to log file - exec "$@" + su datafed -c '"$@"' -- argv0 "$@" fi +