Joshua s brown feature add dev compose

.gitignore

@@ -1,6 +1,7 @@
 /.idea/
 build
 common/proto/common/Version.proto
+compose/.env
 config/datafed.sh
 config/datafed-authz.cfg
 config/datafed-core.cfg

CHANGELOG.md

@@ -3,8 +3,10 @@
 ## MAJOR Breaking changes
 
 ## MINOR Feature
-1. [909] - Added Support for Google Analytics
-2. [916] - Reworked Docker build process to be faster and produce simpler and smaller images
+1. [912] - Adds initial compose file and docker files for python-client
+2. [909] - Added Support for Google Analytics
+3. [916] - Reworked Docker build process to be faster and produce simpler and smaller images
+4. [912] - Adding working compose instance for core metadata services.
 
 ## PATCH Bug fixes/Technical Debt/Documentation
 1. [914] - Improve GitHub template

compose/README.md

@@ -0,0 +1,33 @@
+# Compose Dev environment
+
+The files in this folder are incomplete but are the start for setting up a full
+docker compose instance of datafed.
+
+```bash
+./build_images_for_compose.sh
+```
+
+Create the .env file fill in the missing components that are required.
+```bash
+./generate_env.sh
+```
+
+Stand up the core services.
+```bash
+docker compose -f ./compose_core.yml up
+```
+
+At this point you should be able to navigate in your browser to
+https://localhost
+
+NOTE we are using a self signed certificate so you will have to force your
+browser to allow you to see the page.
+
+Standing up the repo services has been separated because of Globus. You will
+need a machine with firewall exceptions to use it.
+
+## Cleaning up
+
+```bash
+docker compose -f ./compose_core.yml down
+```

compose/build_images_for_compose.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -euf -o pipefail
+
+SCRIPT=$(realpath "$0")
+SOURCE=$(dirname "$SCRIPT")
+PROJECT_ROOT=$(realpath "${SOURCE}/../")
+
+docker build \
+  -f "${PROJECT_ROOT}/docker/Dockerfile.dependencies" \
+  "${PROJECT_ROOT}" \
+  -t datafed-dependencies:latest
+docker build \
+  -f "${PROJECT_ROOT}/docker/Dockerfile.runtime" \
+  "${PROJECT_ROOT}" \
+  -t datafed-runtime:latest
+docker build -f \
+  "${PROJECT_ROOT}/core/docker/Dockerfile" \
+  --build-arg DEPENDENCIES="datafed-dependencies" \
+  --build-arg RUNTIME="datafed-runtime" \
+  "${PROJECT_ROOT}" \
+  -t datafed-core:latest
+docker build -f \
+  "${PROJECT_ROOT}/web/docker/Dockerfile" \
+  --build-arg DEPENDENCIES="datafed-dependencies" \
+  --build-arg RUNTIME="datafed-runtime" \
+  "${PROJECT_ROOT}" \
+  -t datafed-web:latest
+docker build -f \
+  "${PROJECT_ROOT}/repository/docker/Dockerfile" \
+  --build-arg DEPENDENCIES="datafed-dependencies" \
+  --build-arg RUNTIME="datafed-runtime" \
+  "${PROJECT_ROOT}" \
+  -t datafed-repo:latest
+docker build -f \
+  "${PROJECT_ROOT}/docker/Dockerfile.foxx" \
+  --build-arg DEPENDENCIES="datafed-dependencies" \
+  --build-arg RUNTIME="datafed-runtime" \
+  "${PROJECT_ROOT}" \
+  -t datafed-foxx:latest
+

compose/compose_core.yml

@@ -0,0 +1,91 @@
+version: '3.9'
+
+
+services:
+
+  datafed-web:
+    depends_on: ["datafed-core"]
+    environment:
+      DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}"
+      DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}"
+      DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}"
+      DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}"
+      DATAFED_DOMAIN: "${DATAFED_DOMAIN}"
+      DATAFED_HTTPS_SERVER_PORT: "${DATAFED_HTTPS_SERVER_PORT}"
+      DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}"
+      DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}"
+      DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}"
+      DATAFED_CORE_ADDRESS_PORT_INTERNAL: "datafed-core:7513"
+      UID: "${DATAFED_UID}"
+    image: datafed-web:latest
+    ports:
+      - 443:443 # This must be the same port that is mapped to the host for redirects to work
+    volumes:
+      - ./keys:/opt/datafed/keys
+    networks:
+      - datafed-internal
+
+  datafed-core:
+    image: datafed-core:latest
+    depends_on:
+      datafed-foxx:
+        condition: service_healthy
+    environment:
+      DATAFED_GLOBUS_APP_SECRET: "${DATAFED_GLOBUS_APP_SECRET}"
+      DATAFED_GLOBUS_APP_ID: "${DATAFED_GLOBUS_APP_ID}"
+      DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}"
+      DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}"
+      DATAFED_DOMAIN: "${DATAFED_DOMAIN}"
+      DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}"
+      DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}"
+      DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}"
+      UID: "${DATAFED_UID}"
+      DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}"
+      DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}"
+      DATAFED_DATABASE_IP_ADDRESS_PORT: "${DATAFED_DATABASE_IP_ADDRESS}:${DATAFED_DATABASE_PORT}"
+    ports:
+      - 7513 # Communication web server
+      - 7512 # Secure core server communication
+    volumes:
+      - ./keys:/opt/datafed/keys
+    networks:
+      - datafed-core-secure-api
+      - datafed-internal
+
+  datafed-foxx:
+    image: datafed-foxx:latest
+    depends_on: ["arango"]
+    environment:
+      DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}"
+      DATAFED_DOMAIN: "${DATAFED_DOMAIN}"
+      DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}"
+      UID: "${DATAFED_UID}"
+      DATAFED_DATABASE_PASSWORD: "${DATAFED_DATABASE_PASSWORD}"
+      DATAFED_DATABASE_IP_ADDRESS: "${DATAFED_DATABASE_IP_ADDRESS}"
+      DATAFED_DATABASE_HOST: "arango"
+    healthcheck:
+      test: ["CMD", "/bin/bash", "-c", "[ -f /tmp/.foxx_is_installed ]"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+    networks:
+      - datafed-internal
+
+  arango:
+    image: arangodb
+    environment:
+      ARANGO_ROOT_PASSWORD: "${DATAFED_DATABASE_PASSWORD}"
+    ports:
+      - 8529:8529 # Arangodb web UI
+    networks:
+      - datafed-internal
+
+volumes:
+  keys:
+
+networks:
+  datafed-core-secure-api:
+    driver: bridge
+  datafed-internal:
+    driver: bridge
+

compose/generate_env.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+SCRIPT=$(realpath "$0")
+SOURCE=$(dirname "$SCRIPT")
+PROJECT_ROOT=$(realpath "${SOURCE}/..")
+
+if [ -f ".env" ]
+then
+  echo ".env already exist! Will not overwrite!"
+  exit 1
+fi
+
+local_DATAFED_WEB_KEY_DIR="${PROJECT_ROOT}/compose/keys"
+if [ ! -d "$local_DATAFED_WEB_KEY_DIR" ]
+then
+  mkdir -p "$local_DATAFED_WEB_KEY_DIR"
+fi
+
+if [ -z "${DATAFED_COMPOSE_DOMAIN}" ]
+then
+  local_DATAFED_COMPOSE_DOMAIN="localhost"
+else
+  local_DATAFED_COMPOSE_DOMAIN=$(printenv DATAFED_COMPOSE_DOMAIN)
+fi
+
+local_DATAFED_WEB_CERT_NAME="cert.crt"
+local_DATAFED_WEB_KEY_NAME="cert.key"
+
+local_DATAFED_WEB_CERT_PATH="${local_DATAFED_WEB_KEY_DIR}/${local_DATAFED_WEB_CERT_NAME}"
+local_DATAFED_WEB_CSR_PATH="${local_DATAFED_WEB_KEY_DIR}/cert.csr"
+local_DATAFED_WEB_KEY_PATH="${local_DATAFED_WEB_KEY_DIR}/${local_DATAFED_WEB_KEY_NAME}"
+
+if [ ! -e "$local_DATAFED_WEB_CERT_PATH" ] || [ ! -e "$local_DATAFED_WEB_KEY_PATH" ]
+then
+  if [ -e "$local_DATAFED_WEB_CERT_PATH" ]
+  then
+    rm "${local_DATAFED_WEB_CERT_PATH}"
+  fi
+  if [ -e "$local_DATAFED_WEB_KEY_PATH" ]
+  then
+    rm "${local_DATAFED_WEB_KEY_PATH}"
+  fi
+  if [ -e "$local_DATAFED_WEB_CSR_PATH" ]
+  then
+    rm "${local_DATAFED_WEB_CSR_PATH}"
+  fi
+  openssl genrsa -out "$local_DATAFED_WEB_KEY_PATH" 2048
+  openssl req -new -key "$local_DATAFED_WEB_KEY_PATH" \
+    -out "${local_DATAFED_WEB_CSR_PATH}" \
+    -subj "/C=US/ST=TN/L=Oak Ridge/O=ORNL/OU=DLT/CN=${local_DATAFED_COMPOSE_DOMAIN}"
+  openssl x509 -req -days 3650 \
+     -in "${local_DATAFED_WEB_CSR_PATH}" \
+     -signkey "$local_DATAFED_WEB_KEY_PATH" \
+     -out "$local_DATAFED_WEB_CERT_PATH"
+fi
+
+if [ -z "${DATAFED_COMPOSE_USER89_PASSWORD}" ]
+then
+  local_DATAFED_COMPOSE_USER89_PASSWORD="" # For End to end testing
+else
+  local_DATAFED_COMPOSE_USER89_PASSWORD=$(printenv DATAFED_COMPOSE_USER89_PASSWORD)
+fi
+if [ -z "${DATAFED_COMPOSE_REPO_FORM_PATH}" ]
+then
+  local_DATAFED_COMPOSE_REPO_FORM_PATH="" # Where the repo form is located also needed for testing
+else
+  local_DATAFED_COMPOSE_REPO_FORM_PATH=$(printenv DATAFED_COMPOSE_REPO_FORM_PATH)
+fi
+if [ -z "${DATAFED_COMPOSE_GLOBUS_APP_SECRET}" ]
+then
+  local_DATAFED_COMPOSE_GLOBUS_APP_SECRET=""
+else
+  local_DATAFED_COMPOSE_GLOBUS_APP_SECRET=$(printenv DATAFED_COMPOSE_GLOBUS_APP_SECRET)
+fi
+if [ -z "${DATAFED_COMPOSE_GLOBUS_APP_ID}" ]
+then
+  local_DATAFED_COMPOSE_GLOBUS_APP_ID=""
+else
+  local_DATAFED_COMPOSE_GLOBUS_APP_ID=$(printenv DATAFED_COMPOSE_GLOBUS_APP_ID)
+fi
+if [ -z "${DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET}" ]
+then
+  local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET=""
+else
+  local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET=$(printenv DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET)
+fi
+if [ -z "${DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET}" ]
+then
+  local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET=""
+else
+  local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET=$(printenv DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET)
+fi
+if [ -z "${DATAFED_COMPOSE_HTTPS_SERVER_PORT}" ]
+then
+  local_DATAFED_COMPOSE_HTTPS_SERVER_PORT="443"
+else
+  local_DATAFED_COMPOSE_HTTPS_SERVER_PORT=$(printenv DATAFED_COMPOSE_HTTPS_SERVER_PORT)
+fi
+if [ -z "${DATAFED_COMPOSE_CONTAINER_LOG_PATH}" ]
+then
+  local_DATAFED_COMPOSE_CONTAINER_LOG_PATH="/opt/datafed/logs"
+else
+  local_DATAFED_COMPOSE_CONTAINER_LOG_PATH=$(printenv DATAFED_COMPOSE_CONTAINER_LOG_PATH)
+fi
+if [ -z "${DATAFED_COMPOSE_DATABASE_PASSWORD}" ]
+then
+  local_DATAFED_COMPOSE_DATABASE_PASSWORD="butterscotch"
+else
+  local_DATAFED_COMPOSE_DATABASE_PASSWORD=$(printenv DATAFED_COMPOSE_DATABASE_PASSWORD)
+fi
+
+if [ -z "${DATAFED_COMPOSE_DATABASE_IP_ADDRESS}" ]
+then
+  local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS="http://arango"
+else
+  local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS=$(printenv DATAFED_COMPOSE_DATABASE_IP_ADDRESS)
+fi
+
+if [ -z "${DATAFED_COMPOSE_DATABASE_PORT}" ]
+then
+  local_DATAFED_COMPOSE_DATABASE_PORT="8529"
+else
+  local_DATAFED_COMPOSE_DATABASE_PORT=$(printenv DATAFED_COMPOSE_DATABASE_PORT)
+fi
+
+cat << EOF > ".env"
+DATAFED_USER89_PASSWORD="${local_DATAFED_COMPOSE_USER89_PASSWORD}" # For End to end testing
+DATAFED_REPO_FORM_PATH="${local_DATAFED_COMPOSE_REPO_FORM_PATH}" # Where the repo form is located also needed for testing
+DATAFED_GLOBUS_APP_SECRET="${local_DATAFED_COMPOSE_GLOBUS_APP_SECRET}"
+DATAFED_GLOBUS_APP_ID="${local_DATAFED_COMPOSE_GLOBUS_APP_ID}"
+DATAFED_ZEROMQ_SESSION_SECRET="${local_DATAFED_COMPOSE_ZEROMQ_SESSION_SECRET}"
+DATAFED_ZEROMQ_SYSTEM_SECRET="${local_DATAFED_COMPOSE_ZEROMQ_SYSTEM_SECRET}"
+DATAFED_DOMAIN="${local_DATAFED_COMPOSE_DOMAIN}"
+DATAFED_HTTPS_SERVER_PORT="${local_DATAFED_COMPOSE_HTTPS_SERVER_PORT}"
+DATAFED_WEB_CERT_PATH="/opt/datafed/keys/${local_DATAFED_WEB_CERT_NAME}"
+DATAFED_WEB_KEY_PATH="/opt/datafed/keys/${local_DATAFED_WEB_KEY_NAME}"
+DATAFED_CONTAINER_LOG_PATH="${local_DATAFED_COMPOSE_CONTAINER_LOG_PATH}"
+DATAFED_DATABASE_PASSWORD="${local_DATAFED_COMPOSE_DATABASE_PASSWORD}"
+DATAFED_DATABASE_IP_ADDRESS="${local_DATAFED_COMPOSE_DATABASE_IP_ADDRESS}"
+DATAFED_DATABASE_PORT="${local_DATAFED_COMPOSE_DATABASE_PORT}"
+DATAFED_UID="$(id -u)"
+EOF

core/docker/Dockerfile

@@ -7,15 +7,18 @@ ARG         DEPENDENCIES="dependencies"
 ARG              RUNTIME="runtime"
 ARG          DATAFED_DIR="/datafed"
 ARG DATAFED_INSTALL_PATH="/opt/datafed"
+ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies"
 ARG            GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal"
 ARG            BUILD_DIR="$DATAFED_DIR/source"
-ARG              NVM_DIR="$DATAFED_DIR/.nvm"
-ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
-ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
+#ARG              NVM_DIR="$DATAFED_DIR/.nvm"
+#ARG              NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node"
+#ARG              NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin"
 ARG              LIB_DIR="/usr/local/lib"
 
 FROM ${DEPENDENCIES} AS core-build
 
+SHELL ["/bin/bash", "-c"]
+
 ARG DATAFED_DIR
 ARG BUILD_DIR
 ARG DATAFED_INSTALL_PATH
@@ -49,13 +52,16 @@ RUN cmake --build build --target install
 
 FROM ${RUNTIME} AS core
 
+SHELL ["/bin/bash", "-c"]
+
 ARG DATAFED_DIR
 ARG DATAFED_INSTALL_PATH
 ARG BUILD_DIR
 ARG LIB_DIR
 
 # The above should also be available at runtime
 ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH"
+ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}"
 ENV          DATAFED_DIR="$DATAFED_DIR"
 ENV            BUILD_DIR="$BUILD_DIR"
 ENV              LIB_DIR="$LIB_DIR"
@@ -81,5 +87,13 @@ COPY --chown=datafed:root ./cmake/Version.cmake             ${DATAFED_DIR}/cmake
 COPY --from=core-build --chown=datafed:root ${BUILD_DIR}/core/docker/entrypoint.sh    ${BUILD_DIR}/core/entrypoint.sh
 COPY --from=core-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/core/datafed-core ${DATAFED_INSTALL_PATH}/core/datafed-core
 
-ENTRYPOINT ["${BUILD_DIR}/core/entrypoint.sh"]
-CMD ["${DATAFED_INSTALL_PATH}/core/datafed-core","--cfg","${DATAFED_INSTALL_PATH}/core/datafed-core.cfg"]
+USER root
+
+RUN chown -R datafed:root /datafed
+RUN chown -R datafed:root /opt
+
+#SHELL ["/bin/bash"]
+# NOTE docker does not recognize curly braces around variables in entrypoint
+# and CMD
+ENTRYPOINT [ "/datafed/source/core/entrypoint.sh", "/opt/datafed/core/datafed-core","--cfg", "/opt/datafed/core/datafed-core.cfg"]
+#CMD ["$DATAFED_INSTALL_PATH/core/datafed-core","--cfg","$DATAFED_INSTALL_PATH/core/datafed-core.cfg"]

core/docker/Dockerfile.core-base.ubuntu

@@ -11,6 +11,7 @@ COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/
 COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/
 COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/
 COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/
+COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/
 
 RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo
 RUN ${BUILD_DIR}/scripts/generate_datafed.sh