From 19ad17e6502e32e4435c9c80f9bfede988c1ae4d Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Mon, 18 Mar 2024 15:01:02 -0400 Subject: [PATCH 01/84] Fix variable --- compose/build_repo_images_for_compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/build_repo_images_for_compose.sh b/compose/build_repo_images_for_compose.sh index bccdf05ef..65cdb967e 100755 --- a/compose/build_repo_images_for_compose.sh +++ b/compose/build_repo_images_for_compose.sh @@ -8,7 +8,7 @@ PROJECT_ROOT=$(realpath "${SOURCE}/../") . "${PROJECT_ROOT}/scripts/dependency_versions.sh" cd ${PROJECT_ROOT}/external/globus-connect-server-deploy/docker -git checkout "DATAFED_GCS_SUBMODULE_VERSION" +git checkout "$DATAFED_GCS_SUBMODULE_VERSION" docker build --progress plain --tag "gcs-ubuntu-base:latest" - < "./docker-files/Dockerfile.ubuntu-20.04" cd ${PROJECT_ROOT} docker build \ From 7fd201deed819fb987bff7ffa540b81c54dcab01 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:30:05 -0400 Subject: [PATCH 02/84] Add option for building authz library without syslog --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 93110d0f7..b7bb3b6f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,7 @@ if( NOT DEFINED GLOBUS_VERSION ) set(GLOBUS_VERSION 5) endif() OPTION(BUILD_AUTHZ "Build DataFed Authz library" FALSE) +OPTION(BUILD_AUTHZ_WITH_SYSLOG "Build Authz without syslog" TRUE) OPTION(BUILD_CORE_SERVER "Build DataFed Core Server" TRUE) OPTION(BUILD_COMMON "Build DataFed common library" TRUE) OPTION(BUILD_DOCS "Build documentation" TRUE) From a6a1ea420e0f87b58c436fd2428504e73b973ff7 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:30:39 -0400 Subject: [PATCH 03/84] Update command for running Docker.gcs" --- compose/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/compose/README.md b/compose/README.md index ccfbcae3e..75eebc1a7 100644 --- a/compose/README.md +++ b/compose/README.md @@ -50,5 +50,12 @@ Make sure port 80 is not already bound. Also note that the repo server keys should exist in the keys folder before running the gcs instance. ```bash -docker run --env-file .env --network=host --entrypoint /bin/bash -v /home/cloud/compose_collection:/mnt/datafed -v ./globus:/opt/datafed/globus -v ./keys:/opt/datafed/keys -it datafed-gcs:latest +docker run --env-file .env \ + --network=host \ + --entrypoint /bin/bash \ + -v /home/cloud/compose_collection:/mnt/datafed \ + -v ./globus:/opt/datafed/globus \ + -v ./logs:/datafed/logs \ + -v ./keys:/opt/datafed/keys \ + -it datafed-gcs:latest ``` From d98b537b574a52149153887b68af567e0af35ffb Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:31:15 -0400 Subject: [PATCH 04/84] Separate out the different dependencies to reduce build time --- docker/Dockerfile.dependencies | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.dependencies b/docker/Dockerfile.dependencies index f01607fc0..2120c96cc 100644 --- a/docker/Dockerfile.dependencies +++ b/docker/Dockerfile.dependencies @@ -32,11 +32,7 @@ COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh -COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_docs_dependencies.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ -COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ COPY ./cmake/Web.cmake ${BUILD_DIR}/cmake/Web.cmake COPY ./cmake/Version.cmake ${BUILD_DIR}/cmake/Version.cmake COPY ./web/package.json.in ${BUILD_DIR}/web/package.json.in @@ -47,9 +43,21 @@ RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo # run build scripts RUN ${BUILD_DIR}/scripts/generate_datafed.sh + +# Web dependencies +COPY ./scripts/install_ws_dependencies.sh ${BUILD_DIR}/scripts/ RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh -a -r -z -c + +# Core dependencies +COPY ./scripts/install_core_dependencies.sh ${BUILD_DIR}/scripts/ RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh -a -r -z -w + +# Repo dependencies +COPY ./scripts/install_repo_dependencies.sh ${BUILD_DIR}/scripts/ RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh -a -z -w -c + +# Authz dependencies +COPY ./scripts/install_authz_dependencies.sh ${BUILD_DIR}/scripts/ RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_dependencies.sh -a -w -c -r COPY ./scripts/copy_dependency.sh ${BUILD_DIR}/scripts/ From c1dea588f281cb1edd312fe19b1d24c469094f77 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:32:46 -0400 Subject: [PATCH 05/84] Build in container without syslog --- repository/docker/Dockerfile.gcs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs index d555220c6..65f6dedc0 100644 --- a/repository/docker/Dockerfile.gcs +++ b/repository/docker/Dockerfile.gcs @@ -29,16 +29,16 @@ ENV GCS_COLLECTION_ROOT_PATH="/mnt/datafed" ENV DATAFED_GCS_COLLECTION_ROOT_PATH="$GCS_COLLECTION_ROOT_PATH" ENV DATAFED_GLOBUS_CRED_FILE_PATH=/opt/datafed/globus/client_cred.json ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -ENV DATAFED_DIR="$DATAFED_DIR" +ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" ENV DATAFED_GLOBUS_REPO_USER="datafed" RUN mkdir -p ${BUILD_DIR} -RUN mkdir -p ${BUILD_DIR}/logs +RUN mkdir -p ${DATAFED_INSTALL_PATH}/logs RUN mkdir -p ${BUILD_DIR}/common/proto RUN mkdir -p ${DATAFED_INSTALL_PATH}/authz -RUN mkdir -p ${DATAFED_DIR}/collections/mapped +RUN mkdir -p /mnt/datafed # For communicating with public EXPOSE 443 @@ -73,6 +73,7 @@ COPY --chown=datafed:root ./cmake ${BUILD_DIR}/c COPY --chown=datafed:root ./repository/CMakeLists.txt ${BUILD_DIR}/repository/CMakeLists.txt COPY --chown=datafed:root ./repository/gridftp/CMakeLists.txt ${BUILD_DIR}/repository/gridftp/CMakeLists.txt COPY --chown=datafed:root ./scripts/globus/setup_globus.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh +COPY --chown=datafed:root ./scripts/globus/clean_globus.sh ${BUILD_DIR}/scripts/globus/clean_globus.sh COPY --chown=datafed:root ./scripts/globus/generate_repo_form.sh ${BUILD_DIR}/scripts/globus/generate_repo_form.sh COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh COPY --chown=datafed:root ./common ${BUILD_DIR}/common @@ -84,7 +85,7 @@ COPY --chown=datafed:root ./scripts/globus/__init__.py ${BUILD_DIR}/scripts/glo # These extra copy statements are necessary as the authz binary is built in the GCS container # and not the dependencies container so we must copy the build dependencies as well as the # runtime depedencies -COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} COPY --from=dependencies /usr/bin/make /usr/bin/make COPY --from=dependencies /usr/bin/g++ /usr/bin/g++ @@ -104,7 +105,7 @@ COPY --from=dependencies /usr/include /usr/include COPY --from=dependencies /usr/lib /usr/lib COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotobuf.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotobuf.a -COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a @@ -113,16 +114,18 @@ RUN apt-get install vim -y RUN apt-get reinstall globus-connect-server54 -y RUN ${BUILD_DIR}/scripts/generate_gsi-authz_config.sh -RUN ${BUILD_DIR}/scripts/generate_authz_config.sh && \ - ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build \ - -DBUILD_REPO_SERVER=False \ - -DBUILD_AUTHZ=True \ - -DBUILD_CORE_SERVER=False \ - -DBUILD_WEB_SERVER=False \ - -DBUILD_DOCS=False \ - -DBUILD_PYTHON_CLIENT=False \ +# Don't build with syslog the container does not seem to support syslog +RUN ${BUILD_DIR}/scripts/generate_authz_config.sh && \ + ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build \ + -DBUILD_REPO_SERVER=False \ + -DBUILD_AUTHZ=True \ + -DBUILD_AUTHZ_WITH_SYSLOG=False \ + -DBUILD_CORE_SERVER=False \ + -DBUILD_WEB_SERVER=False \ + -DBUILD_DOCS=False \ + -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False -RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 +RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build -j 8 RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install # Get the file that is not a link and give it the right permissions From a50baa0227418a4e5b9d539d7207bad1b023dd9e Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:34:11 -0400 Subject: [PATCH 06/84] Build without syslog and with corrected log path --- repository/docker/Dockerfile.gcs-authz.ubuntu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/docker/Dockerfile.gcs-authz.ubuntu b/repository/docker/Dockerfile.gcs-authz.ubuntu index 043a1b7ec..efc2e0ea8 100644 --- a/repository/docker/Dockerfile.gcs-authz.ubuntu +++ b/repository/docker/Dockerfile.gcs-authz.ubuntu @@ -12,7 +12,7 @@ ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" ENV GCS_COLLECTION_ROOT_PATH="/mnt/datafed" ENV DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" -RUN mkdir -p ${BUILD_DIR}/logs +RUN mkdir -p ${DATAFED_INSTALL_PATH}/logs RUN mkdir -p ${BUILD_DIR}/common/proto RUN mkdir -p ${DATAFED_INSTALL_PATH}/authz RUN mkdir -p ${DATAFED_DIR}/collections/mapped @@ -56,6 +56,7 @@ RUN ${BUILD_DIR}/scripts/generate_datafed.sh RUN ${BUILD_DIR}/scripts/generate_authz_config.sh &&\ ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build -DBUILD_REPO_SERVER=False -DBUILD_AUTHZ=True \ + -DBUILD_AUTHZ_WITH_SYSLOG=False \ -DBUILD_CORE_SERVER=False -DBUILD_WEB_SERVER=False \ -DBUILD_DOCS=False -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=False && \ From 9b86b74685c7892c1279fe22c538eb4c45bbe360 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:34:56 -0400 Subject: [PATCH 07/84] Add compile definitions to pass define to cmake build for authz --- repository/gridftp/globus5/authz/source/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/gridftp/globus5/authz/source/CMakeLists.txt b/repository/gridftp/globus5/authz/source/CMakeLists.txt index 0daec480c..309d80b04 100644 --- a/repository/gridftp/globus5/authz/source/CMakeLists.txt +++ b/repository/gridftp/globus5/authz/source/CMakeLists.txt @@ -11,7 +11,9 @@ target_compile_options(datafed-authz PRIVATE -fPIC) SET_TARGET_PROPERTIES( datafed-authz PROPERTIES LINKER_LANGUAGE CXX ) set_target_properties(datafed-authz PROPERTIES POSITION_INDEPENDENT_CODE ON SOVERSION ${DATAFED_AUTHZ_MAJOR} VERSION ${DATAFED_AUTHZ_MAJOR}.${DATAFED_AUTHZ_MINOR}.${DATAFED_AUTHZ_PATCH}) add_dependencies( datafed-authz common) - +if ( NOT BUILD_AUTHZ_WITH_SYSLOG ) + target_compile_definitions(datafed-authz PRIVATE DONT_USE_SYSLOG) +endif() if(BUILD_SHARED_LIBS) target_link_libraries( datafed-authz common Threads::Threads libzmq From 15436075f058a99e1cca3221a210561496c420c9 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:36:55 -0400 Subject: [PATCH 08/84] Swap out syslog with LOG Macro --- .../gridftp/globus5/authz/source/libauthz.c | 180 ++++++++++++------ 1 file changed, 124 insertions(+), 56 deletions(-) diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index 8bea6e112..2dd6b1b7b 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -10,9 +10,49 @@ // Standard includes #include #include +#include #include #include -#include + +// Define LOG_LEVEL and USE_SYSLOG +#define LOG_LEVEL 1 +#define DONT_USE_SYSLOG + +// Define logging macros +#if defined(DONT_USE_SYSLOG) + FILE *log_file = NULL; + bool write_to_file = true; + #define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { if (LOG_LEVEL >= 1) fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0); \ + do { if (LOG_LEVEL >= 1 && write_to_file ) fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0) + #define AUTHZ_LOG_INFO(fmt, ...) \ + do { if (LOG_LEVEL >= 2) fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); } while (0); \ + do { if (LOG_LEVEL >= 2 && write_to_file ) fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); } while (0) + #define AUTHZ_LOG_ERROR(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 3) fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ + if (LOG_LEVEL >= 3 && write_to_file ) fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ + } while(0) + #define AUTHZ_LOG_INIT(file_path) \ + log_file = fopen(file_path, "a"); \ + if (log_file != NULL) { write_to_file = true; } + #define AUTHZ_LOG_CLOSE() \ + if (log_file != NULL) { fclose(log_file); } +#else + #include + #define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { if (LOG_LEVEL >= 1) syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); } while (0) + #define AUTHZ_LOG_INFO(fmt, ...) \ + do { if (LOG_LEVEL >= 2) syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); } while (0) + #define AUTHZ_LOG_ERROR(fmt, ...) \ + do { if (LOG_LEVEL >= 3) syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); } while (0) + #define AUTHZ_LOG_INIT(file_path) \ + openlog("gsi_authz", 0, LOG_AUTH); + #define AUTHZ_LOG_CLOSE() \ + closelog(); +#endif + + typedef void *globus_gsi_authz_handle_t; typedef void (*globus_gsi_authz_cb_t)(void *callback_arg, @@ -149,13 +189,12 @@ bool setConfigVal(const char *a_label, char *a_dest, char *a_src, size_t len = strlen(a_src); if (len == 0) { - syslog(LOG_ERR, "DataFed - '%s' value not set.", a_label); + AUTHZ_LOG_ERROR("DataFed - '%s' value not set.\n", a_label); return true; } if (len > a_max_len) { - syslog(LOG_ERR, - "DataFed - '%s' value too long in authz config file (max %lu).", + AUTHZ_LOG_ERROR("DataFed - '%s' value too long in authz config file (max %lu).\n", a_label, a_max_len); return true; } @@ -169,12 +208,12 @@ bool loadKeyFile(char *a_dest, char *a_filename) { FILE *inf = fopen(a_filename, "r"); if (!inf) { - syslog(LOG_ERR, "DataFed - Could not open key file: %s", a_filename); + AUTHZ_LOG_ERROR("DataFed - Could not open key file: %s\n", a_filename); return true; } if (!fgets(a_dest, MAX_KEY_LEN, inf)) { - syslog(LOG_ERR, "DataFed - Error reading key from file: %s", a_filename); + AUTHZ_LOG_ERROR("DataFed - Error reading key from file: %s\n", a_filename); fclose(inf); return true; } @@ -197,13 +236,13 @@ bool loadConfig() { // If env variable is not set check default location inf = fopen("/opt/datafed/authz/datafed-authz.cfg", "r"); if (!inf) { - syslog(LOG_ERR, "DataFed - DATAFED_AUTHZ_CFG_FILE env variable not set, " + AUTHZ_LOG_ERROR("DataFed - DATAFED_AUTHZ_CFG_FILE env variable not set, " "and datafed-authz.cfg is not located in default " - "location /opt/datafed/authz"); + "location /opt/datafed/authz\n"); return true; } } else { - syslog(LOG_INFO, "DataFed - Loading authz config file: %s", cfg_file); + AUTHZ_LOG_INFO("DataFed - Loading authz config file: %s\n", cfg_file); inf = fopen(cfg_file, "r"); } if (inf) { @@ -229,8 +268,8 @@ bool loadConfig() { // Content is formatted as "key=value" (no spaces) val = strchr(buf, '='); if (!val) { - syslog(LOG_ERR, - "DataFed - Syntax error in authz config file at line %i.", lc); + AUTHZ_LOG_ERROR( + "DataFed - Syntax error in authz config file at line %i.\n", lc); return true; } else { *val = 0; @@ -247,9 +286,10 @@ bool loadConfig() { MAX_ADDR_LEN); else if (strcmp(buf, "user") == 0) err = setConfigVal("user", g_config.user, val, MAX_ID_LEN); - else if (strcmp(buf, "log_path") == 0) + else if (strcmp(buf, "log_path") == 0) { err = setConfigVal("log_path", g_config.log_path, val, MAX_PATH_LEN); - else if (strcmp(buf, "test_path") == 0) + AUTHZ_LOG_INIT(g_config.log_path); + } else if (strcmp(buf, "test_path") == 0) err = setConfigVal("test_path", g_config.test_path, val, MAX_PATH_LEN); else if (strcmp(buf, "globus-collection-path") == 0) err = setConfigVal("globus-collection-path", @@ -264,8 +304,8 @@ bool loadConfig() { g_config.timeout = atoi(val); else { err = true; - syslog(LOG_ERR, - "DataFed - Invalid key, '%s', in authz config file at line %i.", + AUTHZ_LOG_ERROR( + "DataFed - Invalid key, '%s', in authz config file at line %i.\n", buf, lc); } @@ -295,25 +335,37 @@ bool loadConfig() { if (g_config.server_key[0] == 0) strcat(miss, " server_key"); + AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" Release, version %s\n", + getReleaseVersion()); + if (miss[0] != 0) { - syslog(LOG_ERR, "DataFed - Missing required authz config items:%s", miss); + AUTHZ_LOG_ERROR("DataFed - Missing required authz config items:%s\n", miss); return true; } } else { - syslog(LOG_ERR, "DataFed - Could not open authz config file."); + + AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" Release, version %s\n", + getReleaseVersion()); + AUTHZ_LOG_ERROR("DataFed - Could not open authz config file.\n"); + return true; } + AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" Release, version %s\n", + getReleaseVersion()); + return false; } // The same globus_result_t gsi_authz_init() { - openlog("gsi_authz", 0, LOG_AUTH); - syslog(LOG_INFO, "DataFed Authz module started, version %s", getVersion()); - syslog(LOG_INFO, " API, version %s", getAPIVersion()); - syslog(LOG_INFO, " Release, version %s", - getReleaseVersion()); + //openlog("gsi_authz", 0, LOG_AUTH); memset(g_active_contexts, 0, sizeof(g_active_contexts)); // This line is different @@ -325,14 +377,14 @@ globus_result_t gsi_authz_init() { // The same globus_result_t gsi_authz_destroy() { - syslog(LOG_DEBUG, "gsi_authz_destroy"); - + AUTHZ_LOG_DEBUG("gsi_authz_destroy\n"); + AUTHZ_LOG_CLOSE(); return 0; } // The same globus_result_t gsi_authz_handle_init(va_list ap) { - syslog(LOG_DEBUG, "gsi_authz_handle_init"); + AUTHZ_LOG_DEBUG("gsi_authz_handle_init\n"); globus_result_t result = GLOBUS_FAILURE; globus_gsi_authz_handle_t *handle = va_arg(ap, globus_gsi_authz_handle_t *); @@ -348,9 +400,9 @@ globus_result_t gsi_authz_handle_init(va_list ap) { if (setContext(*handle, context) == false) result = GLOBUS_SUCCESS; else - syslog(LOG_ERR, "gsi_authz_handle_init out of handle context space"); + AUTHZ_LOG_ERROR("gsi_authz_handle_init out of handle context space\n"); } else { - syslog(LOG_ERR, "gsi_authz_handle_init context handle already initialized"); + AUTHZ_LOG_DEBUG("gsi_authz_handle_init context handle already initialized\n"); } callback(callback_arg, callback_arg, result); @@ -360,7 +412,7 @@ globus_result_t gsi_authz_handle_init(va_list ap) { // The same globus_result_t gsi_authz_handle_destroy(va_list ap) { - syslog(LOG_INFO, "gsi_authz_handle_destroy"); + AUTHZ_LOG_INFO("gsi_authz_handle_destroy\n"); globus_result_t result = GLOBUS_FAILURE; globus_gsi_authz_handle_t handle = va_arg(ap, globus_gsi_authz_handle_t); @@ -373,7 +425,7 @@ globus_result_t gsi_authz_handle_destroy(va_list ap) { if (clearContext(handle) == false) { result = GLOBUS_SUCCESS; } else { - syslog(LOG_ERR, "gsi_authz_handle_destroy context handle lookup FAILED"); + AUTHZ_LOG_ERROR("gsi_authz_handle_destroy context handle lookup FAILED\n"); } callback(callback_arg, handle, result); @@ -382,7 +434,7 @@ globus_result_t gsi_authz_handle_destroy(va_list ap) { } globus_result_t gsi_authz_authorize_async(va_list ap) { - syslog(LOG_INFO, "gsi_authz_authorize_async"); + AUTHZ_LOG_INFO("gsi_authz_authorize_async\n"); globus_result_t result = GLOBUS_FAILURE; globus_gsi_authz_handle_t handle = va_arg(ap, globus_gsi_authz_handle_t); @@ -393,16 +445,24 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { // void * authz_system_state = va_arg(ap, void *); char *callout_ids1 = getenv("GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS"); + char *callout_username_mapped1 = getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); + char *callout_id_mapped1 = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); - syslog(LOG_DEBUG, "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", + AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", callout_ids1); + AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", + callout_id_mapped1); + AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", + g_config.globus_collection_path, action, object); if (strcmp(action, "lookup") == 0 || strcmp(action, "chdir") == 0) { + AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", + g_config.globus_collection_path, action, object); result = GLOBUS_SUCCESS; callback(callback_arg, handle, result); return result; } - syslog(LOG_ERR, "gsi_authz_authorize_async, handle: %p, act: %s, obj: %s", + AUTHZ_LOG_ERROR("gsi_authz_authorize_async, handle: %p, act: %s, obj: %s\n", handle, action, object); OM_uint32 min_stat; @@ -425,7 +485,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { maj_stat = gss_display_name(&min_stat, target, &target_buf, &target_type); if (maj_stat == GSS_S_COMPLETE) { - syslog(LOG_INFO, "Auth client: %s, file: %s, action: %s", + AUTHZ_LOG_INFO("Auth client: %s, file: %s, action: %s\n", (char *)client_buf.value, object, action); // Testing hack @@ -442,7 +502,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { // Consortium/OU=Globus Connect User/CN=", 52 ) != 0 ) if (strncmp((char *)client_buf.value, "/C=US/O=Globus Consortium/OU=Globus", 35) != 0) { - syslog(LOG_ERR, "Invalid certificate subject prefix: %s", + AUTHZ_LOG_ERROR("Invalid certificate subject prefix: %s\n", (char *)client_buf.value); } else { /* Note: For some reason, globus will provide the CN as either a @@ -457,36 +517,44 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { // TODO Should check client uuid str len to make sure it won't // overflow if (strncmp((char *)client_buf.value + 52, "u_", 2) == 0) { - syslog(LOG_INFO, "Globus user prefix detected, decode UUID"); + AUTHZ_LOG_INFO("Globus user prefix detected, decode UUID\n"); client_id = malloc(40); if (!decodeUUID((char *)client_buf.value + 54, client_id)) { - syslog(LOG_ERR, "Failed to decode subject UUID: %s", + AUTHZ_LOG_ERROR("Failed to decode subject UUID: %s\n", (char *)client_buf.value + 54); free(client_id); client_id = 0; } } else { - syslog(LOG_INFO, "Using client CN for authz"); + AUTHZ_LOG_INFO("Using client CN for authz\n"); // Find "/CN=" in cert DN const char *cn = strstr(client_buf.value, "/CN="); - if (!cn) - syslog(LOG_ERR, "Common Name not found in client DN"); - else + if (!cn) { + AUTHZ_LOG_ERROR("Common Name not found in client DN\n"); + } else { client_id = strdup((char *)cn + 4); + } char *callout_ids = getenv("GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS"); + char *callout_username_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); + char *callout_id_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); if (callout_ids != NULL) { - syslog(LOG_DEBUG, + AUTHZ_LOG_DEBUG( "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", callout_ids); client_id = strdup(callout_ids); - syslog(LOG_INFO, "libauthz.c client_id(s): %s\n", client_id); + AUTHZ_LOG_INFO("libauthz.c client_id(s): %s\n", client_id); + } else if ( callout_id_mapped != NULL ) { + AUTHZ_LOG_DEBUG( + "libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", + callout_id_mapped); + client_id = strdup(callout_id_mapped); } else { - syslog(LOG_ERR, + AUTHZ_LOG_ERROR( "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS.\n"); } } @@ -497,10 +565,10 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { result = GLOBUS_SUCCESS; } else { - syslog(LOG_INFO, - "libauthz.c Auth client_id: %s, file: %s, action: %s", + AUTHZ_LOG_INFO( + "libauthz.c Auth client_id: %s, file: %s, action: %s\n", client_id, object, action); - syslog(LOG_INFO, "libauthz.c checkAuthorization FAIL.\n"); + AUTHZ_LOG_INFO("libauthz.c checkAuthorization FAIL.\n"); } free(client_id); @@ -509,51 +577,51 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { gss_release_buffer(&min_stat, &target_buf); } else { - syslog(LOG_ERR, "gss_display_name target FAILED, maj: %d, min: %d", + AUTHZ_LOG_ERROR("gss_display_name target FAILED, maj: %d, min: %d\n", maj_stat, min_stat); } gss_release_buffer(&min_stat, &client_buf); } else { - syslog(LOG_ERR, "gss_display_name source FAILED, maj: %d, min: %d", + AUTHZ_LOG_ERROR("gss_display_name source FAILED, maj: %d, min: %d\n", maj_stat, min_stat); } } else { - syslog(LOG_ERR, "gss_inquire_context FAILED, maj: %d, min: %d", maj_stat, + AUTHZ_LOG_ERROR("gss_inquire_context FAILED, maj: %d, min: %d\n", maj_stat, min_stat); } } else { - syslog(LOG_ERR, "context handle lookup FAILED"); + AUTHZ_LOG_ERROR("context handle lookup FAILED\n"); } if (result != GLOBUS_SUCCESS) { globus_object_t *error = globus_error_construct_no_authentication(0, 0); - syslog(LOG_INFO, "Authz: FAILED"); + AUTHZ_LOG_INFO("Authz: FAILED\n"); result = globus_error_put(error); } else { - syslog(LOG_ERR, "Authz: PASSED"); + AUTHZ_LOG_ERROR("Authz: PASSED\n"); callback(callback_arg, handle, result); } - syslog(LOG_ERR, "Authz returning"); + AUTHZ_LOG_ERROR("Authz returning\n"); return result; } // The same globus_result_t gsi_authz_cancel() { - syslog(LOG_DEBUG, "gsi_authz_cancel\n"); + AUTHZ_LOG_DEBUG("gsi_authz_cancel\n"); return 0; } // The same globus_result_t gsi_authz_identify() { - syslog(LOG_DEBUG, "gsi_authz_identify\n"); + AUTHZ_LOG_DEBUG("gsi_authz_identify\n"); return 0; } globus_result_t gsi_map_user(va_list Ap) { - syslog(LOG_DEBUG, "gsi_map_user"); + AUTHZ_LOG_DEBUG("gsi_map_user\n"); char *service = NULL; char *desired_identity = NULL; From 62f6ebce0ffac32f1c7bdea4ede72dfc3193926d Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:43:03 -0400 Subject: [PATCH 09/84] Correct default log path --- scripts/generate_authz_config.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/generate_authz_config.sh b/scripts/generate_authz_config.sh index ba4c4f25c..e9eab44da 100755 --- a/scripts/generate_authz_config.sh +++ b/scripts/generate_authz_config.sh @@ -30,6 +30,13 @@ Help() REPO_ID="datafed-home" +if [ -z "${DATAFED_DEFAULT_LOG_PATH}" ] +then + local_DATAFED_LOG_PATH="/var/log/datafed" +else + local_DATAFED_LOG_PATH=$(printenv DATAFED_DEFAULT_LOG_PATH) +fi + if [ -z "DATAFED_DOMAIN" ] then local_DATAFED_DOMAIN="datafed.ornl.gov" @@ -115,6 +122,7 @@ server_key=${DATAFED_INSTALL_PATH}/keys/datafed-core-key.pub repo_id=repo/$DATAFED_REPO_ID_AND_DIR pub_key=${DATAFED_INSTALL_PATH}/keys/datafed-repo-key.pub priv_key=${DATAFED_INSTALL_PATH}/keys/datafed-repo-key.priv +log_path=${local_DATAFED_LOG_PATH}/datafed-gsi-authz.log user=$local_DATAFED_AUTHZ_USER globus-collection-path=$local_GCS_COLLECTION_ROOT_PATH EOF From 0208e9fb5795ea053f365aaa165281ca1bea08f2 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:43:32 -0400 Subject: [PATCH 10/84] Add default log path --- repository/docker/Dockerfile.gcs | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs index 65f6dedc0..14a76d14d 100644 --- a/repository/docker/Dockerfile.gcs +++ b/repository/docker/Dockerfile.gcs @@ -33,6 +33,7 @@ ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" ENV DATAFED_GLOBUS_REPO_USER="datafed" +ENV DATAFED_DEFAULT_LOG_PATH="$DATAFED_INSTALL_PATH/logs" RUN mkdir -p ${BUILD_DIR} RUN mkdir -p ${DATAFED_INSTALL_PATH}/logs From f3b956bab1ed37d2cf61ec8dafb2df5b2d1472f2 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:49:54 -0400 Subject: [PATCH 11/84] Remove groups --- scripts/globus/globus_cleanup.py | 145 +++++++++++++++++-------------- 1 file changed, 79 insertions(+), 66 deletions(-) diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 764bdc36c..6e81980b7 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,6 +1,7 @@ import globus_sdk -from globus_sdk import AuthClient, AccessTokenAuthorizer +from globus_sdk import AuthClient,GroupsClient, AccessTokenAuthorizer +from globus_sdk.scopes import GroupsScopes import subprocess import json import sys @@ -54,7 +55,12 @@ client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project -client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +group_scope = GroupsScopes.make_mutable("all") +client.oauth2_start_flow(requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + + str(group_scope), + refresh_tokens=True) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -67,9 +73,14 @@ print(token_response) refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) + +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, + client) ac_rt = AuthClient(authorizer=rt_authorizer) +gr_rt = GroupsClient(authorizer=rt_authorizer_groups) userinfo = ac_rt.oauth2_userinfo() @@ -82,98 +93,100 @@ # check if project exists already project_exists = utils.projectExists(ac_rt, PROJECT_NAME) -if project_exists is False: - print(f"No project with name {PROJECT_NAME} exists! No cleanup is necessary") - sys.exit(0) - -projects = ac_rt.get_projects() -project_id = utils.getProjectId(projects, PROJECT_NAME) +if project_exists: -clients_in_project = utils.getClientsInProject(ac_rt, project_id) + projects = ac_rt.get_projects() + project_id = utils.getProjectId(projects, PROJECT_NAME) + print(f"project id is {project_id}") + clients_in_project = utils.getClientsInProject(ac_rt, project_id) -if len(clients_in_project) == 0: - print("No clients were detected in the project we can just delete the" - "project and be done.") -else: + if len(clients_in_project) == 0: + print("No clients were detected in the project we can just delete the" + "project and be done.") + else: # Check if the deployment key exists if it does read it and verify that the # client exists for the globus connect server if it does not then we will # call the setup command - gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile(DEPLOYMENT_KEY_PATH) + gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile(DEPLOYMENT_KEY_PATH) - valid_key = utils.isGCSDeploymentKeyValid(ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key) + valid_key = utils.isGCSDeploymentKeyValid(ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key) - all_gcs_client_ids = utils.getAllGCSClientIds(ac_rt, project_id, ENDPOINT_NAME) + all_gcs_client_ids = utils.getAllGCSClientIds(ac_rt, project_id, ENDPOINT_NAME) - if valid_key is False and len(all_gcs_client_ids) > 0: - print("Looks like gcs client does not exist in the cloud" - f" for the project: {project_id}." - "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") - sys.exit(1) + if valid_key is False and len(all_gcs_client_ids) > 0: + print("Looks like gcs client does not exist in the cloud" + f" for the project: {project_id}." + "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}") + sys.exit(1) - if gcs_id_from_deployment_key is None and len(all_gcs_client_ids) > 0: - print("Looks like deployment key does not exist, please either " - "add the correct deployment." - f" cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") - sys.exit(1) + if gcs_id_from_deployment_key is None and len(all_gcs_client_ids) > 0: + print("Looks like deployment key does not exist, please either " + "add the correct deployment." + f" cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}") + sys.exit(1) - if len(all_gcs_client_ids) > 0: + if len(all_gcs_client_ids) > 0: - if utils.command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") - sys.exit(1) + if utils.command_exists("globus-connect-server") is False: + print("Cannot create deployment key, we require globus-connect-server to be installed") + sys.exit(1) - else: + else: - print("Now that we know a GCS instance exists we have to make sure" - "we have valid credentials to run the globus-connect-server command" - "non interatively, this means we have to create credentials and a" - "client if they don't exist and when we are done with everything" - "delete them.") + print("Now that we know a GCS instance exists we have to make sure" + "we have valid credentials to run the globus-connect-server command" + "non interatively, this means we have to create credentials and a" + "client if they don't exist and when we are done with everything" + "delete them.") - client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) + client_id, client_secret = utils.createClient( + ac_rt, + CLIENT_NAME, + project_id, + CRED_NAME, + CRED_FILE_PATH) - ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) + ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+="globus-connect-server endpoint cleanup " - bash_command+=f" --deployment-key \"{DEPLOYMENT_KEY_PATH}\" " - bash_command+=" --agree-to-delete-endpoint" - print("Bash command to run") - print(bash_command) - - proc = subprocess.Popen(bash_command, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, shell=True, text=True) + bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " + bash_command+="globus-connect-server endpoint cleanup " + bash_command+=f" --deployment-key \"{DEPLOYMENT_KEY_PATH}\" " + bash_command+=" --agree-to-delete-endpoint" + print("Bash command to run") + print(bash_command) + + proc = subprocess.Popen(bash_command, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, shell=True, text=True) - output, error = proc.communicate(input="yes\n") + output, error = proc.communicate(input="yes\n") - # Print the output and error, if any - print("Output:", output) - print("Error:", error) + # Print the output and error, if any + print("Output:", output) + print("Error:", error) - # Now we can try to delete the remaining clients that are in the project - # Get all of the clients that are not gcs clients and delete them + # Now we can try to delete the remaining clients that are in the project + # Get all of the clients that are not gcs clients and delete them - utils.deleteAllNonGCSClients(ac_rt, project_id) - + utils.deleteAllNonGCSClients(ac_rt, project_id) + # CLOSE - if len(clients_in_project) == 0: # Try to remove project this will only work if there are no other clients in # the project -print(f"Attempting to remove project {project_id}") -project_remove = ac_rt.delete_project(project_id) -print(project_remove) + print(f"Attempting to remove project {project_id}") + project_remove = ac_rt.delete_project(project_id) + print(project_remove) + +# Now trying to clean up groups +group_display_name = f"{DATAFED_GCS_ROOT_NAME} Group Creator: {username}" +deleteGroup(group_display_name) From 4c4667e5eae9c0d31bf552cf026e4b0977fa184e Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:56:11 -0400 Subject: [PATCH 12/84] Add groupExists getGroupId and deleteGroup functions --- scripts/globus/utils.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 84194498a..3a617e674 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -92,6 +92,31 @@ def getCredentialID(auth_client, client_id, cred_name): return cred['id'] return None +def groupExists(client, group_name): + my_groups = client.get_my_groups() + for group in my_groups: + if group["name"] == group_name: + return True + return False + +def getGroupId(client, group_name): + my_groups = client.get_my_groups() + for group in my_groups: + if group["name"] == group_name: + return group["id"] + return None + + + +def deleteGroup(client, group_name): + my_groups = client.get_my_groups() + for group in my_groups: + if group["name"] == group_name: + result = client.delete_group(group["id"]) + print(f"Removing group: {group_name} with id: {group['id']}") + print(result) + + def validFile(file_name): file_exists = False file_empty = True From d3e34ea59a0feb12a8223160c868dfb1f9d1de35 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 22:56:38 -0400 Subject: [PATCH 13/84] Add ability to add client to subscription --- scripts/globus/initialize_globus_endpoint.py | 60 +++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index a45684fbb..12900979d 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -2,7 +2,9 @@ import globus_sdk import subprocess import utils -from globus_sdk import AuthClient, AccessTokenAuthorizer +from globus_sdk import AuthClient, GroupsClient, AccessTokenAuthorizer +from globus_sdk.scopes import GroupsScopes + import json import os import sys @@ -67,7 +69,13 @@ # manage_projects scope to create a project # view_identities to user information for creating GCS server -client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) + +group_scope = GroupsScopes.make_mutable("all") +client.oauth2_start_flow(requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + + str(group_scope), + refresh_tokens=True) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -76,15 +84,24 @@ token_response = client.oauth2_exchange_code_for_tokens(auth_code) # Extract the token refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) + +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, + client) # auth_client_refresh_token ac_rt = AuthClient(authorizer=rt_authorizer) +gr_rt = GroupsClient(authorizer=rt_authorizer_groups) userinfo = ac_rt.oauth2_userinfo() # Will get the primary email and id identity_id = userinfo["sub"] email = userinfo["email"] username = userinfo["preferred_username"] +print("username") +print(username) +print("userinfo") +print(userinfo) organization = userinfo["identity_provider_display_name"] # Need to determine the project uuid @@ -125,3 +142,42 @@ DATAFED_GLOBUS_SUBSCRIPTION, userinfo) +# Create subscription subgroup +results = gr_rt.get_group_by_subscription_id(DATAFED_GLOBUS_SUBSCRIPTION) + +parent_group_id=results["group_id"] +print("Groups by sub") +print(results) +group_name = f"{DATAFED_GCS_ROOT_NAME} Group Creator: {username}" + +if groupExists(gr_rt, group_name): + group_id = getGroupId(gr_rt, group_name) +else: + package = { + "name": group_name, + "description": "DataFed Repository Subscription Group, used for" + "granting access to the application client to setup the repository in " + "Globus", + "parent_id": str(parent_group_id) + } + + + result = gr_rt.create_group(package) + group_id = result['id'] + +print("group id") +print(group_id) + +batch = globus_sdk.BatchMembershipActions() +batch.add_members(client_id, role="admin") +result = gr_rt.batch_membership_action(group_id, batch) + +print("membership_action") +print(result) +package = { + "subscription_id": DATAFED_GLOBUS_SUBSCRIPTION + } +result = gr_rt.update_group(group_id, package) +print("update group") +print(result) + From d86de13c9f7b01b5bce12f1e0a56f2625e4c06b2 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 23:07:29 -0400 Subject: [PATCH 14/84] Adjustments to setup_globus.sh to account for subscription --- scripts/globus/setup_globus.sh | 36 +++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/scripts/globus/setup_globus.sh b/scripts/globus/setup_globus.sh index 48f626e5d..93d99d779 100755 --- a/scripts/globus/setup_globus.sh +++ b/scripts/globus/setup_globus.sh @@ -33,6 +33,14 @@ else CRED_FILE_PATH="$DATAFED_GLOBUS_CRED_FILE_PATH" fi +if [ -z "${DATAFED_GLOBUS_SUBSCRIPTION}" ] +then + echo "DATAFED_GLOBUS_SUBSCRIPTION not defined" + DATAFED_GLOBUS_SUBSCRIPTION="" +else + DATAFED_GLOBUS_SUBSCRIPTION=$(printenv DATAFED_GLOBUS_SUBSCRIPTION) +fi + if [ -f "$CRED_FILE_PATH" ]; then echo "File exists! $CRED_FILE_PATH" else @@ -70,7 +78,9 @@ cat << EOF > mapping.json EOF #DOMAINS="--domain ornl.gov --domain clients.auth.globus.org --domain gmail.com" -DOMAINS="--domain ornl.gov" +# For the Globus client to create the guest collection need to allow client +# from the domain +DOMAINS="--domain ornl.gov --domain clients.auth.globus.org" echo "{" > path_restriction.json echo " \"DATA_TYPE\": \"path_restrictions#1.0.0\"," >> path_restriction.json @@ -110,28 +120,36 @@ spaces_in_name=$(echo $GATEWAY_NAME | awk '{print gsub("[ \t]",""); exit}') columns=$(( $spaces_in_name + 3 )) uuid_of_storage_gateway=$( globus-connect-server storage-gateway list | grep "$GATEWAY_NAME" | awk -v col=$columns '{ print $col }') +# If not empty +extra_collection_arg="" +if [ -n "${DATAFED_GLOBUS_SUBSCRIPTION}" ] +then + globus-connect-server endpoint set-subscription-id "${DATAFED_GLOBUS_SUBSCRIPTION}" + extra_collection_arg="--allow-guest-collections" +fi + if [ -z "$collection_line" ] then + # NOTE enable-anonymous-writes is allowed without a subscription + # NOTE allow-guest-collections requires a subscription globus-connect-server collection create \ "$uuid_of_storage_gateway" \ "/" \ "$COLLECTION_NAME" \ - --enable-anonymous-writes \ - --allow-guest-collections \ --default-directory "${GCS_COLLECTION_ROOT_PATH}" \ - --disable-https + --enable-anonymous-writes \ + --disable-https "$extra_collection_arg" else uuid_of_collection=$( globus-connect-server collection list | grep "$COLLECTION_NAME" | awk '{ print $1 }') - + # NOTE enable-anonymous-writes is allowed without a subscription + # NOTE allow-guest-collections requires a subscription globus-connect-server collection update \ "$uuid_of_collection" \ - --enable-anonymous-writes \ - --allow-guest-collections \ --default-directory "${GCS_COLLECTION_ROOT_PATH}" \ - --disable-https - + --enable-anonymous-writes \ + --disable-https "$extra_collection_arg" fi echo "When creating a guest collection it must be created in: $GCS_COLLECTION_ROOT_PATH" From 17a6b42cf2bee04ce7a112e1b836398c6e4d4f57 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 23:07:57 -0400 Subject: [PATCH 15/84] add namespace to function --- scripts/globus/globus_cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 6e81980b7..fa16651fe 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -188,5 +188,5 @@ # Now trying to clean up groups group_display_name = f"{DATAFED_GCS_ROOT_NAME} Group Creator: {username}" -deleteGroup(group_display_name) +utils.deleteGroup(group_display_name) From 36958de764b3206da9269b427de1b03ade67755c Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 23:08:28 -0400 Subject: [PATCH 16/84] Allow for building with and or without subscription --- scripts/globus/initialize_globus_endpoint.py | 64 ++++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index 12900979d..fb3749954 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -63,7 +63,7 @@ if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: DATAFED_GLOBUS_SUBSCRIPTION=os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") else: - DATAFED_GLOBUS_SUBSCRIPTION="" + DATAFED_GLOBUS_SUBSCRIPTION=None client = globus_sdk.NativeAppAuthClient(CLIENT_ID) @@ -139,45 +139,45 @@ DEPLOYMENT_KEY_PATH, ENDPOINT_NAME, DATAFED_GLOBUS_CONTROL_PORT, - DATAFED_GLOBUS_SUBSCRIPTION, userinfo) +if DATAFED_GLOBUS_SUBSCRIPTION is not None: # Create subscription subgroup -results = gr_rt.get_group_by_subscription_id(DATAFED_GLOBUS_SUBSCRIPTION) + results = gr_rt.get_group_by_subscription_id(DATAFED_GLOBUS_SUBSCRIPTION) -parent_group_id=results["group_id"] -print("Groups by sub") -print(results) -group_name = f"{DATAFED_GCS_ROOT_NAME} Group Creator: {username}" + parent_group_id=results["group_id"] + print("Groups by sub") + print(results) + group_name = f"{DATAFED_GCS_ROOT_NAME} Group Creator: {username}" -if groupExists(gr_rt, group_name): - group_id = getGroupId(gr_rt, group_name) -else: - package = { - "name": group_name, - "description": "DataFed Repository Subscription Group, used for" - "granting access to the application client to setup the repository in " - "Globus", - "parent_id": str(parent_group_id) - } + if utils.groupExists(gr_rt, group_name): + group_id = utils.getGroupId(gr_rt, group_name) + else: + package = { + "name": group_name, + "description": "DataFed Repository Subscription Group, used for" + "granting access to the application client to setup the repository in " + "Globus", + "parent_id": str(parent_group_id) + } - result = gr_rt.create_group(package) - group_id = result['id'] + result = gr_rt.create_group(package) + group_id = result['id'] -print("group id") -print(group_id) + print("group id") + print(group_id) -batch = globus_sdk.BatchMembershipActions() -batch.add_members(client_id, role="admin") -result = gr_rt.batch_membership_action(group_id, batch) + batch = globus_sdk.BatchMembershipActions() + batch.add_members(client_id, role="admin") + result = gr_rt.batch_membership_action(group_id, batch) -print("membership_action") -print(result) -package = { - "subscription_id": DATAFED_GLOBUS_SUBSCRIPTION - } -result = gr_rt.update_group(group_id, package) -print("update group") -print(result) + print("membership_action") + print(result) + package = { + "subscription_id": DATAFED_GLOBUS_SUBSCRIPTION + } + result = gr_rt.update_group(group_id, package) + print("update group") + print(result) From a3c5b05bab7464acbfe3495bd34e57d285d2bab1 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 23:08:58 -0400 Subject: [PATCH 17/84] Remove subscription argument from create endpoint --- scripts/globus/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 3a617e674..a34f2b904 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -292,7 +292,6 @@ def createGCSEndpoint( deployment_key_file, endpoint_name, control_port, - subscription_id, userinfo): identity_id = userinfo["sub"] From 4d96acd1adca22f02b2b685890b09cedf0fc22b6 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 23:09:43 -0400 Subject: [PATCH 18/84] Make sure that globus_sdk is installed on the authz container --- scripts/install_authz_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_authz_dependencies.sh b/scripts/install_authz_dependencies.sh index 9fe40dc9e..ce0cc0224 100755 --- a/scripts/install_authz_dependencies.sh +++ b/scripts/install_authz_dependencies.sh @@ -13,7 +13,7 @@ source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "build-essential" "g++" "gcc" "autoconf" "automake" "make" "git" "python3-pkg-resources" "python3-pip" "pkg-config" "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev") -pip_packages=("setuptools" "distro" "jwt") +pip_packages=("setuptools" "distro" "jwt" "globus_sdk") externals=("cmake" "protobuf" "libsodium" "libzmq" ) local_UNIFY=false From 491a1c5f2eab53c26cc360b1b212a074f3cc2df4 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 21 Mar 2024 23:23:44 -0400 Subject: [PATCH 19/84] Add creation of guest collection --- scripts/globus/create_guest_collection.py | 284 ++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 scripts/globus/create_guest_collection.py diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py new file mode 100644 index 000000000..d656b1ba0 --- /dev/null +++ b/scripts/globus/create_guest_collection.py @@ -0,0 +1,284 @@ + +import globus_sdk +import subprocess +import utils +from globus_sdk import AuthClient, AccessTokenAuthorizer +import json +import os +import sys + + +# The Globus project the GCS endpoint will be created in +if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: + DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") +else: + DATAFED_GCS_ROOT_NAME="DataFed Repo" + +if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: + PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") +else: + PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" + +# This is for confidential client +if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: + CLIENT_NAME = os.getenv("DATAFED_GLOBUS_CLIENT_NAME") +else: + CLIENT_NAME = DATAFED_GCS_ROOT_NAME + " Setup Client" + +# Name of the client secret used by the confidential client +if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: + CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") +else: + CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" + +# Name of the file where we will store confidential client credentials +if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: + CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") +else: + CRED_FILE_PATH="./client_cred.json" + +if os.getenv("GCS_CLI_ENDPOINT_ID") is not None: + ENDPOINT_ID = os.getenv("GCS_CLI_ENDPOINT_ID") +else: + raise Exception("GCS_CLI_ENDPOINT_ID must be defined") + +# Name to give to endpoint +if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: + ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") +else: + ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" + +# Path to deployment key +if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: + DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") +else: + DEPLOYMENT_KEY_PATH="./deployment-key.json" + +# Path to deployment key +if os.getenv("DATAFED_GLOBUS_CONTROL_PORT") is not None: + DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT") +else: + DATAFED_GLOBUS_CONTROL_PORT="443" + +if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: + DATAFED_GLOBUS_SUBSCRIPTION=os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") +else: + DATAFED_GLOBUS_SUBSCRIPTION="" + +if os.getenv("DATAFED_GCS_URL") is not None: + DATAFED_GCS_URL=os.getenv("DATAFED_GCS_URL") +else: + DATAFED_GCS_URL="" + +if os.getenv("GCS_CLI_CLIENT_ID") is not None: + client_id = os.getenv("GCS_CLI_CLIENT_ID") +else: + client_id = None + +if os.getenv("GCS_CLI_CLIENT_SECRET") is not None: + client_secret = os.getenv("GCS_CLI_CLIENT_SECRET") +else: + client_secret = None + +if os.getenv("MAPPED_COLLECTION_ID") is not None: + mapped_collection_id = os.getenv("MAPPED_COLLECTION_ID") +else: + mapped_collection_id = None + +if os.getenv("DATAFED_GCS_COLLECTION_MAPPED") is not None: + mapped_collection_name = os.getenv("DATAFED_GCS_COLLECTION_MAPPED") +else: + mapped_collection_name = f"{DATAFED_GCS_ROOT_NAME} Collection Mapped" + +if os.getenv("DATAFED_GCS_COLLECTION_GUEST") is not None: + guest_collection_name = os.getenv("DATAFED_GCS_COLLECTION_GUEST") +else: + guest_collection_name = f"{DATAFED_GCS_ROOT_NAME} Collection Guest" + +if os.getenv("STORAGE_GATEWAY_ID") is not None: + storage_gateway_id = os.getenv("STORAGE_GATEWAY_ID") +else: + storage_gateway_id = None + + +if os.getenv("DATAFED_GCS_STORAGE_GATEWAY") is not None: + storage_gateway_name = os.getenv("DATAFED_GCS_STORAGE_GATEWAY") +else: + storage_gateway_name = f"{DATAFED_GCS_ROOT_NAME} Storage Gateway" + + +if os.getenv("DATAFED_REPO_USER") is not None: + local_username = os.getenv("DATAFED_REPO_USER") +else: + raise Exception("DATAFED_REPO_USER is not defined.") + +#client = globus_sdk.NativeAppAuthClient(CLIENT_ID) + +# manage_projects scope to create a project +# view_identities to user information for creating GCS server +#client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +# +#authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) +#print("Please go to this URL and login: \n", authorize_url) +#auth_code = input("Please enter the authorization code: ") +# +#token_response = client.oauth2_exchange_code_for_tokens(auth_code) +## Extract the token +#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +## auth_client_refresh_token +#ac_rt = AuthClient(authorizer=rt_authorizer) +# +#userinfo = ac_rt.oauth2_userinfo() +## Will get the primary email and id +#identity_id = userinfo["sub"] +#email = userinfo["email"] +#username = userinfo["preferred_username"] +#organization = userinfo["identity_provider_display_name"] + +if client_id is None: + client_id = getClientIdFromCredFile(CRED_FILE_PATH) + +if client_secret is None: + client_secret = getCredentialFromFile(CRED_FILE_PATH, client_id) + +client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) + +scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" +authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) +#cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, +# scopes) + +#token_response = client.oauth2_client_credentials_tokens() + +#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +# the useful values that you want at the end of this +#globus_auth_data = token_response.by_resource_server["auth.globus.org"] +#globus_transfer_data = +#token_response.by_resource_server["transfer.api.globus.org"] +#globus_auth_token = globus_auth_data["access_token"] +#globus_transfer_token = globus_transfer_data["access_token"] + +gcs_client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) + + +import globus_sdk +from globus_sdk import scopes + +# constants +#endpoint_hostname = "https://ecf8ed.08cc.data.globus.org" +#endpoint_id = "769c7ed0-744a-41b5-b4a8-db37b10b1ac9" +#mapped_collection_id = "580ecb92-de56-42ee-a5ec-3d3886767b94" +#storage_gateway_id = "3fdd7f41-4a05-4856-8fcd-2fb50066c590" + +# client credentials +# This client identity must have the needed permissions to create a guest +# collection on the mapped collection, and a valid mapping to a local account +# on the storage gateway that matches the local_username +# If using user tokens, the user must be the one with the correct permissions +# and identity mapping. +#client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" +#client_secret = "*redacted*" +#local_username = "datafed" + +# The scope the client will need, note that primary scope is for the endpoint, +# but it has a dependency on the mapped collection's data_access scope + +# Build a GCSClient to act as the client by using a ClientCredentialsAuthorizor +confidential_client = globus_sdk.ConfidentialAppAuthClient( + client_id=client_id, client_secret=client_secret +) +scope = scopes.GCSEndpointScopeBuilder(GCS_CLI_ENDPOINT_ID).make_mutable("manage_collections") +authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, scopes=scope) +client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) + +collection_list = client.get_collection_list() +print("collection_list") +print(collection_list) + +mapped_collection_found = False + +for item in collection_list["data"] + print(item["display_name"]) + if mapped_collection_id is not None: + if item["id"] == mapped_collection_id: + mapped_collection_found = True + if item["display_name"] != mapped_collection_name: + raise Exception("Expected display name is different from what " + "is expected for mapped collection " + f"{mapped_collection_id}, if using non standard" + " display name for mapped collection " + f"{mapped_collection_name} then the " + "MAPPED_COLLECTION_NAME env variable must be " + "set.") + break + elif item["display_name"] == mapped_collection_name: + mapped_collection_found = True + mapped_collection_id = item["id"] + break + +if mapped_collection_found == False: + raise Exception("Missing required mapped collection") + +storage_gateway_found = False +storage_gateway_list = client.get_storage_gateway_list() +for item in storage_gateway_list["data"] + print(item["display_name"]) + if storage_gateway_id is not None: + if item["id"] == storage_gateway_id: + storage_gateway_found = True + if item["display_name"] != storage_gateway_name: + raise Exception("Expected display name is different from what " + "is expected for storage gateway " + f"{storage_gateway_id}, if using non standard" + " display name for storage gateway " + f"{storage_gateway_name} then the " + "DATAFED_GCS_STORAGE_GATEWAY env variable must be " + "set.") + break + elif item["display_name"] == storage_gateway_name: + storage_gateway_found = True + storage_gateway_id = item["id"] + break + +if storage_gateway_found == False: + raise Exception("Missing required storage gateway") + + +# Get the mapped collection id +scope.add_dependency(scopes.GCSCollectionScopeBuilder(mapped_collection_id).data_access) +authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, scopes=scope) +client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) + +guest_collection_found = False + +for item in collection_list["data"] + print(item["display_name"]) + if item["display_name"] == guest_collection_name: + guest_collection_found = True + guest_collection_id = item["id"] + break + +# https://github.com/globus/globus-sdk-python/blob/main/docs/examples/guest_collection_creation.rst +if guest_collection_found == False: + credential_document = globus_sdk.UserCredentialDocument( + storage_gateway_id=storage_gateway_id, + identity_id=client_id, + username=local_username, + ) + client.create_user_credential(credential_document) + +# Create the collection + collection_document = globus_sdk.GuestCollectionDocument( + public="True", + collection_base_path="/", + display_name=guest_collection_name, + mapped_collection_id=mapped_collection_id, + ) + response = client.create_collection(collection_document) + guest_collection_id = response["id"] + print(f"guest collection {guest_collection_id} created") + + +# For guest collection creation see From 80dee35555e6cadd2e838bac3ce6135d802ba8b8 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 3 Apr 2024 12:59:25 -0400 Subject: [PATCH 20/84] Pass GCS ROOT NAME variable to globus cleanup python script in entrypoint --- compose/cleanup_globus_files.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/cleanup_globus_files.sh b/compose/cleanup_globus_files.sh index 44ceb76b9..b6eda3ade 100755 --- a/compose/cleanup_globus_files.sh +++ b/compose/cleanup_globus_files.sh @@ -31,6 +31,7 @@ fi sudo globus-connect-server node cleanup +DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \ python3 "${PROJECT_ROOT}/scripts/globus/globus_cleanup.py" #rm -rf "${PROJECT_ROOT}/compose/globus" From a6e0515c3e27d5370e6dc6ccceb61fa589594bdc Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 3 Apr 2024 13:00:15 -0400 Subject: [PATCH 21/84] Add missing variable from compose file --- compose/compose_repo.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/compose_repo.yml b/compose/compose_repo.yml index f0c027682..8aa22e71c 100644 --- a/compose/compose_repo.yml +++ b/compose/compose_repo.yml @@ -11,6 +11,7 @@ services: DATAFED_HTTPS_SERVER_PORT: "${DATAFED_HTTPS_SERVER_PORT}" DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" DATAFED_CORE_ADDRESS_PORT_INTERNAL: "datafed-core:7513" + DATAFED_GCS_COLLECTION_ROOT_PATH: "/mnt/datafed" UID: "${DATAFED_UID}" HOST_HOSTNAME: "localhost" network_mode: host @@ -32,7 +33,7 @@ services: UID: "${DATAFED_UID}" HOST_HOSTNAME: "localhost" DATAFED_AUTHZ_USER: "datafed" - GCS_COLLECTION_ROOT_PATH: "/mnt/datafed" + DATAFED_GCS_COLLECTION_ROOT_PATH: "/mnt/datafed" network_mode: host image: datafed-gcs:latest volumes: From 735eb0f911346e2af24d21033d9c0053f6af5cf4 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Wed, 3 Apr 2024 13:03:28 -0400 Subject: [PATCH 22/84] Standardize env variable name --- compose/generate_globus_files.sh | 1 + doc_source/source/admin/general.rst | 2 +- docs/_sources/admin/general.rst.txt | 2 +- docs/admin/general.html | 4 ++-- external/globus-connect-server-deploy | 2 +- repository/docker/Dockerfile | 1 + repository/docker/Dockerfile.gcs | 3 +-- repository/docker/Dockerfile.gcs-authz.ubuntu | 2 +- repository/docker/entrypoint_authz.sh | 4 ++-- scripts/generate_authz_config.sh | 10 +++++----- scripts/generate_datafed.sh | 6 +++--- scripts/generate_repo_config.sh | 10 +++++----- scripts/globus/generate_repo_form.sh | 6 +++--- scripts/globus/globus_cleanup.py | 8 ++++---- scripts/globus/initialize_globus_endpoint.py | 9 +++++++-- scripts/globus/setup_globus.sh | 18 +++++++++--------- scripts/globus/utils.py | 2 ++ 17 files changed, 49 insertions(+), 41 deletions(-) diff --git a/compose/generate_globus_files.sh b/compose/generate_globus_files.sh index 4f932a730..0ed182846 100755 --- a/compose/generate_globus_files.sh +++ b/compose/generate_globus_files.sh @@ -25,4 +25,5 @@ DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH="$DATAFED_HOST_DEPLOYMENT_KEY_PATH" \ DATAFED_GLOBUS_CRED_FILE_PATH="$DATAFED_HOST_CRED_FILE_PATH" \ DATAFED_GLOBUS_CONTROL_PORT="$DATAFED_GLOBUS_CONTROL_PORT" \ DATAFED_GLOBUS_SUBSCRIPTION="$DATAFED_GLOBUS_SUBSCRIPTION" \ +DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \ python3 ${PROJECT_ROOT}/scripts/globus/initialize_globus_endpoint.py diff --git a/doc_source/source/admin/general.rst b/doc_source/source/admin/general.rst index 1b9062e20..6f4a57fed 100644 --- a/doc_source/source/admin/general.rst +++ b/doc_source/source/admin/general.rst @@ -98,7 +98,7 @@ of the configuration options: 10. DATAFED_SERVER_PORT - Needed by repo server 11. DATAFED_DOMAIN - Needed by repo, web and authz 12. DATAFED_GCS_ROOT_NAME - Needed by repo and authz by Globus -13. GCS_COLLECTION_ROOT_PATH - Needed by repo and authz for Globus +13. DATAFED_GCS_COLLECTION_ROOT_PATH - Needed by repo and authz for Globus 14. DATAFED_REPO_ID_AND_DIR - Needed for repo and authz Descriptions of what these variables are can also be found in the ./config/datafed.sh file. Once the diff --git a/docs/_sources/admin/general.rst.txt b/docs/_sources/admin/general.rst.txt index 9b33e9d1e..c780e1c4e 100644 --- a/docs/_sources/admin/general.rst.txt +++ b/docs/_sources/admin/general.rst.txt @@ -98,7 +98,7 @@ of the configuration options: 10. DATAFED_SERVER_PORT - Needed by repo server 11. DATAFED_DOMAIN - Needed by repo, web and authz 12. DATAFED_GCS_ROOT_NAME - Needed by repo and authz by Globus -13. GCS_COLLECTION_ROOT_PATH - Needed by repo and authz for Globus +13. DATAFED_GCS_COLLECTION_ROOT_PATH - Needed by repo and authz for Globus 14. DATAFED_REPO_ID_AND_DIR - Needed for repo and authz Descriptions of what these variables are can also be found in the ./config/datafed.sh file. Once the diff --git a/docs/admin/general.html b/docs/admin/general.html index 370024abd..ef58285ea 100644 --- a/docs/admin/general.html +++ b/docs/admin/general.html @@ -203,7 +203,7 @@

Downloading DataFed & Installing Dependencies

  • DATAFED_SERVER_PORT - Needed by repo server

  • DATAFED_DOMAIN - Needed by repo, web and authz

  • DATAFED_GCS_ROOT_NAME - Needed by repo and authz by Globus

    • -

  • GCS_COLLECTION_ROOT_PATH - Needed by repo and authz for Globus

    • +

  • DATAFED_GCS_COLLECTION_ROOT_PATH - Needed by repo and authz for Globus

  • DATAFED_REPO_ID_AND_DIR - Needed for repo and authz

    • Descriptions of what these variables are can also be found in the ./config/datafed.sh file. Once the @@ -445,4 +445,4 @@

    Networking

    - \ No newline at end of file + diff --git a/external/globus-connect-server-deploy b/external/globus-connect-server-deploy index 1d42a0df7..38a40b7a0 160000 --- a/external/globus-connect-server-deploy +++ b/external/globus-connect-server-deploy @@ -1 +1 @@ -Subproject commit 1d42a0df7cd8389718205b038621677ebb575306 +Subproject commit 38a40b7a00f86b2355663680c328c1fef3311f6a diff --git a/repository/docker/Dockerfile b/repository/docker/Dockerfile index ac48545c5..74a3d5249 100644 --- a/repository/docker/Dockerfile +++ b/repository/docker/Dockerfile @@ -61,6 +61,7 @@ ARG DATAFED_DEPENDENCIES_INSTALL_PATH # The above should also be available at runtime ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" +ENV DATAFED_GCS_COLLECTION_ROOT_PATH="/mnt/datafed" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs index 14a76d14d..b09fdd64b 100644 --- a/repository/docker/Dockerfile.gcs +++ b/repository/docker/Dockerfile.gcs @@ -25,8 +25,7 @@ ARG DATAFED_DEPENDENCIES_INSTALL_PATH ARG LIB_DIR ENV DATAFED_DEPENDENCIES_INSTALL_PATH="$DATAFED_DEPENDENCIES_INSTALL_PATH" -ENV GCS_COLLECTION_ROOT_PATH="/mnt/datafed" -ENV DATAFED_GCS_COLLECTION_ROOT_PATH="$GCS_COLLECTION_ROOT_PATH" +ENV DATAFED_GCS_COLLECTION_ROOT_PATH="/mnt/datafed" ENV DATAFED_GLOBUS_CRED_FILE_PATH=/opt/datafed/globus/client_cred.json ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" ENV DATAFED_DIR="$DATAFED_DIR" diff --git a/repository/docker/Dockerfile.gcs-authz.ubuntu b/repository/docker/Dockerfile.gcs-authz.ubuntu index efc2e0ea8..3b904f544 100644 --- a/repository/docker/Dockerfile.gcs-authz.ubuntu +++ b/repository/docker/Dockerfile.gcs-authz.ubuntu @@ -9,7 +9,7 @@ ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -ENV GCS_COLLECTION_ROOT_PATH="/mnt/datafed" +ENV DATAFED_GCS_COLLECTION_ROOT_PATH="/mnt/datafed" ENV DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" RUN mkdir -p ${DATAFED_INSTALL_PATH}/logs diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index a41406552..b3489b7aa 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -9,7 +9,7 @@ SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath ${SOURCE}/../..) # Translate datafed env variables to globus env variables -export GCS_COLLECTION_ROOT_PATH=/mnt/datafed +export DATAFED_GCS_COLLECTION_ROOT_PATH=/mnt/datafed # This env variables are needed for running globus-connect-server without # logging in @@ -23,7 +23,7 @@ export GLOBUS_CLIENT_ID=$(cat /opt/datafed/globus/client_cred.json | jq -r .cli export GLOBUS_CLIENT_SECRET=$(cat /opt/datafed/globus/client_cred.json | jq -r .secret) export DEPLOYMENT_KEY=$(cat "$DEPLOYMENT_KEY_PATH" ) -chown -R datafed:root ${GCS_COLLECTION_ROOT_PATH} +chown -R datafed:root ${DATAFED_GCS_COLLECTION_ROOT_PATH} "${PROJECT_ROOT}/scripts/generate_datafed.sh" diff --git a/scripts/generate_authz_config.sh b/scripts/generate_authz_config.sh index e9eab44da..374e81974 100755 --- a/scripts/generate_authz_config.sh +++ b/scripts/generate_authz_config.sh @@ -53,11 +53,11 @@ else local_DATAFED_SERVER_PORT=$(printenv DATAFED_SERVER_PORT) fi -if [ -z "${GCS_COLLECTION_ROOT_PATH}" ] +if [ -z "${DATAFED_GCS_COLLECTION_ROOT_PATH}" ] then - local_GCS_COLLECTION_ROOT_PATH="/mnt/datafed-repo/mapped" + local_DATAFED_GCS_COLLECTION_ROOT_PATH="/mnt/datafed-repo/mapped" else - local_GCS_COLLECTION_ROOT_PATH=$(printenv GCS_COLLECTION_ROOT_PATH) + local_DATAFED_GCS_COLLECTION_ROOT_PATH=$(printenv DATAFED_GCS_COLLECTION_ROOT_PATH) fi if [ -z $DATAFED_GLOBUS_REPO_USER ] @@ -101,7 +101,7 @@ while [ : ]; do ;; -g | --globus-collection-path) echo "Processing 'Globus Collection Path' option. Input argument is '$2'" - local_GCS_COLLECTION_ROOT_PATH=$2 + local_DATAFED_GCS_COLLECTION_ROOT_PATH=$2 shift 2 ;; --) shift; @@ -124,7 +124,7 @@ pub_key=${DATAFED_INSTALL_PATH}/keys/datafed-repo-key.pub priv_key=${DATAFED_INSTALL_PATH}/keys/datafed-repo-key.priv log_path=${local_DATAFED_LOG_PATH}/datafed-gsi-authz.log user=$local_DATAFED_AUTHZ_USER -globus-collection-path=$local_GCS_COLLECTION_ROOT_PATH +globus-collection-path=$local_DATAFED_GCS_COLLECTION_ROOT_PATH EOF echo diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index 2dc142e7d..6617069af 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -295,18 +295,18 @@ export DATAFED_GCS_ROOT_NAME="$local_DATAFED_GCS_ROOT_NAME" # # The path will be created if it does not exist # -# \$GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR" +# \$DATAFED_GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR" # # So if these variables are defined as: # DATAFED_GCS_ROOT_NAME="datafed-home" -# GCS_COLLECTION_ROOT_PATH="/home/cades/collections/mapped/" +# DATAFED_GCS_COLLECTION_ROOT_PATH="/home/cades/collections/mapped/" # # A folder named # # "/home/cades/collections/mapped/datafed-home" # # Will be created -export GCS_COLLECTION_ROOT_PATH="$local_DATAFED_GCS_COLLECTION_ROOT_PATH" +export DATAFED_GCS_COLLECTION_ROOT_PATH="$local_DATAFED_GCS_COLLECTION_ROOT_PATH" # The DataFed repo id, this also must be the name # of the directory that will be placed in Globus # collection, avoid using spaces in the name. diff --git a/scripts/generate_repo_config.sh b/scripts/generate_repo_config.sh index f11d45b7b..bfc2e30a1 100755 --- a/scripts/generate_repo_config.sh +++ b/scripts/generate_repo_config.sh @@ -38,11 +38,11 @@ else local_DATAFED_DOMAIN=$(printenv DATAFED_DOMAIN) fi -if [ -z "${GCS_COLLECTION_ROOT_PATH}" ] +if [ -z "${DATAFED_GCS_COLLECTION_ROOT_PATH}" ] then - local_GCS_COLLECTION_ROOT_PATH="/mnt/datafed-repo/mapped" + local_DATAFED_GCS_COLLECTION_ROOT_PATH="/mnt/datafed-repo/mapped" else - local_GCS_COLLECTION_ROOT_PATH=$(printenv GCS_COLLECTION_ROOT_PATH) + local_DATAFED_GCS_COLLECTION_ROOT_PATH=$(printenv DATAFED_GCS_COLLECTION_ROOT_PATH) fi local_DATAFED_LOG_PATH="" @@ -91,7 +91,7 @@ while [ : ]; do ;; -g | --globus-collection-path) echo "Processing 'Globus Collection Path' option. Input argument is '$2'" - local_GCS_COLLECTION_ROOT_PATH=$2 + local_DATAFED_GCS_COLLECTION_ROOT_PATH=$2 shift 2 ;; --) shift; @@ -116,7 +116,7 @@ cred-dir=$local_DATAFED_CRED_DIR server=tcp://$local_DATAFED_DOMAIN:${local_DATAFED_PORT} port=$local_DATAFED_REPO_PORT threads=$local_DATAFED_REPO_THREADS -globus-collection-path=$local_GCS_COLLECTION_ROOT_PATH +globus-collection-path=$local_DATAFED_GCS_COLLECTION_ROOT_PATH EOF echo diff --git a/scripts/globus/generate_repo_form.sh b/scripts/globus/generate_repo_form.sh index ba4516dff..c57f8dfa9 100755 --- a/scripts/globus/generate_repo_form.sh +++ b/scripts/globus/generate_repo_form.sh @@ -27,9 +27,9 @@ then exit 1 fi -if [ -z "$GCS_COLLECTION_ROOT_PATH" ] +if [ -z "$DATAFED_GCS_COLLECTION_ROOT_PATH" ] then - echo "GCS_COLLECTION_ROOT_PATH is not defined cannot run $SCRIPT" + echo "DATAFED_GCS_COLLECTION_ROOT_PATH is not defined cannot run $SCRIPT" exit 1 fi @@ -93,7 +93,7 @@ public_key=$(cat ${DATAFED_INSTALL_PATH}/keys/datafed-repo-key.pub) GATEWAY_NAME="${DATAFED_GCS_ROOT_NAME} Storage Gateway" GUEST_COLLECTION_NAME="${DATAFED_GCS_ROOT_NAME} Collection Guest" MAPPED_COLLECTION_NAME="${DATAFED_GCS_ROOT_NAME} Collection Mapped" -PATH_TO_GUEST_ROOT="${GCS_COLLECTION_ROOT_PATH}" +PATH_TO_GUEST_ROOT="${DATAFED_GCS_COLLECTION_ROOT_PATH}" uuid_of_collection=$( globus-connect-server collection list | grep "$GUEST_COLLECTION_NAME" | awk '{ print $1 }') diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index fa16651fe..4f0cba90f 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -167,8 +167,8 @@ output, error = proc.communicate(input="yes\n") # Print the output and error, if any - print("Output:", output) - print("Error:", error) + print(output) + print(error) # Now we can try to delete the remaining clients that are in the project @@ -186,7 +186,7 @@ print(project_remove) # Now trying to clean up groups -group_display_name = f"{DATAFED_GCS_ROOT_NAME} Group Creator: {username}" +group_display_name = f"{DATAFED_GCS_ROOT_NAME} Group" -utils.deleteGroup(group_display_name) +utils.deleteGroup(gr_rt, group_display_name) diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index fb3749954..d2985b916 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -105,7 +105,10 @@ organization = userinfo["identity_provider_display_name"] # Need to determine the project uuid -project_id = utils.createProject(ac_rt, PROJECT_NAME, userinfo) +if utils.projectExists(ac_rt, PROJECT_NAME) == False: + project_id = utils.createProject(ac_rt, PROJECT_NAME, userinfo) +else: + project_id = utils.getProjectId(ac_rt, PROJECT_NAME, userinfo) count = utils.countProjects(ac_rt, PROJECT_NAME) @@ -148,11 +151,13 @@ parent_group_id=results["group_id"] print("Groups by sub") print(results) - group_name = f"{DATAFED_GCS_ROOT_NAME} Group Creator: {username}" + group_name = f"{DATAFED_GCS_ROOT_NAME} Group" if utils.groupExists(gr_rt, group_name): + print("Group exists already") group_id = utils.getGroupId(gr_rt, group_name) else: + print(f"Group does not exist {group_name}") package = { "name": group_name, "description": "DataFed Repository Subscription Group, used for" diff --git a/scripts/globus/setup_globus.sh b/scripts/globus/setup_globus.sh index 93d99d779..4f19be298 100755 --- a/scripts/globus/setup_globus.sh +++ b/scripts/globus/setup_globus.sh @@ -13,9 +13,9 @@ then exit 1 fi -if [ -z "$GCS_COLLECTION_ROOT_PATH" ] +if [ -z "$DATAFED_GCS_COLLECTION_ROOT_PATH" ] then - echo "GCS_COLLECTION_ROOT_PATH is not defined cannot run $SCRIPT" + echo "DATAFED_GCS_COLLECTION_ROOT_PATH is not defined cannot run $SCRIPT" exit 1 fi @@ -84,7 +84,7 @@ DOMAINS="--domain ornl.gov --domain clients.auth.globus.org" echo "{" > path_restriction.json echo " \"DATA_TYPE\": \"path_restrictions#1.0.0\"," >> path_restriction.json -echo " \"read_write\": [\"${GCS_COLLECTION_ROOT_PATH}\"]" >> path_restriction.json +echo " \"read_write\": [\"${DATAFED_GCS_COLLECTION_ROOT_PATH}\"]" >> path_restriction.json echo "}" >> path_restriction.json if [ -z "$gateway_line" ] @@ -111,8 +111,8 @@ else fi # Create project/ and /user folders -mkdir -p "${GCS_COLLECTION_ROOT_PATH}/${DATAFED_REPO_ID_AND_DIR}/user" -mkdir -p "${GCS_COLLECTION_ROOT_PATH}/${DATAFED_REPO_ID_AND_DIR}/project" +mkdir -p "${DATAFED_GCS_COLLECTION_ROOT_PATH}/${DATAFED_REPO_ID_AND_DIR}/user" +mkdir -p "${DATAFED_GCS_COLLECTION_ROOT_PATH}/${DATAFED_REPO_ID_AND_DIR}/project" collection_line=$( globus-connect-server collection list | grep "$COLLECTION_NAME" ) @@ -137,7 +137,7 @@ then "$uuid_of_storage_gateway" \ "/" \ "$COLLECTION_NAME" \ - --default-directory "${GCS_COLLECTION_ROOT_PATH}" \ + --default-directory "${DATAFED_GCS_COLLECTION_ROOT_PATH}" \ --enable-anonymous-writes \ --disable-https "$extra_collection_arg" else @@ -147,14 +147,14 @@ else # NOTE allow-guest-collections requires a subscription globus-connect-server collection update \ "$uuid_of_collection" \ - --default-directory "${GCS_COLLECTION_ROOT_PATH}" \ + --default-directory "${DATAFED_GCS_COLLECTION_ROOT_PATH}" \ --enable-anonymous-writes \ --disable-https "$extra_collection_arg" fi -echo "When creating a guest collection it must be created in: $GCS_COLLECTION_ROOT_PATH" +echo "When creating a guest collection it must be created in: $DATAFED_GCS_COLLECTION_ROOT_PATH" echo "And the display name should be exactly: $GUEST_COLLECTION_NAME" echo "You will also need to add permissions for all Globus users so that they have write access." echo "" echo "When registering the repository with DataFed the ID must be: $DATAFED_REPO_ID_AND_DIR" -echo "When registering the repository with DataFed path is abs to the mapped collection and must be listed as: ${GCS_COLLECTION_ROOT_PATH}/${DATAFED_REPO_ID_AND_DIR}" +echo "When registering the repository with DataFed path is abs to the mapped collection and must be listed as: ${DATAFED_GCS_COLLECTION_ROOT_PATH}/${DATAFED_REPO_ID_AND_DIR}" diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index a34f2b904..64bc381f5 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -94,6 +94,8 @@ def getCredentialID(auth_client, client_id, cred_name): def groupExists(client, group_name): my_groups = client.get_my_groups() + print("My groups") + print(my_groups) for group in my_groups: if group["name"] == group_name: return True From f054c315079f1df28a984c40022977a1e424b344 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:14:57 -0400 Subject: [PATCH 23/84] Update README.md to include instructions for building repo compose --- compose/README.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/compose/README.md b/compose/README.md index 75eebc1a7..0609610bf 100644 --- a/compose/README.md +++ b/compose/README.md @@ -1,16 +1,23 @@ # Compose Dev environment -The files in this folder are incomplete but are the start for setting up a full -docker compose instance of datafed. +## Generating configuration varaibles + +Create the .env file fill in the missing components that are required. ```bash -./build_images_for_compose.sh +./generate_env.sh ``` -Create the .env file fill in the missing components that are required. +Both the repo compose file and the core compose file will use the .env file +that is generated from this step. + +## Building Core Services + +The files in this folder are incomplete but are the start for setting up a full +docker compose instance of datafed. ```bash -./generate_env.sh +./build_images_for_compose.sh ``` Stand up the core services. @@ -28,6 +35,18 @@ browser to allow you to see the page. Standing up the repo services has been separated because of Globus. You will need a machine with firewall exceptions to use it. +## Building Repo & Authz-Globus Services + +```bash +./build_repo_images_for_compose.sh +``` + +Standing up the repo services. NOTE gcs-authz container will use host network +due to the large numbers of ports that are needed by Globus. + +```bash +docker compose -f ./compose_repo.yml up +``` ## Running individual containers From a27d74dde7bea6891d19474b5452dc7cda00f08e Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:19:04 -0400 Subject: [PATCH 24/84] Fix env variables so they are pulled from .env file --- compose/compose_repo.yml | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/compose/compose_repo.yml b/compose/compose_repo.yml index 8aa22e71c..dc9b1a61d 100644 --- a/compose/compose_repo.yml +++ b/compose/compose_repo.yml @@ -1,48 +1,59 @@ -version: '3.9' - +# WARNING +# +# Any env variable that must be provided and overwrite what is in the container +# Must be explicitly listed in the environment section of the specific service +# --env file variables will not be default exist in the container. services: datafed-repo: environment: DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" - DATAFED_DOMAIN: "datafed-core" + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" DATAFED_HTTPS_SERVER_PORT: "${DATAFED_HTTPS_SERVER_PORT}" DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" - DATAFED_CORE_ADDRESS_PORT_INTERNAL: "datafed-core:7513" - DATAFED_GCS_COLLECTION_ROOT_PATH: "/mnt/datafed" + DATAFED_CORE_ADDRESS_PORT_INTERNAL: "${DATAFED_DOMAIN}:7513" + DATAFED_GCS_COLLECTION_ROOT_PATH: "${DATAFED_GCS_COLLECTION_ROOT_PATH}" UID: "${DATAFED_UID}" HOST_HOSTNAME: "localhost" network_mode: host image: datafed-repo:latest volumes: - ./keys:/opt/datafed/keys + - ./logs:/datafed/logs + - ${DATAFED_HOST_COLLECTION_MOUNT}:${DATAFED_GCS_COLLECTION_ROOT_PATH} ports: - 9000:9000 # Communication core server - # Needs host port 80 +# Needs host port 80 datafed-gcs: environment: DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" DATAFED_ZEROMQ_SYSTEM_SECRET: "${DATAFED_ZEROMQ_SYSTEM_SECRET}" - DATAFED_DOMAIN: "datafed-core" + DATAFED_DOMAIN: "${DATAFED_DOMAIN}" DATAFED_HTTPS_SERVER_PORT: "${DATAFED_HTTPS_SERVER_PORT}" DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" DATAFED_CORE_ADDRESS_PORT_INTERNAL: "datafed-core:7513" + DATAFED_GCS_ROOT_NAME: "${DATAFED_GCS_ROOT_NAME}" + DATAFED_REPO_ID_AND_DIR: "${DATAFED_REPO_ID_AND_DIR}" + DATAFED_GLOBUS_SUBSCRIPTION: "${DATAFED_GLOBUS_SUBSCRIPTION}" + DATAFED_GLOBUS_CONTROL_PORT: "${DATAFED_GLOBUS_CONTROL_PORT}" + DATAFED_GCS_COLLECTION_ROOT_PATH: "${DATAFED_GCS_COLLECTION_ROOT_PATH}" + DATAFED_REPO_USER: "${DATAFED_REPO_USER}" UID: "${DATAFED_UID}" HOST_HOSTNAME: "localhost" DATAFED_AUTHZ_USER: "datafed" - DATAFED_GCS_COLLECTION_ROOT_PATH: "/mnt/datafed" network_mode: host image: datafed-gcs:latest volumes: - ./keys:/opt/datafed/keys - ./globus:/opt/datafed/globus - - "${DATAFED_HOST_COLLECTION_MOUNT}:${DATAFED_GCS_COLLECTION_ROOT_PATH}" + - ./logs:/datafed/logs + - ${DATAFED_HOST_COLLECTION_MOUNT}:${DATAFED_GCS_COLLECTION_ROOT_PATH} # External true indicates that the network is created by a # separate compose instance -networks: - datafed-core-secure-api: - external: true +#networks: +# datafed-core-secure-api: +# external: true From bea43aaaa7304b072b7ad7f74ad79bf7e08ebb52 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:19:33 -0400 Subject: [PATCH 25/84] Make sure that the generate_env.sh file created a log folder --- compose/generate_env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/generate_env.sh b/compose/generate_env.sh index df17b4303..549671a47 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -162,6 +162,8 @@ else fi +# Make the logs folder if it doesn't exist +mkdir -p "${PROJECT_ROOT}/compose/logs" # Do not put " around anything and do not add comments in the .env file cat << EOF > ".env" From bdeec69dfe0d5f170c58a54895909357ff84a646 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:20:24 -0400 Subject: [PATCH 26/84] Add entrypoint command and copy in python env from dependencies into gcs container --- repository/docker/Dockerfile.gcs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs index b09fdd64b..5c4d58cc5 100644 --- a/repository/docker/Dockerfile.gcs +++ b/repository/docker/Dockerfile.gcs @@ -108,6 +108,8 @@ COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotobuf.a COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a +# Copy python env over +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/python/datafed ${DATAFED_DEPENDENCIES_INSTALL_PATH}/python/datafed RUN apt update RUN apt-get install vim -y @@ -141,3 +143,7 @@ USER root RUN sed -i '/--client-id/d' /entrypoint.sh WORKDIR ${DATAFED_INSTALL_PATH}/authz + +# NOTE If you do not overwrite the entrypoint file it will use the GCS instance +# prorivded one +ENTRYPOINT ["/datafed/source/repository/docker/entrypoint_authz.sh"] From 986a7f8ae68e840c73a4a4e57974bf0877f27938 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:23:08 -0400 Subject: [PATCH 27/84] Add python env to entrypoint before running guest collection creation script --- repository/docker/entrypoint_authz.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index b3489b7aa..5f33208a3 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -19,6 +19,7 @@ export GCS_CLI_ENDPOINT_ID=$(cat /opt/datafed/globus/deployment-key.json | jq - export DEPLOYMENT_KEY_PATH="/opt/datafed/globus/deployment-key.json" # These env variables are for running the gcs entrypoint file + export GLOBUS_CLIENT_ID=$(cat /opt/datafed/globus/client_cred.json | jq -r .client) export GLOBUS_CLIENT_SECRET=$(cat /opt/datafed/globus/client_cred.json | jq -r .secret) export DEPLOYMENT_KEY=$(cat "$DEPLOYMENT_KEY_PATH" ) @@ -117,5 +118,16 @@ fi "${BUILD_DIR}/scripts/globus/setup_globus.sh" +source "${DATAFED_PYTHON_ENV}/bin/activate" +# Must be passed in directly +GCS_CLI_ENDPOINT_ID="$GCS_CLI_ENDPOINT_ID" \ +DATAFED_GCS_URL="$DATAFED_GCS_URL" \ +GCS_CLI_CLIENT_ID="$GCS_CLI_CLIENT_ID" \ +GCS_CLI_CLIENT_SECRET="$GCS_CLI_CLIENT_SECRET" \ +DATAFED_REPO_USER="$DATAFED_REPO_USER" \ + python3 "${BUILD_DIR}/scripts/globus/create_guest_collection.py" + +echo "Container is running." +sleep infinity #"$@" -- argv0 "$@" From 16affdbc3be5494aced8baae29d572b501d1269e Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:23:55 -0400 Subject: [PATCH 28/84] Add python env setup to dependency installation --- scripts/dependency_install_functions.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index 56bd5d5f7..102eabeb1 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -59,6 +59,12 @@ else fi fi +init_python() { + if [ ! -e "$DATAFED_DEPENDENCIES_INSTALL_PATH" ] || [ ! -d "$DATAFED_PYTHON_DEPENDENCIES_DIR" ]; then + mkdir -p "$DATAFED_PYTHON_DEPENDENCIES_DIR" + fi + python3 -m venv "${DATAFED_PYTHON_ENV}" +} install_cmake() { if [ ! -e "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.cmake_installed-${DATAFED_CMAKE_VERSION}" ]; then @@ -132,10 +138,14 @@ install_protobuf() { #fi cd python + init_python + source "${DATAFED_PYTHON_ENV}/bin/activate" LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 -m pip install numpy LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py build LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py test - LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py install --user + # Because we have activaited a venv we don't want to use the --user flag + # with the install command + LD_LIBRARY_PATH="$LD_LIBRARY_PATH" PATH="$PATH" python3 setup.py install cd ../ # Cleanup build file with root ownership if [ -f build/install_manifest.txt ] From 16cadf2c4d9750ce11c6585fea3eb0f80bb9800f Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:24:29 -0400 Subject: [PATCH 29/84] Add python dependency install variables to datafed.sh --- scripts/generate_datafed.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index 6617069af..d2ea3357f 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -207,6 +207,8 @@ export DATAFED_INSTALL_PATH="$local_DATAFED_INSTALL_PATH" # by default it will install to: # /opt/datafed/dependencies export DATAFED_DEPENDENCIES_INSTALL_PATH="$local_DATAFED_DEPENDENCIES_INSTALL_PATH" +export DATAFED_PYTHON_DEPENDENCIES_DIR="${local_DATAFED_DEPENDENCIES_INSTALL_PATH}/python" +export DATAFED_PYTHON_ENV="${local_DATAFED_DEPENDENCIES_INSTALL_PATH}/python/datafed" # ************************************************ # Env Variables for Core & Web Server # ************************************************ From 34af7fdb0e78bb580a1ecb18c9fc405393ff7fb2 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:27:32 -0400 Subject: [PATCH 30/84] Simplify getenv calls and tab spacing as well as syntax errors with missing : around for loops --- scripts/globus/create_guest_collection.py | 170 +++++++--------------- 1 file changed, 50 insertions(+), 120 deletions(-) diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index d656b1ba0..59f6131fe 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -9,107 +9,37 @@ # The Globus project the GCS endpoint will be created in -if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: - DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") -else: - DATAFED_GCS_ROOT_NAME="DataFed Repo" - -if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: - PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") -else: - PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" - +DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME", "DataFed Repo") +PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME", DATAFED_GCS_ROOT_NAME + " Project") # This is for confidential client -if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: - CLIENT_NAME = os.getenv("DATAFED_GLOBUS_CLIENT_NAME") -else: - CLIENT_NAME = DATAFED_GCS_ROOT_NAME + " Setup Client" - +CLIENT_NAME = os.getenv("DATAFED_GLOBUS_CLIENT_NAME", DATAFED_GCS_ROOT_NAME + " Setup Client") # Name of the client secret used by the confidential client -if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: - CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") -else: - CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" - +CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME",DATAFED_GCS_ROOT_NAME + " Cred") # Name of the file where we will store confidential client credentials -if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: - CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") -else: - CRED_FILE_PATH="./client_cred.json" - -if os.getenv("GCS_CLI_ENDPOINT_ID") is not None: - ENDPOINT_ID = os.getenv("GCS_CLI_ENDPOINT_ID") -else: - raise Exception("GCS_CLI_ENDPOINT_ID must be defined") - -# Name to give to endpoint -if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: - ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") -else: - ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" - +CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH","./client_cred.json") +ENDPOINT_ID = os.getenv("GCS_CLI_ENDPOINT_ID") +ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME",DATAFED_GCS_ROOT_NAME + " Endpoint") # Path to deployment key -if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: - DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") -else: - DEPLOYMENT_KEY_PATH="./deployment-key.json" +DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH","./deployment-key.json") # Path to deployment key -if os.getenv("DATAFED_GLOBUS_CONTROL_PORT") is not None: - DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT") -else: - DATAFED_GLOBUS_CONTROL_PORT="443" - -if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: - DATAFED_GLOBUS_SUBSCRIPTION=os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") -else: - DATAFED_GLOBUS_SUBSCRIPTION="" - -if os.getenv("DATAFED_GCS_URL") is not None: - DATAFED_GCS_URL=os.getenv("DATAFED_GCS_URL") -else: - DATAFED_GCS_URL="" - -if os.getenv("GCS_CLI_CLIENT_ID") is not None: - client_id = os.getenv("GCS_CLI_CLIENT_ID") -else: - client_id = None - -if os.getenv("GCS_CLI_CLIENT_SECRET") is not None: - client_secret = os.getenv("GCS_CLI_CLIENT_SECRET") -else: - client_secret = None - -if os.getenv("MAPPED_COLLECTION_ID") is not None: - mapped_collection_id = os.getenv("MAPPED_COLLECTION_ID") -else: - mapped_collection_id = None - -if os.getenv("DATAFED_GCS_COLLECTION_MAPPED") is not None: - mapped_collection_name = os.getenv("DATAFED_GCS_COLLECTION_MAPPED") -else: - mapped_collection_name = f"{DATAFED_GCS_ROOT_NAME} Collection Mapped" - -if os.getenv("DATAFED_GCS_COLLECTION_GUEST") is not None: - guest_collection_name = os.getenv("DATAFED_GCS_COLLECTION_GUEST") -else: - guest_collection_name = f"{DATAFED_GCS_ROOT_NAME} Collection Guest" - -if os.getenv("STORAGE_GATEWAY_ID") is not None: - storage_gateway_id = os.getenv("STORAGE_GATEWAY_ID") -else: - storage_gateway_id = None - - -if os.getenv("DATAFED_GCS_STORAGE_GATEWAY") is not None: - storage_gateway_name = os.getenv("DATAFED_GCS_STORAGE_GATEWAY") -else: - storage_gateway_name = f"{DATAFED_GCS_ROOT_NAME} Storage Gateway" - - -if os.getenv("DATAFED_REPO_USER") is not None: - local_username = os.getenv("DATAFED_REPO_USER") -else: +DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT", "443") +DATAFED_GCS_URL=os.getenv("DATAFED_GCS_URL") +client_id = os.getenv("GCS_CLI_CLIENT_ID") +client_secret = os.getenv("GCS_CLI_CLIENT_SECRET") +mapped_collection_id = os.getenv("MAPPED_COLLECTION_ID") +mapped_collection_name = os.getenv("DATAFED_GCS_COLLECTION_MAPPED", f"{DATAFED_GCS_ROOT_NAME} Collection Mapped") +guest_collection_name = os.getenv("DATAFED_GCS_COLLECTION_GUEST",f"{DATAFED_GCS_ROOT_NAME} Collection Guest") +storage_gateway_id = os.getenv("STORAGE_GATEWAY_ID") +storage_gateway_name = os.getenv("DATAFED_GCS_STORAGE_GATEWAY",f"{DATAFED_GCS_ROOT_NAME} Storage Gateway") +local_username = os.getenv("DATAFED_REPO_USER") + +if ENDPOINT_ID is None: + raise Exception("GCS_CLI_ENDPOINT_ID must be defined as an env varaible") +if DATAFED_GCS_URL is None: + raise Exception("Unable to create guest collection, DATAFED_GCS_URL is not" + " defined.") +if local_username is None: raise Exception("DATAFED_REPO_USER is not defined.") #client = globus_sdk.NativeAppAuthClient(CLIENT_ID) @@ -189,7 +119,7 @@ confidential_client = globus_sdk.ConfidentialAppAuthClient( client_id=client_id, client_secret=client_secret ) -scope = scopes.GCSEndpointScopeBuilder(GCS_CLI_ENDPOINT_ID).make_mutable("manage_collections") +scope = scopes.GCSEndpointScopeBuilder(ENDPOINT_ID).make_mutable("manage_collections") authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, scopes=scope) client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) @@ -199,11 +129,11 @@ mapped_collection_found = False -for item in collection_list["data"] +for item in collection_list["data"]: print(item["display_name"]) if mapped_collection_id is not None: if item["id"] == mapped_collection_id: - mapped_collection_found = True + mapped_collection_found = True if item["display_name"] != mapped_collection_name: raise Exception("Expected display name is different from what " "is expected for mapped collection " @@ -214,20 +144,20 @@ "set.") break elif item["display_name"] == mapped_collection_name: - mapped_collection_found = True - mapped_collection_id = item["id"] - break - + mapped_collection_found = True + mapped_collection_id = item["id"] + break + if mapped_collection_found == False: raise Exception("Missing required mapped collection") storage_gateway_found = False storage_gateway_list = client.get_storage_gateway_list() -for item in storage_gateway_list["data"] +for item in storage_gateway_list["data"]: print(item["display_name"]) if storage_gateway_id is not None: if item["id"] == storage_gateway_id: - storage_gateway_found = True + storage_gateway_found = True if item["display_name"] != storage_gateway_name: raise Exception("Expected display name is different from what " "is expected for storage gateway " @@ -238,10 +168,10 @@ "set.") break elif item["display_name"] == storage_gateway_name: - storage_gateway_found = True - storage_gateway_id = item["id"] - break - + storage_gateway_found = True + storage_gateway_id = item["id"] + break + if storage_gateway_found == False: raise Exception("Missing required storage gateway") @@ -253,28 +183,28 @@ guest_collection_found = False -for item in collection_list["data"] +for item in collection_list["data"]: print(item["display_name"]) if item["display_name"] == guest_collection_name: - guest_collection_found = True - guest_collection_id = item["id"] - break - + guest_collection_found = True + guest_collection_id = item["id"] + break + # https://github.com/globus/globus-sdk-python/blob/main/docs/examples/guest_collection_creation.rst if guest_collection_found == False: credential_document = globus_sdk.UserCredentialDocument( - storage_gateway_id=storage_gateway_id, - identity_id=client_id, - username=local_username, + storage_gateway_id=storage_gateway_id, + identity_id=client_id, + username=local_username, ) client.create_user_credential(credential_document) # Create the collection collection_document = globus_sdk.GuestCollectionDocument( - public="True", - collection_base_path="/", - display_name=guest_collection_name, - mapped_collection_id=mapped_collection_id, + public="True", + collection_base_path="/", + display_name=guest_collection_name, + mapped_collection_id=mapped_collection_id, ) response = client.create_collection(collection_document) guest_collection_id = response["id"] From 905329dd791e9483f1409f934bc484a6de5e2138 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 4 Apr 2024 10:28:08 -0400 Subject: [PATCH 31/84] add python venv functionality to dependency install scripts --- scripts/install_authz_dependencies.sh | 4 +++- scripts/install_client_dependencies.sh | 5 +++++ scripts/install_core_dependencies.sh | 4 +++- scripts/install_dependencies.sh | 2 ++ scripts/install_docs_dependencies.sh | 4 +++- scripts/install_python_client_dependencies.sh | 4 +++- scripts/install_repo_dependencies.sh | 4 +++- 7 files changed, 22 insertions(+), 5 deletions(-) diff --git a/scripts/install_authz_dependencies.sh b/scripts/install_authz_dependencies.sh index ce0cc0224..68142b5a4 100755 --- a/scripts/install_authz_dependencies.sh +++ b/scripts/install_authz_dependencies.sh @@ -12,7 +12,7 @@ source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" packages=("libtool" "build-essential" "g++" "gcc" "autoconf" "automake" "make" "git" "python3-pkg-resources" "python3-pip" "pkg-config" - "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev") + "libglobus-common-dev" "wget" "jq" "sudo" "libboost-all-dev" "python3-venv") pip_packages=("setuptools" "distro" "jwt" "globus_sdk") externals=("cmake" "protobuf" "libsodium" "libzmq" ) @@ -45,6 +45,8 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" apt-get update "$SUDO_CMD" dpkg --configure -a "$SUDO_CMD" apt-get install -y "${packages[@]}" + init_python + source "${DATAFED_PYTHON_ENV}/bin/activate" python3 -m pip install --upgrade pip python3 -m pip install "${pip_packages[@]}" diff --git a/scripts/install_client_dependencies.sh b/scripts/install_client_dependencies.sh index cea8afb38..2c475fa4b 100755 --- a/scripts/install_client_dependencies.sh +++ b/scripts/install_client_dependencies.sh @@ -10,11 +10,16 @@ PROJECT_ROOT=$(realpath "${SOURCE}/..") source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" +packages=("python3-pkg-resources" "python3-pip" "pkg-config" "python3-venv") + sudo_command # This script will install all of the dependencies needed by DataFed 1.0 "$SUDO_CMD" apt-get update "$SUDO_CMD" dpkg --configure -a +"$SUDO_CMD" apt-get install -y "${packages[@]}" +init_python +source "${DATAFED_PYTHON_ENV}/bin/activate" python3 -m pip install -r "${PROJECT_ROOT}/python/datafed_pkg/requirements.txt" install_protobuf diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index fe78bb3f2..c79ee55d2 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -10,7 +10,7 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..) source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip") +packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "wget" "rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip" "python3-venv") pip_packages=("setuptools") # NOTE the order matters here externals=("cmake" "nlohmann_json" "json_schema_validator" "protobuf" @@ -44,6 +44,8 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" apt-get update "$SUDO_CMD" dpkg --configure -a "$SUDO_CMD" apt-get install -y "${packages[@]}" + init_python + source "${DATAFED_PYTHON_ENV}/bin/activate" python3 -m pip install --upgrade pip python3 -m pip install "${pip_packages[@]}" diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh index c3b5c9cec..c6b1621b3 100755 --- a/scripts/install_dependencies.sh +++ b/scripts/install_dependencies.sh @@ -115,6 +115,8 @@ all_pip_packages=$(cat "$pip_file_path") IFS=' ' read -ra all_pip_packages_array <<< "$all_pip_packages" if [ ${#all_pip_packages_array[@]} -gt 0 ]; then echo "DEPENDENCIES (${all_pip_packages_array[@]})" + init_python + source "${DATAFED_PYTHON_ENV}/bin/activate" python3 -m pip install "${all_pip_packages_array[@]}" fi diff --git a/scripts/install_docs_dependencies.sh b/scripts/install_docs_dependencies.sh index c0f2b3dfe..06ee7c3de 100755 --- a/scripts/install_docs_dependencies.sh +++ b/scripts/install_docs_dependencies.sh @@ -10,7 +10,7 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..) source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("g++" "gcc" "make" "python3-pkg-resources" "python3-pip" "pkg-config") +packages=("g++" "gcc" "make" "python3-pkg-resources" "python3-pip" "pkg-config" "python3-venv") externals=("cmake" "protobuf") pip_packages=("setuptools" "sphinx" "sphinx-rtd-theme" "sphinx-autoapi") @@ -42,6 +42,8 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" apt-get update "$SUDO_CMD" dpkg --configure -a "$SUDO_CMD" apt-get install -y "${packages[@]}" + init_python + source "${DATAFED_PYTHON_ENV}/bin/activate" python3 -m pip install --upgrade pip python3 -m pip install "${pip_packages[@]}" diff --git a/scripts/install_python_client_dependencies.sh b/scripts/install_python_client_dependencies.sh index e16fa04a4..a8467a3f0 100755 --- a/scripts/install_python_client_dependencies.sh +++ b/scripts/install_python_client_dependencies.sh @@ -17,7 +17,7 @@ sudo_command "$SUDO_CMD" dpkg --configure -a "$SUDO_CMD" apt-get install -y libtool build-essential g++ gcc make libboost-all-dev \ pkg-config autoconf automake unzip libcurl4-openssl-dev wget \ -rapidjson-dev libkrb5-dev git python3-pkg-resources python3-pip libssl-dev +rapidjson-dev libkrb5-dev git python3-pkg-resources python3-pip python3-venv libssl-dev cd ~ install_cmake @@ -25,6 +25,8 @@ cd ~ # Install cmake 3.17 +init_python +source "${DATAFED_PYTHON_ENV}/bin/activate" python3 -m pip install --upgrade pip python3 -m pip install setuptools diff --git a/scripts/install_repo_dependencies.sh b/scripts/install_repo_dependencies.sh index b1ceff8c9..723a3ad81 100755 --- a/scripts/install_repo_dependencies.sh +++ b/scripts/install_repo_dependencies.sh @@ -10,7 +10,7 @@ PROJECT_ROOT=$(realpath ${SOURCE}/..) source "${PROJECT_ROOT}/scripts/utils.sh" source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh" -packages=("libtool" "wget" "build-essential" "g++" "gcc" "libboost-all-dev" "pkg-config" "autoconf" "automake" "make" "unzip" "git" "python3-pkg-resources" "python3-pip") +packages=("libtool" "wget" "build-essential" "g++" "gcc" "libboost-all-dev" "pkg-config" "autoconf" "automake" "make" "unzip" "git" "python3-pkg-resources" "python3-pip" "python3-venv") pip_packages=("setuptools") externals=("cmake" "protobuf" "libsodium" "libzmq") @@ -44,6 +44,8 @@ if [[ $local_UNIFY = false ]]; then "$SUDO_CMD" dpkg --configure -a "$SUDO_CMD" apt-get install -y "${packages[@]}" + init_python + source "${DATAFED_PYTHON_ENV}/bin/activate" python3 -m pip install --upgrade pip python3 -m pip install "${pip_packages[@]}" From 2c7452f7825f150be301208fd4ea34ede31b5c94 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Sat, 6 Apr 2024 00:05:29 -0400 Subject: [PATCH 32/84] Add additional error checking before using unique_pointer --- common/source/servers/Proxy.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/common/source/servers/Proxy.cpp b/common/source/servers/Proxy.cpp index e3905d9fc..c78ceff7f 100644 --- a/common/source/servers/Proxy.cpp +++ b/common/source/servers/Proxy.cpp @@ -123,8 +123,14 @@ void Proxy::run() { // public if (resp_from_client_socket.error == false and resp_from_client_socket.time_out == false) { - m_communicators[SocketRole::SERVER]->send( - *resp_from_client_socket.message); + if(not resp_from_server_socket.message ) { + DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to send to server."); + } else { + m_communicators[SocketRole::SERVER]->send( + *resp_from_client_socket.message); + } } // If there are operations that need to happen on incoming messages, @@ -137,11 +143,17 @@ void Proxy::run() { // ... - Serv Sock - Proxy ------ Client Sock - Serv Sock - Inter App if (resp_from_server_socket.error == false and resp_from_server_socket.time_out == false) { - for (auto &in_operator : m_incoming_operators) { - in_operator->execute(*resp_from_server_socket.message); + if(not resp_from_server_socket.message ) { + DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to operate and send to client."); + } else { + for (auto &in_operator : m_incoming_operators) { + in_operator->execute(*resp_from_server_socket.message); + } + m_communicators[SocketRole::CLIENT]->send( + *resp_from_server_socket.message); } - m_communicators[SocketRole::CLIENT]->send( - *resp_from_server_socket.message); } } catch (TraceException &e) { From 19397832085bc9679daedfd0dfe58b2afc4c858a Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Sat, 6 Apr 2024 00:07:30 -0400 Subject: [PATCH 33/84] Add additional checks before using unique pointer --- core/server/ClientWorker.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/server/ClientWorker.cpp b/core/server/ClientWorker.cpp index 91ba2f5e1..40e57f00b 100644 --- a/core/server/ClientWorker.cpp +++ b/core/server/ClientWorker.cpp @@ -346,6 +346,9 @@ void ClientWorker::workerThread(LogContext log_context) { ICommunicator::Response response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); if (response.time_out == false and response.error == false) { + if ( not response.message ) { + DL_ERROR(message_log_context, "No timeout or error was reported but message is not defined.") + } IMessage &message = *response.message; uint16_t msg_type = std::get( From b516efe0e686ceb3ec082ff9b5c9049a626e86c6 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Sat, 6 Apr 2024 00:08:32 -0400 Subject: [PATCH 34/84] Fix error where unique pointer was being used before checking if it was valid --- .../gridftp/globus5/authz/source/AuthzWorker.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index 86ddb120a..4bc7261bd 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -226,8 +226,10 @@ class AuthzWorker { << client->address()); auto response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); - log_context.correlation_id = std::get( - response.message->get(MessageAttribute::CORRELATION_ID)); + if (response.message ) { // Make sure the message exists before we try to access it + log_context.correlation_id = std::get( + response.message->get(MessageAttribute::CORRELATION_ID)); + } if (response.time_out) { std::string error_msg = "AuthWorker.cpp Core service did not respond within timeout."; @@ -250,6 +252,11 @@ class AuthzWorker { "communicating with the core service: " << response.error_msg); } else { + + if( not response.message ) { + DL_ERROR(log_context, "No error was reported and no time out occured but message is not defined."); + } + auto payload = std::get(response.message->getPayload()); Anon::NackReply *nack = dynamic_cast(payload); From 5e3280f213eff99475285d79778c306bab943fba Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Sat, 6 Apr 2024 00:12:29 -0400 Subject: [PATCH 35/84] Fixed bug with accessing invalid messages and added error checks in the case message is not defined --- core/server/TaskWorker.cpp | 3 ++ repository/server/RepoServer.cpp | 15 +++++++--- repository/server/RequestWorker.cpp | 43 ++++++++++++++++------------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/core/server/TaskWorker.cpp b/core/server/TaskWorker.cpp index 3660c8308..87dd0f1ad 100644 --- a/core/server/TaskWorker.cpp +++ b/core/server/TaskWorker.cpp @@ -339,6 +339,9 @@ TaskWorker::cmdRawDataUpdateSize(TaskWorker &me, const Value &a_task_params, if (response.time_out == true) { return response; } else if (response.error == false) { + if( not response.message ) { + DL_ERROR(log_context, "No timeout or error was reported but no message was defined."); + } auto proto_msg = std::get(response.message->getPayload()); auto size_reply = dynamic_cast(proto_msg); diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index 69cbf49dc..d851eb0b7 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -101,9 +101,8 @@ void Server::run() { } void Server::checkServerVersion() { - DL_INFO(m_log_context, "Checking core server connection and version"); + DL_INFO(m_log_context, "Checking core server connection and version at " << m_config.core_server); - auto msg = std::make_unique(); // msg; // Generate random security keys for anon version request to core server KeyGenerator generator; auto local_keys = @@ -148,6 +147,9 @@ void Server::checkServerVersion() { MessageFactory msg_factory; for (int i = 0; i < 10; i++) { + DL_INFO(m_log_context, "Attempt " << i << " to initialize communication " + << " with core server at " << m_config.core_server); + auto msg = std::make_unique(); auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); message->setPayload(std::move(msg)); message->set(MessageAttribute::KEY, @@ -159,14 +161,19 @@ void Server::checkServerVersion() { client->send(*message); auto response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); - msg_log_context.correlation_id = std::get( - response.message->get(MessageAttribute::CORRELATION_ID)); if (response.time_out) { DL_ERROR(msg_log_context, "Timeout waiting for response from core server: " << m_config.core_server); + } else if(response.error) { + DL_ERROR(msg_log_context, + "Error encountered waiting for core server: " + << m_config.core_server << " msg " << response.error_msg); } else { + + msg_log_context.correlation_id = std::get( + response.message->get(MessageAttribute::CORRELATION_ID)); auto payload = std::get(response.message->getPayload()); VersionReply *ver_reply = dynamic_cast(payload); diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index 5ee08f0cd..be08293fd 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -146,33 +146,38 @@ void RequestWorker::workerThread(LogContext log_context) { DL_TRACE(message_log_context, "Checking timeouts: " << response.time_out); if (response.time_out == false and response.error == false) { - // May not have a correlation id if the message timed out - DL_TRACE(log_context, "Getting correlation_id."); - if (response.message->exists(MessageAttribute::CORRELATION_ID)) { - message_log_context.correlation_id = std::get( - response.message->get(MessageAttribute::CORRELATION_ID)); - } + if( not response.message) { + DL_ERROR(log_context, "Error: No error or timeout occurred but the" + << " message does not exist."); + } else { + // May not have a correlation id if the message timed out + DL_TRACE(log_context, "Getting correlation_id."); + if (response.message->exists(MessageAttribute::CORRELATION_ID)) { + message_log_context.correlation_id = std::get( + response.message->get(MessageAttribute::CORRELATION_ID)); + } - IMessage &message = *response.message; - uint16_t msg_type = std::get( - message.get(constants::message::google::MSG_TYPE)); + IMessage &message = *response.message; + uint16_t msg_type = std::get( + message.get(constants::message::google::MSG_TYPE)); - DL_TRACE(message_log_context, "Received msg of type: " << msg_type); + DL_TRACE(message_log_context, "Received msg of type: " << msg_type); - if (m_msg_handlers.count(msg_type)) { - map::iterator handler = + if (m_msg_handlers.count(msg_type)) { + map::iterator handler = m_msg_handlers.find(msg_type); - DL_TRACE(message_log_context, "Calling handler"); + DL_TRACE(message_log_context, "Calling handler"); - auto send_message = + auto send_message = (this->*handler->second)(std::move(response.message)); - client->send(*(send_message)); + client->send(*(send_message)); - DL_TRACE(message_log_context, "Reply sent."); - } else { - DL_ERROR(message_log_context, - "Received unregistered msg type: " << msg_type); + DL_TRACE(message_log_context, "Reply sent."); + } else { + DL_ERROR(message_log_context, + "Received unregistered msg type: " << msg_type); + } } } else if (response.error) { DL_DEBUG(message_log_context, "Error detected: " << response.error_msg); From 71e39e2eb360b9042406a42e5b9b696d37274a9f Mon Sep 17 00:00:00 2001 From: par-hermes Date: Sat, 6 Apr 2024 04:44:45 +0000 Subject: [PATCH 36/84] cpp-py-formatter --- repository/gridftp/globus5/authz/source/URL.c | 105 +++++---- repository/gridftp/globus5/authz/source/URL.h | 5 +- .../globus5/authz/tests/unit/test_URL.cpp | 69 +++--- scripts/globus/globus_cleanup.py | 125 +++++----- scripts/globus/initialize_globus_endpoint.py | 75 +++--- scripts/globus/utils.py | 220 ++++++++++-------- 6 files changed, 331 insertions(+), 268 deletions(-) diff --git a/repository/gridftp/globus5/authz/source/URL.c b/repository/gridftp/globus5/authz/source/URL.c index ad898dd44..7d2d93cd5 100644 --- a/repository/gridftp/globus5/authz/source/URL.c +++ b/repository/gridftp/globus5/authz/source/URL.c @@ -5,41 +5,53 @@ // Function to extract the relative path from an FTP URL // Returns 1 on success, 0 on failure -int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength) { +int ftpExtractRelativePath(const char *url, char *relativePath, + size_t maxLength) { size_t len_of_prefix = strlen("ftp://"); size_t len_of_url = strlen(url); - + // Step 1. Check that the URL starts with "ftp://" - if (strncmp(url, "ftp://", len_of_prefix) != 0) { - fprintf(stderr, "Error: URL must start with 'ftp:// but you have provided %s'\n", url); - return 0; - } + if (strncmp(url, "ftp://", len_of_prefix) != 0) { + fprintf(stderr, + "Error: URL must start with 'ftp:// but you have provided %s'\n", + url); + return 0; + } - if( len_of_url == len_of_prefix ) { - // This means we have ftp:// but with no relative path and missing - // the final / separating the domain from the relative path. - fprintf(stderr, "Error: Invalid URL format expected ftp://domain/ instead received %s\n", url); - return 0; - } else if ( url[len_of_prefix] == '/' ) { - // If they are not equal the url must be greater because we already - // compared the prefix. Let's make sure we don't have - // ftp:/// where no domain is given this is invalid as well - // - // NOTE the third / will appear at index 6 not 7 - fprintf(stderr, "Error: Invalid URL format missing domain name expected ftp://domain/ instead received %s\n", url); - return 0; + if (len_of_url == len_of_prefix) { + // This means we have ftp:// but with no relative path and missing + // the final / separating the domain from the relative path. + fprintf(stderr, + "Error: Invalid URL format expected ftp://domain/ instead received " + "%s\n", + url); + return 0; + } else if (url[len_of_prefix] == '/') { + // If they are not equal the url must be greater because we already + // compared the prefix. Let's make sure we don't have + // ftp:/// where no domain is given this is invalid as well + // + // NOTE the third / will appear at index 6 not 7 + fprintf(stderr, + "Error: Invalid URL format missing domain name expected " + "ftp://domain/ instead received %s\n", + url); + return 0; } - // Find the position of the third slash ('/') after "ftp://" - const char *slashPtr = strchr(url + len_of_prefix, '/'); - if (slashPtr == NULL) { - if( len_of_url == len_of_prefix ) { + // Find the position of the third slash ('/') after "ftp://" + const char *slashPtr = strchr(url + len_of_prefix, '/'); + if (slashPtr == NULL) { + if (len_of_url == len_of_prefix) { // This means we have ftp:// but with no relative path and missing // the final / separating the domain from the relative path. - fprintf(stderr, "Error: Invalid URL format expected ftp://domain/ instead received %s\n", url); + fprintf(stderr, + "Error: Invalid URL format expected ftp://domain/ instead " + "received %s\n", + url); return 0; } else { // This means we have ftp://domain but with no relative path and missing - // the final / separating the domain from the relative path. We will + // the final / separating the domain from the relative path. We will // report this as a success and return a slash relativePath[0] = '/'; relativePath[1] = '\0'; @@ -48,32 +60,33 @@ int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength } printf("slashPtr is %s\n", slashPtr); - // Calculate the length of the relative path - size_t pathLength = strlen(slashPtr); + // Calculate the length of the relative path + size_t pathLength = strlen(slashPtr); - // Check if the provided buffer is sufficient - if (pathLength >= maxLength) { - fprintf(stderr, "Error: Insufficient buffer size max size is %ld actual size is %ld\n", maxLength, pathLength); - return 0; - } + // Check if the provided buffer is sufficient + if (pathLength >= maxLength) { + fprintf( + stderr, + "Error: Insufficient buffer size max size is %ld actual size is %ld\n", + maxLength, pathLength); + return 0; + } - // Copy the relative path to the output buffer - strcpy(relativePath, slashPtr); + // Copy the relative path to the output buffer + strcpy(relativePath, slashPtr); - return 1; // Success + return 1; // Success } int comparePrefix(const char *str1, const char *str2, size_t prefix_length) { - size_t len1 = strlen(str1); - size_t len2 = strlen(str2); - - // Ensure the prefix length is not longer than the shortest string length - if (prefix_length > len1 || prefix_length > len2) { - return -1; // Prefix length is longer than one or both of the strings - } - - // Compare the prefixes - return strncmp(str1, str2, prefix_length); -} + size_t len1 = strlen(str1); + size_t len2 = strlen(str2); + // Ensure the prefix length is not longer than the shortest string length + if (prefix_length > len1 || prefix_length > len2) { + return -1; // Prefix length is longer than one or both of the strings + } + // Compare the prefixes + return strncmp(str1, str2, prefix_length); +} diff --git a/repository/gridftp/globus5/authz/source/URL.h b/repository/gridftp/globus5/authz/source/URL.h index f9e007406..47dbceb85 100644 --- a/repository/gridftp/globus5/authz/source/URL.h +++ b/repository/gridftp/globus5/authz/source/URL.h @@ -15,12 +15,13 @@ extern "C" { * ftp://domain Returns 1 for success * else it will return 0 for failure **/ -int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength); +int ftpExtractRelativePath(const char *url, char *relativePath, + size_t maxLength); /** * Will compare two strings and ensure that prefixes are equivalent * - * On success will return the results of strncmp which will be 0 if they + * On success will return the results of strncmp which will be 0 if they * match. **/ int comparePrefix(const char *str1, const char *str2, size_t prefix_length); diff --git a/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp b/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp index 9c371a2cc..3861d34da 100644 --- a/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp +++ b/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp @@ -12,8 +12,8 @@ BOOST_AUTO_TEST_SUITE(BufferTest) BOOST_AUTO_TEST_CASE(test_1_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url1 = "ftp://domain/relative_path"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url1 = "ftp://domain/relative_path"; int rv = ftpExtractRelativePath(url1, relative_path, sizeof(relative_path)); // Should pass @@ -22,100 +22,97 @@ BOOST_AUTO_TEST_CASE(test_1_ftpExtractRelativePath) { } BOOST_AUTO_TEST_CASE(test_2_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp://domain"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp://domain"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should not throw an error - BOOST_CHECK( rv == 1); - BOOST_CHECK( strcmp(relative_path, "/") == 0); + BOOST_CHECK(rv == 1); + BOOST_CHECK(strcmp(relative_path, "/") == 0); } - BOOST_AUTO_TEST_CASE(test_3_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = ""; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = ""; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should throw an error - BOOST_CHECK( rv == 0); + BOOST_CHECK(rv == 0); } - BOOST_AUTO_TEST_CASE(test_4_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp:///"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp:///"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should throw an error - BOOST_CHECK( rv == 0); + BOOST_CHECK(rv == 0); } - BOOST_AUTO_TEST_CASE(test_5_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp:/domain///path"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp:/domain///path"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should throw an error because prefix is incorrect - BOOST_CHECK( rv == 0); + BOOST_CHECK(rv == 0); } BOOST_AUTO_TEST_CASE(test_6_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp://domain///path"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp://domain///path"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should not throw an error - BOOST_CHECK( rv == 1); + BOOST_CHECK(rv == 1); printf("Relative path is %s\n", relative_path); BOOST_CHECK(strcmp(relative_path, "///path") == 0); } -BOOST_AUTO_TEST_CASE(test_1_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_1_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path=""; + const char *relative_path = ""; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should fail BOOST_CHECK(rv != 0); } -BOOST_AUTO_TEST_CASE(test_2_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_2_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/"; + const char *relative_path = "/"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should fail BOOST_CHECK(rv != 0); } -BOOST_AUTO_TEST_CASE(test_3_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_3_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/mnt/storage/globus/"; + const char *relative_path = "/mnt/storage/globus/"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should pass BOOST_CHECK(rv == 0); } -BOOST_AUTO_TEST_CASE(test_4_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_4_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/mnt/storage/globus"; + const char *relative_path = "/mnt/storage/globus"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should pass BOOST_CHECK(rv == 0); } -BOOST_AUTO_TEST_CASE(test_5_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_5_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/mnt/storage/globu"; + const char *relative_path = "/mnt/storage/globu"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should fail diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 764bdc36c..e74f69a81 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,4 +1,3 @@ - import globus_sdk from globus_sdk import AuthClient, AccessTokenAuthorizer import subprocess @@ -8,18 +7,18 @@ import utils # Define your client ID and client secret -CLIENT_ID = 'f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8' # NATIVE +CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" # NATIVE # The Globus project the GCS endpoint will be created in if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") else: - DATAFED_GCS_ROOT_NAME="DataFed Repo" + DATAFED_GCS_ROOT_NAME = "DataFed Repo" if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: - PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") + PROJECT_NAME = os.getenv("DATAFED_GLOBUS_PROJECT_NAME") else: - PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" + PROJECT_NAME = DATAFED_GCS_ROOT_NAME + " Project" # This is for confidential client if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: @@ -29,32 +28,35 @@ # Name of the client secret used by the confidential client if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: - CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") + CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME") else: - CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" + CRED_NAME = DATAFED_GCS_ROOT_NAME + " Cred" # Name of the file where we will store confidential client credentials if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: - CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") + CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") else: - CRED_FILE_PATH="./client_cred.json" + CRED_FILE_PATH = "./client_cred.json" # Name to give to endpoint if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") else: - ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" + ENDPOINT_NAME = DATAFED_GCS_ROOT_NAME + " Endpoint" # Path to deployment key if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: - DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") + DEPLOYMENT_KEY_PATH = os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") else: - DEPLOYMENT_KEY_PATH="./deployment-key.json" + DEPLOYMENT_KEY_PATH = "./deployment-key.json" client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project -client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -66,7 +68,9 @@ print(token_response) -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) ac_rt = AuthClient(authorizer=rt_authorizer) @@ -92,68 +96,84 @@ clients_in_project = utils.getClientsInProject(ac_rt, project_id) if len(clients_in_project) == 0: - print("No clients were detected in the project we can just delete the" - "project and be done.") + print( + "No clients were detected in the project we can just delete the" + "project and be done." + ) else: -# Check if the deployment key exists if it does read it and verify that the -# client exists for the globus connect server if it does not then we will -# call the setup command + # Check if the deployment key exists if it does read it and verify that the + # client exists for the globus connect server if it does not then we will + # call the setup command - gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile(DEPLOYMENT_KEY_PATH) + gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile( + DEPLOYMENT_KEY_PATH + ) - valid_key = utils.isGCSDeploymentKeyValid(ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key) + valid_key = utils.isGCSDeploymentKeyValid( + ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key + ) all_gcs_client_ids = utils.getAllGCSClientIds(ac_rt, project_id, ENDPOINT_NAME) if valid_key is False and len(all_gcs_client_ids) > 0: - print("Looks like gcs client does not exist in the cloud" - f" for the project: {project_id}." - "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like gcs client does not exist in the cloud" + f" for the project: {project_id}." + "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if gcs_id_from_deployment_key is None and len(all_gcs_client_ids) > 0: - print("Looks like deployment key does not exist, please either " - "add the correct deployment." - f" cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like deployment key does not exist, please either " + "add the correct deployment." + f" cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if len(all_gcs_client_ids) > 0: if utils.command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - print("Now that we know a GCS instance exists we have to make sure" - "we have valid credentials to run the globus-connect-server command" - "non interatively, this means we have to create credentials and a" - "client if they don't exist and when we are done with everything" - "delete them.") + print( + "Now that we know a GCS instance exists we have to make sure" + "we have valid credentials to run the globus-connect-server command" + "non interatively, this means we have to create credentials and a" + "client if they don't exist and when we are done with everything" + "delete them." + ) client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) - + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH + ) - ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) + ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+="globus-connect-server endpoint cleanup " - bash_command+=f" --deployment-key \"{DEPLOYMENT_KEY_PATH}\" " - bash_command+=" --agree-to-delete-endpoint" + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += "globus-connect-server endpoint cleanup " + bash_command += f' --deployment-key "{DEPLOYMENT_KEY_PATH}" ' + bash_command += " --agree-to-delete-endpoint" print("Bash command to run") print(bash_command) - - proc = subprocess.Popen(bash_command, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, shell=True, text=True) + + proc = subprocess.Popen( + bash_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True, + text=True, + ) output, error = proc.communicate(input="yes\n") @@ -161,12 +181,11 @@ print("Output:", output) print("Error:", error) - # Now we can try to delete the remaining clients that are in the project # Get all of the clients that are not gcs clients and delete them utils.deleteAllNonGCSClients(ac_rt, project_id) - + # CLOSE - if len(clients_in_project) == 0: @@ -175,5 +194,3 @@ print(f"Attempting to remove project {project_id}") project_remove = ac_rt.delete_project(project_id) print(project_remove) - - diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index a45684fbb..b9e540202 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess import utils @@ -9,18 +8,18 @@ # Hard coded Native Client ID -CLIENT_ID = 'f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8' +CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" # The Globus project the GCS endpoint will be created in if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") else: - DATAFED_GCS_ROOT_NAME="DataFed Repo" + DATAFED_GCS_ROOT_NAME = "DataFed Repo" if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: - PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") + PROJECT_NAME = os.getenv("DATAFED_GLOBUS_PROJECT_NAME") else: - PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" + PROJECT_NAME = DATAFED_GCS_ROOT_NAME + " Project" # This is for confidential client if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: @@ -30,44 +29,47 @@ # Name of the client secret used by the confidential client if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: - CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") + CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME") else: - CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" + CRED_NAME = DATAFED_GCS_ROOT_NAME + " Cred" # Name of the file where we will store confidential client credentials if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: - CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") + CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") else: - CRED_FILE_PATH="./client_cred.json" + CRED_FILE_PATH = "./client_cred.json" # Name to give to endpoint if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") else: - ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" + ENDPOINT_NAME = DATAFED_GCS_ROOT_NAME + " Endpoint" # Path to deployment key if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: - DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") + DEPLOYMENT_KEY_PATH = os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") else: - DEPLOYMENT_KEY_PATH="./deployment-key.json" + DEPLOYMENT_KEY_PATH = "./deployment-key.json" # Path to deployment key if os.getenv("DATAFED_GLOBUS_CONTROL_PORT") is not None: - DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT") + DATAFED_GLOBUS_CONTROL_PORT = os.getenv("DATAFED_GLOBUS_CONTROL_PORT") else: - DATAFED_GLOBUS_CONTROL_PORT="443" + DATAFED_GLOBUS_CONTROL_PORT = "443" if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: - DATAFED_GLOBUS_SUBSCRIPTION=os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") + DATAFED_GLOBUS_SUBSCRIPTION = os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") else: - DATAFED_GLOBUS_SUBSCRIPTION="" + DATAFED_GLOBUS_SUBSCRIPTION = "" client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project # view_identities to user information for creating GCS server -client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -75,7 +77,9 @@ token_response = client.oauth2_exchange_code_for_tokens(auth_code) # Extract the token -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) # auth_client_refresh_token ac_rt = AuthClient(authorizer=rt_authorizer) @@ -93,20 +97,19 @@ count = utils.countProjects(ac_rt, PROJECT_NAME) if count != 1: - print("Something is wrong there should be at least one project with name" - f" {PROJECT_NAME} instead there are {count} with that name") + print( + "Something is wrong there should be at least one project with name" + f" {PROJECT_NAME} instead there are {count} with that name" + ) sys.exit(1) print(f"Project id is {project_id}") client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH +) # Add the globus client as an admin to the project -ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) +ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) # Get clients in project clients_in_project = utils.getClientsInProject(ac_rt, project_id) @@ -115,13 +118,13 @@ # client exists for the globus connect server if it does not then we will # call the setup command utils.createGCSEndpoint( - ac_rt, - client_id, - client_secret, - project_id, - DEPLOYMENT_KEY_PATH, - ENDPOINT_NAME, - DATAFED_GLOBUS_CONTROL_PORT, - DATAFED_GLOBUS_SUBSCRIPTION, - userinfo) - + ac_rt, + client_id, + client_secret, + project_id, + DEPLOYMENT_KEY_PATH, + ENDPOINT_NAME, + DATAFED_GLOBUS_CONTROL_PORT, + DATAFED_GLOBUS_SUBSCRIPTION, + userinfo, +) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 84194498a..b42b17411 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess from globus_sdk import AuthClient, AccessTokenAuthorizer @@ -6,12 +5,14 @@ import os import sys + def getProjectId(projects, project_name): for project in projects: - if project['display_name'] == project_name: - return project['id'] + if project["display_name"] == project_name: + return project["id"] return None + def projectExists(auth_client, project_name): projects = auth_client.get_projects() project_id = getProjectId(projects, project_name) @@ -21,6 +22,7 @@ def projectExists(auth_client, project_name): project_exists = False return project_exists + def createProject(auth_client, project_name, userinfo): identity_id = userinfo["sub"] @@ -30,35 +32,40 @@ def createProject(auth_client, project_name, userinfo): if project_exists is False: project_create_result = auth_client.create_project( - project_name, - contact_email=email, - admin_ids=[identity_id]) - return project_create_result['project']['id'] + project_name, contact_email=email, admin_ids=[identity_id] + ) + return project_create_result["project"]["id"] projects = auth_client.get_projects() return getProjectId(projects, project_name) + def countProjects(auth_client, project_name): projects = auth_client.get_projects() count = 0 for project in projects: - if project['display_name'] == project_name: + if project["display_name"] == project_name: count += 1 return count + def getClientId(auth_client, client_name, project_id): get_client_result = auth_client.get_clients() - for client in get_client_result['clients']: - if client['name'] == client_name and client['project'] == project_id: - return client['id'] + for client in get_client_result["clients"]: + if client["name"] == client_name and client["project"] == project_id: + return client["id"] return None + def getAllGCSClientIds(auth_client, project_id, endpoint_name): clients_in_project = getClientsInProject(auth_client, project_id) all_gcs_client_ids = [] for client in clients_in_project: - if client['client_type'] == "globus_connect_server" and client['name'] == endpoint_name: - all_gcs_client_ids.append(client['id']) + if ( + client["client_type"] == "globus_connect_server" + and client["name"] == endpoint_name + ): + all_gcs_client_ids.append(client["id"]) return all_gcs_client_ids @@ -66,8 +73,8 @@ def getClientsInProject(auth_client, project_id): # Get clients in project get_client_result = auth_client.get_clients() clients_in_project = [] - for client in get_client_result['clients']: - if client['project'] == project_id: + for client in get_client_result["clients"]: + if client["project"] == project_id: clients_in_project.append(client) return clients_in_project @@ -80,18 +87,22 @@ def createNewClient(auth_client, client_name, project_id): client_exists = True if client_exists is False: - result = auth_client.create_client(client_name, project=project_id, public_client=False) + result = auth_client.create_client( + client_name, project=project_id, public_client=False + ) client_id = result["client"]["id"] return client_id + def getCredentialID(auth_client, client_id, cred_name): get_client_cred_result = auth_client.get_client_credentials(client_id) - for cred in get_client_cred_result['credentials']: - if cred['name'] == cred_name: - return cred['id'] + for cred in get_client_cred_result["credentials"]: + if cred["name"] == cred_name: + return cred["id"] return None + def validFile(file_name): file_exists = False file_empty = True @@ -109,59 +120,57 @@ def getCredentialFromFile(cred_file_name, cred_id): # name cred_exists_locally, cred_empty = validFile(cred_file_name) if cred_empty is False: - with open(cred_file_name, 'r') as f: + with open(cred_file_name, "r") as f: loaded_data = json.load(f) - if loaded_data['client'] == cred_id: - return loaded_data['secret'] + if loaded_data["client"] == cred_id: + return loaded_data["secret"] return None + def getClientIdFromCredFile(cred_file_name): # Check to see if the local secret is the same id and not just the same # name cred_exists_locally, cred_empty = validFile(cred_file_name) if cred_empty is False: - with open(cred_file_name, 'r') as f: + with open(cred_file_name, "r") as f: loaded_data = json.load(f) - return loaded_data['client'] + return loaded_data["client"] return None + def getEndpointIdFromFile(deployment_key_file_path): # Check to see if the local secret is the same id and not just the same # name exists_locally, empty = validFile(deployment_key_file_path) if empty is False: - with open(deployment_key_file_path, 'r') as f: + with open(deployment_key_file_path, "r") as f: loaded_data = json.load(f) - return loaded_data['client_id'] + return loaded_data["client_id"] return None def createNewCredential(auth_client, client_id, cred_name, cred_file): get_client_cred_result = auth_client.get_client_credentials(client_id) - for cred in get_client_cred_result['credentials']: + for cred in get_client_cred_result["credentials"]: # Should have stored secret locally - auth_client.delete_client_credential(client_id, cred['id']) + auth_client.delete_client_credential(client_id, cred["id"]) cred_result = auth_client.create_client_credential(client_id, cred_name) - # Have to change this to a dict + # Have to change this to a dict obj = { - 'client': cred_result['credential']['client'], - 'id': cred_result['credential']['id'], - 'name': cred_result['credential']['name'], - 'secret': cred_result['credential']['secret'] - } - with open(cred_file, 'w') as f: + "client": cred_result["credential"]["client"], + "id": cred_result["credential"]["id"], + "name": cred_result["credential"]["name"], + "secret": cred_result["credential"]["secret"], + } + with open(cred_file, "w") as f: json.dump(obj, f) - return cred_result['credential']['secret'] + return cred_result["credential"]["secret"] -def getClientSecret( - auth_client, - client_id, - cred_name, - cred_id, - cred_file): + +def getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file): client_secret = getCredentialFromFile(cred_file, cred_id) @@ -171,7 +180,7 @@ def getClientSecret( if client_secret: create_new_credential = False remove_cached_credential = False - remove_old_credential = True + remove_old_credential = True if remove_old_credential: auth_client.delete_client_credential(client_id, cred_id) @@ -183,11 +192,8 @@ def getClientSecret( if create_new_credential: # Remove credentials from cloud client_secret = createNewCredential( - auth_client, - client_id, - cred_name, - cred_file - ) + auth_client, client_id, cred_name, cred_file + ) return client_secret @@ -203,8 +209,11 @@ def createClient(auth_client, client_name, project_id, cred_name, cred_file): cred_exists_locally, cred_empty = validFile(cred_file) - client_secret = getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file) - return client_id, client_secret + client_secret = getClientSecret( + auth_client, client_id, cred_name, cred_id, cred_file + ) + return client_id, client_secret + def getGCSClientIDFromDeploymentFile(deployment_key_file): deployment_key_exists, deployment_key_empty = validFile(deployment_key_file) @@ -216,9 +225,9 @@ def getGCSClientIDFromDeploymentFile(deployment_key_file): os.remove(deployment_key_file) else: # If it is not empty get the client id - with open(deployment_key_file, 'r') as f: + with open(deployment_key_file, "r") as f: loaded_data = json.load(f) - return loaded_data['client_id'] + return loaded_data["client_id"] return None @@ -231,44 +240,53 @@ def command_exists(command): # 'which' command returns non-zero exit status if the command is not found return False + def isGCSDeploymentKeyValid(auth_client, project_id, endpoint_name, gcs_id): clients_in_project = getClientsInProject(auth_client, project_id) # Check if the deployment key is valid for the project for client in clients_in_project: - if client['client_type'] == "globus_connect_server" and client['name'] == endpoint_name: + if ( + client["client_type"] == "globus_connect_server" + and client["name"] == endpoint_name + ): if gcs_id: # If gcs_id exists see if it is found remotely - if client['id'] == gcs_id: + if client["id"] == gcs_id: print("Deployment key endpoint is still valid found in cloud") return True else: # Found a globus_connect_server but did not find local deployment # key if deployment_key_empty: - print("Found globus_connect_server already registered but did" - " not find deployment key locally.") + print( + "Found globus_connect_server already registered but did" + " not find deployment key locally." + ) return False def deleteAllNonGCSClients(auth_client, project_id): clients = getClientsInProject(auth_client, project_id) for client in clients: - if client['project'] == project_id and client['client_type'] != "globus_connect_server": - auth_client.delete_client(client['id']) - + if ( + client["project"] == project_id + and client["client_type"] != "globus_connect_server" + ): + auth_client.delete_client(client["id"]) def createGCSEndpoint( - auth_client, - client_id, - client_secret, - project_id, - deployment_key_file, - endpoint_name, - control_port, - subscription_id, - userinfo): + auth_client, + client_id, + client_secret, + project_id, + deployment_key_file, + endpoint_name, + control_port, + subscription_id, + userinfo, +): identity_id = userinfo["sub"] email = userinfo["email"] @@ -277,53 +295,69 @@ def createGCSEndpoint( gcs_id_from_deployment_key = getGCSClientIDFromDeploymentFile(deployment_key_file) - valid_key = isGCSDeploymentKeyValid(auth_client, project_id, endpoint_name, gcs_id_from_deployment_key) + valid_key = isGCSDeploymentKeyValid( + auth_client, project_id, endpoint_name, gcs_id_from_deployment_key + ) if valid_key is False and gcs_id_from_deployment_key: - print("Looks like deployment key exists but does not contain credentials " - f"in the cloud for the project: {project_id}, please either " - "add the correct deployment key or remove the gcs instance" - "registered in the project") + print( + "Looks like deployment key exists but does not contain credentials " + f"in the cloud for the project: {project_id}, please either " + "add the correct deployment key or remove the gcs instance" + "registered in the project" + ) sys.exit(1) -# Create gcs_instance + # Create gcs_instance if valid_key is False: if command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" " - bash_command+=f" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+=f" globus-connect-server endpoint setup \"{endpoint_name}\" " - bash_command+=f" --organization \"{organization}\" " - bash_command+=f" --project-id \"{project_id}\" " - bash_command+=" --agree-to-letsencrypt-tos " - bash_command+=f" --project-admin \"{username}\" " - bash_command+=f" --owner \"{client_id}@clients.auth.globus.org\" " - bash_command+=f" --contact-email \"{email}\" " - bash_command+=f" --deployment-key \"{deployment_key_file}\" " + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" ' + bash_command += f' GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += f' globus-connect-server endpoint setup "{endpoint_name}" ' + bash_command += f' --organization "{organization}" ' + bash_command += f' --project-id "{project_id}" ' + bash_command += " --agree-to-letsencrypt-tos " + bash_command += f' --project-admin "{username}" ' + bash_command += f' --owner "{client_id}@clients.auth.globus.org" ' + bash_command += f' --contact-email "{email}" ' + bash_command += f' --deployment-key "{deployment_key_file}" ' print("Bash command to run") print(bash_command) - - process = subprocess.Popen(bash_command, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + + process = subprocess.Popen( + bash_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) # Print the output for line in process.stdout: - print(line, end='') + print(line, end="") deployment_key_exists, deployment_key_empty = validFile(deployment_key_file) if deployment_key_exists is False: - print(f"Something is wrong deployment key does not exist {deployment_key_file} ") + print( + f"Something is wrong deployment key does not exist {deployment_key_file} " + ) sys.exit(1) if deployment_key_empty: - print(f"Something is wrong deployment key is empty {deployment_key_file} ") + print( + f"Something is wrong deployment key is empty {deployment_key_file} " + ) sys.exit(1) # WARNING!!!!!! # This will not work if a node does not first exist, I think at least one # node must be running. - #if len(subscription_id) != 0: + # if len(subscription_id) != 0: # if command_exists("globus-connect-server") is False: # print("Cannot create deployment key, we require globus-connect-server to be installed") # sys.exit(1) @@ -336,10 +370,8 @@ def createGCSEndpoint( # bash_command+=" globus-connect-server endpoint update " # bash_command+=f" --subscription-id \"{subscription_id}\" " # print(bash_command) - # + # # process = subprocess.Popen(bash_command, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) # # Print the output # for line in process.stdout: # print(line, end='') - - From c7a3b9f09cca36c45d95fde70008cd788e838f8a Mon Sep 17 00:00:00 2001 From: par-hermes Date: Sat, 6 Apr 2024 04:57:40 +0000 Subject: [PATCH 37/84] cpp-py-formatter --- common/source/servers/Proxy.cpp | 20 ++- core/server/ClientWorker.cpp | 6 +- core/server/TaskWorker.cpp | 5 +- .../globus5/authz/source/AuthzWorker.cpp | 10 +- .../gridftp/globus5/authz/source/libauthz.c | 164 +++++++++++------- repository/server/RepoServer.cpp | 18 +- repository/server/RequestWorker.cpp | 10 +- scripts/globus/create_guest_collection.py | 142 ++++++++------- scripts/globus/globus_cleanup.py | 120 +++++++------ scripts/globus/initialize_globus_endpoint.py | 64 +++---- scripts/globus/utils.py | 21 ++- 11 files changed, 331 insertions(+), 249 deletions(-) diff --git a/common/source/servers/Proxy.cpp b/common/source/servers/Proxy.cpp index c78ceff7f..e88bd3df0 100644 --- a/common/source/servers/Proxy.cpp +++ b/common/source/servers/Proxy.cpp @@ -123,10 +123,12 @@ void Proxy::run() { // public if (resp_from_client_socket.error == false and resp_from_client_socket.time_out == false) { - if(not resp_from_server_socket.message ) { - DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " - << "response is not defined but no timeouts or errors were " - << "triggered, unable to send to server."); + if (not resp_from_server_socket.message) { + DL_ERROR( + m_log_context, + "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to send to server."); } else { m_communicators[SocketRole::SERVER]->send( *resp_from_client_socket.message); @@ -143,10 +145,12 @@ void Proxy::run() { // ... - Serv Sock - Proxy ------ Client Sock - Serv Sock - Inter App if (resp_from_server_socket.error == false and resp_from_server_socket.time_out == false) { - if(not resp_from_server_socket.message ) { - DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " - << "response is not defined but no timeouts or errors were " - << "triggered, unable to operate and send to client."); + if (not resp_from_server_socket.message) { + DL_ERROR( + m_log_context, + "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to operate and send to client."); } else { for (auto &in_operator : m_incoming_operators) { in_operator->execute(*resp_from_server_socket.message); diff --git a/core/server/ClientWorker.cpp b/core/server/ClientWorker.cpp index 40e57f00b..2c3084ee0 100644 --- a/core/server/ClientWorker.cpp +++ b/core/server/ClientWorker.cpp @@ -346,8 +346,10 @@ void ClientWorker::workerThread(LogContext log_context) { ICommunicator::Response response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); if (response.time_out == false and response.error == false) { - if ( not response.message ) { - DL_ERROR(message_log_context, "No timeout or error was reported but message is not defined.") + if (not response.message) { + DL_ERROR( + message_log_context, + "No timeout or error was reported but message is not defined.") } IMessage &message = *response.message; diff --git a/core/server/TaskWorker.cpp b/core/server/TaskWorker.cpp index 87dd0f1ad..95c55ead2 100644 --- a/core/server/TaskWorker.cpp +++ b/core/server/TaskWorker.cpp @@ -339,8 +339,9 @@ TaskWorker::cmdRawDataUpdateSize(TaskWorker &me, const Value &a_task_params, if (response.time_out == true) { return response; } else if (response.error == false) { - if( not response.message ) { - DL_ERROR(log_context, "No timeout or error was reported but no message was defined."); + if (not response.message) { + DL_ERROR(log_context, + "No timeout or error was reported but no message was defined."); } auto proto_msg = std::get(response.message->getPayload()); diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index 4bc7261bd..6a6eaa7e0 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -226,7 +226,8 @@ class AuthzWorker { << client->address()); auto response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); - if (response.message ) { // Make sure the message exists before we try to access it + if (response.message) { // Make sure the message exists before we try to + // access it log_context.correlation_id = std::get( response.message->get(MessageAttribute::CORRELATION_ID)); } @@ -252,9 +253,10 @@ class AuthzWorker { "communicating with the core service: " << response.error_msg); } else { - - if( not response.message ) { - DL_ERROR(log_context, "No error was reported and no time out occured but message is not defined."); + + if (not response.message) { + DL_ERROR(log_context, "No error was reported and no time out occured " + "but message is not defined."); } auto payload = diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index 2dd6b1b7b..fd03bb7b2 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -20,40 +20,63 @@ // Define logging macros #if defined(DONT_USE_SYSLOG) - FILE *log_file = NULL; - bool write_to_file = true; - #define AUTHZ_LOG_DEBUG(fmt, ...) \ - do { if (LOG_LEVEL >= 1) fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0); \ - do { if (LOG_LEVEL >= 1 && write_to_file ) fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INFO(fmt, ...) \ - do { if (LOG_LEVEL >= 2) fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); } while (0); \ - do { if (LOG_LEVEL >= 2 && write_to_file ) fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_ERROR(fmt, ...) \ - do { \ - if (LOG_LEVEL >= 3) fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ - if (LOG_LEVEL >= 3 && write_to_file ) fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ - } while(0) - #define AUTHZ_LOG_INIT(file_path) \ - log_file = fopen(file_path, "a"); \ - if (log_file != NULL) { write_to_file = true; } - #define AUTHZ_LOG_CLOSE() \ - if (log_file != NULL) { fclose(log_file); } +FILE *log_file = NULL; +bool write_to_file = true; +#define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 1) \ + fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); \ + } while (0); \ + do { \ + if (LOG_LEVEL >= 1 && write_to_file) \ + fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INFO(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 2) \ + fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); \ + } while (0); \ + do { \ + if (LOG_LEVEL >= 2 && write_to_file) \ + fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_ERROR(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 3) \ + fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ + if (LOG_LEVEL >= 3 && write_to_file) \ + fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INIT(file_path) \ + log_file = fopen(file_path, "a"); \ + if (log_file != NULL) { \ + write_to_file = true; \ + } +#define AUTHZ_LOG_CLOSE() \ + if (log_file != NULL) { \ + fclose(log_file); \ + } #else - #include - #define AUTHZ_LOG_DEBUG(fmt, ...) \ - do { if (LOG_LEVEL >= 1) syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INFO(fmt, ...) \ - do { if (LOG_LEVEL >= 2) syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_ERROR(fmt, ...) \ - do { if (LOG_LEVEL >= 3) syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INIT(file_path) \ - openlog("gsi_authz", 0, LOG_AUTH); - #define AUTHZ_LOG_CLOSE() \ - closelog(); +#include +#define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 1) \ + syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INFO(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 2) \ + syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_ERROR(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 3) \ + syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INIT(file_path) openlog("gsi_authz", 0, LOG_AUTH); +#define AUTHZ_LOG_CLOSE() closelog(); #endif - - typedef void *globus_gsi_authz_handle_t; typedef void (*globus_gsi_authz_cb_t)(void *callback_arg, globus_gsi_authz_handle_t handle, @@ -189,13 +212,14 @@ bool setConfigVal(const char *a_label, char *a_dest, char *a_src, size_t len = strlen(a_src); if (len == 0) { - AUTHZ_LOG_ERROR("DataFed - '%s' value not set.\n", a_label); + AUTHZ_LOG_ERROR("DataFed - '%s' value not set.\n", a_label); return true; } if (len > a_max_len) { - AUTHZ_LOG_ERROR("DataFed - '%s' value too long in authz config file (max %lu).\n", - a_label, a_max_len); + AUTHZ_LOG_ERROR( + "DataFed - '%s' value too long in authz config file (max %lu).\n", + a_label, a_max_len); return true; } @@ -269,7 +293,7 @@ bool loadConfig() { val = strchr(buf, '='); if (!val) { AUTHZ_LOG_ERROR( - "DataFed - Syntax error in authz config file at line %i.\n", lc); + "DataFed - Syntax error in authz config file at line %i.\n", lc); return true; } else { *val = 0; @@ -305,8 +329,8 @@ bool loadConfig() { else { err = true; AUTHZ_LOG_ERROR( - "DataFed - Invalid key, '%s', in authz config file at line %i.\n", - buf, lc); + "DataFed - Invalid key, '%s', in authz config file at line %i.\n", + buf, lc); } if (err) { @@ -336,20 +360,23 @@ bool loadConfig() { strcat(miss, " server_key"); AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); - AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", + getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); if (miss[0] != 0) { - AUTHZ_LOG_ERROR("DataFed - Missing required authz config items:%s\n", miss); + AUTHZ_LOG_ERROR("DataFed - Missing required authz config items:%s\n", + miss); return true; } } else { AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); - AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", + getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); AUTHZ_LOG_ERROR("DataFed - Could not open authz config file.\n"); return true; @@ -358,14 +385,14 @@ bool loadConfig() { AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); return false; } // The same globus_result_t gsi_authz_init() { - //openlog("gsi_authz", 0, LOG_AUTH); + // openlog("gsi_authz", 0, LOG_AUTH); memset(g_active_contexts, 0, sizeof(g_active_contexts)); // This line is different @@ -402,7 +429,8 @@ globus_result_t gsi_authz_handle_init(va_list ap) { else AUTHZ_LOG_ERROR("gsi_authz_handle_init out of handle context space\n"); } else { - AUTHZ_LOG_DEBUG("gsi_authz_handle_init context handle already initialized\n"); + AUTHZ_LOG_DEBUG( + "gsi_authz_handle_init context handle already initialized\n"); } callback(callback_arg, callback_arg, result); @@ -449,21 +477,21 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { char *callout_id_mapped1 = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", - callout_ids1); + callout_ids1); AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", - callout_id_mapped1); + callout_id_mapped1); AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", - g_config.globus_collection_path, action, object); + g_config.globus_collection_path, action, object); if (strcmp(action, "lookup") == 0 || strcmp(action, "chdir") == 0) { AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", - g_config.globus_collection_path, action, object); + g_config.globus_collection_path, action, object); result = GLOBUS_SUCCESS; callback(callback_arg, handle, result); return result; } AUTHZ_LOG_ERROR("gsi_authz_authorize_async, handle: %p, act: %s, obj: %s\n", - handle, action, object); + handle, action, object); OM_uint32 min_stat; gss_name_t client = GSS_C_NO_NAME; @@ -486,7 +514,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { gss_display_name(&min_stat, target, &target_buf, &target_type); if (maj_stat == GSS_S_COMPLETE) { AUTHZ_LOG_INFO("Auth client: %s, file: %s, action: %s\n", - (char *)client_buf.value, object, action); + (char *)client_buf.value, object, action); // Testing hack #if 0 @@ -503,7 +531,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { if (strncmp((char *)client_buf.value, "/C=US/O=Globus Consortium/OU=Globus", 35) != 0) { AUTHZ_LOG_ERROR("Invalid certificate subject prefix: %s\n", - (char *)client_buf.value); + (char *)client_buf.value); } else { /* Note: For some reason, globus will provide the CN as either a * UUID that is linked to the client's account and encoded in @@ -523,7 +551,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { if (!decodeUUID((char *)client_buf.value + 54, client_id)) { AUTHZ_LOG_ERROR("Failed to decode subject UUID: %s\n", - (char *)client_buf.value + 54); + (char *)client_buf.value + 54); free(client_id); client_id = 0; } @@ -539,23 +567,25 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { } char *callout_ids = getenv("GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS"); - char *callout_username_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); - char *callout_id_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); + char *callout_username_mapped = + getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); + char *callout_id_mapped = + getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); if (callout_ids != NULL) { AUTHZ_LOG_DEBUG( - "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", - callout_ids); + "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", + callout_ids); client_id = strdup(callout_ids); AUTHZ_LOG_INFO("libauthz.c client_id(s): %s\n", client_id); - } else if ( callout_id_mapped != NULL ) { + } else if (callout_id_mapped != NULL) { AUTHZ_LOG_DEBUG( - "libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", - callout_id_mapped); + "libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", + callout_id_mapped); client_id = strdup(callout_id_mapped); } else { AUTHZ_LOG_ERROR( - "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS.\n"); + "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS.\n"); } } @@ -566,8 +596,8 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { } else { AUTHZ_LOG_INFO( - "libauthz.c Auth client_id: %s, file: %s, action: %s\n", - client_id, object, action); + "libauthz.c Auth client_id: %s, file: %s, action: %s\n", + client_id, object, action); AUTHZ_LOG_INFO("libauthz.c checkAuthorization FAIL.\n"); } @@ -578,17 +608,17 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { gss_release_buffer(&min_stat, &target_buf); } else { AUTHZ_LOG_ERROR("gss_display_name target FAILED, maj: %d, min: %d\n", - maj_stat, min_stat); + maj_stat, min_stat); } gss_release_buffer(&min_stat, &client_buf); } else { AUTHZ_LOG_ERROR("gss_display_name source FAILED, maj: %d, min: %d\n", - maj_stat, min_stat); + maj_stat, min_stat); } } else { - AUTHZ_LOG_ERROR("gss_inquire_context FAILED, maj: %d, min: %d\n", maj_stat, - min_stat); + AUTHZ_LOG_ERROR("gss_inquire_context FAILED, maj: %d, min: %d\n", + maj_stat, min_stat); } } else { AUTHZ_LOG_ERROR("context handle lookup FAILED\n"); diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index d851eb0b7..e3051e3cc 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -101,7 +101,8 @@ void Server::run() { } void Server::checkServerVersion() { - DL_INFO(m_log_context, "Checking core server connection and version at " << m_config.core_server); + DL_INFO(m_log_context, "Checking core server connection and version at " + << m_config.core_server); // Generate random security keys for anon version request to core server KeyGenerator generator; @@ -148,7 +149,8 @@ void Server::checkServerVersion() { for (int i = 0; i < 10; i++) { DL_INFO(m_log_context, "Attempt " << i << " to initialize communication " - << " with core server at " << m_config.core_server); + << " with core server at " + << m_config.core_server); auto msg = std::make_unique(); auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); message->setPayload(std::move(msg)); @@ -166,14 +168,14 @@ void Server::checkServerVersion() { DL_ERROR(msg_log_context, "Timeout waiting for response from core server: " << m_config.core_server); - } else if(response.error) { - DL_ERROR(msg_log_context, - "Error encountered waiting for core server: " - << m_config.core_server << " msg " << response.error_msg); + } else if (response.error) { + DL_ERROR(msg_log_context, "Error encountered waiting for core server: " + << m_config.core_server << " msg " + << response.error_msg); } else { - + msg_log_context.correlation_id = std::get( - response.message->get(MessageAttribute::CORRELATION_ID)); + response.message->get(MessageAttribute::CORRELATION_ID)); auto payload = std::get(response.message->getPayload()); VersionReply *ver_reply = dynamic_cast(payload); diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index be08293fd..43b0652b5 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -146,9 +146,9 @@ void RequestWorker::workerThread(LogContext log_context) { DL_TRACE(message_log_context, "Checking timeouts: " << response.time_out); if (response.time_out == false and response.error == false) { - if( not response.message) { + if (not response.message) { DL_ERROR(log_context, "Error: No error or timeout occurred but the" - << " message does not exist."); + << " message does not exist."); } else { // May not have a correlation id if the message timed out DL_TRACE(log_context, "Getting correlation_id."); @@ -165,18 +165,18 @@ void RequestWorker::workerThread(LogContext log_context) { if (m_msg_handlers.count(msg_type)) { map::iterator handler = - m_msg_handlers.find(msg_type); + m_msg_handlers.find(msg_type); DL_TRACE(message_log_context, "Calling handler"); auto send_message = - (this->*handler->second)(std::move(response.message)); + (this->*handler->second)(std::move(response.message)); client->send(*(send_message)); DL_TRACE(message_log_context, "Reply sent."); } else { DL_ERROR(message_log_context, - "Received unregistered msg type: " << msg_type); + "Received unregistered msg type: " << msg_type); } } } else if (response.error) { diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index 59f6131fe..0099f34fa 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess import utils @@ -10,61 +9,76 @@ # The Globus project the GCS endpoint will be created in DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME", "DataFed Repo") -PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME", DATAFED_GCS_ROOT_NAME + " Project") +PROJECT_NAME = os.getenv( + "DATAFED_GLOBUS_PROJECT_NAME", DATAFED_GCS_ROOT_NAME + " Project" +) # This is for confidential client -CLIENT_NAME = os.getenv("DATAFED_GLOBUS_CLIENT_NAME", DATAFED_GCS_ROOT_NAME + " Setup Client") +CLIENT_NAME = os.getenv( + "DATAFED_GLOBUS_CLIENT_NAME", DATAFED_GCS_ROOT_NAME + " Setup Client" +) # Name of the client secret used by the confidential client -CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME",DATAFED_GCS_ROOT_NAME + " Cred") +CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME", DATAFED_GCS_ROOT_NAME + " Cred") # Name of the file where we will store confidential client credentials -CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH","./client_cred.json") +CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH", "./client_cred.json") ENDPOINT_ID = os.getenv("GCS_CLI_ENDPOINT_ID") -ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME",DATAFED_GCS_ROOT_NAME + " Endpoint") +ENDPOINT_NAME = os.getenv( + "DATAFED_GLOBUS_ENDPOINT_NAME", DATAFED_GCS_ROOT_NAME + " Endpoint" +) # Path to deployment key -DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH","./deployment-key.json") +DEPLOYMENT_KEY_PATH = os.getenv( + "DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH", "./deployment-key.json" +) # Path to deployment key -DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT", "443") -DATAFED_GCS_URL=os.getenv("DATAFED_GCS_URL") +DATAFED_GLOBUS_CONTROL_PORT = os.getenv("DATAFED_GLOBUS_CONTROL_PORT", "443") +DATAFED_GCS_URL = os.getenv("DATAFED_GCS_URL") client_id = os.getenv("GCS_CLI_CLIENT_ID") client_secret = os.getenv("GCS_CLI_CLIENT_SECRET") mapped_collection_id = os.getenv("MAPPED_COLLECTION_ID") -mapped_collection_name = os.getenv("DATAFED_GCS_COLLECTION_MAPPED", f"{DATAFED_GCS_ROOT_NAME} Collection Mapped") -guest_collection_name = os.getenv("DATAFED_GCS_COLLECTION_GUEST",f"{DATAFED_GCS_ROOT_NAME} Collection Guest") +mapped_collection_name = os.getenv( + "DATAFED_GCS_COLLECTION_MAPPED", f"{DATAFED_GCS_ROOT_NAME} Collection Mapped" +) +guest_collection_name = os.getenv( + "DATAFED_GCS_COLLECTION_GUEST", f"{DATAFED_GCS_ROOT_NAME} Collection Guest" +) storage_gateway_id = os.getenv("STORAGE_GATEWAY_ID") -storage_gateway_name = os.getenv("DATAFED_GCS_STORAGE_GATEWAY",f"{DATAFED_GCS_ROOT_NAME} Storage Gateway") +storage_gateway_name = os.getenv( + "DATAFED_GCS_STORAGE_GATEWAY", f"{DATAFED_GCS_ROOT_NAME} Storage Gateway" +) local_username = os.getenv("DATAFED_REPO_USER") if ENDPOINT_ID is None: raise Exception("GCS_CLI_ENDPOINT_ID must be defined as an env varaible") if DATAFED_GCS_URL is None: - raise Exception("Unable to create guest collection, DATAFED_GCS_URL is not" - " defined.") + raise Exception( + "Unable to create guest collection, DATAFED_GCS_URL is not" " defined." + ) if local_username is None: raise Exception("DATAFED_REPO_USER is not defined.") -#client = globus_sdk.NativeAppAuthClient(CLIENT_ID) +# client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project # view_identities to user information for creating GCS server -#client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +# client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) # -#authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) -#print("Please go to this URL and login: \n", authorize_url) -#auth_code = input("Please enter the authorization code: ") +# authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) +# print("Please go to this URL and login: \n", authorize_url) +# auth_code = input("Please enter the authorization code: ") # -#token_response = client.oauth2_exchange_code_for_tokens(auth_code) +# token_response = client.oauth2_exchange_code_for_tokens(auth_code) ## Extract the token -#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) ## auth_client_refresh_token -#ac_rt = AuthClient(authorizer=rt_authorizer) +# ac_rt = AuthClient(authorizer=rt_authorizer) # -#userinfo = ac_rt.oauth2_userinfo() +# userinfo = ac_rt.oauth2_userinfo() ## Will get the primary email and id -#identity_id = userinfo["sub"] -#email = userinfo["email"] -#username = userinfo["preferred_username"] -#organization = userinfo["identity_provider_display_name"] +# identity_id = userinfo["sub"] +# email = userinfo["email"] +# username = userinfo["preferred_username"] +# organization = userinfo["identity_provider_display_name"] if client_id is None: client_id = getClientIdFromCredFile(CRED_FILE_PATH) @@ -74,21 +88,21 @@ client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) -scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" +scopes = "openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) -#cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, +# cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, # scopes) -#token_response = client.oauth2_client_credentials_tokens() +# token_response = client.oauth2_client_credentials_tokens() -#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) # the useful values that you want at the end of this -#globus_auth_data = token_response.by_resource_server["auth.globus.org"] -#globus_transfer_data = -#token_response.by_resource_server["transfer.api.globus.org"] -#globus_auth_token = globus_auth_data["access_token"] -#globus_transfer_token = globus_transfer_data["access_token"] +# globus_auth_data = token_response.by_resource_server["auth.globus.org"] +# globus_transfer_data = +# token_response.by_resource_server["transfer.api.globus.org"] +# globus_auth_token = globus_auth_data["access_token"] +# globus_transfer_token = globus_transfer_data["access_token"] gcs_client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) @@ -97,10 +111,10 @@ from globus_sdk import scopes # constants -#endpoint_hostname = "https://ecf8ed.08cc.data.globus.org" -#endpoint_id = "769c7ed0-744a-41b5-b4a8-db37b10b1ac9" -#mapped_collection_id = "580ecb92-de56-42ee-a5ec-3d3886767b94" -#storage_gateway_id = "3fdd7f41-4a05-4856-8fcd-2fb50066c590" +# endpoint_hostname = "https://ecf8ed.08cc.data.globus.org" +# endpoint_id = "769c7ed0-744a-41b5-b4a8-db37b10b1ac9" +# mapped_collection_id = "580ecb92-de56-42ee-a5ec-3d3886767b94" +# storage_gateway_id = "3fdd7f41-4a05-4856-8fcd-2fb50066c590" # client credentials # This client identity must have the needed permissions to create a guest @@ -108,9 +122,9 @@ # on the storage gateway that matches the local_username # If using user tokens, the user must be the one with the correct permissions # and identity mapping. -#client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" -#client_secret = "*redacted*" -#local_username = "datafed" +# client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" +# client_secret = "*redacted*" +# local_username = "datafed" # The scope the client will need, note that primary scope is for the endpoint, # but it has a dependency on the mapped collection's data_access scope @@ -135,13 +149,15 @@ if item["id"] == mapped_collection_id: mapped_collection_found = True if item["display_name"] != mapped_collection_name: - raise Exception("Expected display name is different from what " - "is expected for mapped collection " - f"{mapped_collection_id}, if using non standard" - " display name for mapped collection " - f"{mapped_collection_name} then the " - "MAPPED_COLLECTION_NAME env variable must be " - "set.") + raise Exception( + "Expected display name is different from what " + "is expected for mapped collection " + f"{mapped_collection_id}, if using non standard" + " display name for mapped collection " + f"{mapped_collection_name} then the " + "MAPPED_COLLECTION_NAME env variable must be " + "set." + ) break elif item["display_name"] == mapped_collection_name: mapped_collection_found = True @@ -159,19 +175,21 @@ if item["id"] == storage_gateway_id: storage_gateway_found = True if item["display_name"] != storage_gateway_name: - raise Exception("Expected display name is different from what " - "is expected for storage gateway " - f"{storage_gateway_id}, if using non standard" - " display name for storage gateway " - f"{storage_gateway_name} then the " - "DATAFED_GCS_STORAGE_GATEWAY env variable must be " - "set.") + raise Exception( + "Expected display name is different from what " + "is expected for storage gateway " + f"{storage_gateway_id}, if using non standard" + " display name for storage gateway " + f"{storage_gateway_name} then the " + "DATAFED_GCS_STORAGE_GATEWAY env variable must be " + "set." + ) break elif item["display_name"] == storage_gateway_name: storage_gateway_found = True storage_gateway_id = item["id"] break - + if storage_gateway_found == False: raise Exception("Missing required storage gateway") @@ -189,7 +207,7 @@ guest_collection_found = True guest_collection_id = item["id"] break - + # https://github.com/globus/globus-sdk-python/blob/main/docs/examples/guest_collection_creation.rst if guest_collection_found == False: credential_document = globus_sdk.UserCredentialDocument( @@ -199,7 +217,7 @@ ) client.create_user_credential(credential_document) -# Create the collection + # Create the collection collection_document = globus_sdk.GuestCollectionDocument( public="True", collection_base_path="/", diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 133ee9d3b..502c57733 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,5 +1,5 @@ import globus_sdk -from globus_sdk import AuthClient,GroupsClient, AccessTokenAuthorizer +from globus_sdk import AuthClient, GroupsClient, AccessTokenAuthorizer from globus_sdk.scopes import GroupsScopes import subprocess import json @@ -55,11 +55,12 @@ client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project group_scope = GroupsScopes.make_mutable("all") -client.oauth2_start_flow(requested_scopes="openid profile email " - "urn:globus:auth:scope:auth.globus.org:manage_projects " - "urn:globus:auth:scope:auth.globus.org:view_identities " + - str(group_scope), - refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + str(group_scope), + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -71,13 +72,16 @@ print(token_response) -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] +refresh_token_groups = token_response.by_resource_server["groups.api.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, - client) +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, client) ac_rt = AuthClient(authorizer=rt_authorizer) gr_rt = GroupsClient(authorizer=rt_authorizer_groups) @@ -100,68 +104,84 @@ clients_in_project = utils.getClientsInProject(ac_rt, project_id) if len(clients_in_project) == 0: - print("No clients were detected in the project we can just delete the" - "project and be done.") + print( + "No clients were detected in the project we can just delete the" + "project and be done." + ) else: - # Check if the deployment key exists if it does read it and verify that the - # client exists for the globus connect server if it does not then we will - # call the setup command + # Check if the deployment key exists if it does read it and verify that the + # client exists for the globus connect server if it does not then we will + # call the setup command - gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile(DEPLOYMENT_KEY_PATH) + gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile( + DEPLOYMENT_KEY_PATH + ) - valid_key = utils.isGCSDeploymentKeyValid(ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key) + valid_key = utils.isGCSDeploymentKeyValid( + ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key + ) all_gcs_client_ids = utils.getAllGCSClientIds(ac_rt, project_id, ENDPOINT_NAME) if valid_key is False and len(all_gcs_client_ids) > 0: - print("Looks like gcs client does not exist in the cloud" - f" for the project: {project_id}." - "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like gcs client does not exist in the cloud" + f" for the project: {project_id}." + "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if gcs_id_from_deployment_key is None and len(all_gcs_client_ids) > 0: - print("Looks like deployment key does not exist, please either " - "add the correct deployment." - f" cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like deployment key does not exist, please either " + "add the correct deployment." + f" cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if len(all_gcs_client_ids) > 0: if utils.command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - print("Now that we know a GCS instance exists we have to make sure" - "we have valid credentials to run the globus-connect-server command" - "non interatively, this means we have to create credentials and a" - "client if they don't exist and when we are done with everything" - "delete them.") + print( + "Now that we know a GCS instance exists we have to make sure" + "we have valid credentials to run the globus-connect-server command" + "non interatively, this means we have to create credentials and a" + "client if they don't exist and when we are done with everything" + "delete them." + ) client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH + ) + ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) - ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) - - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+="globus-connect-server endpoint cleanup " - bash_command+=f" --deployment-key \"{DEPLOYMENT_KEY_PATH}\" " - bash_command+=" --agree-to-delete-endpoint" + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += "globus-connect-server endpoint cleanup " + bash_command += f' --deployment-key "{DEPLOYMENT_KEY_PATH}" ' + bash_command += " --agree-to-delete-endpoint" print("Bash command to run") print(bash_command) - - proc = subprocess.Popen(bash_command, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, shell=True, text=True) + + proc = subprocess.Popen( + bash_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True, + text=True, + ) output, error = proc.communicate(input="yes\n") @@ -169,17 +189,15 @@ print(output) print(error) - # Now we can try to delete the remaining clients that are in the project # Get all of the clients that are not gcs clients and delete them utils.deleteAllNonGCSClients(ac_rt, project_id) - -# CLOSE - if len(clients_in_project) == 0: + # CLOSE - if len(clients_in_project) == 0: -# Try to remove project this will only work if there are no other clients in -# the project + # Try to remove project this will only work if there are no other clients in + # the project print(f"Attempting to remove project {project_id}") project_remove = ac_rt.delete_project(project_id) print(project_remove) diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index 16afaa519..0f0e79334 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -62,7 +62,7 @@ if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: DATAFED_GLOBUS_SUBSCRIPTION = os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") else: - DATAFED_GLOBUS_SUBSCRIPTION=None + DATAFED_GLOBUS_SUBSCRIPTION = None client = globus_sdk.NativeAppAuthClient(CLIENT_ID) @@ -70,11 +70,12 @@ # view_identities to user information for creating GCS server group_scope = GroupsScopes.make_mutable("all") -client.oauth2_start_flow(requested_scopes="openid profile email " - "urn:globus:auth:scope:auth.globus.org:manage_projects " - "urn:globus:auth:scope:auth.globus.org:view_identities " + - str(group_scope), - refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + str(group_scope), + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -82,12 +83,15 @@ token_response = client.oauth2_exchange_code_for_tokens(auth_code) # Extract the token -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] +refresh_token_groups = token_response.by_resource_server["groups.api.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, - client) +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, client) # auth_client_refresh_token ac_rt = AuthClient(authorizer=rt_authorizer) gr_rt = GroupsClient(authorizer=rt_authorizer_groups) @@ -133,20 +137,21 @@ # client exists for the globus connect server if it does not then we will # call the setup command utils.createGCSEndpoint( - ac_rt, - client_id, - client_secret, - project_id, - DEPLOYMENT_KEY_PATH, - ENDPOINT_NAME, - DATAFED_GLOBUS_CONTROL_PORT, - userinfo) + ac_rt, + client_id, + client_secret, + project_id, + DEPLOYMENT_KEY_PATH, + ENDPOINT_NAME, + DATAFED_GLOBUS_CONTROL_PORT, + userinfo, +) if DATAFED_GLOBUS_SUBSCRIPTION is not None: -# Create subscription subgroup + # Create subscription subgroup results = gr_rt.get_group_by_subscription_id(DATAFED_GLOBUS_SUBSCRIPTION) - parent_group_id=results["group_id"] + parent_group_id = results["group_id"] print("Groups by sub") print(results) group_name = f"{DATAFED_GCS_ROOT_NAME} Group" @@ -157,16 +162,15 @@ else: print(f"Group does not exist {group_name}") package = { - "name": group_name, - "description": "DataFed Repository Subscription Group, used for" - "granting access to the application client to setup the repository in " - "Globus", - "parent_id": str(parent_group_id) - } - + "name": group_name, + "description": "DataFed Repository Subscription Group, used for" + "granting access to the application client to setup the repository in " + "Globus", + "parent_id": str(parent_group_id), + } result = gr_rt.create_group(package) - group_id = result['id'] + group_id = result["id"] print("group id") print(group_id) @@ -177,9 +181,7 @@ print("membership_action") print(result) - package = { - "subscription_id": DATAFED_GLOBUS_SUBSCRIPTION - } + package = {"subscription_id": DATAFED_GLOBUS_SUBSCRIPTION} result = gr_rt.update_group(group_id, package) print("update group") print(result) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 226679c53..d4588b51c 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -102,6 +102,7 @@ def getCredentialID(auth_client, client_id, cred_name): return cred["id"] return None + def groupExists(client, group_name): my_groups = client.get_my_groups() print("My groups") @@ -111,6 +112,7 @@ def groupExists(client, group_name): return True return False + def getGroupId(client, group_name): my_groups = client.get_my_groups() for group in my_groups: @@ -119,7 +121,6 @@ def getGroupId(client, group_name): return None - def deleteGroup(client, group_name): my_groups = client.get_my_groups() for group in my_groups: @@ -128,6 +129,7 @@ def deleteGroup(client, group_name): print(f"Removing group: {group_name} with id: {group['id']}") print(result) + def validFile(file_name): file_exists = False file_empty = True @@ -302,14 +304,15 @@ def deleteAllNonGCSClients(auth_client, project_id): def createGCSEndpoint( - auth_client, - client_id, - client_secret, - project_id, - deployment_key_file, - endpoint_name, - control_port, - userinfo): + auth_client, + client_id, + client_secret, + project_id, + deployment_key_file, + endpoint_name, + control_port, + userinfo, +): identity_id = userinfo["sub"] email = userinfo["email"] From dc0a4102400389eebf4537b3664542c8839bf834 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:08:09 -0400 Subject: [PATCH 38/84] Add more detailed instructions --- compose/README.md | 169 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 148 insertions(+), 21 deletions(-) diff --git a/compose/README.md b/compose/README.md index 0609610bf..20ebed895 100644 --- a/compose/README.md +++ b/compose/README.md @@ -1,80 +1,207 @@ # Compose Dev environment -## Generating configuration varaibles +The compos Dev environment is split into two different compose files. The +"core metadata services" which comprise the web server, core server and database +and the "repo services" which comprise Globus Connect Server running with the +authz library and the DataFed repo service. + +NOTE Standing up the repo services has been separated because of Globus. You will +need a machine with firewall exceptions to use it. + +## Core Compose Services + +The following steps are used to stand up the core compose file from scratch. +Some of steps you only have to do once. + +1. Generating the env variables. +2. Opening the .env file and entering your configuration +3. Building the images +4. Running the compose file +5. Bringing down the compose file. + +Core services only need an external port 443 for https access. I have been +unable to get this to work with other ports due to the redirect URL required +by Globus for authentication. It does not seem to support ports outside of 443. + +### 1. Generating .env configuration varaibles Create the .env file fill in the missing components that are required. ```bash ./generate_env.sh ``` +### 2. Generating .env configuration varaibles -Both the repo compose file and the core compose file will use the .env file -that is generated from this step. +The .env file will be created in the DataFed/compose folder and will be hidden. +The .env file variables can be changed at this point to your configuration. -## Building Core Services +NOTE the .env file will be read verbatim by compose including any spaces or +"#" comments so do not includ anything but the exact text that needs to be +included in the variables. -The files in this folder are incomplete but are the start for setting up a full -docker compose instance of datafed. +### 3. Building Core Services + +The following command will build all the images to run the core metadata +services. ```bash ./build_images_for_compose.sh ``` +### 4. Running the compose file + Stand up the core services. ```bash +source ./unset_env.sh docker compose -f ./compose_core.yml up ``` +NOTE The unset_env.sh script is to make sure you are not accidentially overwriting +what is in the .env with your local shell env. You can check the configuration +before hand by running. + +```bash +docker compose -f compose_core.yml config +``` + +WARNING - docker compose will prioritize env variables in the following priority +1. From you shell env +2. From the .env file +3. Internally from winthin the image +Be sure that you are not accidentally overwriting .env variables. + At this point you should be able to navigate in your browser to NOTE we are using a self signed certificate so you will have to force your browser to allow you to see the page. -Standing up the repo services has been separated because of Globus. You will -need a machine with firewall exceptions to use it. +### 5. Bringing down the compose file + +## Repo Compose Services + +The following steps are used to stand up the repo compose file. NOTE, that +because of how Globus is configured, there is an additional configuration +and teardown step. + +1. Generating the env variables. +2. Opening the .env file and entering your configuration +3. Running the generate_globus_files.sh script +4. Building the images +5. Running the compose file +6. Bringing down the compose file. +7. Running the cleanup_globus_files.sh if you -## Building Repo & Authz-Globus Services +### 1. Generating .env configuration varaibles + +Create the .env file fill in the missing components that are required. + +```bash +./generate_env.sh +``` + +### 2. Generating .env configuration varaibles + +The .env file will be created in the DataFed/compose folder and will be hidden. +The .env file variables can be changed at this point to your configuration. + +NOTE the .env file will be read verbatim by compose including any spaces or +"#" comments so do not includ anything but the exact text that needs to be +included in the variables. + +### 3. Globus configuration + +This step is only required once, after which the necessary files should exist +in DataFed/compose/globus. These files will contain the Globus configuration +needed for additional cycles of "docker compose up" and "docker compose down". + +### 4. Building Repo Services + +The following command will build all the images to run the core metadata +services. ```bash ./build_repo_images_for_compose.sh ``` -Standing up the repo services. NOTE gcs-authz container will use host network -due to the large numbers of ports that are needed by Globus. +### 5. Running the compose file + +Stand up the repo services. ```bash +source ./unset_env.sh docker compose -f ./compose_repo.yml up ``` -## Running individual containers - -If you just want to run a single container at a time with the same configuration -this can also be doen using commands like the following. +NOTE The unset_env.sh script is to make sure you are not accidentially overwriting +what is in the .env with your local shell env. You can check the configuration +before hand by running. Be aware, the 'source' is to apply changes to your +current terminal session. ```bash -docker run -e UID=$(id -u) --env-file .env -it datafed-web:latest /bin/bash +docker compose -f compose_repo.yml config ``` +WARNING - docker compose will prioritize env variables in the following priority +1. From you shell env +2. From the .env file +3. Internally from winthin the image +Be sure that you are not accidentally overwriting .env variables. + +### 6. Bringing down the compose file + ## Cleaning up ```bash docker compose -f ./compose_core.yml down ``` -## Running gcs Docker container +# Running isolated containers -Make sure port 80 is not already bound. Also note that the repo server keys -should exist in the keys folder before running the gcs instance. +If you just want to run a single container at a time with the same configuration +this can also be doen using commands like the following. + +## DataFed Web + +```bash +source ./unset_env.sh +docker run --env-file .env \ + -e UID=$(id -u) \ + -p 443:443 \ + -p 7513:7513 \ + -t datafed-web:latest +``` + +## DataFed GCS ```bash docker run --env-file .env \ --network=host \ - --entrypoint /bin/bash \ -v /home/cloud/compose_collection:/mnt/datafed \ -v ./globus:/opt/datafed/globus \ -v ./logs:/datafed/logs \ -v ./keys:/opt/datafed/keys \ - -it datafed-gcs:latest + -t datafed-gcs:latest ``` + +To interact more directly with the container the '-i' flag can be added and the +entrypoint file can be overwritten by including '--entrypoint /bin/bash' + +# Common Errors + +## Errors during compose up + +Make sure all the ports that are needed are open on the local host. These +include, ports + +443 for the datafed-web and datafed-gcs container +7512 for the datafed-core container +50000-51000 for the datafed-gcs container +9000 for the datafed-repo container +80 for the datafed-gcs container + +Make sure port 80 is not already bound on the host. Also note that the repo server keys +should exist in the keys folder before running the gcs instance. + + From 7d86a247a8ec00006af9563163358f820c170ba4 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:34:43 -0400 Subject: [PATCH 39/84] Move DataFed/compose/logs to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 87697010a..97ed71946 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ common/proto/common/Version.proto compose/.env compose/globus compose/keys +compose/logs compose/unset_env.sh config/datafed.sh config/datafed-authz.cfg From e90f11af46a4a63510fed7df01b1a493fda3b14b Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:35:11 -0400 Subject: [PATCH 40/84] Fix error in Proxy.cpp --- common/source/servers/Proxy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/source/servers/Proxy.cpp b/common/source/servers/Proxy.cpp index c78ceff7f..a61fc2da8 100644 --- a/common/source/servers/Proxy.cpp +++ b/common/source/servers/Proxy.cpp @@ -123,7 +123,7 @@ void Proxy::run() { // public if (resp_from_client_socket.error == false and resp_from_client_socket.time_out == false) { - if(not resp_from_server_socket.message ) { + if(not resp_from_client_socket.message ) { DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " << "response is not defined but no timeouts or errors were " << "triggered, unable to send to server."); From 1ef16dd3c9069d5eed53d57ca5a8bd2fea00a9ad Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:35:26 -0400 Subject: [PATCH 41/84] Add more detailed compose instructions --- compose/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/compose/README.md b/compose/README.md index 20ebed895..7eb12e4fb 100644 --- a/compose/README.md +++ b/compose/README.md @@ -79,6 +79,17 @@ browser to allow you to see the page. ### 5. Bringing down the compose file +To completely remove the compose instance and all state the following should +be run. + +```bash +docker compose -f ./compose_core.yml down --volumes +``` + +NOTE the volumes will remove all cached state. If the '--volumes' flag is +not added then on a subsequent "compose up" the database will not be a clean +install but contain state from previous runs. + ## Repo Compose Services The following steps are used to stand up the repo compose file. NOTE, that @@ -200,6 +211,7 @@ include, ports 50000-51000 for the datafed-gcs container 9000 for the datafed-repo container 80 for the datafed-gcs container +8512 for arangodb web server interface Make sure port 80 is not already bound on the host. Also note that the repo server keys should exist in the keys folder before running the gcs instance. From 673dab2bb0733fc8789883e38cbadd34573b82e7 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:36:18 -0400 Subject: [PATCH 42/84] Open core secure port 7512 and make arangob database persistent with addition of volume --- compose/compose_core.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/compose/compose_core.yml b/compose/compose_core.yml index 18d9b6c62..c7bb2e1da 100644 --- a/compose/compose_core.yml +++ b/compose/compose_core.yml @@ -14,6 +14,7 @@ services: DATAFED_HTTPS_SERVER_PORT: "${DATAFED_HTTPS_SERVER_PORT}" DATAFED_WEB_CERT_PATH: "${DATAFED_WEB_CERT_PATH}" DATAFED_WEB_KEY_PATH: "${DATAFED_WEB_KEY_PATH}" + DATAFED_WEB_USER: "datafed" DATAFED_DEFAULT_LOG_PATH: "${DATAFED_CONTAINER_LOG_PATH}" DATAFED_CORE_ADDRESS_PORT_INTERNAL: "datafed-core:7513" UID: "${DATAFED_UID}" @@ -22,6 +23,7 @@ services: - 443:443 # This must be the same port that is mapped to the host for redirects to work volumes: - ./keys:/opt/datafed/keys + - ./logs:${DATAFED_CONTAINER_LOG_PATH} networks: - datafed-internal @@ -45,11 +47,11 @@ services: DATAFED_DATABASE_IP_ADDRESS_PORT: "${DATAFED_DATABASE_IP_ADDRESS}:${DATAFED_DATABASE_PORT}" ports: - 7513 # Communication web server - - 7512 # Secure core server communication + - 7512:7512 # Secure core server communication must be exposed outside of the container volumes: - ./keys:/opt/datafed/keys + - ./logs:${DATAFED_CONTAINER_LOG_PATH} networks: - - datafed-core-secure-api - datafed-internal datafed-foxx: @@ -67,7 +69,9 @@ services: test: ["CMD", "/bin/bash", "-c", "[ -f /tmp/.foxx_is_installed ]"] interval: 10s timeout: 5s - retries: 10 + retries: 20 + volumes: + - foxx_tmp:/tmp networks: - datafed-internal @@ -75,17 +79,24 @@ services: image: arangodb environment: ARANGO_ROOT_PASSWORD: "${DATAFED_DATABASE_PASSWORD}" + volumes: + - arango_db:/var/lib/arangodb3 ports: - 8529:8529 # Arangodb web UI networks: - datafed-internal +# Fox tmp is used so that if docker compose up is run a second time it won't +# need to overwrite the database it will simply use the existing one hence +# skipping some of the setup steps. The foxx entrypoint script will create +# a file in tmp to indicate that this is the case. +# +# arango_db is used to cache the state of the arangodb database volumes: - keys: + foxx_tmp: + arango_db: networks: - datafed-core-secure-api: - driver: bridge datafed-internal: driver: bridge From 7f98654e5880bbf6b19de3461fb1f313a49b485d Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:36:48 -0400 Subject: [PATCH 43/84] Make log file path configurable in docker file --- core/docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index 029343062..bab738f2a 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -64,6 +64,7 @@ ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" +ENV DATAFED_DEFAULT_LOG_PATH="$DATAFED_INSTALL_PATH/logs" RUN ldconfig From 924bf0e8d1fab95c497ca64b03a895139976b3c8 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:39:25 -0400 Subject: [PATCH 44/84] Add comments and create some default folders --- docker/Dockerfile.foxx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile.foxx b/docker/Dockerfile.foxx index cef331f8d..cdaa7eb4f 100644 --- a/docker/Dockerfile.foxx +++ b/docker/Dockerfile.foxx @@ -26,6 +26,8 @@ ENV BUILD_DIR="${BUILD_DIR}" ENV DATAFED_DIR="${DATAFED_DIR}" ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" ENV PATH=${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin:${PATH} +ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" +ENV DATAFED_DEFAULT_LOG_PATH="$DATAFED_INSTALL_PATH/logs" COPY ./core/CMakeLists.txt ${BUILD_DIR}/core/CMakeLists.txt COPY ./CMakeLists.txt ${BUILD_DIR} @@ -41,21 +43,21 @@ COPY ./scripts/install_foxx.sh ${BUILD_DIR}/scripts/ RUN adduser --disabled-password --gecos "" datafed RUN mkdir -p ${DATAFED_DIR} -RUN mkdir -p /opt/datafed -RUN mkdir -p /var/log/datafed -RUN chown -R datafed:root /opt/datafed -RUN chown -R datafed:root /var/log/datafed -RUN chown -R datafed:root ${DATAFED_DIR} -RUN chown -R datafed:root ${DATAFED_DEPENDENCIES_INSTALL_PATH} +RUN mkdir -p ${DATAFED_INSTALL_PATH} +RUN mkdir -p ${DATAFED_DEFAULT_LOG_PATH} RUN curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key; apt-key add - < Release.key RUN echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list +# If trust the site but the gpg key has expired can add [trusted=yes] +# i.e. +# RUN echo 'deb [trusted=yes] https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list RUN apt update RUN apt install arangodb3-client -y RUN arangosh --version USER root -WORKDIR ${DATAFED_DIR} +WORKDIR ${BUILD_DIR} +#ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ${BUILD_DIR}/docker/entrypoint_foxx.sh From 845b6ea1d6540d1b601a30590783206accf50915 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:56:43 -0400 Subject: [PATCH 45/84] Remove the chmod commands from foxx ephemeral container --- docker/entrypoint_foxx.sh | 41 +++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh index 318381bac..ed8387eb9 100755 --- a/docker/entrypoint_foxx.sh +++ b/docker/entrypoint_foxx.sh @@ -2,28 +2,34 @@ set -euf -o pipefail -if [ -n "$UID" ]; then - usermod -u $UID datafed -fi +# NOTE We do not need to change the user in this container because we should not have +# any log output, and running chmod to update all the folders during runtime is +# expensive! +# +# The following lines are not needed +# if [ -n "$UID" ]; then +# usermod -u $UID datafed +# fi SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/../") -log_path="$DATAFED_DEFAULT_LOG_PATH" -if [ ! -d "${log_path}" ] -then - su -c "mkdir -p ${log_path}" datafed -fi - -cd "${PROJECT_ROOT}" -# Check to see if foxx has previously been installed install_flag="/tmp/.foxx_is_installed" if [ ! -f "$install_flag" ] then - su datafed -c "${PROJECT_ROOT}/scripts/generate_datafed.sh" - - su datafed -c "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build \ + log_path="$DATAFED_DEFAULT_LOG_PATH" + if [ ! -d "${log_path}" ] + then + su -c "mkdir -p ${log_path}" datafed + fi + + # It should be fine to run this as root because it is an ephemeral container anyway + cd "${PROJECT_ROOT}" + # Check to see if foxx has previously been installed + "${PROJECT_ROOT}/scripts/generate_datafed.sh" + + "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" -S. -B build \ -DBUILD_REPO_SERVER=False \ -DBUILD_COMMON=False \ -DBUILD_AUTHZ=False \ @@ -32,14 +38,15 @@ then -DBUILD_DOCS=False \ -DBUILD_PYTHON_CLIENT=False \ -DBUILD_FOXX=True \ - -DINSTALL_FOXX=True" + -DINSTALL_FOXX=True + - su datafed -c "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build" + "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" --build build # Give arango container a minute to initialize # should be replaced with health check at some point sleep 5 - su datafed -c "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target install" + "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" --build build --target install touch "$install_flag" fi From 1d0589ffbbefc94cc555da197b4921c09270df27 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:58:35 -0400 Subject: [PATCH 46/84] Remove chown command from web dockerfile because redundant with what is in the entrypoint file --- web/docker/Dockerfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/web/docker/Dockerfile b/web/docker/Dockerfile index 9ed5a7b8f..7e699672f 100644 --- a/web/docker/Dockerfile +++ b/web/docker/Dockerfile @@ -61,6 +61,7 @@ ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" ENV DATAFED_DEPENDENCIES_INSTALL_PATH="$DATAFED_DEPENDENCIES_INSTALL_PATH" ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" +ENV DATAFED_DEFAULT_LOG_PATH="$DATAFED_INSTALL_PATH/logs" RUN apt install -y python3 make g++ @@ -81,12 +82,7 @@ COPY --from=ws-build --chown=datafed:root ${BUILD_DIR}/web ${BUILD_DIR}/web COPY --from=ws-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/web ${DATAFED_INSTALL_PATH}/web COPY --from=ws-build --chown=datafed:root /usr/bin/curl /usr/bin/curl -RUN chown -R datafed:root ${DATAFED_INSTALL_PATH}/web -RUN chown -R datafed:root ${BUILD_DIR} - -WORKDIR ${DATAFED_DIR} - -RUN echo "${DATAFED_DIR}" +WORKDIR ${BUILD_DIR} USER root From 510cdc9e61876aea3395db321ed0ef89b5bd9d9e Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Mon, 8 Apr 2024 07:59:37 -0400 Subject: [PATCH 47/84] Add chown to entrypoint file for web because UID is changed --- web/docker/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 3ef52ae68..43f33b799 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -7,6 +7,8 @@ if [ -n "$UID" ]; then usermod -u "$UID" datafed fi +chown -R datafed:root ${DATAFED_INSTALL_PATH}/web +chown -R datafed:root ${BUILD_DIR} SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") From a11e663ae7f1094024dce3080c27cc03d6b772e8 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Mon, 8 Apr 2024 12:55:42 -0400 Subject: [PATCH 48/84] Make sure that the collection path is available to the datafed user --- repository/docker/entrypoint_authz.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index 5f33208a3..e72c1d0b3 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -24,7 +24,11 @@ export GLOBUS_CLIENT_ID=$(cat /opt/datafed/globus/client_cred.json | jq -r .cli export GLOBUS_CLIENT_SECRET=$(cat /opt/datafed/globus/client_cred.json | jq -r .secret) export DEPLOYMENT_KEY=$(cat "$DEPLOYMENT_KEY_PATH" ) -chown -R datafed:root ${DATAFED_GCS_COLLECTION_ROOT_PATH} +if [ -n "$UID" ]; then + echo "Switching datafed user to UID: ${UID}" + usermod -u $UID datafed + chown -R datafed:root ${DATAFED_GCS_COLLECTION_ROOT_PATH} +fi "${PROJECT_ROOT}/scripts/generate_datafed.sh" @@ -114,6 +118,7 @@ fi if [ ! -d "${DATAFED_GCS_COLLECTION_ROOT_PATH}" ] then mkdir -p "$DATAFED_GCS_COLLECTION_ROOT_PATH" + chown -R datafed:root ${DATAFED_GCS_COLLECTION_ROOT_PATH} fi "${BUILD_DIR}/scripts/globus/setup_globus.sh" @@ -127,6 +132,8 @@ GCS_CLI_CLIENT_SECRET="$GCS_CLI_CLIENT_SECRET" \ DATAFED_REPO_USER="$DATAFED_REPO_USER" \ python3 "${BUILD_DIR}/scripts/globus/create_guest_collection.py" +"${BUILD_DIR}/scripts/globus/generate_repo_form.sh" + echo "Container is running." sleep infinity From 85cdcc1ad0a181427925ab4f37df1108b01fd206 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Mon, 8 Apr 2024 12:56:23 -0400 Subject: [PATCH 49/84] Change permissions to datafed user in entrypoint file so that the repo server runs as the datafed user --- repository/docker/entrypoint_repo.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index 0ea57ddbb..51bd23c04 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -1,11 +1,6 @@ #!/bin/bash set -euf -o pipefail - -if [ -n "$UID" ]; then - usermod -u $UID datafed -fi - SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/../..") @@ -14,6 +9,16 @@ PROJECT_ROOT=$(realpath "${SOURCE}/../..") "${PROJECT_ROOT}/scripts/generate_repo_config.sh" "${PROJECT_ROOT}/scripts/install_repo.sh" +# This is only part of the solution the other part is running chown +if [ -n "$UID" ]; then + echo "Switching datafed user to UID: ${UID}" + usermod -u $UID datafed + chown -R datafed:root ${PROJECT_ROOT} + chown -R datafed:root /opt/datafed/repo/ + chown -R datafed:root /mnt/datafed +fi + + log_path="$DATAFED_DEFAULT_LOG_PATH" if [ ! -d "${log_path}" ] From a81ab531cab4ad74e19c035b2bfaa275d8ec14f9 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Mon, 8 Apr 2024 12:57:04 -0400 Subject: [PATCH 50/84] Add acl functions to globus_create collections script --- scripts/globus/create_guest_collection.py | 48 +++++++++++++++++++---- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index 59f6131fe..a7d6211c1 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -96,12 +96,6 @@ import globus_sdk from globus_sdk import scopes -# constants -#endpoint_hostname = "https://ecf8ed.08cc.data.globus.org" -#endpoint_id = "769c7ed0-744a-41b5-b4a8-db37b10b1ac9" -#mapped_collection_id = "580ecb92-de56-42ee-a5ec-3d3886767b94" -#storage_gateway_id = "3fdd7f41-4a05-4856-8fcd-2fb50066c590" - # client credentials # This client identity must have the needed permissions to create a guest # collection on the mapped collection, and a valid mapping to a local account @@ -119,6 +113,13 @@ confidential_client = globus_sdk.ConfidentialAppAuthClient( client_id=client_id, client_secret=client_secret ) + +token_response = confidential_client.oauth2_client_credentials_tokens() +globus_transfer_data = token_response.by_resource_server["transfer.api.globus.org"] +globus_transfer_token = globus_transfer_data["access_token"] +authorizer_tc = globus_sdk.AccessTokenAuthorizer(globus_transfer_token) +tc = globus_sdk.TransferClient(authorizer=authorizer_tc) + scope = scopes.GCSEndpointScopeBuilder(ENDPOINT_ID).make_mutable("manage_collections") authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, scopes=scope) client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) @@ -211,4 +212,37 @@ print(f"guest collection {guest_collection_id} created") -# For guest collection creation see +# Create ACL rule for Guest anonymous access +acl_list = tc.endpoint_acl_list(endpoint_id=guest_collection_id) +create_acl=True +update_acl=False +acl_id=None +for item in acl_list["DATA"]: + if item["principal_type"] == "all_authenticated_users": + create_acl = False + acl_id = item["id"] + if item["permissions"] != "rw": + print("Need to update acl for all users permissions are incorrect") + update_acl = True + else: + print("ACL rule already exists for all users") + break + +rule_data = { + "DATA_TYPE": "access", + "path": "/", + "permissions": "rw", + "principal": "", + "principal_type": "all_authenticated_users", + "role_id": None, + "role_type": None + } +if create_acl: + print(f"Creating acl rule for guest_collection {guest_collection_id}") + tc.add_endpoint_acl_rule(endpoint_id=guest_collection_id, rule_data=rule_data) +elif update_acl: + print(f"Updating acl rule ({acl_id}) for guest_collection {guest_collection_id}") + tc.update_endpoint_acl_rule(endpoint_id=guest_collection_id, rule_id=acl_id, rule_data=rule_data) + +acl_list = tc.endpoint_acl_list(endpoint_id=guest_collection_id) +print(acl_list) From 4d02a6cf8c12570a24015f20674a483465dc13fa Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Mon, 8 Apr 2024 12:57:34 -0400 Subject: [PATCH 51/84] Fix form that is created --- scripts/globus/generate_repo_form.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/globus/generate_repo_form.sh b/scripts/globus/generate_repo_form.sh index c57f8dfa9..e279a1cce 100755 --- a/scripts/globus/generate_repo_form.sh +++ b/scripts/globus/generate_repo_form.sh @@ -108,7 +108,7 @@ fi # Probably should grab this from the config file local_DATAFED_REPO_EGRESS_PORT="9000" -repo_domain_name=$(domainname -A | xargs) +repo_domain_name=$(domainname -A | awk '{print $1}') if [ -z "$repo_domain_name" ] then @@ -125,7 +125,7 @@ then echo "export DATAFED_REPO_SERVER_ADDRESS=\"tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT\"" >> ${OUTPUT_SCRIPT_NAME} echo "export DATAFED_REPO_PUBLIC_KEY=\"$public_key\"" >> ${OUTPUT_SCRIPT_NAME} echo "export DATAFED_REPO_ENDPOINT_UUID=\"$uuid_of_collection\"" >> ${OUTPUT_SCRIPT_NAME} - echo "export DATAFED_REPO_RELATIVE_PATH=\"\\$DATAFED_REPO_ID_AND_DIR\"" >> ${OUTPUT_SCRIPT_NAME} + echo "export DATAFED_REPO_RELATIVE_PATH=\"${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR\"" >> ${OUTPUT_SCRIPT_NAME} echo "export DATAFED_REPO_DOMAIN=\"\"" >> ${OUTPUT_SCRIPT_NAME} echo "export DATAFED_REPO_EXPORT_PATH=\"\"" >> ${OUTPUT_SCRIPT_NAME} echo "export DATAFED_REPO_CAPACITY=\"\"" >> ${OUTPUT_SCRIPT_NAME} @@ -141,7 +141,7 @@ then echo "address=\"tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT\"" >> ${OUTPUT_SCRIPT_NAME} echo "pub_key=\"$public_key\"" >> ${OUTPUT_SCRIPT_NAME} echo "endpoint=\"$uuid_of_collection\"" >> ${OUTPUT_SCRIPT_NAME} - echo "path=\"\\$DATAFED_REPO_ID_AND_DIR\"" >> ${OUTPUT_SCRIPT_NAME} + echo "path=\"${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR\"" >> ${OUTPUT_SCRIPT_NAME} echo "domain=\"\"" >> ${OUTPUT_SCRIPT_NAME} echo "exp_path=\"\"" >> ${OUTPUT_SCRIPT_NAME} echo "capacity=\"\"" >> ${OUTPUT_SCRIPT_NAME} @@ -159,7 +159,7 @@ then echo " \"address\": \"tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT\"," >> ${OUTPUT_SCRIPT_NAME} echo " \"pub_key\": \"$public_key\"," >> ${OUTPUT_SCRIPT_NAME} echo " \"endpoint\": \"$uuid_of_collection\"," >> ${OUTPUT_SCRIPT_NAME} - echo " \"path\": \"/$DATAFED_REPO_ID_AND_DIR\"," >> ${OUTPUT_SCRIPT_NAME} + echo " \"path\": \"${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR\"," >> ${OUTPUT_SCRIPT_NAME} echo " \"domain\": \"\"," >> ${OUTPUT_SCRIPT_NAME} echo " \"exp_path\": \"\"," >> ${OUTPUT_SCRIPT_NAME} echo " \"capacity\": 0," >> ${OUTPUT_SCRIPT_NAME} @@ -176,7 +176,7 @@ echo "Description: A description of the repository." echo "Srvr. Address: tcp://$repo_domain_name:$local_DATAFED_REPO_EGRESS_PORT" echo "Public Key: $public_key" echo "End-point ID: $uuid_of_collection" -echo "Path: /$DATAFED_REPO_ID_AND_DIR" +echo "Path: ${PATH_TO_GUEST_ROOT}/$DATAFED_REPO_ID_AND_DIR" echo "Domain: " # I don't know what this is echo "Export Path: " From 1116739aa4992464bb1ded491232525e4163d03f Mon Sep 17 00:00:00 2001 From: par-hermes Date: Mon, 8 Apr 2024 16:58:49 +0000 Subject: [PATCH 52/84] cpp-py-formatter --- common/source/servers/Proxy.cpp | 20 +- core/server/ClientWorker.cpp | 6 +- core/server/TaskWorker.cpp | 5 +- .../globus5/authz/source/AuthzWorker.cpp | 10 +- .../gridftp/globus5/authz/source/libauthz.c | 164 +++++++------ repository/server/RepoServer.cpp | 18 +- repository/server/RequestWorker.cpp | 10 +- scripts/globus/create_guest_collection.py | 160 +++++++------ scripts/globus/globus_cleanup.py | 144 +++++++----- scripts/globus/initialize_globus_endpoint.py | 109 +++++---- scripts/globus/utils.py | 220 ++++++++++-------- 11 files changed, 487 insertions(+), 379 deletions(-) diff --git a/common/source/servers/Proxy.cpp b/common/source/servers/Proxy.cpp index a61fc2da8..11894df0a 100644 --- a/common/source/servers/Proxy.cpp +++ b/common/source/servers/Proxy.cpp @@ -123,10 +123,12 @@ void Proxy::run() { // public if (resp_from_client_socket.error == false and resp_from_client_socket.time_out == false) { - if(not resp_from_client_socket.message ) { - DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " - << "response is not defined but no timeouts or errors were " - << "triggered, unable to send to server."); + if (not resp_from_client_socket.message) { + DL_ERROR( + m_log_context, + "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to send to server."); } else { m_communicators[SocketRole::SERVER]->send( *resp_from_client_socket.message); @@ -143,10 +145,12 @@ void Proxy::run() { // ... - Serv Sock - Proxy ------ Client Sock - Serv Sock - Inter App if (resp_from_server_socket.error == false and resp_from_server_socket.time_out == false) { - if(not resp_from_server_socket.message ) { - DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " - << "response is not defined but no timeouts or errors were " - << "triggered, unable to operate and send to client."); + if (not resp_from_server_socket.message) { + DL_ERROR( + m_log_context, + "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to operate and send to client."); } else { for (auto &in_operator : m_incoming_operators) { in_operator->execute(*resp_from_server_socket.message); diff --git a/core/server/ClientWorker.cpp b/core/server/ClientWorker.cpp index 40e57f00b..2c3084ee0 100644 --- a/core/server/ClientWorker.cpp +++ b/core/server/ClientWorker.cpp @@ -346,8 +346,10 @@ void ClientWorker::workerThread(LogContext log_context) { ICommunicator::Response response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); if (response.time_out == false and response.error == false) { - if ( not response.message ) { - DL_ERROR(message_log_context, "No timeout or error was reported but message is not defined.") + if (not response.message) { + DL_ERROR( + message_log_context, + "No timeout or error was reported but message is not defined.") } IMessage &message = *response.message; diff --git a/core/server/TaskWorker.cpp b/core/server/TaskWorker.cpp index 87dd0f1ad..95c55ead2 100644 --- a/core/server/TaskWorker.cpp +++ b/core/server/TaskWorker.cpp @@ -339,8 +339,9 @@ TaskWorker::cmdRawDataUpdateSize(TaskWorker &me, const Value &a_task_params, if (response.time_out == true) { return response; } else if (response.error == false) { - if( not response.message ) { - DL_ERROR(log_context, "No timeout or error was reported but no message was defined."); + if (not response.message) { + DL_ERROR(log_context, + "No timeout or error was reported but no message was defined."); } auto proto_msg = std::get(response.message->getPayload()); diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index 4bc7261bd..6a6eaa7e0 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -226,7 +226,8 @@ class AuthzWorker { << client->address()); auto response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); - if (response.message ) { // Make sure the message exists before we try to access it + if (response.message) { // Make sure the message exists before we try to + // access it log_context.correlation_id = std::get( response.message->get(MessageAttribute::CORRELATION_ID)); } @@ -252,9 +253,10 @@ class AuthzWorker { "communicating with the core service: " << response.error_msg); } else { - - if( not response.message ) { - DL_ERROR(log_context, "No error was reported and no time out occured but message is not defined."); + + if (not response.message) { + DL_ERROR(log_context, "No error was reported and no time out occured " + "but message is not defined."); } auto payload = diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index 2dd6b1b7b..fd03bb7b2 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -20,40 +20,63 @@ // Define logging macros #if defined(DONT_USE_SYSLOG) - FILE *log_file = NULL; - bool write_to_file = true; - #define AUTHZ_LOG_DEBUG(fmt, ...) \ - do { if (LOG_LEVEL >= 1) fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0); \ - do { if (LOG_LEVEL >= 1 && write_to_file ) fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INFO(fmt, ...) \ - do { if (LOG_LEVEL >= 2) fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); } while (0); \ - do { if (LOG_LEVEL >= 2 && write_to_file ) fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_ERROR(fmt, ...) \ - do { \ - if (LOG_LEVEL >= 3) fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ - if (LOG_LEVEL >= 3 && write_to_file ) fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ - } while(0) - #define AUTHZ_LOG_INIT(file_path) \ - log_file = fopen(file_path, "a"); \ - if (log_file != NULL) { write_to_file = true; } - #define AUTHZ_LOG_CLOSE() \ - if (log_file != NULL) { fclose(log_file); } +FILE *log_file = NULL; +bool write_to_file = true; +#define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 1) \ + fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); \ + } while (0); \ + do { \ + if (LOG_LEVEL >= 1 && write_to_file) \ + fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INFO(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 2) \ + fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); \ + } while (0); \ + do { \ + if (LOG_LEVEL >= 2 && write_to_file) \ + fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_ERROR(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 3) \ + fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ + if (LOG_LEVEL >= 3 && write_to_file) \ + fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INIT(file_path) \ + log_file = fopen(file_path, "a"); \ + if (log_file != NULL) { \ + write_to_file = true; \ + } +#define AUTHZ_LOG_CLOSE() \ + if (log_file != NULL) { \ + fclose(log_file); \ + } #else - #include - #define AUTHZ_LOG_DEBUG(fmt, ...) \ - do { if (LOG_LEVEL >= 1) syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INFO(fmt, ...) \ - do { if (LOG_LEVEL >= 2) syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_ERROR(fmt, ...) \ - do { if (LOG_LEVEL >= 3) syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INIT(file_path) \ - openlog("gsi_authz", 0, LOG_AUTH); - #define AUTHZ_LOG_CLOSE() \ - closelog(); +#include +#define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 1) \ + syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INFO(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 2) \ + syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_ERROR(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 3) \ + syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INIT(file_path) openlog("gsi_authz", 0, LOG_AUTH); +#define AUTHZ_LOG_CLOSE() closelog(); #endif - - typedef void *globus_gsi_authz_handle_t; typedef void (*globus_gsi_authz_cb_t)(void *callback_arg, globus_gsi_authz_handle_t handle, @@ -189,13 +212,14 @@ bool setConfigVal(const char *a_label, char *a_dest, char *a_src, size_t len = strlen(a_src); if (len == 0) { - AUTHZ_LOG_ERROR("DataFed - '%s' value not set.\n", a_label); + AUTHZ_LOG_ERROR("DataFed - '%s' value not set.\n", a_label); return true; } if (len > a_max_len) { - AUTHZ_LOG_ERROR("DataFed - '%s' value too long in authz config file (max %lu).\n", - a_label, a_max_len); + AUTHZ_LOG_ERROR( + "DataFed - '%s' value too long in authz config file (max %lu).\n", + a_label, a_max_len); return true; } @@ -269,7 +293,7 @@ bool loadConfig() { val = strchr(buf, '='); if (!val) { AUTHZ_LOG_ERROR( - "DataFed - Syntax error in authz config file at line %i.\n", lc); + "DataFed - Syntax error in authz config file at line %i.\n", lc); return true; } else { *val = 0; @@ -305,8 +329,8 @@ bool loadConfig() { else { err = true; AUTHZ_LOG_ERROR( - "DataFed - Invalid key, '%s', in authz config file at line %i.\n", - buf, lc); + "DataFed - Invalid key, '%s', in authz config file at line %i.\n", + buf, lc); } if (err) { @@ -336,20 +360,23 @@ bool loadConfig() { strcat(miss, " server_key"); AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); - AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", + getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); if (miss[0] != 0) { - AUTHZ_LOG_ERROR("DataFed - Missing required authz config items:%s\n", miss); + AUTHZ_LOG_ERROR("DataFed - Missing required authz config items:%s\n", + miss); return true; } } else { AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); - AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", + getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); AUTHZ_LOG_ERROR("DataFed - Could not open authz config file.\n"); return true; @@ -358,14 +385,14 @@ bool loadConfig() { AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); return false; } // The same globus_result_t gsi_authz_init() { - //openlog("gsi_authz", 0, LOG_AUTH); + // openlog("gsi_authz", 0, LOG_AUTH); memset(g_active_contexts, 0, sizeof(g_active_contexts)); // This line is different @@ -402,7 +429,8 @@ globus_result_t gsi_authz_handle_init(va_list ap) { else AUTHZ_LOG_ERROR("gsi_authz_handle_init out of handle context space\n"); } else { - AUTHZ_LOG_DEBUG("gsi_authz_handle_init context handle already initialized\n"); + AUTHZ_LOG_DEBUG( + "gsi_authz_handle_init context handle already initialized\n"); } callback(callback_arg, callback_arg, result); @@ -449,21 +477,21 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { char *callout_id_mapped1 = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", - callout_ids1); + callout_ids1); AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", - callout_id_mapped1); + callout_id_mapped1); AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", - g_config.globus_collection_path, action, object); + g_config.globus_collection_path, action, object); if (strcmp(action, "lookup") == 0 || strcmp(action, "chdir") == 0) { AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", - g_config.globus_collection_path, action, object); + g_config.globus_collection_path, action, object); result = GLOBUS_SUCCESS; callback(callback_arg, handle, result); return result; } AUTHZ_LOG_ERROR("gsi_authz_authorize_async, handle: %p, act: %s, obj: %s\n", - handle, action, object); + handle, action, object); OM_uint32 min_stat; gss_name_t client = GSS_C_NO_NAME; @@ -486,7 +514,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { gss_display_name(&min_stat, target, &target_buf, &target_type); if (maj_stat == GSS_S_COMPLETE) { AUTHZ_LOG_INFO("Auth client: %s, file: %s, action: %s\n", - (char *)client_buf.value, object, action); + (char *)client_buf.value, object, action); // Testing hack #if 0 @@ -503,7 +531,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { if (strncmp((char *)client_buf.value, "/C=US/O=Globus Consortium/OU=Globus", 35) != 0) { AUTHZ_LOG_ERROR("Invalid certificate subject prefix: %s\n", - (char *)client_buf.value); + (char *)client_buf.value); } else { /* Note: For some reason, globus will provide the CN as either a * UUID that is linked to the client's account and encoded in @@ -523,7 +551,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { if (!decodeUUID((char *)client_buf.value + 54, client_id)) { AUTHZ_LOG_ERROR("Failed to decode subject UUID: %s\n", - (char *)client_buf.value + 54); + (char *)client_buf.value + 54); free(client_id); client_id = 0; } @@ -539,23 +567,25 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { } char *callout_ids = getenv("GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS"); - char *callout_username_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); - char *callout_id_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); + char *callout_username_mapped = + getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); + char *callout_id_mapped = + getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); if (callout_ids != NULL) { AUTHZ_LOG_DEBUG( - "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", - callout_ids); + "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", + callout_ids); client_id = strdup(callout_ids); AUTHZ_LOG_INFO("libauthz.c client_id(s): %s\n", client_id); - } else if ( callout_id_mapped != NULL ) { + } else if (callout_id_mapped != NULL) { AUTHZ_LOG_DEBUG( - "libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", - callout_id_mapped); + "libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", + callout_id_mapped); client_id = strdup(callout_id_mapped); } else { AUTHZ_LOG_ERROR( - "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS.\n"); + "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS.\n"); } } @@ -566,8 +596,8 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { } else { AUTHZ_LOG_INFO( - "libauthz.c Auth client_id: %s, file: %s, action: %s\n", - client_id, object, action); + "libauthz.c Auth client_id: %s, file: %s, action: %s\n", + client_id, object, action); AUTHZ_LOG_INFO("libauthz.c checkAuthorization FAIL.\n"); } @@ -578,17 +608,17 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { gss_release_buffer(&min_stat, &target_buf); } else { AUTHZ_LOG_ERROR("gss_display_name target FAILED, maj: %d, min: %d\n", - maj_stat, min_stat); + maj_stat, min_stat); } gss_release_buffer(&min_stat, &client_buf); } else { AUTHZ_LOG_ERROR("gss_display_name source FAILED, maj: %d, min: %d\n", - maj_stat, min_stat); + maj_stat, min_stat); } } else { - AUTHZ_LOG_ERROR("gss_inquire_context FAILED, maj: %d, min: %d\n", maj_stat, - min_stat); + AUTHZ_LOG_ERROR("gss_inquire_context FAILED, maj: %d, min: %d\n", + maj_stat, min_stat); } } else { AUTHZ_LOG_ERROR("context handle lookup FAILED\n"); diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index d851eb0b7..e3051e3cc 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -101,7 +101,8 @@ void Server::run() { } void Server::checkServerVersion() { - DL_INFO(m_log_context, "Checking core server connection and version at " << m_config.core_server); + DL_INFO(m_log_context, "Checking core server connection and version at " + << m_config.core_server); // Generate random security keys for anon version request to core server KeyGenerator generator; @@ -148,7 +149,8 @@ void Server::checkServerVersion() { for (int i = 0; i < 10; i++) { DL_INFO(m_log_context, "Attempt " << i << " to initialize communication " - << " with core server at " << m_config.core_server); + << " with core server at " + << m_config.core_server); auto msg = std::make_unique(); auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); message->setPayload(std::move(msg)); @@ -166,14 +168,14 @@ void Server::checkServerVersion() { DL_ERROR(msg_log_context, "Timeout waiting for response from core server: " << m_config.core_server); - } else if(response.error) { - DL_ERROR(msg_log_context, - "Error encountered waiting for core server: " - << m_config.core_server << " msg " << response.error_msg); + } else if (response.error) { + DL_ERROR(msg_log_context, "Error encountered waiting for core server: " + << m_config.core_server << " msg " + << response.error_msg); } else { - + msg_log_context.correlation_id = std::get( - response.message->get(MessageAttribute::CORRELATION_ID)); + response.message->get(MessageAttribute::CORRELATION_ID)); auto payload = std::get(response.message->getPayload()); VersionReply *ver_reply = dynamic_cast(payload); diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index be08293fd..43b0652b5 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -146,9 +146,9 @@ void RequestWorker::workerThread(LogContext log_context) { DL_TRACE(message_log_context, "Checking timeouts: " << response.time_out); if (response.time_out == false and response.error == false) { - if( not response.message) { + if (not response.message) { DL_ERROR(log_context, "Error: No error or timeout occurred but the" - << " message does not exist."); + << " message does not exist."); } else { // May not have a correlation id if the message timed out DL_TRACE(log_context, "Getting correlation_id."); @@ -165,18 +165,18 @@ void RequestWorker::workerThread(LogContext log_context) { if (m_msg_handlers.count(msg_type)) { map::iterator handler = - m_msg_handlers.find(msg_type); + m_msg_handlers.find(msg_type); DL_TRACE(message_log_context, "Calling handler"); auto send_message = - (this->*handler->second)(std::move(response.message)); + (this->*handler->second)(std::move(response.message)); client->send(*(send_message)); DL_TRACE(message_log_context, "Reply sent."); } else { DL_ERROR(message_log_context, - "Received unregistered msg type: " << msg_type); + "Received unregistered msg type: " << msg_type); } } } else if (response.error) { diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index a7d6211c1..5a82814bb 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess import utils @@ -10,61 +9,76 @@ # The Globus project the GCS endpoint will be created in DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME", "DataFed Repo") -PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME", DATAFED_GCS_ROOT_NAME + " Project") +PROJECT_NAME = os.getenv( + "DATAFED_GLOBUS_PROJECT_NAME", DATAFED_GCS_ROOT_NAME + " Project" +) # This is for confidential client -CLIENT_NAME = os.getenv("DATAFED_GLOBUS_CLIENT_NAME", DATAFED_GCS_ROOT_NAME + " Setup Client") +CLIENT_NAME = os.getenv( + "DATAFED_GLOBUS_CLIENT_NAME", DATAFED_GCS_ROOT_NAME + " Setup Client" +) # Name of the client secret used by the confidential client -CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME",DATAFED_GCS_ROOT_NAME + " Cred") +CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME", DATAFED_GCS_ROOT_NAME + " Cred") # Name of the file where we will store confidential client credentials -CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH","./client_cred.json") +CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH", "./client_cred.json") ENDPOINT_ID = os.getenv("GCS_CLI_ENDPOINT_ID") -ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME",DATAFED_GCS_ROOT_NAME + " Endpoint") +ENDPOINT_NAME = os.getenv( + "DATAFED_GLOBUS_ENDPOINT_NAME", DATAFED_GCS_ROOT_NAME + " Endpoint" +) # Path to deployment key -DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH","./deployment-key.json") +DEPLOYMENT_KEY_PATH = os.getenv( + "DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH", "./deployment-key.json" +) # Path to deployment key -DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT", "443") -DATAFED_GCS_URL=os.getenv("DATAFED_GCS_URL") +DATAFED_GLOBUS_CONTROL_PORT = os.getenv("DATAFED_GLOBUS_CONTROL_PORT", "443") +DATAFED_GCS_URL = os.getenv("DATAFED_GCS_URL") client_id = os.getenv("GCS_CLI_CLIENT_ID") client_secret = os.getenv("GCS_CLI_CLIENT_SECRET") mapped_collection_id = os.getenv("MAPPED_COLLECTION_ID") -mapped_collection_name = os.getenv("DATAFED_GCS_COLLECTION_MAPPED", f"{DATAFED_GCS_ROOT_NAME} Collection Mapped") -guest_collection_name = os.getenv("DATAFED_GCS_COLLECTION_GUEST",f"{DATAFED_GCS_ROOT_NAME} Collection Guest") +mapped_collection_name = os.getenv( + "DATAFED_GCS_COLLECTION_MAPPED", f"{DATAFED_GCS_ROOT_NAME} Collection Mapped" +) +guest_collection_name = os.getenv( + "DATAFED_GCS_COLLECTION_GUEST", f"{DATAFED_GCS_ROOT_NAME} Collection Guest" +) storage_gateway_id = os.getenv("STORAGE_GATEWAY_ID") -storage_gateway_name = os.getenv("DATAFED_GCS_STORAGE_GATEWAY",f"{DATAFED_GCS_ROOT_NAME} Storage Gateway") +storage_gateway_name = os.getenv( + "DATAFED_GCS_STORAGE_GATEWAY", f"{DATAFED_GCS_ROOT_NAME} Storage Gateway" +) local_username = os.getenv("DATAFED_REPO_USER") if ENDPOINT_ID is None: raise Exception("GCS_CLI_ENDPOINT_ID must be defined as an env varaible") if DATAFED_GCS_URL is None: - raise Exception("Unable to create guest collection, DATAFED_GCS_URL is not" - " defined.") + raise Exception( + "Unable to create guest collection, DATAFED_GCS_URL is not" " defined." + ) if local_username is None: raise Exception("DATAFED_REPO_USER is not defined.") -#client = globus_sdk.NativeAppAuthClient(CLIENT_ID) +# client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project # view_identities to user information for creating GCS server -#client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +# client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) # -#authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) -#print("Please go to this URL and login: \n", authorize_url) -#auth_code = input("Please enter the authorization code: ") +# authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) +# print("Please go to this URL and login: \n", authorize_url) +# auth_code = input("Please enter the authorization code: ") # -#token_response = client.oauth2_exchange_code_for_tokens(auth_code) +# token_response = client.oauth2_exchange_code_for_tokens(auth_code) ## Extract the token -#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) ## auth_client_refresh_token -#ac_rt = AuthClient(authorizer=rt_authorizer) +# ac_rt = AuthClient(authorizer=rt_authorizer) # -#userinfo = ac_rt.oauth2_userinfo() +# userinfo = ac_rt.oauth2_userinfo() ## Will get the primary email and id -#identity_id = userinfo["sub"] -#email = userinfo["email"] -#username = userinfo["preferred_username"] -#organization = userinfo["identity_provider_display_name"] +# identity_id = userinfo["sub"] +# email = userinfo["email"] +# username = userinfo["preferred_username"] +# organization = userinfo["identity_provider_display_name"] if client_id is None: client_id = getClientIdFromCredFile(CRED_FILE_PATH) @@ -74,21 +88,21 @@ client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) -scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" +scopes = "openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) -#cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, +# cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, # scopes) -#token_response = client.oauth2_client_credentials_tokens() +# token_response = client.oauth2_client_credentials_tokens() -#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) # the useful values that you want at the end of this -#globus_auth_data = token_response.by_resource_server["auth.globus.org"] -#globus_transfer_data = -#token_response.by_resource_server["transfer.api.globus.org"] -#globus_auth_token = globus_auth_data["access_token"] -#globus_transfer_token = globus_transfer_data["access_token"] +# globus_auth_data = token_response.by_resource_server["auth.globus.org"] +# globus_transfer_data = +# token_response.by_resource_server["transfer.api.globus.org"] +# globus_auth_token = globus_auth_data["access_token"] +# globus_transfer_token = globus_transfer_data["access_token"] gcs_client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) @@ -102,9 +116,9 @@ # on the storage gateway that matches the local_username # If using user tokens, the user must be the one with the correct permissions # and identity mapping. -#client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" -#client_secret = "*redacted*" -#local_username = "datafed" +# client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" +# client_secret = "*redacted*" +# local_username = "datafed" # The scope the client will need, note that primary scope is for the endpoint, # but it has a dependency on the mapped collection's data_access scope @@ -136,13 +150,15 @@ if item["id"] == mapped_collection_id: mapped_collection_found = True if item["display_name"] != mapped_collection_name: - raise Exception("Expected display name is different from what " - "is expected for mapped collection " - f"{mapped_collection_id}, if using non standard" - " display name for mapped collection " - f"{mapped_collection_name} then the " - "MAPPED_COLLECTION_NAME env variable must be " - "set.") + raise Exception( + "Expected display name is different from what " + "is expected for mapped collection " + f"{mapped_collection_id}, if using non standard" + " display name for mapped collection " + f"{mapped_collection_name} then the " + "MAPPED_COLLECTION_NAME env variable must be " + "set." + ) break elif item["display_name"] == mapped_collection_name: mapped_collection_found = True @@ -160,19 +176,21 @@ if item["id"] == storage_gateway_id: storage_gateway_found = True if item["display_name"] != storage_gateway_name: - raise Exception("Expected display name is different from what " - "is expected for storage gateway " - f"{storage_gateway_id}, if using non standard" - " display name for storage gateway " - f"{storage_gateway_name} then the " - "DATAFED_GCS_STORAGE_GATEWAY env variable must be " - "set.") + raise Exception( + "Expected display name is different from what " + "is expected for storage gateway " + f"{storage_gateway_id}, if using non standard" + " display name for storage gateway " + f"{storage_gateway_name} then the " + "DATAFED_GCS_STORAGE_GATEWAY env variable must be " + "set." + ) break elif item["display_name"] == storage_gateway_name: storage_gateway_found = True storage_gateway_id = item["id"] break - + if storage_gateway_found == False: raise Exception("Missing required storage gateway") @@ -190,7 +208,7 @@ guest_collection_found = True guest_collection_id = item["id"] break - + # https://github.com/globus/globus-sdk-python/blob/main/docs/examples/guest_collection_creation.rst if guest_collection_found == False: credential_document = globus_sdk.UserCredentialDocument( @@ -200,7 +218,7 @@ ) client.create_user_credential(credential_document) -# Create the collection + # Create the collection collection_document = globus_sdk.GuestCollectionDocument( public="True", collection_base_path="/", @@ -214,9 +232,9 @@ # Create ACL rule for Guest anonymous access acl_list = tc.endpoint_acl_list(endpoint_id=guest_collection_id) -create_acl=True -update_acl=False -acl_id=None +create_acl = True +update_acl = False +acl_id = None for item in acl_list["DATA"]: if item["principal_type"] == "all_authenticated_users": create_acl = False @@ -229,20 +247,22 @@ break rule_data = { - "DATA_TYPE": "access", - "path": "/", - "permissions": "rw", - "principal": "", - "principal_type": "all_authenticated_users", - "role_id": None, - "role_type": None - } + "DATA_TYPE": "access", + "path": "/", + "permissions": "rw", + "principal": "", + "principal_type": "all_authenticated_users", + "role_id": None, + "role_type": None, +} if create_acl: print(f"Creating acl rule for guest_collection {guest_collection_id}") tc.add_endpoint_acl_rule(endpoint_id=guest_collection_id, rule_data=rule_data) elif update_acl: print(f"Updating acl rule ({acl_id}) for guest_collection {guest_collection_id}") - tc.update_endpoint_acl_rule(endpoint_id=guest_collection_id, rule_id=acl_id, rule_data=rule_data) + tc.update_endpoint_acl_rule( + endpoint_id=guest_collection_id, rule_id=acl_id, rule_data=rule_data + ) acl_list = tc.endpoint_acl_list(endpoint_id=guest_collection_id) print(acl_list) diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 4f0cba90f..502c57733 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,6 +1,5 @@ - import globus_sdk -from globus_sdk import AuthClient,GroupsClient, AccessTokenAuthorizer +from globus_sdk import AuthClient, GroupsClient, AccessTokenAuthorizer from globus_sdk.scopes import GroupsScopes import subprocess import json @@ -9,18 +8,18 @@ import utils # Define your client ID and client secret -CLIENT_ID = 'f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8' # NATIVE +CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" # NATIVE # The Globus project the GCS endpoint will be created in if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") else: - DATAFED_GCS_ROOT_NAME="DataFed Repo" + DATAFED_GCS_ROOT_NAME = "DataFed Repo" if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: - PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") + PROJECT_NAME = os.getenv("DATAFED_GLOBUS_PROJECT_NAME") else: - PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" + PROJECT_NAME = DATAFED_GCS_ROOT_NAME + " Project" # This is for confidential client if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: @@ -30,37 +29,38 @@ # Name of the client secret used by the confidential client if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: - CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") + CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME") else: - CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" + CRED_NAME = DATAFED_GCS_ROOT_NAME + " Cred" # Name of the file where we will store confidential client credentials if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: - CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") + CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") else: - CRED_FILE_PATH="./client_cred.json" + CRED_FILE_PATH = "./client_cred.json" # Name to give to endpoint if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") else: - ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" + ENDPOINT_NAME = DATAFED_GCS_ROOT_NAME + " Endpoint" # Path to deployment key if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: - DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") + DEPLOYMENT_KEY_PATH = os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") else: - DEPLOYMENT_KEY_PATH="./deployment-key.json" + DEPLOYMENT_KEY_PATH = "./deployment-key.json" client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project group_scope = GroupsScopes.make_mutable("all") -client.oauth2_start_flow(requested_scopes="openid profile email " - "urn:globus:auth:scope:auth.globus.org:manage_projects " - "urn:globus:auth:scope:auth.globus.org:view_identities " + - str(group_scope), - refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + str(group_scope), + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -72,13 +72,16 @@ print(token_response) -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] +refresh_token_groups = token_response.by_resource_server["groups.api.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, - client) +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, client) ac_rt = AuthClient(authorizer=rt_authorizer) gr_rt = GroupsClient(authorizer=rt_authorizer_groups) @@ -101,68 +104,84 @@ clients_in_project = utils.getClientsInProject(ac_rt, project_id) if len(clients_in_project) == 0: - print("No clients were detected in the project we can just delete the" - "project and be done.") + print( + "No clients were detected in the project we can just delete the" + "project and be done." + ) else: -# Check if the deployment key exists if it does read it and verify that the -# client exists for the globus connect server if it does not then we will -# call the setup command + # Check if the deployment key exists if it does read it and verify that the + # client exists for the globus connect server if it does not then we will + # call the setup command - gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile(DEPLOYMENT_KEY_PATH) + gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile( + DEPLOYMENT_KEY_PATH + ) - valid_key = utils.isGCSDeploymentKeyValid(ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key) + valid_key = utils.isGCSDeploymentKeyValid( + ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key + ) all_gcs_client_ids = utils.getAllGCSClientIds(ac_rt, project_id, ENDPOINT_NAME) if valid_key is False and len(all_gcs_client_ids) > 0: - print("Looks like gcs client does not exist in the cloud" - f" for the project: {project_id}." - "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like gcs client does not exist in the cloud" + f" for the project: {project_id}." + "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if gcs_id_from_deployment_key is None and len(all_gcs_client_ids) > 0: - print("Looks like deployment key does not exist, please either " - "add the correct deployment." - f" cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like deployment key does not exist, please either " + "add the correct deployment." + f" cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if len(all_gcs_client_ids) > 0: if utils.command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - print("Now that we know a GCS instance exists we have to make sure" - "we have valid credentials to run the globus-connect-server command" - "non interatively, this means we have to create credentials and a" - "client if they don't exist and when we are done with everything" - "delete them.") + print( + "Now that we know a GCS instance exists we have to make sure" + "we have valid credentials to run the globus-connect-server command" + "non interatively, this means we have to create credentials and a" + "client if they don't exist and when we are done with everything" + "delete them." + ) client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) - + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH + ) - ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) + ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+="globus-connect-server endpoint cleanup " - bash_command+=f" --deployment-key \"{DEPLOYMENT_KEY_PATH}\" " - bash_command+=" --agree-to-delete-endpoint" + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += "globus-connect-server endpoint cleanup " + bash_command += f' --deployment-key "{DEPLOYMENT_KEY_PATH}" ' + bash_command += " --agree-to-delete-endpoint" print("Bash command to run") print(bash_command) - - proc = subprocess.Popen(bash_command, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, shell=True, text=True) + + proc = subprocess.Popen( + bash_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True, + text=True, + ) output, error = proc.communicate(input="yes\n") @@ -170,17 +189,15 @@ print(output) print(error) - # Now we can try to delete the remaining clients that are in the project # Get all of the clients that are not gcs clients and delete them utils.deleteAllNonGCSClients(ac_rt, project_id) - -# CLOSE - if len(clients_in_project) == 0: + # CLOSE - if len(clients_in_project) == 0: -# Try to remove project this will only work if there are no other clients in -# the project + # Try to remove project this will only work if there are no other clients in + # the project print(f"Attempting to remove project {project_id}") project_remove = ac_rt.delete_project(project_id) print(project_remove) @@ -189,4 +206,3 @@ group_display_name = f"{DATAFED_GCS_ROOT_NAME} Group" utils.deleteGroup(gr_rt, group_display_name) - diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index d2985b916..0f0e79334 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess import utils @@ -11,18 +10,18 @@ # Hard coded Native Client ID -CLIENT_ID = 'f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8' +CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" # The Globus project the GCS endpoint will be created in if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") else: - DATAFED_GCS_ROOT_NAME="DataFed Repo" + DATAFED_GCS_ROOT_NAME = "DataFed Repo" if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: - PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") + PROJECT_NAME = os.getenv("DATAFED_GLOBUS_PROJECT_NAME") else: - PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" + PROJECT_NAME = DATAFED_GCS_ROOT_NAME + " Project" # This is for confidential client if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: @@ -32,38 +31,38 @@ # Name of the client secret used by the confidential client if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: - CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") + CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME") else: - CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" + CRED_NAME = DATAFED_GCS_ROOT_NAME + " Cred" # Name of the file where we will store confidential client credentials if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: - CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") + CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") else: - CRED_FILE_PATH="./client_cred.json" + CRED_FILE_PATH = "./client_cred.json" # Name to give to endpoint if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") else: - ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" + ENDPOINT_NAME = DATAFED_GCS_ROOT_NAME + " Endpoint" # Path to deployment key if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: - DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") + DEPLOYMENT_KEY_PATH = os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") else: - DEPLOYMENT_KEY_PATH="./deployment-key.json" + DEPLOYMENT_KEY_PATH = "./deployment-key.json" # Path to deployment key if os.getenv("DATAFED_GLOBUS_CONTROL_PORT") is not None: - DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT") + DATAFED_GLOBUS_CONTROL_PORT = os.getenv("DATAFED_GLOBUS_CONTROL_PORT") else: - DATAFED_GLOBUS_CONTROL_PORT="443" + DATAFED_GLOBUS_CONTROL_PORT = "443" if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: - DATAFED_GLOBUS_SUBSCRIPTION=os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") + DATAFED_GLOBUS_SUBSCRIPTION = os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") else: - DATAFED_GLOBUS_SUBSCRIPTION=None + DATAFED_GLOBUS_SUBSCRIPTION = None client = globus_sdk.NativeAppAuthClient(CLIENT_ID) @@ -71,11 +70,12 @@ # view_identities to user information for creating GCS server group_scope = GroupsScopes.make_mutable("all") -client.oauth2_start_flow(requested_scopes="openid profile email " - "urn:globus:auth:scope:auth.globus.org:manage_projects " - "urn:globus:auth:scope:auth.globus.org:view_identities " + - str(group_scope), - refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + str(group_scope), + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -83,12 +83,15 @@ token_response = client.oauth2_exchange_code_for_tokens(auth_code) # Extract the token -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] +refresh_token_groups = token_response.by_resource_server["groups.api.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, - client) +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, client) # auth_client_refresh_token ac_rt = AuthClient(authorizer=rt_authorizer) gr_rt = GroupsClient(authorizer=rt_authorizer_groups) @@ -113,20 +116,19 @@ count = utils.countProjects(ac_rt, PROJECT_NAME) if count != 1: - print("Something is wrong there should be at least one project with name" - f" {PROJECT_NAME} instead there are {count} with that name") + print( + "Something is wrong there should be at least one project with name" + f" {PROJECT_NAME} instead there are {count} with that name" + ) sys.exit(1) print(f"Project id is {project_id}") client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH +) # Add the globus client as an admin to the project -ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) +ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) # Get clients in project clients_in_project = utils.getClientsInProject(ac_rt, project_id) @@ -135,20 +137,21 @@ # client exists for the globus connect server if it does not then we will # call the setup command utils.createGCSEndpoint( - ac_rt, - client_id, - client_secret, - project_id, - DEPLOYMENT_KEY_PATH, - ENDPOINT_NAME, - DATAFED_GLOBUS_CONTROL_PORT, - userinfo) + ac_rt, + client_id, + client_secret, + project_id, + DEPLOYMENT_KEY_PATH, + ENDPOINT_NAME, + DATAFED_GLOBUS_CONTROL_PORT, + userinfo, +) if DATAFED_GLOBUS_SUBSCRIPTION is not None: -# Create subscription subgroup + # Create subscription subgroup results = gr_rt.get_group_by_subscription_id(DATAFED_GLOBUS_SUBSCRIPTION) - parent_group_id=results["group_id"] + parent_group_id = results["group_id"] print("Groups by sub") print(results) group_name = f"{DATAFED_GCS_ROOT_NAME} Group" @@ -159,16 +162,15 @@ else: print(f"Group does not exist {group_name}") package = { - "name": group_name, - "description": "DataFed Repository Subscription Group, used for" - "granting access to the application client to setup the repository in " - "Globus", - "parent_id": str(parent_group_id) - } - + "name": group_name, + "description": "DataFed Repository Subscription Group, used for" + "granting access to the application client to setup the repository in " + "Globus", + "parent_id": str(parent_group_id), + } result = gr_rt.create_group(package) - group_id = result['id'] + group_id = result["id"] print("group id") print(group_id) @@ -179,10 +181,7 @@ print("membership_action") print(result) - package = { - "subscription_id": DATAFED_GLOBUS_SUBSCRIPTION - } + package = {"subscription_id": DATAFED_GLOBUS_SUBSCRIPTION} result = gr_rt.update_group(group_id, package) print("update group") print(result) - diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 64bc381f5..d4588b51c 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess from globus_sdk import AuthClient, AccessTokenAuthorizer @@ -6,12 +5,14 @@ import os import sys + def getProjectId(projects, project_name): for project in projects: - if project['display_name'] == project_name: - return project['id'] + if project["display_name"] == project_name: + return project["id"] return None + def projectExists(auth_client, project_name): projects = auth_client.get_projects() project_id = getProjectId(projects, project_name) @@ -21,6 +22,7 @@ def projectExists(auth_client, project_name): project_exists = False return project_exists + def createProject(auth_client, project_name, userinfo): identity_id = userinfo["sub"] @@ -30,35 +32,40 @@ def createProject(auth_client, project_name, userinfo): if project_exists is False: project_create_result = auth_client.create_project( - project_name, - contact_email=email, - admin_ids=[identity_id]) - return project_create_result['project']['id'] + project_name, contact_email=email, admin_ids=[identity_id] + ) + return project_create_result["project"]["id"] projects = auth_client.get_projects() return getProjectId(projects, project_name) + def countProjects(auth_client, project_name): projects = auth_client.get_projects() count = 0 for project in projects: - if project['display_name'] == project_name: + if project["display_name"] == project_name: count += 1 return count + def getClientId(auth_client, client_name, project_id): get_client_result = auth_client.get_clients() - for client in get_client_result['clients']: - if client['name'] == client_name and client['project'] == project_id: - return client['id'] + for client in get_client_result["clients"]: + if client["name"] == client_name and client["project"] == project_id: + return client["id"] return None + def getAllGCSClientIds(auth_client, project_id, endpoint_name): clients_in_project = getClientsInProject(auth_client, project_id) all_gcs_client_ids = [] for client in clients_in_project: - if client['client_type'] == "globus_connect_server" and client['name'] == endpoint_name: - all_gcs_client_ids.append(client['id']) + if ( + client["client_type"] == "globus_connect_server" + and client["name"] == endpoint_name + ): + all_gcs_client_ids.append(client["id"]) return all_gcs_client_ids @@ -66,8 +73,8 @@ def getClientsInProject(auth_client, project_id): # Get clients in project get_client_result = auth_client.get_clients() clients_in_project = [] - for client in get_client_result['clients']: - if client['project'] == project_id: + for client in get_client_result["clients"]: + if client["project"] == project_id: clients_in_project.append(client) return clients_in_project @@ -80,18 +87,22 @@ def createNewClient(auth_client, client_name, project_id): client_exists = True if client_exists is False: - result = auth_client.create_client(client_name, project=project_id, public_client=False) + result = auth_client.create_client( + client_name, project=project_id, public_client=False + ) client_id = result["client"]["id"] return client_id + def getCredentialID(auth_client, client_id, cred_name): get_client_cred_result = auth_client.get_client_credentials(client_id) - for cred in get_client_cred_result['credentials']: - if cred['name'] == cred_name: - return cred['id'] + for cred in get_client_cred_result["credentials"]: + if cred["name"] == cred_name: + return cred["id"] return None + def groupExists(client, group_name): my_groups = client.get_my_groups() print("My groups") @@ -101,6 +112,7 @@ def groupExists(client, group_name): return True return False + def getGroupId(client, group_name): my_groups = client.get_my_groups() for group in my_groups: @@ -109,7 +121,6 @@ def getGroupId(client, group_name): return None - def deleteGroup(client, group_name): my_groups = client.get_my_groups() for group in my_groups: @@ -136,59 +147,57 @@ def getCredentialFromFile(cred_file_name, cred_id): # name cred_exists_locally, cred_empty = validFile(cred_file_name) if cred_empty is False: - with open(cred_file_name, 'r') as f: + with open(cred_file_name, "r") as f: loaded_data = json.load(f) - if loaded_data['client'] == cred_id: - return loaded_data['secret'] + if loaded_data["client"] == cred_id: + return loaded_data["secret"] return None + def getClientIdFromCredFile(cred_file_name): # Check to see if the local secret is the same id and not just the same # name cred_exists_locally, cred_empty = validFile(cred_file_name) if cred_empty is False: - with open(cred_file_name, 'r') as f: + with open(cred_file_name, "r") as f: loaded_data = json.load(f) - return loaded_data['client'] + return loaded_data["client"] return None + def getEndpointIdFromFile(deployment_key_file_path): # Check to see if the local secret is the same id and not just the same # name exists_locally, empty = validFile(deployment_key_file_path) if empty is False: - with open(deployment_key_file_path, 'r') as f: + with open(deployment_key_file_path, "r") as f: loaded_data = json.load(f) - return loaded_data['client_id'] + return loaded_data["client_id"] return None def createNewCredential(auth_client, client_id, cred_name, cred_file): get_client_cred_result = auth_client.get_client_credentials(client_id) - for cred in get_client_cred_result['credentials']: + for cred in get_client_cred_result["credentials"]: # Should have stored secret locally - auth_client.delete_client_credential(client_id, cred['id']) + auth_client.delete_client_credential(client_id, cred["id"]) cred_result = auth_client.create_client_credential(client_id, cred_name) - # Have to change this to a dict + # Have to change this to a dict obj = { - 'client': cred_result['credential']['client'], - 'id': cred_result['credential']['id'], - 'name': cred_result['credential']['name'], - 'secret': cred_result['credential']['secret'] - } - with open(cred_file, 'w') as f: + "client": cred_result["credential"]["client"], + "id": cred_result["credential"]["id"], + "name": cred_result["credential"]["name"], + "secret": cred_result["credential"]["secret"], + } + with open(cred_file, "w") as f: json.dump(obj, f) - return cred_result['credential']['secret'] + return cred_result["credential"]["secret"] + -def getClientSecret( - auth_client, - client_id, - cred_name, - cred_id, - cred_file): +def getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file): client_secret = getCredentialFromFile(cred_file, cred_id) @@ -198,7 +207,7 @@ def getClientSecret( if client_secret: create_new_credential = False remove_cached_credential = False - remove_old_credential = True + remove_old_credential = True if remove_old_credential: auth_client.delete_client_credential(client_id, cred_id) @@ -210,11 +219,8 @@ def getClientSecret( if create_new_credential: # Remove credentials from cloud client_secret = createNewCredential( - auth_client, - client_id, - cred_name, - cred_file - ) + auth_client, client_id, cred_name, cred_file + ) return client_secret @@ -230,8 +236,11 @@ def createClient(auth_client, client_name, project_id, cred_name, cred_file): cred_exists_locally, cred_empty = validFile(cred_file) - client_secret = getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file) - return client_id, client_secret + client_secret = getClientSecret( + auth_client, client_id, cred_name, cred_id, cred_file + ) + return client_id, client_secret + def getGCSClientIDFromDeploymentFile(deployment_key_file): deployment_key_exists, deployment_key_empty = validFile(deployment_key_file) @@ -243,9 +252,9 @@ def getGCSClientIDFromDeploymentFile(deployment_key_file): os.remove(deployment_key_file) else: # If it is not empty get the client id - with open(deployment_key_file, 'r') as f: + with open(deployment_key_file, "r") as f: loaded_data = json.load(f) - return loaded_data['client_id'] + return loaded_data["client_id"] return None @@ -258,43 +267,52 @@ def command_exists(command): # 'which' command returns non-zero exit status if the command is not found return False + def isGCSDeploymentKeyValid(auth_client, project_id, endpoint_name, gcs_id): clients_in_project = getClientsInProject(auth_client, project_id) # Check if the deployment key is valid for the project for client in clients_in_project: - if client['client_type'] == "globus_connect_server" and client['name'] == endpoint_name: + if ( + client["client_type"] == "globus_connect_server" + and client["name"] == endpoint_name + ): if gcs_id: # If gcs_id exists see if it is found remotely - if client['id'] == gcs_id: + if client["id"] == gcs_id: print("Deployment key endpoint is still valid found in cloud") return True else: # Found a globus_connect_server but did not find local deployment # key if deployment_key_empty: - print("Found globus_connect_server already registered but did" - " not find deployment key locally.") + print( + "Found globus_connect_server already registered but did" + " not find deployment key locally." + ) return False def deleteAllNonGCSClients(auth_client, project_id): clients = getClientsInProject(auth_client, project_id) for client in clients: - if client['project'] == project_id and client['client_type'] != "globus_connect_server": - auth_client.delete_client(client['id']) - + if ( + client["project"] == project_id + and client["client_type"] != "globus_connect_server" + ): + auth_client.delete_client(client["id"]) def createGCSEndpoint( - auth_client, - client_id, - client_secret, - project_id, - deployment_key_file, - endpoint_name, - control_port, - userinfo): + auth_client, + client_id, + client_secret, + project_id, + deployment_key_file, + endpoint_name, + control_port, + userinfo, +): identity_id = userinfo["sub"] email = userinfo["email"] @@ -303,53 +321,69 @@ def createGCSEndpoint( gcs_id_from_deployment_key = getGCSClientIDFromDeploymentFile(deployment_key_file) - valid_key = isGCSDeploymentKeyValid(auth_client, project_id, endpoint_name, gcs_id_from_deployment_key) + valid_key = isGCSDeploymentKeyValid( + auth_client, project_id, endpoint_name, gcs_id_from_deployment_key + ) if valid_key is False and gcs_id_from_deployment_key: - print("Looks like deployment key exists but does not contain credentials " - f"in the cloud for the project: {project_id}, please either " - "add the correct deployment key or remove the gcs instance" - "registered in the project") + print( + "Looks like deployment key exists but does not contain credentials " + f"in the cloud for the project: {project_id}, please either " + "add the correct deployment key or remove the gcs instance" + "registered in the project" + ) sys.exit(1) -# Create gcs_instance + # Create gcs_instance if valid_key is False: if command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" " - bash_command+=f" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+=f" globus-connect-server endpoint setup \"{endpoint_name}\" " - bash_command+=f" --organization \"{organization}\" " - bash_command+=f" --project-id \"{project_id}\" " - bash_command+=" --agree-to-letsencrypt-tos " - bash_command+=f" --project-admin \"{username}\" " - bash_command+=f" --owner \"{client_id}@clients.auth.globus.org\" " - bash_command+=f" --contact-email \"{email}\" " - bash_command+=f" --deployment-key \"{deployment_key_file}\" " + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" ' + bash_command += f' GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += f' globus-connect-server endpoint setup "{endpoint_name}" ' + bash_command += f' --organization "{organization}" ' + bash_command += f' --project-id "{project_id}" ' + bash_command += " --agree-to-letsencrypt-tos " + bash_command += f' --project-admin "{username}" ' + bash_command += f' --owner "{client_id}@clients.auth.globus.org" ' + bash_command += f' --contact-email "{email}" ' + bash_command += f' --deployment-key "{deployment_key_file}" ' print("Bash command to run") print(bash_command) - - process = subprocess.Popen(bash_command, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + + process = subprocess.Popen( + bash_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) # Print the output for line in process.stdout: - print(line, end='') + print(line, end="") deployment_key_exists, deployment_key_empty = validFile(deployment_key_file) if deployment_key_exists is False: - print(f"Something is wrong deployment key does not exist {deployment_key_file} ") + print( + f"Something is wrong deployment key does not exist {deployment_key_file} " + ) sys.exit(1) if deployment_key_empty: - print(f"Something is wrong deployment key is empty {deployment_key_file} ") + print( + f"Something is wrong deployment key is empty {deployment_key_file} " + ) sys.exit(1) # WARNING!!!!!! # This will not work if a node does not first exist, I think at least one # node must be running. - #if len(subscription_id) != 0: + # if len(subscription_id) != 0: # if command_exists("globus-connect-server") is False: # print("Cannot create deployment key, we require globus-connect-server to be installed") # sys.exit(1) @@ -362,10 +396,8 @@ def createGCSEndpoint( # bash_command+=" globus-connect-server endpoint update " # bash_command+=f" --subscription-id \"{subscription_id}\" " # print(bash_command) - # + # # process = subprocess.Popen(bash_command, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) # # Print the output # for line in process.stdout: # print(line, end='') - - From affac0e23548ace0ea322c3471eba87d1e711c65 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Mon, 8 Apr 2024 14:30:43 -0400 Subject: [PATCH 53/84] Remove startup attempts just keep looping --- repository/server/RepoServer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index d851eb0b7..0d4a82f97 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -146,8 +146,10 @@ void Server::checkServerVersion() { MessageFactory msg_factory; - for (int i = 0; i < 10; i++) { - DL_INFO(m_log_context, "Attempt " << i << " to initialize communication " + size_t attempt = 0; + while (true) { + ++attempt; + DL_INFO(m_log_context, "Attempt " << attempt << " to initialize communication " << " with core server at " << m_config.core_server); auto msg = std::make_unique(); auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); From 9042f991271e6f806c04fcb3cfbada180154763d Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Mon, 8 Apr 2024 14:31:25 -0400 Subject: [PATCH 54/84] Add functions for being smart about paths --- repository/server/RequestWorker.cpp | 163 ++++++++++++++++++++-------- repository/server/RequestWorker.hpp | 2 + 2 files changed, 121 insertions(+), 44 deletions(-) diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index be08293fd..f90d7046f 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -36,6 +36,121 @@ namespace Repo { map RequestWorker::m_msg_handlers; +bool RequestWorker::prefixesEqual(const std::string& str1, const std::string& str2, size_t length) const { + // Check if either string is shorter than the specified length + if (str1.length() < length || str2.length() < length) { + return false; + } + + // Use std::equal to compare the prefixes + return std::equal(str1.begin(), str1.begin() + length, str2.begin()); +} + +std::string RequestWorker::createSanitizedPath(const std::string& path) const { + + string sanitized_request_path = path; + while (!sanitized_request_path.empty()) { + if (sanitized_request_path.back() == '/') { + sanitized_request_path.pop_back(); + } else { + break; + } + } + /** + * When item is provided it is allowed to be provided as + * + * path = '/relative_path/to/file' + * + * or + * + * path = 'globus_collection_path/relative_path/to/file + * + * If a relative path is applied then the config settings are used and + * the globus collection path is prepended. Otherwise if the the absolute + * path is provided then no change is made. + * + * E.g. + * + * Assuming + * path = "/datafed-home" + * globus_collection_path = "/mnt/datafed" + * + * Then + * local_path = "/mnt/datafed/datafed-home" + * + * Else if + * + * path = "/mnt/datafed/datafed-home" + * globus_collection_path = "/mnt/datafed" + * + * Then it won't prepend + * local_path = "/mnt/datafed/datafed-home" + **/ + string local_path; + if ( prefixesEqual( + sanitized_request_path, + m_config.globus_collection_path, + m_config.globus_collection_path.length()) ) { + + /** + * If both paths exist throw an error indicating there is ambiguity. I.e. + * + * If both + * /datafed/datafed/file1.txt + * /datafed/file1.txt + * + * exist and the variables are + * + * globus_collectin_path = "/datafed" + * path = "/datafed/file1.txt" + * + * First off something with the configuration is likely off and secondly + * It's impossible to determine which file is correct. + * + **/ + std::string local_path_1 = m_config.globus_collection_path; + std::string local_path_2 = ""; + if (sanitized_request_path.front() != '/') { + if (local_path_1.back() != '/') { + local_path_1 += "/" + sanitized_request_path; + } else { + local_path_1 += sanitized_request_path; + } + local_path_2 += "/" + sanitized_request_path; + } else { + local_path_1 += sanitized_request_path; + local_path_2 += sanitized_request_path; + } + + boost::filesystem::path data_path_1(local_path_1); // long + boost::filesystem::path data_path_2(local_path_2); // shorter + + if (boost::filesystem::exists(data_path_1) and boost::filesystem::exists(data_path_2) ){ + // If they are the exact same then ignore else throw an error + // + // i.e. if globus_collection_path is / + if( local_path_1 != local_path_2 ){ + DL_ERROR(m_log_context, + "RequestWorker::createSanitizedPath, exception: something is really wrong both path " << local_path_1 << " and path " << local_path_2 << " exists, which makes unambiguously determining the correct paths impossible."); + throw; + } + + } + + // Use the longer path + local_path = local_path_1; + } else { + local_path = m_config.globus_collection_path; + if (sanitized_request_path.front() != '/') { + local_path += "/" + sanitized_request_path; + } else { + local_path += sanitized_request_path; + } + } + + return local_path; +} + RequestWorker::RequestWorker(size_t a_tid, LogContext log_context) : m_config(Config::getInstance()), m_tid(a_tid), m_run(true), m_log_context(log_context) { @@ -316,21 +431,8 @@ RequestWorker::procDataGetSizeRequest(std::unique_ptr &&msg_request) { for (int i = 0; i < request->loc_size(); i++) { const RecordDataLocation &item = request->loc(i); - string sanitized_request_path = item.path(); - while (!sanitized_request_path.empty()) { - if (sanitized_request_path.back() == '/') { - sanitized_request_path.pop_back(); - } else { - break; - } - } - - string local_path = m_config.globus_collection_path; - if (sanitized_request_path.front() != '/') { - local_path += "/" + sanitized_request_path; - } else { - local_path += sanitized_request_path; - } + std::string local_path = createSanitizedPath(item.path()); + boost::filesystem::path data_path(local_path); data_sz = reply.add_size(); @@ -357,20 +459,7 @@ std::unique_ptr RequestWorker::procPathCreateRequest(std::unique_ptr &&msg_request) { PROC_MSG_BEGIN(Auth::RepoPathCreateRequest, Anon::AckReply) - string sanitized_request_path = request->path(); - while (!sanitized_request_path.empty()) { - if (sanitized_request_path.back() == '/') { - sanitized_request_path.pop_back(); - } else { - break; - } - } - string local_path = m_config.globus_collection_path; - if (sanitized_request_path.front() != '/') { - local_path += "/" + sanitized_request_path; - } else { - local_path += sanitized_request_path; - } + std::string local_path = createSanitizedPath(request->path()); boost::filesystem::path data_path(local_path); DL_DEBUG(message_log_context, @@ -391,21 +480,7 @@ RequestWorker::procPathDeleteRequest(std::unique_ptr &&msg_request) { DL_DEBUG(message_log_context, "Relative path delete request: " << request->path()); - string sanitized_request_path = request->path(); - while (!sanitized_request_path.empty()) { - if (sanitized_request_path.back() == '/') { - sanitized_request_path.pop_back(); - } else { - break; - } - } - - string local_path = m_config.globus_collection_path; - if (sanitized_request_path.front() != '/') { - local_path += "/" + sanitized_request_path; - } else { - local_path += sanitized_request_path; - } + std::string local_path = createSanitizedPath(request->path()); boost::filesystem::path data_path(local_path); DL_TRACE(message_log_context, diff --git a/repository/server/RequestWorker.hpp b/repository/server/RequestWorker.hpp index 74bef01b8..0dc3529d8 100644 --- a/repository/server/RequestWorker.hpp +++ b/repository/server/RequestWorker.hpp @@ -33,6 +33,8 @@ class RequestWorker { private: void setupMsgHandlers(); void workerThread(LogContext log_context); + bool prefixesEqual(const std::string& str1, const std::string& str2, size_t length) const; + std::string createSanitizedPath(const std::string& path) const; std::unique_ptr procVersionRequest(std::unique_ptr &&); std::unique_ptr procDataDeleteRequest(std::unique_ptr &&); From 0f5db9da6a4291045bfee40f468c9e3ee086f2ba Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 10:33:03 -0400 Subject: [PATCH 55/84] Add unsigned int as type for LogLevel enum --- common/include/common/DynaLog.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/include/common/DynaLog.hpp b/common/include/common/DynaLog.hpp index ba755b44f..e02cbeffe 100644 --- a/common/include/common/DynaLog.hpp +++ b/common/include/common/DynaLog.hpp @@ -73,7 +73,7 @@ namespace SDMS { -enum class LogLevel { CRITICAL, ERROR, WARNING, INFO, DEBUG, TRACE }; +enum class LogLevel : unsigned int { CRITICAL, ERROR, WARNING, INFO, DEBUG, TRACE }; std::string toString(const LogLevel level); int toSysLog(const LogLevel level); From b04e529b22b657ffed9b0b7f53e5857eeaab17cf Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 10:33:33 -0400 Subject: [PATCH 56/84] Cast to unsigned int before comparing enum numeric values --- common/source/DynaLog.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/source/DynaLog.cpp b/common/source/DynaLog.cpp index 255611070..79a1814aa 100644 --- a/common/source/DynaLog.cpp +++ b/common/source/DynaLog.cpp @@ -114,37 +114,37 @@ void Logger::addStream(std::ostream &stream) { void Logger::trace(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (m_log_level >= LogLevel::TRACE) { + if (static_cast(m_log_level) >= static_cast(LogLevel::TRACE)) { output(LogLevel::TRACE, file, func, line_num, context, message); } } void Logger::debug(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (m_log_level >= LogLevel::DEBUG) { + if (static_cast(m_log_level) >= static_cast(LogLevel::DEBUG)) { output(LogLevel::DEBUG, file, func, line_num, context, message); } } void Logger::info(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (m_log_level >= LogLevel::INFO) { + if (static_cast(m_log_level) >= static_cast(LogLevel::INFO)) { output(LogLevel::INFO, file, func, line_num, context, message); } } void Logger::warning(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (m_log_level >= LogLevel::WARNING) { + if (static_cast(m_log_level) >= static_cast(LogLevel::WARNING)) { output(LogLevel::WARNING, file, func, line_num, context, message); } } void Logger::error(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (m_log_level >= LogLevel::ERROR) { + if (static_cast(m_log_level) >= static_cast(LogLevel::ERROR)) { output(LogLevel::ERROR, file, func, line_num, context, message); } } void Logger::critical(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (m_log_level >= LogLevel::CRITICAL) { + if (static_cast(m_log_level) >= static_cast(LogLevel::CRITICAL)) { output(LogLevel::CRITICAL, file, func, line_num, context, message); } } From 4e6ff399fd3d4b06aa635ec8218ca818d871958b Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 10:34:02 -0400 Subject: [PATCH 57/84] Fix log file path in compose file --- compose/compose_repo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/compose_repo.yml b/compose/compose_repo.yml index dc9b1a61d..3e4cf7b8a 100644 --- a/compose/compose_repo.yml +++ b/compose/compose_repo.yml @@ -21,7 +21,7 @@ services: image: datafed-repo:latest volumes: - ./keys:/opt/datafed/keys - - ./logs:/datafed/logs + - ./logs:${DATAFED_CONTAINER_LOG_PATH} - ${DATAFED_HOST_COLLECTION_MOUNT}:${DATAFED_GCS_COLLECTION_ROOT_PATH} ports: - 9000:9000 # Communication core server @@ -49,7 +49,7 @@ services: volumes: - ./keys:/opt/datafed/keys - ./globus:/opt/datafed/globus - - ./logs:/datafed/logs + - ./logs:${DATAFED_CONTAINER_LOG_PATH} - ${DATAFED_HOST_COLLECTION_MOUNT}:${DATAFED_GCS_COLLECTION_ROOT_PATH} # External true indicates that the network is created by a From b7b28a1a9bbf3e3e942c4cd4f0cff6666e284f79 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 10:38:03 -0400 Subject: [PATCH 58/84] Removed cached files from docker compose repo --- repository/docker/entrypoint_authz.sh | 30 ++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index e72c1d0b3..309a02bd7 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -49,7 +49,28 @@ cp "$PROJECT_ROOT/config/datafed-authz.cfg" ${DATAFED_INSTALL_PATH}/authz # the globus-connect-server node setup command attempts to use systemctl which # is not installed in the container -# Run in background +# Make sure files did not persist from a previous docker compose run +# This code block is used for cleaning up files that might have been cached by +# docker compose. These files are not always appropraitely removed when the +# Globus container exits. +# +# List of files to delete +files=("/run/gcs_manager" "/run/gcs_manager/pid" "/var/run/apache2/apache2.pid" + "/var/run/httpd/httpd.pid" "/run/globus-gridftp-server.pid") +# Loop over the list of files +for file in "${files[@]}"; do + if [ -e "$file" ]; then + echo "Removing $file" + rm -rf "$file" + fi +done +link="/run/gcs_manager.sock" +if [ -L "$link" ]; then + echo "Removing symbolic link $link" + rm "$link" +fi + +# Run the GCS entrypoint file in the background /entrypoint.sh & # NOTE - can only change the tcp control port after setting up the end point @@ -102,11 +123,9 @@ do printf "\r${EraseToEOL}" minutes=$((minutes + 1)) - #sleep 60 HTTP_CODE=$(${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/curl -s -o /dev/null -w "%{http_code}\n" -I "https://${DATAFED_GCS_URL}/api/info") done printf "\n" -#export DATAFED_GCS_URL=$(globus-connect-server endpoint show --format json | jq -r .gcs_manager_url) log_path="$DATAFED_DEFAULT_LOG_PATH" @@ -135,6 +154,7 @@ DATAFED_REPO_USER="$DATAFED_REPO_USER" \ "${BUILD_DIR}/scripts/globus/generate_repo_form.sh" echo "Container is running." -sleep infinity -#"$@" -- argv0 "$@" +# Return to last file +tail -f "${DATAFED_DEFAULT_LOG_PATH}/datafed-gsi-authz.log" +sleep infinity From 41d1be396bf06eaae6c639c614ca662b01eacc33 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 11:33:32 -0400 Subject: [PATCH 59/84] Add comments explaining intent and log file info --- .../globus5/authz/source/AuthzWorker.cpp | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index 6a6eaa7e0..c1e142770 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -29,7 +29,6 @@ using namespace std; using namespace SDMS::Anon; using namespace SDMS::Auth; -// using namespace SDMS::authz::version; namespace { std::string randomAlphaNumericCode() { @@ -141,8 +140,27 @@ class AuthzWorker { EXCEPT(1, "Path to data item is not within the collection"); } - auto sanitized_path = local_path.substr(prefix.length()); - + /** + * TODO Separation of Concerns Repo vs Core services + * + * The following line is currenctly commented out because it will require + * some actual work and testing to get it working correctly. + * + * Ideally, when a DataFed repository is registered with the DataFed core + * services it should not need to know anything about the DataFed repository + * i.e. posix path to the actual collection. So when a user makes a request + * to do something on the DataFed managed endpoint the repo service should + * not need to send the full POSIX path back to the core service. + * + * This is not currently working but that is what the sanitized path should + * be. + * auto sanitized_path = local_path.substr(prefix.length()); + * + * Changes will need to be made anywhere the path is used. And when a + * transfer document is created it should be provided with a path relative + * to the endpoint not to the absolute of the filesystem. + **/ + auto sanitized_path = local_path; std::unique_ptr m_sec_ctx; std::unordered_map cred_options; @@ -312,9 +330,21 @@ const char *getReleaseVersion() { // The same int checkAuthorization(char *client_id, char *object, char *action, struct Config *config) { +#if defined(DONT_USE_SYSLOG) + SDMS::global_logger.setSysLog(false); +#else SDMS::global_logger.setSysLog(true); - SDMS::global_logger.setLevel(SDMS::LogLevel::TRACE); +#endif + SDMS::global_logger.setLevel(SDMS::LogLevel::INFO); SDMS::global_logger.addStream(std::cerr); + auto log_path_authz = std::string(config->log_path); + if( log_path_authz.length() > 0 ) { + // Append to the existing path because we don't want the C++ and C code + // trying to write to the same file + log_path_authz.append("_authz"); + std::ofstream log_file_worker(log_path_authz); + SDMS::global_logger.addStream(log_file_worker); + } SDMS::LogContext log_context; log_context.thread_name = "authz_check"; From 30bf22281da899b92d28901b0674b4da71a416bb Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 11:35:01 -0400 Subject: [PATCH 60/84] Fix bug in log macros, initialize path with null terminator change ERROR to debug --- .../gridftp/globus5/authz/source/libauthz.c | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index fd03bb7b2..04f497dcf 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -1,5 +1,6 @@ // Local private includes +#include "Config.h" #include "AuthzWorker.h" // Globus third party includes @@ -16,35 +17,38 @@ // Define LOG_LEVEL and USE_SYSLOG #define LOG_LEVEL 1 + +#ifndef DONT_USE_SYSLOG #define DONT_USE_SYSLOG +#endif // Define logging macros #if defined(DONT_USE_SYSLOG) FILE *log_file = NULL; -bool write_to_file = true; +bool write_to_file = false; #define AUTHZ_LOG_DEBUG(fmt, ...) \ do { \ - if (LOG_LEVEL >= 1) \ + if (LOG_LEVEL <= 1) \ fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); \ } while (0); \ do { \ - if (LOG_LEVEL >= 1 && write_to_file) \ + if (LOG_LEVEL <= 1 && write_to_file) \ fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); \ } while (0) #define AUTHZ_LOG_INFO(fmt, ...) \ do { \ - if (LOG_LEVEL >= 2) \ + if (LOG_LEVEL <= 2) \ fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); \ } while (0); \ do { \ - if (LOG_LEVEL >= 2 && write_to_file) \ + if (LOG_LEVEL <= 2 && write_to_file) \ fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); \ } while (0) #define AUTHZ_LOG_ERROR(fmt, ...) \ do { \ - if (LOG_LEVEL >= 3) \ + if (LOG_LEVEL <= 3) \ fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ - if (LOG_LEVEL >= 3 && write_to_file) \ + if (LOG_LEVEL <= 3 && write_to_file) \ fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ } while (0) #define AUTHZ_LOG_INIT(file_path) \ @@ -60,17 +64,17 @@ bool write_to_file = true; #include #define AUTHZ_LOG_DEBUG(fmt, ...) \ do { \ - if (LOG_LEVEL >= 1) \ + if (LOG_LEVEL <= 1) \ syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); \ } while (0) #define AUTHZ_LOG_INFO(fmt, ...) \ do { \ - if (LOG_LEVEL >= 2) \ + if (LOG_LEVEL <= 2) \ syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); \ } while (0) #define AUTHZ_LOG_ERROR(fmt, ...) \ do { \ - if (LOG_LEVEL >= 3) \ + if (LOG_LEVEL <= 3) \ syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); \ } while (0) #define AUTHZ_LOG_INIT(file_path) openlog("gsi_authz", 0, LOG_AUTH); @@ -302,6 +306,7 @@ bool loadConfig() { // Default values g_config.timeout = 10000; + g_config.log_path[0] = '\0'; if (strcmp(buf, "repo_id") == 0) err = setConfigVal("repo_id", g_config.repo_id, val, MAX_ID_LEN); @@ -629,7 +634,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { AUTHZ_LOG_INFO("Authz: FAILED\n"); result = globus_error_put(error); } else { - AUTHZ_LOG_ERROR("Authz: PASSED\n"); + AUTHZ_LOG_DEBUG("Authz: PASSED\n"); callback(callback_arg, handle, result); } From 69edf35b683d93134f0ce88f41a84c97752692aa Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 11:36:04 -0400 Subject: [PATCH 61/84] Correct default in sanitized path and add function where it was missing --- repository/server/RequestWorker.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index c3b284ebf..9ff0b14c6 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -137,8 +137,8 @@ std::string RequestWorker::createSanitizedPath(const std::string& path) const { } - // Use the longer path - local_path = local_path_1; + // Use the shorter path + local_path = local_path_2; } else { local_path = m_config.globus_collection_path; if (sanitized_request_path.front() != '/') { @@ -408,7 +408,8 @@ RequestWorker::procDataDeleteRequest(std::unique_ptr &&msg_request) { if (request->loc_size()) { for (int i = 0; i < request->loc_size(); i++) { - string local_path = request->loc(i).path(); + std::string local_path = createSanitizedPath(request->loc(i).path()); + DL_DEBUG(message_log_context, "Delete " << request->loc_size() << " file(s), path: " << local_path); From b36321ef98cfbac84c782d8c64f8e521639d513f Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Tue, 9 Apr 2024 11:36:58 -0400 Subject: [PATCH 62/84] Alter default globus dir to root --- scripts/globus/setup_globus.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/globus/setup_globus.sh b/scripts/globus/setup_globus.sh index 4f19be298..158f8aeb2 100755 --- a/scripts/globus/setup_globus.sh +++ b/scripts/globus/setup_globus.sh @@ -137,8 +137,8 @@ then "$uuid_of_storage_gateway" \ "/" \ "$COLLECTION_NAME" \ - --default-directory "${DATAFED_GCS_COLLECTION_ROOT_PATH}" \ --enable-anonymous-writes \ + --default-directory "/" \ --disable-https "$extra_collection_arg" else @@ -147,8 +147,8 @@ else # NOTE allow-guest-collections requires a subscription globus-connect-server collection update \ "$uuid_of_collection" \ - --default-directory "${DATAFED_GCS_COLLECTION_ROOT_PATH}" \ --enable-anonymous-writes \ + --default-directory "/" \ --disable-https "$extra_collection_arg" fi From 771302b1b7cb9688c7e91bcdcbe2c9414d57fa3b Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 11:40:14 -0400 Subject: [PATCH 63/84] Cleanup some of the log debug statements in gridftp --- core/database/foxx/api/authz_router.js | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/core/database/foxx/api/authz_router.js b/core/database/foxx/api/authz_router.js index c8771a003..a74f4f7fb 100644 --- a/core/database/foxx/api/authz_router.js +++ b/core/database/foxx/api/authz_router.js @@ -15,23 +15,16 @@ router.get('/gridftp', function(req, res) { const client = g_lib.getUserFromClientID_noexcept(req.queryParams.client); - console.log("1"); var idx = req.queryParams.file.lastIndexOf("/"); - console.log("2"); var data_key = req.queryParams.file.substr(idx + 1); - console.log("3"); var data_id = "d/" + data_key; - console.log("4"); - console.log("client is ", client); // Special case - allow unknown client to read a publicly accessible record - console.log(req.queryParams); if (!client) { if (req.queryParams.act != "read" || !g_lib.hasPublicRead(data_id)) { console.log("Permission to read denied!"); throw g_lib.ERR_PERM_DENIED; } - console.log("5"); console.log("allow anon read of public record"); } else { console.log("client:", client); @@ -62,13 +55,10 @@ router.get('/gridftp', function(req, res) { throw [g_lib.ERR_INVALID_PARAM, "Invalid gridFTP action: ", req.queryParams.act]; } - console.log("6 client: ", client, " data_id: ", data_id); + console.log("client: ", client, " data_id: ", data_id); if (!g_lib.hasAdminPermObject(client, data_id)) { - console.log("7"); var data = g_db.d.document(data_id); - console.log("8"); if (!g_lib.hasPermissions(client, data, req_perm)) { - console.log("9"); console.log("Client: ", client, " does not have permission!"); throw g_lib.ERR_PERM_DENIED; } @@ -78,9 +68,7 @@ router.get('/gridftp', function(req, res) { // Verify repo and path are correct for record // Note: only managed records have an allocations and this gridftp auth call is only made for managed records //var path = req.queryParams.file.substr( req.queryParams.file.indexOf("/",8)); - console.log("10"); var path = req.queryParams.file; - console.log("11"); console.log("data_id is, ", data_id); var loc = g_db.loc.firstExample({ _from: data_id @@ -91,7 +79,6 @@ router.get('/gridftp', function(req, res) { console.log("Permission denied data is not managed by DataFed. This can happen if you try to do a transfer directly from Globus.") throw g_lib.ERR_PERM_DENIED; } - console.log("12"); var alloc = g_db.alloc.firstExample({ _from: loc.uid, _to: loc._to @@ -107,22 +94,25 @@ router.get('/gridftp', function(req, res) { } console.log("path:", path, " alloc path:", alloc.path + data_key, " loc: ", loc); - console.log("13"); if (alloc.path + data_key != path) { // This may be due to an alloc/owner change - // Allow IF new path matches + // Allow If new path matches console.log("authz loc info:", loc); if (!loc.new_repo) { + console.log("Throw a permission denied error"); throw g_lib.ERR_PERM_DENIED; } + console.log("Creating alloc"); alloc = g_db.alloc.firstExample({ _from: loc.new_owner ? loc.new_owner : loc.uid, _to: loc.new_repo }); + console.log("alloc is "); + console.log(alloc); if (!alloc || (alloc.path + data_key != path)) { throw g_lib.ERR_PERM_DENIED; } From 39777bb81d922720b79f91cc74988cf39b4b3c3a Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 11:44:24 -0400 Subject: [PATCH 64/84] Add log message to database api --- core/server/DatabaseAPI.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/server/DatabaseAPI.cpp b/core/server/DatabaseAPI.cpp index 396e4b42d..7ad56684e 100644 --- a/core/server/DatabaseAPI.cpp +++ b/core/server/DatabaseAPI.cpp @@ -2507,6 +2507,7 @@ void DatabaseAPI::repoAuthz(const Auth::RepoAuthzRequest &a_request, (void)a_reply; Value result; + DL_INFO(log_context, "authz/gridftp repo: " << a_reply.repo() << " file " << a_request.file() << " act " << a_request.action()); dbGet("authz/gridftp", {{"repo", a_request.repo()}, {"file", a_request.file()}, From a1852afbc9d3cbc8af46d9960518d3e23a74dff6 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 11:44:57 -0400 Subject: [PATCH 65/84] Add additional comment to AuthzWorker.cpp --- repository/gridftp/globus5/authz/source/AuthzWorker.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index c1e142770..37e87ab94 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -159,6 +159,9 @@ class AuthzWorker { * Changes will need to be made anywhere the path is used. And when a * transfer document is created it should be provided with a path relative * to the endpoint not to the absolute of the filesystem. + * + * Changes will also need to be made when entering information in the repo + * form if this is done. **/ auto sanitized_path = local_path; std::unique_ptr m_sec_ctx; From cca87c0f43038245bd93aea91c45514e338bf262 Mon Sep 17 00:00:00 2001 From: par-hermes Date: Tue, 9 Apr 2024 15:46:33 +0000 Subject: [PATCH 66/84] cpp-py-formatter --- common/include/common/DynaLog.hpp | 9 +- common/source/DynaLog.cpp | 18 +- core/server/DatabaseAPI.cpp | 4 +- .../globus5/authz/source/AuthzWorker.cpp | 4 +- .../gridftp/globus5/authz/source/libauthz.c | 2 +- repository/server/RepoServer.cpp | 5 +- repository/server/RequestWorker.cpp | 212 +++++++++--------- repository/server/RequestWorker.hpp | 5 +- 8 files changed, 140 insertions(+), 119 deletions(-) diff --git a/common/include/common/DynaLog.hpp b/common/include/common/DynaLog.hpp index e02cbeffe..38f76f513 100644 --- a/common/include/common/DynaLog.hpp +++ b/common/include/common/DynaLog.hpp @@ -73,7 +73,14 @@ namespace SDMS { -enum class LogLevel : unsigned int { CRITICAL, ERROR, WARNING, INFO, DEBUG, TRACE }; +enum class LogLevel : unsigned int { + CRITICAL, + ERROR, + WARNING, + INFO, + DEBUG, + TRACE +}; std::string toString(const LogLevel level); int toSysLog(const LogLevel level); diff --git a/common/source/DynaLog.cpp b/common/source/DynaLog.cpp index 79a1814aa..f89ccee95 100644 --- a/common/source/DynaLog.cpp +++ b/common/source/DynaLog.cpp @@ -114,37 +114,43 @@ void Logger::addStream(std::ostream &stream) { void Logger::trace(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (static_cast(m_log_level) >= static_cast(LogLevel::TRACE)) { + if (static_cast(m_log_level) >= + static_cast(LogLevel::TRACE)) { output(LogLevel::TRACE, file, func, line_num, context, message); } } void Logger::debug(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (static_cast(m_log_level) >= static_cast(LogLevel::DEBUG)) { + if (static_cast(m_log_level) >= + static_cast(LogLevel::DEBUG)) { output(LogLevel::DEBUG, file, func, line_num, context, message); } } void Logger::info(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (static_cast(m_log_level) >= static_cast(LogLevel::INFO)) { + if (static_cast(m_log_level) >= + static_cast(LogLevel::INFO)) { output(LogLevel::INFO, file, func, line_num, context, message); } } void Logger::warning(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (static_cast(m_log_level) >= static_cast(LogLevel::WARNING)) { + if (static_cast(m_log_level) >= + static_cast(LogLevel::WARNING)) { output(LogLevel::WARNING, file, func, line_num, context, message); } } void Logger::error(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (static_cast(m_log_level) >= static_cast(LogLevel::ERROR)) { + if (static_cast(m_log_level) >= + static_cast(LogLevel::ERROR)) { output(LogLevel::ERROR, file, func, line_num, context, message); } } void Logger::critical(std::string file, std::string func, int line_num, const LogContext &context, const std::string &message) { - if (static_cast(m_log_level) >= static_cast(LogLevel::CRITICAL)) { + if (static_cast(m_log_level) >= + static_cast(LogLevel::CRITICAL)) { output(LogLevel::CRITICAL, file, func, line_num, context, message); } } diff --git a/core/server/DatabaseAPI.cpp b/core/server/DatabaseAPI.cpp index 7ad56684e..2c81f8a69 100644 --- a/core/server/DatabaseAPI.cpp +++ b/core/server/DatabaseAPI.cpp @@ -2507,7 +2507,9 @@ void DatabaseAPI::repoAuthz(const Auth::RepoAuthzRequest &a_request, (void)a_reply; Value result; - DL_INFO(log_context, "authz/gridftp repo: " << a_reply.repo() << " file " << a_request.file() << " act " << a_request.action()); + DL_INFO(log_context, "authz/gridftp repo: " << a_reply.repo() << " file " + << a_request.file() << " act " + << a_request.action()); dbGet("authz/gridftp", {{"repo", a_request.repo()}, {"file", a_request.file()}, diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index 37e87ab94..fcae2dd09 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -142,7 +142,7 @@ class AuthzWorker { /** * TODO Separation of Concerns Repo vs Core services - * + * * The following line is currenctly commented out because it will require * some actual work and testing to get it working correctly. * @@ -341,7 +341,7 @@ int checkAuthorization(char *client_id, char *object, char *action, SDMS::global_logger.setLevel(SDMS::LogLevel::INFO); SDMS::global_logger.addStream(std::cerr); auto log_path_authz = std::string(config->log_path); - if( log_path_authz.length() > 0 ) { + if (log_path_authz.length() > 0) { // Append to the existing path because we don't want the C++ and C code // trying to write to the same file log_path_authz.append("_authz"); diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index 04f497dcf..482921e73 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -1,7 +1,7 @@ // Local private includes -#include "Config.h" #include "AuthzWorker.h" +#include "Config.h" // Globus third party includes #include diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index 1110457aa..addbbd7cd 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -150,8 +150,9 @@ void Server::checkServerVersion() { size_t attempt = 0; while (true) { ++attempt; - DL_INFO(m_log_context, "Attempt " << attempt << " to initialize communication " - << " with core server at " << m_config.core_server); + DL_INFO(m_log_context, + "Attempt " << attempt << " to initialize communication " + << " with core server at " << m_config.core_server); auto msg = std::make_unique(); auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); message->setPayload(std::move(msg)); diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index 9ff0b14c6..49bb9186e 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -36,119 +36,123 @@ namespace Repo { map RequestWorker::m_msg_handlers; -bool RequestWorker::prefixesEqual(const std::string& str1, const std::string& str2, size_t length) const { - // Check if either string is shorter than the specified length - if (str1.length() < length || str2.length() < length) { - return false; - } - - // Use std::equal to compare the prefixes - return std::equal(str1.begin(), str1.begin() + length, str2.begin()); +bool RequestWorker::prefixesEqual(const std::string &str1, + const std::string &str2, + size_t length) const { + // Check if either string is shorter than the specified length + if (str1.length() < length || str2.length() < length) { + return false; + } + + // Use std::equal to compare the prefixes + return std::equal(str1.begin(), str1.begin() + length, str2.begin()); } -std::string RequestWorker::createSanitizedPath(const std::string& path) const { +std::string RequestWorker::createSanitizedPath(const std::string &path) const { - string sanitized_request_path = path; - while (!sanitized_request_path.empty()) { - if (sanitized_request_path.back() == '/') { - sanitized_request_path.pop_back(); - } else { - break; - } + string sanitized_request_path = path; + while (!sanitized_request_path.empty()) { + if (sanitized_request_path.back() == '/') { + sanitized_request_path.pop_back(); + } else { + break; } - /** - * When item is provided it is allowed to be provided as - * - * path = '/relative_path/to/file' + } + /** + * When item is provided it is allowed to be provided as + * + * path = '/relative_path/to/file' + * + * or + * + * path = 'globus_collection_path/relative_path/to/file + * + * If a relative path is applied then the config settings are used and + * the globus collection path is prepended. Otherwise if the the absolute + * path is provided then no change is made. + * + * E.g. + * + * Assuming + * path = "/datafed-home" + * globus_collection_path = "/mnt/datafed" + * + * Then + * local_path = "/mnt/datafed/datafed-home" + * + * Else if + * + * path = "/mnt/datafed/datafed-home" + * globus_collection_path = "/mnt/datafed" + * + * Then it won't prepend + * local_path = "/mnt/datafed/datafed-home" + **/ + string local_path; + if (prefixesEqual(sanitized_request_path, m_config.globus_collection_path, + m_config.globus_collection_path.length())) { + + /** + * If both paths exist throw an error indicating there is ambiguity. I.e. * - * or + * If both + * /datafed/datafed/file1.txt + * /datafed/file1.txt * - * path = 'globus_collection_path/relative_path/to/file + * exist and the variables are * - * If a relative path is applied then the config settings are used and - * the globus collection path is prepended. Otherwise if the the absolute - * path is provided then no change is made. - * - * E.g. - * - * Assuming - * path = "/datafed-home" - * globus_collection_path = "/mnt/datafed" + * globus_collectin_path = "/datafed" + * path = "/datafed/file1.txt" * - * Then - * local_path = "/mnt/datafed/datafed-home" - * - * Else if - * - * path = "/mnt/datafed/datafed-home" - * globus_collection_path = "/mnt/datafed" + * First off something with the configuration is likely off and secondly + * It's impossible to determine which file is correct. * - * Then it won't prepend - * local_path = "/mnt/datafed/datafed-home" **/ - string local_path; - if ( prefixesEqual( - sanitized_request_path, - m_config.globus_collection_path, - m_config.globus_collection_path.length()) ) { - - /** - * If both paths exist throw an error indicating there is ambiguity. I.e. - * - * If both - * /datafed/datafed/file1.txt - * /datafed/file1.txt - * - * exist and the variables are - * - * globus_collectin_path = "/datafed" - * path = "/datafed/file1.txt" - * - * First off something with the configuration is likely off and secondly - * It's impossible to determine which file is correct. - * - **/ - std::string local_path_1 = m_config.globus_collection_path; - std::string local_path_2 = ""; - if (sanitized_request_path.front() != '/') { - if (local_path_1.back() != '/') { - local_path_1 += "/" + sanitized_request_path; - } else { - local_path_1 += sanitized_request_path; - } - local_path_2 += "/" + sanitized_request_path; - } else { - local_path_1 += sanitized_request_path; - local_path_2 += sanitized_request_path; - } - - boost::filesystem::path data_path_1(local_path_1); // long - boost::filesystem::path data_path_2(local_path_2); // shorter - - if (boost::filesystem::exists(data_path_1) and boost::filesystem::exists(data_path_2) ){ - // If they are the exact same then ignore else throw an error - // - // i.e. if globus_collection_path is / - if( local_path_1 != local_path_2 ){ - DL_ERROR(m_log_context, - "RequestWorker::createSanitizedPath, exception: something is really wrong both path " << local_path_1 << " and path " << local_path_2 << " exists, which makes unambiguously determining the correct paths impossible."); - throw; - } + std::string local_path_1 = m_config.globus_collection_path; + std::string local_path_2 = ""; + if (sanitized_request_path.front() != '/') { + if (local_path_1.back() != '/') { + local_path_1 += "/" + sanitized_request_path; + } else { + local_path_1 += sanitized_request_path; + } + local_path_2 += "/" + sanitized_request_path; + } else { + local_path_1 += sanitized_request_path; + local_path_2 += sanitized_request_path; + } - } + boost::filesystem::path data_path_1(local_path_1); // long + boost::filesystem::path data_path_2(local_path_2); // shorter + + if (boost::filesystem::exists(data_path_1) and + boost::filesystem::exists(data_path_2)) { + // If they are the exact same then ignore else throw an error + // + // i.e. if globus_collection_path is / + if (local_path_1 != local_path_2) { + DL_ERROR(m_log_context, + "RequestWorker::createSanitizedPath, exception: something is " + "really wrong both path " + << local_path_1 << " and path " << local_path_2 + << " exists, which makes unambiguously determining the " + "correct paths impossible."); + throw; + } + } - // Use the shorter path - local_path = local_path_2; - } else { - local_path = m_config.globus_collection_path; - if (sanitized_request_path.front() != '/') { - local_path += "/" + sanitized_request_path; - } else { - local_path += sanitized_request_path; - } - } + // Use the shorter path + local_path = local_path_2; + } else { + local_path = m_config.globus_collection_path; + if (sanitized_request_path.front() != '/') { + local_path += "/" + sanitized_request_path; + } else { + local_path += sanitized_request_path; + } + } - return local_path; + return local_path; } RequestWorker::RequestWorker(size_t a_tid, LogContext log_context) @@ -432,8 +436,8 @@ RequestWorker::procDataGetSizeRequest(std::unique_ptr &&msg_request) { for (int i = 0; i < request->loc_size(); i++) { const RecordDataLocation &item = request->loc(i); - std::string local_path = createSanitizedPath(item.path()); - + std::string local_path = createSanitizedPath(item.path()); + boost::filesystem::path data_path(local_path); data_sz = reply.add_size(); @@ -460,7 +464,7 @@ std::unique_ptr RequestWorker::procPathCreateRequest(std::unique_ptr &&msg_request) { PROC_MSG_BEGIN(Auth::RepoPathCreateRequest, Anon::AckReply) - std::string local_path = createSanitizedPath(request->path()); + std::string local_path = createSanitizedPath(request->path()); boost::filesystem::path data_path(local_path); DL_DEBUG(message_log_context, @@ -481,7 +485,7 @@ RequestWorker::procPathDeleteRequest(std::unique_ptr &&msg_request) { DL_DEBUG(message_log_context, "Relative path delete request: " << request->path()); - std::string local_path = createSanitizedPath(request->path()); + std::string local_path = createSanitizedPath(request->path()); boost::filesystem::path data_path(local_path); DL_TRACE(message_log_context, diff --git a/repository/server/RequestWorker.hpp b/repository/server/RequestWorker.hpp index 0dc3529d8..12a04a0f2 100644 --- a/repository/server/RequestWorker.hpp +++ b/repository/server/RequestWorker.hpp @@ -33,8 +33,9 @@ class RequestWorker { private: void setupMsgHandlers(); void workerThread(LogContext log_context); - bool prefixesEqual(const std::string& str1, const std::string& str2, size_t length) const; - std::string createSanitizedPath(const std::string& path) const; + bool prefixesEqual(const std::string &str1, const std::string &str2, + size_t length) const; + std::string createSanitizedPath(const std::string &path) const; std::unique_ptr procVersionRequest(std::unique_ptr &&); std::unique_ptr procDataDeleteRequest(std::unique_ptr &&); From ebcb9777ee1eba00f6ed31c8e5e4217d638b5e71 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 11:56:11 -0400 Subject: [PATCH 67/84] Add Changelog comment --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e7e96143..c878eca29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ smaller images 4. [912] - Adding working compose instance for core metadata services. 5. [937] - Working metadata services running together as part of CI +6. [946] - Added docker compose for DataFed Repository and for Metadata Services ## PATCH Bug fixes/Technical Debt/Documentation From 5a98ca109ca41fa87cb1c9ba6f3df03eb84a035d Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 12:00:06 -0400 Subject: [PATCH 68/84] Address Readme concerns --- compose/README.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/compose/README.md b/compose/README.md index 7eb12e4fb..e508e796d 100644 --- a/compose/README.md +++ b/compose/README.md @@ -5,8 +5,8 @@ The compos Dev environment is split into two different compose files. The and the "repo services" which comprise Globus Connect Server running with the authz library and the DataFed repo service. -NOTE Standing up the repo services has been separated because of Globus. You will -need a machine with firewall exceptions to use it. +NOTE Standing up the repo services has been separated because of Globus. You +will need a machine with firewall exceptions to use it. ## Core Compose Services @@ -23,14 +23,14 @@ Core services only need an external port 443 for https access. I have been unable to get this to work with other ports due to the redirect URL required by Globus for authentication. It does not seem to support ports outside of 443. -### 1. Generating .env configuration varaibles +### 1. Generating .env configuration varaibles for Core Services Create the .env file fill in the missing components that are required. ```bash ./generate_env.sh ``` -### 2. Generating .env configuration varaibles +### 2. Fill in the needed .env variables for the Core Services The .env file will be created in the DataFed/compose folder and will be hidden. The .env file variables can be changed at this point to your configuration. @@ -48,7 +48,7 @@ services. ./build_images_for_compose.sh ``` -### 4. Running the compose file +### 4. Running the core compose file Stand up the core services. @@ -57,9 +57,9 @@ source ./unset_env.sh docker compose -f ./compose_core.yml up ``` -NOTE The unset_env.sh script is to make sure you are not accidentially overwriting -what is in the .env with your local shell env. You can check the configuration -before hand by running. +NOTE The unset_env.sh script is to make sure you are not accidentially +overwriting what is in the .env with your local shell env. You can check the +configuration before hand by running. ```bash docker compose -f compose_core.yml config @@ -77,7 +77,7 @@ At this point you should be able to navigate in your browser to NOTE we are using a self signed certificate so you will have to force your browser to allow you to see the page. -### 5. Bringing down the compose file +### 5. Bringing down the core compose file To completely remove the compose instance and all state the following should be run. @@ -104,7 +104,7 @@ and teardown step. 6. Bringing down the compose file. 7. Running the cleanup_globus_files.sh if you -### 1. Generating .env configuration varaibles +### 1. Generating .env configuration varaibles for the Repo Services Create the .env file fill in the missing components that are required. @@ -112,7 +112,7 @@ Create the .env file fill in the missing components that are required. ./generate_env.sh ``` -### 2. Generating .env configuration varaibles +### 2. Enter the .env varaibles for the Repo Services The .env file will be created in the DataFed/compose folder and will be hidden. The .env file variables can be changed at this point to your configuration. @@ -136,7 +136,7 @@ services. ./build_repo_images_for_compose.sh ``` -### 5. Running the compose file +### 5. Running the Repo compose file Stand up the repo services. @@ -145,10 +145,10 @@ source ./unset_env.sh docker compose -f ./compose_repo.yml up ``` -NOTE The unset_env.sh script is to make sure you are not accidentially overwriting -what is in the .env with your local shell env. You can check the configuration -before hand by running. Be aware, the 'source' is to apply changes to your -current terminal session. +NOTE The unset_env.sh script is to make sure you are not accidentially +overwriting what is in the .env with your local shell env. You can check the +configuration before hand by running. Be aware, the 'source' is to apply +changes to your current terminal session. ```bash docker compose -f compose_repo.yml config @@ -168,12 +168,12 @@ Be sure that you are not accidentally overwriting .env variables. docker compose -f ./compose_core.yml down ``` -# Running isolated containers +## Running isolated containers If you just want to run a single container at a time with the same configuration this can also be doen using commands like the following. -## DataFed Web +### DataFed Web ```bash source ./unset_env.sh @@ -184,7 +184,7 @@ docker run --env-file .env \ -t datafed-web:latest ``` -## DataFed GCS +### DataFed GCS ```bash docker run --env-file .env \ @@ -199,9 +199,9 @@ docker run --env-file .env \ To interact more directly with the container the '-i' flag can be added and the entrypoint file can be overwritten by including '--entrypoint /bin/bash' -# Common Errors +## Common Errors -## Errors during compose up +### Errors during compose up Make sure all the ports that are needed are open on the local host. These include, ports @@ -213,7 +213,7 @@ include, ports 80 for the datafed-gcs container 8512 for arangodb web server interface -Make sure port 80 is not already bound on the host. Also note that the repo server keys -should exist in the keys folder before running the gcs instance. +Make sure port 80 is not already bound on the host. Also note that the repo +server keys should exist in the keys folder before running the gcs instance. From 0ee01562eea8ea5e398a6c2b9db6434e251577c0 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 12:01:37 -0400 Subject: [PATCH 69/84] Address static code analysis for cleanup_globus_files.sh --- compose/cleanup_globus_files.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compose/cleanup_globus_files.sh b/compose/cleanup_globus_files.sh index b6eda3ade..afdc0b472 100755 --- a/compose/cleanup_globus_files.sh +++ b/compose/cleanup_globus_files.sh @@ -13,20 +13,20 @@ then exit 1 fi -. ${PROJECT_ROOT}/compose/.env +. "${PROJECT_ROOT}/compose/.env" export DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH="$DATAFED_HOST_DEPLOYMENT_KEY_PATH" export DATAFED_GLOBUS_CRED_FILE_PATH="$DATAFED_HOST_CRED_FILE_PATH" if [ -f "$DATAFED_HOST_CRED_FILE_PATH" ] then - export GCS_CLI_CLIENT_ID=$(cat ${DATAFED_HOST_CRED_FILE_PATH} | jq -r .client) - export GCS_CLI_CLIENT_SECRET=$(cat ${DATAFED_HOST_CRED_FILE_PATH} | jq -r .secret) + export GCS_CLI_CLIENT_ID=$(cat "${DATAFED_HOST_CRED_FILE_PATH}" | jq -r .client) + export GCS_CLI_CLIENT_SECRET=$(cat "${DATAFED_HOST_CRED_FILE_PATH}" | jq -r .secret) fi if [ -f "$DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH" ] then - export GCS_CLI_ENDPOINT_ID=$(cat ${DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH} | jq -r .client_id) + export GCS_CLI_ENDPOINT_ID=$(cat "${DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH}" | jq -r .client_id) fi sudo globus-connect-server node cleanup From 0b2f49b4d11887c119f282cf4aa00a25dc71c483 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 12:02:56 -0400 Subject: [PATCH 70/84] Remove unused variable --- core/database/foxx/api/user_router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/database/foxx/api/user_router.js b/core/database/foxx/api/user_router.js index 302d625a6..1a75dbb18 100644 --- a/core/database/foxx/api/user_router.js +++ b/core/database/foxx/api/user_router.js @@ -74,7 +74,7 @@ router.get('/create', function(req, res) { throw [g_lib.ERR_AUTHN_FAILED, "Invalid system credentials"]; } - var i, c, + var i, time = Math.floor(Date.now() / 1000), name = req.queryParams.name.trim(), idx = name.lastIndexOf(" "); From 755708970836a62de6836407caca703d2fdfbdba Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 12:11:05 -0400 Subject: [PATCH 71/84] Cleanup entrypoint_authz.sh file --- repository/docker/entrypoint_authz.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index 309a02bd7..2f46877b8 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -13,15 +13,15 @@ export DATAFED_GCS_COLLECTION_ROOT_PATH=/mnt/datafed # This env variables are needed for running globus-connect-server without # logging in -export GCS_CLI_CLIENT_ID=$(cat /opt/datafed/globus/client_cred.json | jq -r .client) -export GCS_CLI_CLIENT_SECRET=$(cat /opt/datafed/globus/client_cred.json | jq -r .secret) -export GCS_CLI_ENDPOINT_ID=$(cat /opt/datafed/globus/deployment-key.json | jq -r .client_id) +export GCS_CLI_CLIENT_ID=$(jq -r .client < /opt/datafed/globus/client_cred.json) +export GCS_CLI_CLIENT_SECRET=$(jq -r .secret < /opt/datafed/globus/client_cred.json) +export GCS_CLI_ENDPOINT_ID=$(jq -r .client_id < /opt/datafed/globus/deployment-key.json) export DEPLOYMENT_KEY_PATH="/opt/datafed/globus/deployment-key.json" # These env variables are for running the gcs entrypoint file -export GLOBUS_CLIENT_ID=$(cat /opt/datafed/globus/client_cred.json | jq -r .client) -export GLOBUS_CLIENT_SECRET=$(cat /opt/datafed/globus/client_cred.json | jq -r .secret) +export GLOBUS_CLIENT_ID=$(jq -r .client < /opt/datafed/globus/client_cred.json) +export GLOBUS_CLIENT_SECRET=$(jq -r .secret < /opt/datafed/globus/client_cred.json) export DEPLOYMENT_KEY=$(cat "$DEPLOYMENT_KEY_PATH" ) if [ -n "$UID" ]; then @@ -32,18 +32,18 @@ fi "${PROJECT_ROOT}/scripts/generate_datafed.sh" -source ${PROJECT_ROOT}/config/datafed.sh +source "${PROJECT_ROOT}/config/datafed.sh" # After datafed.sh has been run created "${PROJECT_ROOT}/scripts/generate_authz_config.sh" # Make sure paths exist -mkdir -p ${DATAFED_INSTALL_PATH}/keys -mkdir -p ${DATAFED_DEFAULT_LOG_PATH} +mkdir -p "${DATAFED_INSTALL_PATH}/keys" +mkdir -p "${DATAFED_DEFAULT_LOG_PATH}" # Copy configuration files cp "$PROJECT_ROOT/config/gsi-authz.conf" /etc/grid-security -cp "$PROJECT_ROOT/config/datafed-authz.cfg" ${DATAFED_INSTALL_PATH}/authz +cp "$PROJECT_ROOT/config/datafed-authz.cfg" "${DATAFED_INSTALL_PATH}/authz" # Run node setup command we have to use the entrypoint file for this because # the globus-connect-server node setup command attempts to use systemctl which @@ -96,9 +96,9 @@ done echo "globus-gridftp-server pid file found!" # Need to wait until the domain name is properly registered -DATAFED_GCS_URL=$(cat /var/lib/globus-connect-server/info.json | jq -r .domain_name) +DATAFED_GCS_URL=$(jq -r .domain_name < /var/lib/globus-connect-server/info.json) -HTTP_CODE=$(${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/curl -s -o /dev/null -w "%{http_code}\n" -I "https://${DATAFED_GCS_URL}/api/info") +HTTP_CODE=$("${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/curl" -s -o /dev/null -w "%{http_code}\n" -I "https://${DATAFED_GCS_URL}/api/info") echo "Waiting for domain name (https://${DATAFED_GCS_URL}) to be registered! Code: $HTTP_CODE" printf "\n" minutes=0 From 5c1dbd0928a1ea9c0292ae30ccc7e1380dd2f413 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 12:52:09 -0400 Subject: [PATCH 72/84] Address static analyzer issues --- compose/build_repo_images_for_compose.sh | 4 +- scripts/dependency_install_functions.sh | 6 +-- scripts/generate_gsi-authz_config.sh | 4 +- scripts/globus/create_guest_collection.py | 57 ++------------------ scripts/globus/globus_cleanup.py | 3 +- scripts/globus/initialize_globus_endpoint.py | 5 +- 6 files changed, 12 insertions(+), 67 deletions(-) diff --git a/compose/build_repo_images_for_compose.sh b/compose/build_repo_images_for_compose.sh index 65cdb967e..3ab2994ce 100755 --- a/compose/build_repo_images_for_compose.sh +++ b/compose/build_repo_images_for_compose.sh @@ -7,10 +7,10 @@ SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/../") . "${PROJECT_ROOT}/scripts/dependency_versions.sh" -cd ${PROJECT_ROOT}/external/globus-connect-server-deploy/docker +cd "${PROJECT_ROOT}/external/globus-connect-server-deploy/docker" git checkout "$DATAFED_GCS_SUBMODULE_VERSION" docker build --progress plain --tag "gcs-ubuntu-base:latest" - < "./docker-files/Dockerfile.ubuntu-20.04" -cd ${PROJECT_ROOT} +cd "${PROJECT_ROOT}" docker build \ -f "${PROJECT_ROOT}/docker/Dockerfile.dependencies" \ "${PROJECT_ROOT}" \ diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index 102eabeb1..5b97d98c3 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -7,10 +7,10 @@ source "${SOURCE}/utils.sh" sudo_command # these are the dependencies to be installed by apt -apt_file_path="${PROJECT_ROOT}/tmp/apt_deps" -pip_file_path="${PROJECT_ROOT}/tmp/pip_deps" +export apt_file_path="${PROJECT_ROOT}/tmp/apt_deps" +export pip_file_path="${PROJECT_ROOT}/tmp/pip_deps" # these are the dependencies to be installed and built via cmake -ext_file_path="${PROJECT_ROOT}/tmp/ext_deps" +export ext_file_path="${PROJECT_ROOT}/tmp/ext_deps" if [ ! -d "${PROJECT_ROOT}/tmp" ]; then mkdir -p "${PROJECT_ROOT}/tmp" diff --git a/scripts/generate_gsi-authz_config.sh b/scripts/generate_gsi-authz_config.sh index 6abab4e6e..3482ce297 100755 --- a/scripts/generate_gsi-authz_config.sh +++ b/scripts/generate_gsi-authz_config.sh @@ -5,8 +5,8 @@ set -ef -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/..) -source ${PROJECT_ROOT}/config/datafed.sh +PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/config/datafed.sh" PATH_TO_CONFIG_DIR=$(realpath "$SOURCE/../config") CONFIG_FILE_NAME="gsi-authz.conf" diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index 5a82814bb..872468a15 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -1,8 +1,6 @@ import globus_sdk -import subprocess +from globus_sdk import AccessTokenAuthorizer import utils -from globus_sdk import AuthClient, AccessTokenAuthorizer -import json import os import sys @@ -56,70 +54,21 @@ if local_username is None: raise Exception("DATAFED_REPO_USER is not defined.") -# client = globus_sdk.NativeAppAuthClient(CLIENT_ID) - -# manage_projects scope to create a project -# view_identities to user information for creating GCS server -# client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) -# -# authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) -# print("Please go to this URL and login: \n", authorize_url) -# auth_code = input("Please enter the authorization code: ") -# -# token_response = client.oauth2_exchange_code_for_tokens(auth_code) -## Extract the token -# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -## auth_client_refresh_token -# ac_rt = AuthClient(authorizer=rt_authorizer) -# -# userinfo = ac_rt.oauth2_userinfo() -## Will get the primary email and id -# identity_id = userinfo["sub"] -# email = userinfo["email"] -# username = userinfo["preferred_username"] -# organization = userinfo["identity_provider_display_name"] - if client_id is None: - client_id = getClientIdFromCredFile(CRED_FILE_PATH) + client_id = utils.getClientIdFromCredFile(CRED_FILE_PATH) if client_secret is None: - client_secret = getCredentialFromFile(CRED_FILE_PATH, client_id) + client_secret = utils.getCredentialFromFile(CRED_FILE_PATH, client_id) client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) scopes = "openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) -# cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, -# scopes) - -# token_response = client.oauth2_client_credentials_tokens() - -# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -# the useful values that you want at the end of this -# globus_auth_data = token_response.by_resource_server["auth.globus.org"] -# globus_transfer_data = -# token_response.by_resource_server["transfer.api.globus.org"] -# globus_auth_token = globus_auth_data["access_token"] -# globus_transfer_token = globus_transfer_data["access_token"] gcs_client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) - -import globus_sdk from globus_sdk import scopes -# client credentials -# This client identity must have the needed permissions to create a guest -# collection on the mapped collection, and a valid mapping to a local account -# on the storage gateway that matches the local_username -# If using user tokens, the user must be the one with the correct permissions -# and identity mapping. -# client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" -# client_secret = "*redacted*" -# local_username = "datafed" - # The scope the client will need, note that primary scope is for the endpoint, # but it has a dependency on the mapped collection's data_access scope diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 502c57733..0583f916b 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,8 +1,7 @@ import globus_sdk -from globus_sdk import AuthClient, GroupsClient, AccessTokenAuthorizer +from globus_sdk import AuthClient, GroupsClient from globus_sdk.scopes import GroupsScopes import subprocess -import json import sys import os import utils diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index 0f0e79334..b6fe6887a 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -1,14 +1,11 @@ import globus_sdk -import subprocess import utils -from globus_sdk import AuthClient, GroupsClient, AccessTokenAuthorizer +from globus_sdk import AuthClient, GroupsClient from globus_sdk.scopes import GroupsScopes -import json import os import sys - # Hard coded Native Client ID CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" From 8592941b4d26fbcc5ddcdd2eb1dc9b8921142443 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 14:00:27 -0400 Subject: [PATCH 73/84] Fix typo --- core/server/DatabaseAPI.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/server/DatabaseAPI.cpp b/core/server/DatabaseAPI.cpp index 2c81f8a69..5f3991d48 100644 --- a/core/server/DatabaseAPI.cpp +++ b/core/server/DatabaseAPI.cpp @@ -2507,7 +2507,7 @@ void DatabaseAPI::repoAuthz(const Auth::RepoAuthzRequest &a_request, (void)a_reply; Value result; - DL_INFO(log_context, "authz/gridftp repo: " << a_reply.repo() << " file " + DL_INFO(log_context, "authz/gridftp repo: " << a_request.repo() << " file " << a_request.file() << " act " << a_request.action()); dbGet("authz/gridftp", From e8bf07ee958ac73c5c58ba77894481a7c0f5df21 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 14:24:53 -0400 Subject: [PATCH 74/84] Fixed static code analysis issues --- compose/cleanup_globus_files.sh | 6 ++--- compose/generate_globus_files.sh | 4 ++-- repository/docker/entrypoint_authz.sh | 2 +- repository/docker/entrypoint_repo.sh | 4 ++-- .../gridftp/globus5/authz/source/libauthz.c | 6 ++--- scripts/dependency_install_functions.sh | 2 +- scripts/generate_authz_config.sh | 2 +- scripts/globus/create_guest_collection.py | 2 -- scripts/globus/globus_cleanup.py | 2 +- scripts/globus/initialize_globus_endpoint.py | 2 +- scripts/globus/utils.py | 24 ++++++------------- web/docker/entrypoint.sh | 4 ++-- 12 files changed, 24 insertions(+), 36 deletions(-) diff --git a/compose/cleanup_globus_files.sh b/compose/cleanup_globus_files.sh index afdc0b472..d4ed2a72b 100755 --- a/compose/cleanup_globus_files.sh +++ b/compose/cleanup_globus_files.sh @@ -20,13 +20,13 @@ export DATAFED_GLOBUS_CRED_FILE_PATH="$DATAFED_HOST_CRED_FILE_PATH" if [ -f "$DATAFED_HOST_CRED_FILE_PATH" ] then - export GCS_CLI_CLIENT_ID=$(cat "${DATAFED_HOST_CRED_FILE_PATH}" | jq -r .client) - export GCS_CLI_CLIENT_SECRET=$(cat "${DATAFED_HOST_CRED_FILE_PATH}" | jq -r .secret) + export GCS_CLI_CLIENT_ID=$(jq -r .client < "${DATAFED_HOST_CRED_FILE_PATH}") + export GCS_CLI_CLIENT_SECRET=$(jq -r .secret < "${DATAFED_HOST_CRED_FILE_PATH}") fi if [ -f "$DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH" ] then - export GCS_CLI_ENDPOINT_ID=$(cat "${DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH}" | jq -r .client_id) + export GCS_CLI_ENDPOINT_ID=$(jq -r .client_id < "${DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH}") fi sudo globus-connect-server node cleanup diff --git a/compose/generate_globus_files.sh b/compose/generate_globus_files.sh index 0ed182846..82f5dba89 100755 --- a/compose/generate_globus_files.sh +++ b/compose/generate_globus_files.sh @@ -19,11 +19,11 @@ then mkdir -p "$local_DATAFED_GLOBUS_KEY_DIR" fi -. ${PROJECT_ROOT}/compose/.env +. "${PROJECT_ROOT}/compose/.env" DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH="$DATAFED_HOST_DEPLOYMENT_KEY_PATH" \ DATAFED_GLOBUS_CRED_FILE_PATH="$DATAFED_HOST_CRED_FILE_PATH" \ DATAFED_GLOBUS_CONTROL_PORT="$DATAFED_GLOBUS_CONTROL_PORT" \ DATAFED_GLOBUS_SUBSCRIPTION="$DATAFED_GLOBUS_SUBSCRIPTION" \ DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \ - python3 ${PROJECT_ROOT}/scripts/globus/initialize_globus_endpoint.py + python3 "${PROJECT_ROOT}/scripts/globus/initialize_globus_endpoint.py" diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index 2f46877b8..af23b3cf8 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -114,7 +114,7 @@ do msg='.' sleep 5 - HTTP_CODE=$(${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/curl -s -o /dev/null -w "%{http_code}\n" -I "https://${DATAFED_GCS_URL}/api/info") + HTTP_CODE=$("${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/curl" -s -o /dev/null -w "%{http_code}\n" -I "https://${DATAFED_GCS_URL}/api/info") if [ "$HTTP_CODE" == "200" ] then break diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index 51bd23c04..df730abe6 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -12,8 +12,8 @@ PROJECT_ROOT=$(realpath "${SOURCE}/../..") # This is only part of the solution the other part is running chown if [ -n "$UID" ]; then echo "Switching datafed user to UID: ${UID}" - usermod -u $UID datafed - chown -R datafed:root ${PROJECT_ROOT} + usermod -u "$UID" datafed + chown -R datafed:root "${PROJECT_ROOT}" chown -R datafed:root /opt/datafed/repo/ chown -R datafed:root /mnt/datafed fi diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index 482921e73..629d64178 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -222,7 +222,7 @@ bool setConfigVal(const char *a_label, char *a_dest, char *a_src, if (len > a_max_len) { AUTHZ_LOG_ERROR( - "DataFed - '%s' value too long in authz config file (max %lu).\n", + "DataFed - '%s' value too long in authz config file (max %zu).\n", a_label, a_max_len); return true; } @@ -483,6 +483,8 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", callout_ids1); + AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_MAPPED_USERNAME: %s\n", + callout_username_mapped1); AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", callout_id_mapped1); AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", @@ -572,8 +574,6 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { } char *callout_ids = getenv("GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS"); - char *callout_username_mapped = - getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); char *callout_id_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index 5b97d98c3..8020bd770 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -495,7 +495,6 @@ install_libcurl() { mkdir -p "${PROJECT_ROOT}/external/libcurl" tar -xf "curl-${DATAFED_LIBCURL}.tar.gz" -C "${PROJECT_ROOT}/external/libcurl" cd "${PROJECT_ROOT}/external/libcurl/curl-${DATAFED_LIBCURL}" - PKG_CONFIG_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/pkgconfig" \ # Making third party features and dependencies explicit # OpenSSL is needed for HTTPS encryption @@ -503,6 +502,7 @@ install_libcurl() { # GNUTLS - HTTPS support session management certificate verification etc # NOTE: NSS - Network Security Services for HTTP support is deprecated # NOTE: metalink - is no longer supported and not a valid argument + PKG_CONFIG_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/pkgconfig" \ ./configure --with-ssl="${DATAFED_DEPENDENCIES_INSTALL_PATH}" --with-gnutls --with-zlib \ --enable-file --disable-shared \ --disable-ldap --disable-ldaps --disable-rtsp --disable-dict \ diff --git a/scripts/generate_authz_config.sh b/scripts/generate_authz_config.sh index 374e81974..e7e5f25ab 100755 --- a/scripts/generate_authz_config.sh +++ b/scripts/generate_authz_config.sh @@ -60,7 +60,7 @@ else local_DATAFED_GCS_COLLECTION_ROOT_PATH=$(printenv DATAFED_GCS_COLLECTION_ROOT_PATH) fi -if [ -z $DATAFED_GLOBUS_REPO_USER ] +if [ -z "${DATAFED_GLOBUS_REPO_USER}" ] then local_DATAFED_AUTHZ_USER="$DATAFED_GLOBUS_REPO_USER" else diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index 872468a15..400379992 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -1,8 +1,6 @@ import globus_sdk -from globus_sdk import AccessTokenAuthorizer import utils import os -import sys # The Globus project the GCS endpoint will be created in diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 0583f916b..4d01a20ac 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,5 +1,5 @@ import globus_sdk -from globus_sdk import AuthClient, GroupsClient +from globus_sdk import AuthClient, GroupsClient from globus_sdk.scopes import GroupsScopes import subprocess import sys diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index b6fe6887a..db45be7a1 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -1,6 +1,6 @@ import globus_sdk import utils -from globus_sdk import AuthClient, GroupsClient +from globus_sdk import AuthClient, GroupsClient from globus_sdk.scopes import GroupsScopes import os diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index d4588b51c..226546292 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -1,6 +1,4 @@ -import globus_sdk import subprocess -from globus_sdk import AuthClient, AccessTokenAuthorizer import json import os import sys @@ -145,7 +143,7 @@ def validFile(file_name): def getCredentialFromFile(cred_file_name, cred_id): # Check to see if the local secret is the same id and not just the same # name - cred_exists_locally, cred_empty = validFile(cred_file_name) + _, cred_empty = validFile(cred_file_name) if cred_empty is False: with open(cred_file_name, "r") as f: loaded_data = json.load(f) @@ -157,7 +155,7 @@ def getCredentialFromFile(cred_file_name, cred_id): def getClientIdFromCredFile(cred_file_name): # Check to see if the local secret is the same id and not just the same # name - cred_exists_locally, cred_empty = validFile(cred_file_name) + _, cred_empty = validFile(cred_file_name) if cred_empty is False: with open(cred_file_name, "r") as f: loaded_data = json.load(f) @@ -168,7 +166,7 @@ def getClientIdFromCredFile(cred_file_name): def getEndpointIdFromFile(deployment_key_file_path): # Check to see if the local secret is the same id and not just the same # name - exists_locally, empty = validFile(deployment_key_file_path) + _, empty = validFile(deployment_key_file_path) if empty is False: with open(deployment_key_file_path, "r") as f: loaded_data = json.load(f) @@ -230,12 +228,6 @@ def createClient(auth_client, client_name, project_id, cred_name, cred_file): cred_id = getCredentialID(auth_client, client_id, cred_name) - cred_exists_on_cloud = False - if cred_id: - cred_exists_on_cloud = True - - cred_exists_locally, cred_empty = validFile(cred_file) - client_secret = getClientSecret( auth_client, client_id, cred_name, cred_id, cred_file ) @@ -285,11 +277,10 @@ def isGCSDeploymentKeyValid(auth_client, project_id, endpoint_name, gcs_id): else: # Found a globus_connect_server but did not find local deployment # key - if deployment_key_empty: - print( - "Found globus_connect_server already registered but did" - " not find deployment key locally." - ) + print( + "Found globus_connect_server already registered but did" + " not find deployment key locally." + ) return False @@ -314,7 +305,6 @@ def createGCSEndpoint( userinfo, ): - identity_id = userinfo["sub"] email = userinfo["email"] username = userinfo["preferred_username"] organization = userinfo["identity_provider_display_name"] diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 43f33b799..4c538bd2f 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -7,8 +7,8 @@ if [ -n "$UID" ]; then usermod -u "$UID" datafed fi -chown -R datafed:root ${DATAFED_INSTALL_PATH}/web -chown -R datafed:root ${BUILD_DIR} +chown -R datafed:root "${DATAFED_INSTALL_PATH}/web" +chown -R datafed:root "${BUILD_DIR}" SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") From 3449bbfab9db150895b8c93f7b8d800161120d0f Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 22:47:53 -0400 Subject: [PATCH 75/84] Cleanup --- CMakeLists.txt | 10 ++- cmake/curl_version.cpp | 5 ++ cmake/zlib_version.cpp | 5 ++ common/include/common/CommunicatorFactory.hpp | 2 +- common/include/common/DynaLog.hpp | 11 +++ common/include/common/GSSAPI_Utils.hpp | 6 +- common/include/common/ServerFactory.hpp | 2 +- common/include/common/SmartTokenizer.hpp | 4 +- common/include/common/libjson.hpp | 18 ++--- common/source/Util.cpp | 6 +- .../communicators/ZeroMQCommunicator.hpp | 2 +- common/source/servers/ProxyBasicZMQ.cpp | 6 +- .../security/tcp_secure/test_tcp_secure.sh | 2 +- common/tests/unit/test_DynaLog.cpp | 9 +++ common/tests/unit/test_ProtoBufMap.cpp | 2 + common/tests/unit/test_SocketFactory.cpp | 2 +- compose/README.md | 13 ++-- compose/cleanup_globus_files.sh | 2 - compose/compose_repo.yml | 11 +-- compose/generate_env.sh | 2 +- core/database/backup/datafed-backup.sh | 4 +- core/database/foxx/api/user_router.js | 3 + core/database/tests/test_setup.sh | 18 ++--- core/database/tests/test_teardown.sh | 8 +- core/server/AuthenticationManager.cpp | 6 +- core/server/AuthenticationManager.hpp | 4 +- core/server/DatabaseAPI.cpp | 2 +- docker/Dockerfile.foxx | 34 ++++---- repository/docker/Dockerfile.gcs | 78 +++++++++---------- repository/docker/entrypoint_authz.sh | 3 +- repository/docker/entrypoint_repo.sh | 3 +- repository/server/RepoServer.cpp | 8 ++ scripts/dependency_install_functions.sh | 24 ------ scripts/generate_datafed.sh | 13 +++- scripts/globus/globus_cleanup.py | 2 +- scripts/globus/setup_globus.sh | 17 ++-- scripts/globus/utils.py | 22 ------ 37 files changed, 187 insertions(+), 182 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b7bb3b6f8..5e1ca2e68 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,9 +26,13 @@ OPTION(BUILD_PYTHON_CLIENT "Build python client" TRUE) OPTION(BUILD_TESTS "Build Tests" TRUE) OPTION(BUILD_WEB_SERVER "Build DataFed Web Server" TRUE) OPTION(ENABLE_UNIT_TESTS "Enable unit tests" TRUE) -OPTION(BUILD_SHARED_LIBS "Build using shared libraries. By default this is off. -With the exception of the libdatafed-authz which must be a shared library. This -includes also attempting to link with shared libraries instead of static ones" OFF) +OPTION(BUILD_SHARED_LIBS "By default DataFed tries to build static libraries +with the exception of libdatafed-authz which must always be a shared library, +it will also try to link with as many static libraries as possible. However, +building with static depencies is not completely possible because some system +libraries must be shared libraries for DataFed to be interoperable. If this +setting is turned on DataFed will build it's libraries as shared and try to +link to shared libraries." OFF) set(INSTALL_REPO_SERVER ${BUILD_REPO_SERVER}) set(INSTALL_AUTHZ ${BUILD_AUTHZ}) diff --git a/cmake/curl_version.cpp b/cmake/curl_version.cpp index 3fe6fa2dd..1846a2db4 100644 --- a/cmake/curl_version.cpp +++ b/cmake/curl_version.cpp @@ -1,6 +1,11 @@ #include #include +/** + * This script is used to show what vesion of curl is being used with the + * rest of the build process and will print the version number of the curl + * library. + **/ int main() { std::cout << curl_version() << std::endl; return 0; diff --git a/cmake/zlib_version.cpp b/cmake/zlib_version.cpp index caf2682c5..9544bd0a2 100644 --- a/cmake/zlib_version.cpp +++ b/cmake/zlib_version.cpp @@ -1,6 +1,11 @@ #include #include +/** + * This little file is used to make sure that we are compiling with the version + * of a library that we think we are. This one when run will print the zlib + * version numbers. + **/ int main() { std::cout << zlibVersion() << std::endl; return 0; diff --git a/common/include/common/CommunicatorFactory.hpp b/common/include/common/CommunicatorFactory.hpp index 6dac56f24..7c4a607b5 100644 --- a/common/include/common/CommunicatorFactory.hpp +++ b/common/include/common/CommunicatorFactory.hpp @@ -18,7 +18,7 @@ class CommunicatorFactory { LogContext m_log_context; public: - CommunicatorFactory(LogContext log_context) : m_log_context(log_context){}; + CommunicatorFactory(const LogContext & log_context) : m_log_context(log_context){}; std::unique_ptr create(const SocketOptions &socket_options, const ICredentials &credentials, diff --git a/common/include/common/DynaLog.hpp b/common/include/common/DynaLog.hpp index 38f76f513..0c85f65e5 100644 --- a/common/include/common/DynaLog.hpp +++ b/common/include/common/DynaLog.hpp @@ -73,6 +73,17 @@ namespace SDMS { +/** + * Here unsigned int is used so that we can compare the numeric values when + * choosing whether to print a log message. + * + * CRITICAL = 0 + * ERROR = 1 + * WARNING = 2 + * INFO = 3 + * DEBUG = 4 + * TRACE = 5 + **/ enum class LogLevel : unsigned int { CRITICAL, ERROR, diff --git a/common/include/common/GSSAPI_Utils.hpp b/common/include/common/GSSAPI_Utils.hpp index 331e9a15e..f8e654a89 100644 --- a/common/include/common/GSSAPI_Utils.hpp +++ b/common/include/common/GSSAPI_Utils.hpp @@ -21,21 +21,21 @@ class gssString { m_gss_buf.length = 0; } - gssString(const std::string &a_src) { + explicit gssString(const std::string &a_src) { m_gss_buf.value = 0; m_gss_buf.length = 0; set(a_src); } - gssString(const char *a_src) { + explicit gssString(const char *a_src) { m_gss_buf.value = 0; m_gss_buf.length = 0; set(a_src); } - gssString(gss_name_t a_src) { + explicit gssString(gss_name_t a_src) { m_gss_buf.value = 0; m_gss_buf.length = 0; diff --git a/common/include/common/ServerFactory.hpp b/common/include/common/ServerFactory.hpp index f469377e5..a16f736a3 100644 --- a/common/include/common/ServerFactory.hpp +++ b/common/include/common/ServerFactory.hpp @@ -30,7 +30,7 @@ class ServerFactory { LogContext m_log_context; public: - ServerFactory(LogContext log_context) : m_log_context(log_context){}; + ServerFactory(const LogContext & log_context) : m_log_context(log_context){}; std::unique_ptr create( ServerType server_type, diff --git a/common/include/common/SmartTokenizer.hpp b/common/include/common/SmartTokenizer.hpp index 01de3f22c..8608a3949 100644 --- a/common/include/common/SmartTokenizer.hpp +++ b/common/include/common/SmartTokenizer.hpp @@ -12,7 +12,7 @@ template class SmartTokenizer { SmartTokenizer() {} - SmartTokenizer(const std::string &a_input_str) { + explicit SmartTokenizer(const std::string &a_input_str) { m_buffer.reserve(a_input_str.size()); _parse(a_input_str.c_str(), a_input_str.size()); } @@ -133,4 +133,4 @@ template class SmartTokenizer { std::vector m_tokens; }; -#endif \ No newline at end of file +#endif diff --git a/common/include/common/libjson.hpp b/common/include/common/libjson.hpp index 4c353a663..b12a30f79 100644 --- a/common/include/common/libjson.hpp +++ b/common/include/common/libjson.hpp @@ -389,17 +389,17 @@ class Value { Value() : m_type(VT_NULL), m_value({0}) {} - Value(bool a_value) : m_type(VT_BOOL) { m_value.b = a_value; } + explicit Value(bool a_value) : m_type(VT_BOOL) { m_value.b = a_value; } - Value(double a_value) : m_type(VT_NUMBER) { m_value.n = a_value; } + explicit Value(double a_value) : m_type(VT_NUMBER) { m_value.n = a_value; } - Value(int a_value) : m_type(VT_NUMBER) { m_value.n = a_value; } + explicit Value(int a_value) : m_type(VT_NUMBER) { m_value.n = a_value; } - Value(const std::string &a_value) : m_type(VT_STRING) { + explicit Value(const std::string &a_value) : m_type(VT_STRING) { m_value.s = new String(a_value); } - Value(const char *a_value) : m_type(VT_STRING) { + explicit Value(const char *a_value) : m_type(VT_STRING) { m_value.s = new String(a_value); } @@ -410,7 +410,7 @@ class Value { a_source.m_value.o = 0; } - Value(ValueType a_type) : m_type(a_type) { + explicit Value(ValueType a_type) : m_type(a_type) { if (m_type == VT_OBJECT) { m_value.o = new Object(); } else if (m_type == VT_ARRAY) { @@ -837,7 +837,7 @@ class Value { switch (m_type) { case VT_OBJECT: a_buffer.append("{"); - for (ObjectIter i = m_value.o->begin(); i != m_value.o->end(); i++) { + for (ObjectIter i = m_value.o->begin(); i != m_value.o->end(); ++i) { if (i != m_value.o->begin()) a_buffer.append(",\""); else @@ -851,7 +851,7 @@ class Value { break; case VT_ARRAY: a_buffer.append("["); - for (ArrayIter i = m_value.a->begin(); i != m_value.a->end(); i++) { + for (ArrayIter i = m_value.a->begin(); i != m_value.a->end(); ++i) { if (i != m_value.a->begin()) a_buffer.append(","); i->toStringRecurse(a_buffer); @@ -883,7 +883,7 @@ class Value { a_buffer.append("\""); - for (c = a_value.begin(); c != a_value.end(); c++) { + for (c = a_value.begin(); c != a_value.end(); ++c) { if (*c < 0x20) { a_buffer.append(a, c); a = c + 1; diff --git a/common/source/Util.cpp b/common/source/Util.cpp index 0ed047fc3..1170e265d 100644 --- a/common/source/Util.cpp +++ b/common/source/Util.cpp @@ -79,11 +79,11 @@ void hexDump(const char *a_buffer, const char *a_buffer_end, ostream &a_out) { const unsigned char *e = (unsigned char *)a_buffer_end; bool done = false; - int l = 0, i = 0; + int l = 0; while (!done) { a_out << setw(4) << dec << l << ": "; - for (i = 0; i < 16; ++i) { + for (int i = 0; i < 16; ++i) { if (i == 8) a_out << " "; @@ -102,7 +102,7 @@ void hexDump(const char *a_buffer, const char *a_buffer_end, ostream &a_out) { a_out << " "; - for (i = 0; i < 16; ++i) { + for (int i = 0; i < 16; ++i) { if (p + i != e) { if (isprint(*(p + i))) a_out << *(p + i); diff --git a/common/source/communicators/ZeroMQCommunicator.hpp b/common/source/communicators/ZeroMQCommunicator.hpp index 3dd063ae0..af3a94ae5 100644 --- a/common/source/communicators/ZeroMQCommunicator.hpp +++ b/common/source/communicators/ZeroMQCommunicator.hpp @@ -43,7 +43,7 @@ class ZeroMQCommunicator : public ICommunicator { public: /** To be used by children*/ - ZeroMQCommunicator(const LogContext &log_context) + explicit ZeroMQCommunicator(const LogContext &log_context) : m_log_context(log_context){}; ZeroMQCommunicator(const SocketOptions &socket_options, diff --git a/common/source/servers/ProxyBasicZMQ.cpp b/common/source/servers/ProxyBasicZMQ.cpp index 7b30bf34c..785b81f62 100644 --- a/common/source/servers/ProxyBasicZMQ.cpp +++ b/common/source/servers/ProxyBasicZMQ.cpp @@ -19,8 +19,8 @@ namespace SDMS { -std::string sanitize(std::string val, const std::string pattern, - const std::string replacement) { +std::string sanitize(std::string val, const std::string & pattern, + const std::string & replacement) { for (auto at = val.find(pattern, 0); at != std::string::npos; at = val.find(pattern, at + replacement.length())) { @@ -186,7 +186,7 @@ void ProxyBasicZMQ::run() { * loop. **/ auto terminate_call = [](std::chrono::duration duration, - const std::string address, int thread_id, + const std::string & address, int thread_id, LogContext log_context) { log_context.thread_name += "-terminate_after_timeout"; log_context.thread_id = thread_id; diff --git a/common/tests/security/tcp_secure/test_tcp_secure.sh b/common/tests/security/tcp_secure/test_tcp_secure.sh index 0c385a090..e0e1b6053 100755 --- a/common/tests/security/tcp_secure/test_tcp_secure.sh +++ b/common/tests/security/tcp_secure/test_tcp_secure.sh @@ -34,7 +34,7 @@ echo "TIMEOUT: ${TIMEOUT_CMD}" echo "MAX_TEST_TIME: ${MAX_TEST_TIME_SEC}" # Grab the first 30 packets sent on the loop back interface (127.0.0.1) and port 7515 -match=$( ${TIMEOUT_CMD} ${MAX_TEST_TIME_SEC} ${TCPDUMP_CMD} -vvv -A port 7515 -i lo | grep token) +match=$( "${TIMEOUT_CMD}" "${MAX_TEST_TIME_SEC}" "${TCPDUMP_CMD}" -vvv -A port 7515 -i lo | grep token) echo "Content of grep ${match}" # If '.magic_token' is returned from the network sniffer then we know that diff --git a/common/tests/unit/test_DynaLog.cpp b/common/tests/unit/test_DynaLog.cpp index d89a960d5..dc7494483 100644 --- a/common/tests/unit/test_DynaLog.cpp +++ b/common/tests/unit/test_DynaLog.cpp @@ -16,6 +16,15 @@ using namespace SDMS; BOOST_AUTO_TEST_SUITE(LogTest) +BOOST_AUTO_TEST_CASE(testing_Enum) { + BOOST_CHECK(static_cast(SDMS::LogLevel::TRACE) == 5); + BOOST_CHECK(static_cast(SDMS::LogLevel::DEBUG) == 4); + BOOST_CHECK(static_cast(SDMS::LogLevel::INFO) == 3); + BOOST_CHECK(static_cast(SDMS::LogLevel::WARNING) == 2); + BOOST_CHECK(static_cast(SDMS::LogLevel::ERROR) == 1); + BOOST_CHECK(static_cast(SDMS::LogLevel::CRITICAL) == 0); +} + BOOST_AUTO_TEST_CASE(testing_LogOutput) { std::string file_name = "./log_output_test1.txt"; diff --git a/common/tests/unit/test_ProtoBufMap.cpp b/common/tests/unit/test_ProtoBufMap.cpp index 79d503a1d..b780d8b2c 100644 --- a/common/tests/unit/test_ProtoBufMap.cpp +++ b/common/tests/unit/test_ProtoBufMap.cpp @@ -22,7 +22,9 @@ BOOST_AUTO_TEST_CASE(testing_ProtoBufFactory_ProtocolID) { uint8_t proto_id = proto_map.getProtocolID(MessageProtocol::GOOGLE_ANONONYMOUS); + BOOST_CHECK(proto_id == 1); proto_id = proto_map.getProtocolID(MessageProtocol::GOOGLE_AUTHORIZED); + BOOST_CHECK(proto_id == 2); } BOOST_AUTO_TEST_CASE(testing_ProtoBufFactory) { diff --git a/common/tests/unit/test_SocketFactory.cpp b/common/tests/unit/test_SocketFactory.cpp index c8147f71e..f7f333883 100644 --- a/common/tests/unit/test_SocketFactory.cpp +++ b/common/tests/unit/test_SocketFactory.cpp @@ -29,7 +29,7 @@ BOOST_AUTO_TEST_SUITE(SocketFactoryTest) class DummyCredential : public ICredentials { public: - DummyCredential(const std::string &pub_key) : m_pub_key(pub_key){}; + explicit DummyCredential(const std::string &pub_key) : m_pub_key(pub_key){}; private: std::string m_pub_key = ""; diff --git a/compose/README.md b/compose/README.md index e508e796d..8337b51f8 100644 --- a/compose/README.md +++ b/compose/README.md @@ -1,6 +1,6 @@ # Compose Dev environment -The compos Dev environment is split into two different compose files. The +The compose Dev environment is split into two different compose files. The "core metadata services" which comprise the web server, core server and database and the "repo services" which comprise Globus Connect Server running with the authz library and the DataFed repo service. @@ -102,7 +102,8 @@ and teardown step. 4. Building the images 5. Running the compose file 6. Bringing down the compose file. -7. Running the cleanup_globus_files.sh if you +7. Running the cleanup_globus_files.sh if you want to remove the deployment key + and start completely from scratch. ### 1. Generating .env configuration varaibles for the Repo Services @@ -118,7 +119,7 @@ The .env file will be created in the DataFed/compose folder and will be hidden. The .env file variables can be changed at this point to your configuration. NOTE the .env file will be read verbatim by compose including any spaces or -"#" comments so do not includ anything but the exact text that needs to be +"#" comments so do not include anything but the exact text that needs to be included in the variables. ### 3. Globus configuration @@ -145,10 +146,12 @@ source ./unset_env.sh docker compose -f ./compose_repo.yml up ``` +Be aware, the 'source' is to apply changes to the environment of your current +terminal session. + NOTE The unset_env.sh script is to make sure you are not accidentially overwriting what is in the .env with your local shell env. You can check the -configuration before hand by running. Be aware, the 'source' is to apply -changes to your current terminal session. +configuration before hand by running. ```bash docker compose -f compose_repo.yml config diff --git a/compose/cleanup_globus_files.sh b/compose/cleanup_globus_files.sh index d4ed2a72b..f262cef05 100755 --- a/compose/cleanup_globus_files.sh +++ b/compose/cleanup_globus_files.sh @@ -33,5 +33,3 @@ sudo globus-connect-server node cleanup DATAFED_GCS_ROOT_NAME="$DATAFED_GCS_ROOT_NAME" \ python3 "${PROJECT_ROOT}/scripts/globus/globus_cleanup.py" - -#rm -rf "${PROJECT_ROOT}/compose/globus" diff --git a/compose/compose_repo.yml b/compose/compose_repo.yml index 3e4cf7b8a..3527eefca 100644 --- a/compose/compose_repo.yml +++ b/compose/compose_repo.yml @@ -17,7 +17,6 @@ services: DATAFED_GCS_COLLECTION_ROOT_PATH: "${DATAFED_GCS_COLLECTION_ROOT_PATH}" UID: "${DATAFED_UID}" HOST_HOSTNAME: "localhost" - network_mode: host image: datafed-repo:latest volumes: - ./keys:/opt/datafed/keys @@ -26,7 +25,9 @@ services: ports: - 9000:9000 # Communication core server -# Needs host port 80 +# Needs host port 80 for apache +# Needs ports 50000 - 51000 for GridFTP +# Needs port 443 for control port datafed-gcs: environment: DATAFED_ZEROMQ_SESSION_SECRET: "${DATAFED_ZEROMQ_SESSION_SECRET}" @@ -51,9 +52,3 @@ services: - ./globus:/opt/datafed/globus - ./logs:${DATAFED_CONTAINER_LOG_PATH} - ${DATAFED_HOST_COLLECTION_MOUNT}:${DATAFED_GCS_COLLECTION_ROOT_PATH} - -# External true indicates that the network is created by a -# separate compose instance -#networks: -# datafed-core-secure-api: -# external: true diff --git a/compose/generate_env.sh b/compose/generate_env.sh index 549671a47..7e608b6d0 100755 --- a/compose/generate_env.sh +++ b/compose/generate_env.sh @@ -199,7 +199,7 @@ while IFS='=' read -r key value; do # Check if the line contains the '=' sign if [ -n "$value" ]; then # Print the content before the '=' sign - echo "unset $key" >> "${unset_env_file_name}" + echo "unset $key" >> "${unset_env_file_name}" fi done < ".env" diff --git a/core/database/backup/datafed-backup.sh b/core/database/backup/datafed-backup.sh index 50e7e04f3..2d8a3276d 100755 --- a/core/database/backup/datafed-backup.sh +++ b/core/database/backup/datafed-backup.sh @@ -12,10 +12,10 @@ systemctl stop arangodb3.service backup_file=DataFed_DB_Backup_$(date +"%Y_%m_%d").tar.gz # Tar contents of arangodb directory without full path -tar -C /var/lib/arangodb3 -cvzf ${backup_file} . +tar -C /var/lib/arangodb3 -cvzf "${backup_file}" . # Move backup file to storage location -mv ${backup_file} /data/backups +mv "${backup_file}" /data/backups # Restart DataFed services systemctl start arangodb3.service diff --git a/core/database/foxx/api/user_router.js b/core/database/foxx/api/user_router.js index 1a75dbb18..111e379a4 100644 --- a/core/database/foxx/api/user_router.js +++ b/core/database/foxx/api/user_router.js @@ -86,6 +86,9 @@ router.get('/create', function(req, res) { fname = name.substr(0, idx).trim(); var is_admin = req.queryParams.is_admin; + // It is assumed that if this is the first user to ever log + // into the database they are by default made the admin. + // This will simplify the setup process. if ( g_db.u.count() === 0 ) { is_admin = true; } diff --git a/core/database/tests/test_setup.sh b/core/database/tests/test_setup.sh index 5aecb1801..fbb85a3c6 100755 --- a/core/database/tests/test_setup.sh +++ b/core/database/tests/test_setup.sh @@ -6,8 +6,8 @@ set -f -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../../../) -source ${PROJECT_ROOT}/config/datafed.sh +PROJECT_ROOT=$(realpath "${SOURCE}/../../../") +source "${PROJECT_ROOT}/config/datafed.sh" Help() { @@ -123,11 +123,11 @@ if [[ "$output" =~ .*"sdms".* ]]; then echo "SDMS already exists do nothing" else echo "Creating SDMS" - arangosh --server.password ${local_DATAFED_DATABASE_PASSWORD} --server.username ${local_DATABASE_USER} --javascript.execute ${PROJECT_ROOT}/core/database/foxx/db_create.js + arangosh --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute "${PROJECT_ROOT}/core/database/foxx/db_create.js" # Give time for the database to be created sleep 2 - arangosh --server.password ${local_DATAFED_DATABASE_PASSWORD} --server.username ${local_DATABASE_USER} --javascript.execute-string 'db._useDatabase("sdms"); db.config.insert({"_key": "msg_daily", "msg" : "DataFed servers will be off-line for regular maintenance every Sunday night from 11:45 pm until 12:15 am EST Monday morning."}, {overwrite: true});' - arangosh --server.password ${local_DATAFED_DATABASE_PASSWORD} --server.username ${local_DATABASE_USER} --javascript.execute-string "db._useDatabase(\"sdms\"); db.config.insert({ \"_key\": \"system\", \"_id\": \"config/system\", \"secret\": \"${local_DATAFED_ZEROMQ_SYSTEM_SECRET}\"}, {overwrite: true } );" + arangosh --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute-string 'db._useDatabase("sdms"); db.config.insert({"_key": "msg_daily", "msg" : "DataFed servers will be off-line for regular maintenance every Sunday night from 11:45 pm until 12:15 am EST Monday morning."}, {overwrite: true});' + arangosh --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute-string "db._useDatabase(\"sdms\"); db.config.insert({ \"_key\": \"system\", \"_id\": \"config/system\", \"secret\": \"${local_DATAFED_ZEROMQ_SYSTEM_SECRET}\"}, {overwrite: true } );" fi # There are apparently 3 different ways to deploy Foxx microservices, @@ -151,16 +151,16 @@ NODE_VERSION="v14.21.3" export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm -nvm install $NODE_VERSION -nvm use $NODE_VERSION +nvm install "$NODE_VERSION" +nvm use "$NODE_VERSION" PATH_TO_PASSWD_FILE=${SOURCE}/database_temp.password # Install foxx service node module $NVM_DIR/nvm-exec npm install --global foxx-cli -echo "$local_DATAFED_DATABASE_PASSWORD" > ${PATH_TO_PASSWD_FILE} +echo "$local_DATAFED_DATABASE_PASSWORD" > "${PATH_TO_PASSWD_FILE}" # Check if database foxx services have already been installed -existing_services=$(foxx list -a -u $local_DATABASE_USER -p ${PATH_TO_PASSWD_FILE} --database $local_DATABASE_NAME) +existing_services=$(foxx list -a -u "$local_DATABASE_USER" -p "${PATH_TO_PASSWD_FILE}" --database "$local_DATABASE_NAME") echo "existing services ${existing_services}" if [[ "$existing_services" =~ .*"DataFed".* ]] diff --git a/core/database/tests/test_teardown.sh b/core/database/tests/test_teardown.sh index e307d2cd3..09ce68a17 100755 --- a/core/database/tests/test_teardown.sh +++ b/core/database/tests/test_teardown.sh @@ -5,8 +5,8 @@ set -uf -o pipefail SCRIPT=$(realpath "$0") SOURCE=$(dirname "$SCRIPT") -PROJECT_ROOT=$(realpath ${SOURCE}/../../../) -source ${PROJECT_ROOT}/config/datafed.sh +PROJECT_ROOT=$(realpath "${SOURCE}/../../../") +source "${PROJECT_ROOT}/config/datafed.sh" -PATH_TO_PASSWD_FILE=${SOURCE}/database_temp.password -rm ${PATH_TO_PASSWD_FILE} +PATH_TO_PASSWD_FILE="${SOURCE}/database_temp.password" +rm "${PATH_TO_PASSWD_FILE}" diff --git a/core/server/AuthenticationManager.cpp b/core/server/AuthenticationManager.cpp index 8617904a4..7166d26cc 100644 --- a/core/server/AuthenticationManager.cpp +++ b/core/server/AuthenticationManager.cpp @@ -104,9 +104,9 @@ std::string AuthenticationManager::getUID(const std::string &public_key) const { EXCEPT(1, "Unrecognized public_key during execution of getUID."); } -void AuthenticationManager::addKey(const PublicKeyType pub_key_type, - const std::string public_key, - const std::string uid) { +void AuthenticationManager::addKey(const PublicKeyType & pub_key_type, + const std::string & public_key, + const std::string & uid) { std::lock_guard lock(m_lock); m_auth_mapper.addKey(pub_key_type, public_key, uid); } diff --git a/core/server/AuthenticationManager.hpp b/core/server/AuthenticationManager.hpp index f0ae28ae6..9eb41753f 100644 --- a/core/server/AuthenticationManager.hpp +++ b/core/server/AuthenticationManager.hpp @@ -79,8 +79,8 @@ class AuthenticationManager : public IAuthenticationManager { **/ virtual bool hasKey(const std::string &pub_key) const final; - void addKey(const PublicKeyType pub_key_type, const std::string public_key, - const std::string uid); + void addKey(const PublicKeyType & pub_key_type, const std::string & public_key, + const std::string & uid); /** * Will the id or throw an error diff --git a/core/server/DatabaseAPI.cpp b/core/server/DatabaseAPI.cpp index 5f3991d48..98890d7ee 100644 --- a/core/server/DatabaseAPI.cpp +++ b/core/server/DatabaseAPI.cpp @@ -3106,7 +3106,7 @@ void DatabaseAPI::taskAbort(const std::string &a_task_id, const std::string &a_msg, libjson::Value &a_task_reply, LogContext log_context) { - libjson::Value doc = a_msg; + libjson::Value doc(a_msg); string body = doc.toString(); dbPost("task/abort", {{"task_id", a_task_id}}, &body, a_task_reply, diff --git a/docker/Dockerfile.foxx b/docker/Dockerfile.foxx index cdaa7eb4f..c1a7726c8 100644 --- a/docker/Dockerfile.foxx +++ b/docker/Dockerfile.foxx @@ -2,18 +2,18 @@ # cd ${PROJECT_ROOT} or cd DataFed # docker build -f core/docker/Dockerfile . -ARG BUILD_BASE="ubuntu:focal" -ARG DEPENDENCIES="dependencies" -ARG RUNTIME="runtime" -ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG BUILD_BASE="ubuntu:focal" +ARG DEPENDENCIES="dependencies" +ARG RUNTIME="runtime" +ARG DATAFED_DIR="/datafed" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" -ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" -ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" -ARG LIB_DIR="/usr/local/lib" +ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" +ARG BUILD_DIR="$DATAFED_DIR/source" +ARG NVM_DIR="$DATAFED_DIR/.nvm" +ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS foxx-build @@ -22,10 +22,10 @@ ARG BUILD_DIR ARG DATAFED_INSTALL_PATH ARG DATAFED_DEPENDENCIES_INSTALL_PATH -ENV BUILD_DIR="${BUILD_DIR}" -ENV DATAFED_DIR="${DATAFED_DIR}" +ENV BUILD_DIR="${BUILD_DIR}" +ENV DATAFED_DIR="${DATAFED_DIR}" ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" -ENV PATH=${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin:${PATH} +ENV PATH=${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin:${PATH} ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" ENV DATAFED_DEFAULT_LOG_PATH="$DATAFED_INSTALL_PATH/logs" @@ -47,10 +47,11 @@ RUN mkdir -p ${DATAFED_INSTALL_PATH} RUN mkdir -p ${DATAFED_DEFAULT_LOG_PATH} RUN curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key; apt-key add - < Release.key -RUN echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list + +# RUN echo 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list # If trust the site but the gpg key has expired can add [trusted=yes] # i.e. -# RUN echo 'deb [trusted=yes] https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list +RUN echo 'deb [trusted=yes] https://download.arangodb.com/arangodb38/DEBIAN/ /' | tee /etc/apt/sources.list.d/arangodb.list RUN apt update RUN apt install arangodb3-client -y RUN arangosh --version @@ -59,5 +60,4 @@ USER root WORKDIR ${BUILD_DIR} -#ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ${BUILD_DIR}/docker/entrypoint_foxx.sh diff --git a/repository/docker/Dockerfile.gcs b/repository/docker/Dockerfile.gcs index 5c4d58cc5..717d2a0f8 100644 --- a/repository/docker/Dockerfile.gcs +++ b/repository/docker/Dockerfile.gcs @@ -1,18 +1,18 @@ # NOTE this image must be built with respect to the base of the project i.e. # cd ${PROJECT_ROOT} or cd DataFed # docker build -f repository/docker/Dockerfile.gcs . -ARG BUILD_BASE="ubuntu:focal" -ARG DEPENDENCIES="dependencies" -ARG RUNTIME="runtime" -ARG DATAFED_DIR="/datafed" -ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG BUILD_BASE="ubuntu:focal" +ARG DEPENDENCIES="dependencies" +ARG RUNTIME="runtime" +ARG DATAFED_DIR="/datafed" +ARG DATAFED_INSTALL_PATH="/opt/datafed" ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" -ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" -ARG BUILD_DIR="$DATAFED_DIR/source" -ARG NVM_DIR="$DATAFED_DIR/.nvm" -ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" -ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" -ARG LIB_DIR="/usr/local/lib" +ARG GCS_IMAGE="code.ornl.gov:4567/dlsw/datafed/gcs-ubuntu-focal" +ARG BUILD_DIR="$DATAFED_DIR/source" +ARG NVM_DIR="$DATAFED_DIR/.nvm" +ARG NVM_INC="$DATAFED_DIR/.nvm/versions/node/v13.14.0/include/node" +ARG NVM_BIN="$DATAFED_DIR/.nvm/versions/node/v13.14.0/bin" +ARG LIB_DIR="/usr/local/lib" FROM ${DEPENDENCIES} AS dependencies @@ -28,7 +28,7 @@ ENV DATAFED_DEPENDENCIES_INSTALL_PATH="$DATAFED_DEPENDENCIES_INSTALL_PATH" ENV DATAFED_GCS_COLLECTION_ROOT_PATH="/mnt/datafed" ENV DATAFED_GLOBUS_CRED_FILE_PATH=/opt/datafed/globus/client_cred.json ENV DATAFED_INSTALL_PATH="$DATAFED_INSTALL_PATH" -ENV DATAFED_DIR="$DATAFED_DIR" +ENV DATAFED_DIR="$DATAFED_DIR" ENV BUILD_DIR="$BUILD_DIR" ENV LIB_DIR="$LIB_DIR" ENV DATAFED_GLOBUS_REPO_USER="datafed" @@ -52,7 +52,7 @@ RUN adduser --disabled-password --gecos "" datafed RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo && chmod +x /usr/bin/sudo -COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ COPY ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh COPY ./scripts/dependency_install_functions.sh ${BUILD_DIR}/scripts/ @@ -65,27 +65,27 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${BUILD_DIR}/scripts/install_authz # All files should be owned by the datafed user RUN chown -R datafed:root ${DATAFED_DIR} -COPY --chown=datafed:root ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ -COPY --chown=datafed:root ./scripts/generate_authz_config.sh ${BUILD_DIR}/scripts/generate_authz_config.sh -COPY --chown=datafed:root ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh -COPY --chown=datafed:root ./CMakeLists.txt ${BUILD_DIR} -COPY --chown=datafed:root ./cmake ${BUILD_DIR}/cmake -COPY --chown=datafed:root ./repository/CMakeLists.txt ${BUILD_DIR}/repository/CMakeLists.txt -COPY --chown=datafed:root ./repository/gridftp/CMakeLists.txt ${BUILD_DIR}/repository/gridftp/CMakeLists.txt -COPY --chown=datafed:root ./scripts/globus/setup_globus.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh -COPY --chown=datafed:root ./scripts/globus/clean_globus.sh ${BUILD_DIR}/scripts/globus/clean_globus.sh -COPY --chown=datafed:root ./scripts/globus/generate_repo_form.sh ${BUILD_DIR}/scripts/globus/generate_repo_form.sh -COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh -COPY --chown=datafed:root ./common ${BUILD_DIR}/common -COPY --chown=datafed:root ./repository/gridftp/globus5 ${BUILD_DIR}/repository/gridftp/globus5 -COPY --chown=datafed:root ./scripts/generate_gsi-authz_config.sh ${BUILD_DIR}/scripts/generate_gsi-authz_config.sh -COPY --chown=datafed:root ./scripts/globus/create_guest_collection.py ${BUILD_DIR}/scripts/globus/create_guest_collection.py -COPY --chown=datafed:root ./scripts/globus/utils.py ${BUILD_DIR}/scripts/globus/utils.py -COPY --chown=datafed:root ./scripts/globus/__init__.py ${BUILD_DIR}/scripts/globus/__init__.py +COPY --chown=datafed:root ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ +COPY --chown=datafed:root ./scripts/generate_authz_config.sh ${BUILD_DIR}/scripts/generate_authz_config.sh +COPY --chown=datafed:root ./scripts/utils.sh ${BUILD_DIR}/scripts/utils.sh +COPY --chown=datafed:root ./CMakeLists.txt ${BUILD_DIR} +COPY --chown=datafed:root ./cmake ${BUILD_DIR}/cmake +COPY --chown=datafed:root ./repository/CMakeLists.txt ${BUILD_DIR}/repository/CMakeLists.txt +COPY --chown=datafed:root ./repository/gridftp/CMakeLists.txt ${BUILD_DIR}/repository/gridftp/CMakeLists.txt +COPY --chown=datafed:root ./scripts/globus/setup_globus.sh ${BUILD_DIR}/scripts/globus/setup_globus.sh +COPY --chown=datafed:root ./scripts/globus/clean_globus.sh ${BUILD_DIR}/scripts/globus/clean_globus.sh +COPY --chown=datafed:root ./scripts/globus/generate_repo_form.sh ${BUILD_DIR}/scripts/globus/generate_repo_form.sh +COPY --chown=datafed:root ./repository/docker/entrypoint_authz.sh ${BUILD_DIR}/repository/docker/entrypoint_authz.sh +COPY --chown=datafed:root ./common ${BUILD_DIR}/common +COPY --chown=datafed:root ./repository/gridftp/globus5 ${BUILD_DIR}/repository/gridftp/globus5 +COPY --chown=datafed:root ./scripts/generate_gsi-authz_config.sh ${BUILD_DIR}/scripts/generate_gsi-authz_config.sh +COPY --chown=datafed:root ./scripts/globus/create_guest_collection.py ${BUILD_DIR}/scripts/globus/create_guest_collection.py +COPY --chown=datafed:root ./scripts/globus/utils.py ${BUILD_DIR}/scripts/globus/utils.py +COPY --chown=datafed:root ./scripts/globus/__init__.py ${BUILD_DIR}/scripts/globus/__init__.py # These extra copy statements are necessary as the authz binary is built in the GCS container # and not the dependencies container so we must copy the build dependencies as well as the # runtime depedencies -COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH} ${DATAFED_DEPENDENCIES_INSTALL_PATH} COPY --from=dependencies /usr/bin/make /usr/bin/make COPY --from=dependencies /usr/bin/g++ /usr/bin/g++ @@ -94,20 +94,20 @@ COPY --from=dependencies /usr/bin/ar /usr/bin/ar COPY --from=dependencies /usr/bin/as /usr/bin/as COPY --from=dependencies /usr/bin/ld /usr/bin/ld COPY --from=dependencies /usr/bin/jq /usr/bin/jq -COPY --from=dependencies /usr/bin/sudo /usr/bin/sudo +COPY --from=dependencies /usr/bin/sudo /usr/bin/sudo COPY --from=dependencies /etc/sudoers /etc/sudoers COPY --from=dependencies /usr/lib/gcc /usr/lib/gcc -COPY --from=dependencies /lib /lib +COPY --from=dependencies /lib /lib COPY --from=dependencies /usr/include /usr/include -COPY --from=dependencies /usr/lib /usr/lib +COPY --from=dependencies /usr/lib /usr/lib -COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotobuf.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotobuf.a -COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a -COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a -COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotobuf.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotobuf.a +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libprotoc.a +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libzmq.a +COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a ${DATAFED_DEPENDENCIES_INSTALL_PATH}/lib/libsodium.a # Copy python env over COPY --from=dependencies ${DATAFED_DEPENDENCIES_INSTALL_PATH}/python/datafed ${DATAFED_DEPENDENCIES_INSTALL_PATH}/python/datafed @@ -145,5 +145,5 @@ RUN sed -i '/--client-id/d' /entrypoint.sh WORKDIR ${DATAFED_INSTALL_PATH}/authz # NOTE If you do not overwrite the entrypoint file it will use the GCS instance -# prorivded one +# provided one ENTRYPOINT ["/datafed/source/repository/docker/entrypoint_authz.sh"] diff --git a/repository/docker/entrypoint_authz.sh b/repository/docker/entrypoint_authz.sh index af23b3cf8..0d5aaf93a 100755 --- a/repository/docker/entrypoint_authz.sh +++ b/repository/docker/entrypoint_authz.sh @@ -123,7 +123,7 @@ do printf "\r${EraseToEOL}" minutes=$((minutes + 1)) - HTTP_CODE=$(${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/curl -s -o /dev/null -w "%{http_code}\n" -I "https://${DATAFED_GCS_URL}/api/info") + HTTP_CODE=$("${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/curl" -s -o /dev/null -w "%{http_code}\n" -I "https://${DATAFED_GCS_URL}/api/info") done printf "\n" @@ -157,4 +157,3 @@ echo "Container is running." # Return to last file tail -f "${DATAFED_DEFAULT_LOG_PATH}/datafed-gsi-authz.log" -sleep infinity diff --git a/repository/docker/entrypoint_repo.sh b/repository/docker/entrypoint_repo.sh index df730abe6..23ea4c4e7 100755 --- a/repository/docker/entrypoint_repo.sh +++ b/repository/docker/entrypoint_repo.sh @@ -38,5 +38,6 @@ else su datafed -c '"$@"' -- argv0 "$@" fi -echo "Give a few minutes to debug the problem" +# Allow the container to exist for a bit in case we need to jump in and look +# around sleep 10000 diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index addbbd7cd..53a99f41e 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -148,6 +148,14 @@ void Server::checkServerVersion() { MessageFactory msg_factory; size_t attempt = 0; + + /** + * Here the repo server will continually try to make a connection with the + * core services. This was changed from making a fixed number of attempts. + * The reasoning was that doing so would make the repo service dependent + * on the core actually running. By continually looping the coupling between + * the core services and the repo service is reduced. + **/ while (true) { ++attempt; DL_INFO(m_log_context, diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh index 8020bd770..ce6924ee9 100644 --- a/scripts/dependency_install_functions.sh +++ b/scripts/dependency_install_functions.sh @@ -124,18 +124,6 @@ install_protobuf() { # Don't build shared, it messes up the static library linking because the # cmake file installed are not compatible # WARNING - static library will break if build with shared options on - #cmake -S . -B build \ - # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - # -DBUILD_SHARED_LIBS=ON \ - # -Dprotobuf_BUILD_TESTS=OFF \ - # -DABSL_PROPAGATE_CXX_STD=ON \ - # -DCMAKE_INSTALL_PREFIX="${DATAFED_DEPENDENCIES_INSTALL_PATH}" - #cmake --build build -j 8 - #if [ -w "${DATAFED_DEPENDENCIES_INSTALL_PATH}" ]; then - # cmake --build build --target install - #else - # "$SUDO_CMD" cmake --build build --target install - #fi cd python init_python @@ -322,18 +310,6 @@ install_json_schema_validator() { # WARNING building shared library will overwrite cmake file for static # library, does not appear to support both targets at the same time, similar # to protobuf - # - # Build shared - #cmake -S . -B build \ - # -DBUILD_SHARED_LIBS=ON \ - # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - # -DCMAKE_INSTALL_PREFIX="${DATAFED_DEPENDENCIES_INSTALL_PATH}" - #cmake --build build -j 8 - #if [ -w "${DATAFED_DEPENDENCIES_INSTALL_PATH}" ]; then - # cmake --build build --target install - #else - # "$SUDO_CMD" cmake --build build --target install - #fi # Mark json-schema-validator as installed touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.json_schema_validator_installed-${DATAFED_JSON_SCHEMA_VALIDATOR_VERSION}" cd "$original_dir" diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index d2ea3357f..e829ee9fe 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -181,6 +181,14 @@ else local_DATAFED_GLOBUS_CONTROL_PORT=$(printenv DATAFED_GLOBUS_CONTROL_PORT) fi +if [ -z "${DATAFED_GLOBUS_ALLOWED_DOMAINS}" ] +then + local_DATAFED_GLOBUS_ALLOWED_DOMAINS="globusid.org" +else + local_DATAFED_GLOBUS_ALLOWED_DOMAINS=$(printenv DATAFED_GLOBUS_ALLOWED_DOMAINS) +fi + + if [ ! -d "$PATH_TO_CONFIG_DIR" ] then mkdir -p "$PATH_TO_CONFIG_DIR" @@ -316,8 +324,9 @@ export DATAFED_GCS_COLLECTION_ROOT_PATH="$local_DATAFED_GCS_COLLECTION_ROOT_PATH export DATAFED_REPO_ID_AND_DIR="$local_DATAFED_REPO_ID_AND_DIR" # Institutionally allowed domains, users that have accounts in these domains # will have the ability to store data on the repository. -# i.e. ornl.gov, or cu.edu -export DATAFED_GLOBUS_ALLOWED_DOMAINS="" +# i.e. ornl.gov or cu.edu or gmail.com by default clients.auth.globus.org +# must be allowed to allow automatic setup. +export DATAFED_GLOBUS_ALLOWED_DOMAINS="$local_DATAFED_GLOBUS_ALLOWED_DOMAINS" # Globus control port default is 443, might want to change if hosting # a web server on the same machine. export DATAFED_GLOBUS_CONTROL_PORT="$local_DATAFED_GLOBUS_CONTROL_PORT" diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 4d01a20ac..aed189272 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -6,7 +6,7 @@ import os import utils -# Define your client ID and client secret +# Native Client ID for the DataFed Globus Setup Scripts CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" # NATIVE # The Globus project the GCS endpoint will be created in diff --git a/scripts/globus/setup_globus.sh b/scripts/globus/setup_globus.sh index 158f8aeb2..421fbbd65 100755 --- a/scripts/globus/setup_globus.sh +++ b/scripts/globus/setup_globus.sh @@ -41,6 +41,12 @@ else DATAFED_GLOBUS_SUBSCRIPTION=$(printenv DATAFED_GLOBUS_SUBSCRIPTION) fi +if [ -z "$DATAFED_GLOBUS_ALLOWED_DOMAINS" ] +then + echo "DATAFED_GLOBUS_ALLOWED_DOMAINS is not defined shoudl be i.e. globusid.org or gmail.com or ornl.gov or cu.edu or something of the sort." + exit 1 +fi + if [ -f "$CRED_FILE_PATH" ]; then echo "File exists! $CRED_FILE_PATH" else @@ -54,13 +60,7 @@ COLLECTION_NAME="${DATAFED_GCS_ROOT_NAME} Collection Mapped" GUEST_COLLECTION_NAME="${DATAFED_GCS_ROOT_NAME} Collection Guest" gateway_line=$( globus-connect-server storage-gateway list | grep "$GATEWAY_NAME" ) -# { -# "source": "{id}", -# "match": "${DATAFED_GLOBUS_APP_ID}@clients.auth.globus.org", -# "output": "${DATAFED_GLOBUS_REPO_USER}", -# "ignore_case": false, -# "literal": true -# } + echo "$DATAFED_GLOBUS_REPO_USER" cat << EOF > mapping.json { @@ -77,10 +77,9 @@ cat << EOF > mapping.json } EOF -#DOMAINS="--domain ornl.gov --domain clients.auth.globus.org --domain gmail.com" # For the Globus client to create the guest collection need to allow client # from the domain -DOMAINS="--domain ornl.gov --domain clients.auth.globus.org" +DOMAINS="--domain $DATAFED_GLOBUS_ALLOWED_DOMAINS --domain clients.auth.globus.org" echo "{" > path_restriction.json echo " \"DATA_TYPE\": \"path_restrictions#1.0.0\"," >> path_restriction.json diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 226546292..424afd7b4 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -369,25 +369,3 @@ def createGCSEndpoint( f"Something is wrong deployment key is empty {deployment_key_file} " ) sys.exit(1) - - # WARNING!!!!!! - # This will not work if a node does not first exist, I think at least one - # node must be running. - # if len(subscription_id) != 0: - # if command_exists("globus-connect-server") is False: - # print("Cannot create deployment key, we require globus-connect-server to be installed") - # sys.exit(1) - # else: - # endpoint_id = getEndpointIdFromFile(deployment_key_file) - - # bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" " - # bash_command+=f" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - # bash_command+=f" GCS_CLI_ENDPOINT_ID=\"{endpoint_id}\" " - # bash_command+=" globus-connect-server endpoint update " - # bash_command+=f" --subscription-id \"{subscription_id}\" " - # print(bash_command) - # - # process = subprocess.Popen(bash_command, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - # # Print the output - # for line in process.stdout: - # print(line, end='') From 8ec199ebbadee7b1ba4b3b5850c5d9e95d048dc8 Mon Sep 17 00:00:00 2001 From: par-hermes Date: Wed, 10 Apr 2024 03:02:30 +0000 Subject: [PATCH 76/84] cpp-py-formatter --- cmake/curl_version.cpp | 4 ++-- common/include/common/CommunicatorFactory.hpp | 3 ++- common/include/common/DynaLog.hpp | 4 ++-- common/include/common/ServerFactory.hpp | 2 +- common/source/servers/ProxyBasicZMQ.cpp | 6 +++--- core/server/AuthenticationManager.cpp | 6 +++--- core/server/AuthenticationManager.hpp | 4 ++-- repository/server/RepoServer.cpp | 2 +- 8 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cmake/curl_version.cpp b/cmake/curl_version.cpp index 1846a2db4..18ad1ab6c 100644 --- a/cmake/curl_version.cpp +++ b/cmake/curl_version.cpp @@ -2,8 +2,8 @@ #include /** - * This script is used to show what vesion of curl is being used with the - * rest of the build process and will print the version number of the curl + * This script is used to show what vesion of curl is being used with the + * rest of the build process and will print the version number of the curl * library. **/ int main() { diff --git a/common/include/common/CommunicatorFactory.hpp b/common/include/common/CommunicatorFactory.hpp index 7c4a607b5..a5723b2ac 100644 --- a/common/include/common/CommunicatorFactory.hpp +++ b/common/include/common/CommunicatorFactory.hpp @@ -18,7 +18,8 @@ class CommunicatorFactory { LogContext m_log_context; public: - CommunicatorFactory(const LogContext & log_context) : m_log_context(log_context){}; + CommunicatorFactory(const LogContext &log_context) + : m_log_context(log_context){}; std::unique_ptr create(const SocketOptions &socket_options, const ICredentials &credentials, diff --git a/common/include/common/DynaLog.hpp b/common/include/common/DynaLog.hpp index 0c85f65e5..0f8814a98 100644 --- a/common/include/common/DynaLog.hpp +++ b/common/include/common/DynaLog.hpp @@ -75,14 +75,14 @@ namespace SDMS { /** * Here unsigned int is used so that we can compare the numeric values when - * choosing whether to print a log message. + * choosing whether to print a log message. * * CRITICAL = 0 * ERROR = 1 * WARNING = 2 * INFO = 3 * DEBUG = 4 - * TRACE = 5 + * TRACE = 5 **/ enum class LogLevel : unsigned int { CRITICAL, diff --git a/common/include/common/ServerFactory.hpp b/common/include/common/ServerFactory.hpp index a16f736a3..3ca5e5e73 100644 --- a/common/include/common/ServerFactory.hpp +++ b/common/include/common/ServerFactory.hpp @@ -30,7 +30,7 @@ class ServerFactory { LogContext m_log_context; public: - ServerFactory(const LogContext & log_context) : m_log_context(log_context){}; + ServerFactory(const LogContext &log_context) : m_log_context(log_context){}; std::unique_ptr create( ServerType server_type, diff --git a/common/source/servers/ProxyBasicZMQ.cpp b/common/source/servers/ProxyBasicZMQ.cpp index 785b81f62..eda7f3bfe 100644 --- a/common/source/servers/ProxyBasicZMQ.cpp +++ b/common/source/servers/ProxyBasicZMQ.cpp @@ -19,8 +19,8 @@ namespace SDMS { -std::string sanitize(std::string val, const std::string & pattern, - const std::string & replacement) { +std::string sanitize(std::string val, const std::string &pattern, + const std::string &replacement) { for (auto at = val.find(pattern, 0); at != std::string::npos; at = val.find(pattern, at + replacement.length())) { @@ -186,7 +186,7 @@ void ProxyBasicZMQ::run() { * loop. **/ auto terminate_call = [](std::chrono::duration duration, - const std::string & address, int thread_id, + const std::string &address, int thread_id, LogContext log_context) { log_context.thread_name += "-terminate_after_timeout"; log_context.thread_id = thread_id; diff --git a/core/server/AuthenticationManager.cpp b/core/server/AuthenticationManager.cpp index 7166d26cc..44bb35542 100644 --- a/core/server/AuthenticationManager.cpp +++ b/core/server/AuthenticationManager.cpp @@ -104,9 +104,9 @@ std::string AuthenticationManager::getUID(const std::string &public_key) const { EXCEPT(1, "Unrecognized public_key during execution of getUID."); } -void AuthenticationManager::addKey(const PublicKeyType & pub_key_type, - const std::string & public_key, - const std::string & uid) { +void AuthenticationManager::addKey(const PublicKeyType &pub_key_type, + const std::string &public_key, + const std::string &uid) { std::lock_guard lock(m_lock); m_auth_mapper.addKey(pub_key_type, public_key, uid); } diff --git a/core/server/AuthenticationManager.hpp b/core/server/AuthenticationManager.hpp index 9eb41753f..d648da8ab 100644 --- a/core/server/AuthenticationManager.hpp +++ b/core/server/AuthenticationManager.hpp @@ -79,8 +79,8 @@ class AuthenticationManager : public IAuthenticationManager { **/ virtual bool hasKey(const std::string &pub_key) const final; - void addKey(const PublicKeyType & pub_key_type, const std::string & public_key, - const std::string & uid); + void addKey(const PublicKeyType &pub_key_type, const std::string &public_key, + const std::string &uid); /** * Will the id or throw an error diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index 53a99f41e..c58acb23a 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -155,7 +155,7 @@ void Server::checkServerVersion() { * The reasoning was that doing so would make the repo service dependent * on the core actually running. By continually looping the coupling between * the core services and the repo service is reduced. - **/ + **/ while (true) { ++attempt; DL_INFO(m_log_context, From bb493bef4bbe85d3b167ea74ca424dd2fca5b67e Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 23:03:25 -0400 Subject: [PATCH 77/84] Make methods explicit and remove unused varaible --- common/include/common/CommunicatorFactory.hpp | 2 +- common/include/common/ServerFactory.hpp | 2 +- common/source/communicators/ZeroMQCommunicator.hpp | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/common/include/common/CommunicatorFactory.hpp b/common/include/common/CommunicatorFactory.hpp index 7c4a607b5..d7451af6a 100644 --- a/common/include/common/CommunicatorFactory.hpp +++ b/common/include/common/CommunicatorFactory.hpp @@ -18,7 +18,7 @@ class CommunicatorFactory { LogContext m_log_context; public: - CommunicatorFactory(const LogContext & log_context) : m_log_context(log_context){}; + explicit CommunicatorFactory(const LogContext & log_context) : m_log_context(log_context){}; std::unique_ptr create(const SocketOptions &socket_options, const ICredentials &credentials, diff --git a/common/include/common/ServerFactory.hpp b/common/include/common/ServerFactory.hpp index a16f736a3..64ca41804 100644 --- a/common/include/common/ServerFactory.hpp +++ b/common/include/common/ServerFactory.hpp @@ -30,7 +30,7 @@ class ServerFactory { LogContext m_log_context; public: - ServerFactory(const LogContext & log_context) : m_log_context(log_context){}; + explicit ServerFactory(const LogContext & log_context) : m_log_context(log_context){}; std::unique_ptr create( ServerType server_type, diff --git a/common/source/communicators/ZeroMQCommunicator.hpp b/common/source/communicators/ZeroMQCommunicator.hpp index af3a94ae5..ac131d5b1 100644 --- a/common/source/communicators/ZeroMQCommunicator.hpp +++ b/common/source/communicators/ZeroMQCommunicator.hpp @@ -25,7 +25,6 @@ namespace SDMS { class ZeroMQCommunicator : public ICommunicator { protected: std::unique_ptr m_socket; - uint16_t m_zmq_context; void *m_zmq_socket = nullptr; int m_zmq_socket_type; void *m_zmq_ctx = nullptr; From 30c0909d7f747551992ebc3c007223908e9d580f Mon Sep 17 00:00:00 2001 From: par-hermes Date: Wed, 10 Apr 2024 03:08:58 +0000 Subject: [PATCH 78/84] cpp-py-formatter --- common/include/common/CommunicatorFactory.hpp | 2 +- common/include/common/ServerFactory.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/include/common/CommunicatorFactory.hpp b/common/include/common/CommunicatorFactory.hpp index 93683abc8..8b5c66c7a 100644 --- a/common/include/common/CommunicatorFactory.hpp +++ b/common/include/common/CommunicatorFactory.hpp @@ -18,7 +18,7 @@ class CommunicatorFactory { LogContext m_log_context; public: - explicit CommunicatorFactory(const LogContext & log_context) + explicit CommunicatorFactory(const LogContext &log_context) : m_log_context(log_context){}; std::unique_ptr create(const SocketOptions &socket_options, diff --git a/common/include/common/ServerFactory.hpp b/common/include/common/ServerFactory.hpp index 34d395dc4..7398cb272 100644 --- a/common/include/common/ServerFactory.hpp +++ b/common/include/common/ServerFactory.hpp @@ -30,7 +30,7 @@ class ServerFactory { LogContext m_log_context; public: - explicit ServerFactory(const LogContext & log_context) + explicit ServerFactory(const LogContext &log_context) : m_log_context(log_context){}; std::unique_ptr create( From c5872f6a3936f2416bb843a1044c7fe168841587 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 23:16:21 -0400 Subject: [PATCH 79/84] Spelling of compose in Readme --- compose/README.md | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/compose/README.md b/compose/README.md index 8337b51f8..cf4edc6a3 100644 --- a/compose/README.md +++ b/compose/README.md @@ -1,6 +1,6 @@ # Compose Dev environment -The compose Dev environment is split into two different compose files. The +The Compose Dev environment is split into two different Compose files. The "core metadata services" which comprise the web server, core server and database and the "repo services" which comprise Globus Connect Server running with the authz library and the DataFed repo service. @@ -10,14 +10,14 @@ will need a machine with firewall exceptions to use it. ## Core Compose Services -The following steps are used to stand up the core compose file from scratch. +The following steps are used to stand up the core Compose file from scratch. Some of steps you only have to do once. 1. Generating the env variables. 2. Opening the .env file and entering your configuration 3. Building the images -4. Running the compose file -5. Bringing down the compose file. +4. Running the Compose file +5. Bringing down the Compose file. Core services only need an external port 443 for https access. I have been unable to get this to work with other ports due to the redirect URL required @@ -32,10 +32,10 @@ Create the .env file fill in the missing components that are required. ``` ### 2. Fill in the needed .env variables for the Core Services -The .env file will be created in the DataFed/compose folder and will be hidden. +The .env file will be created in the DataFed/Compose folder and will be hidden. The .env file variables can be changed at this point to your configuration. -NOTE the .env file will be read verbatim by compose including any spaces or +NOTE the .env file will be read verbatim by Compose including any spaces or "#" comments so do not includ anything but the exact text that needs to be included in the variables. @@ -48,7 +48,7 @@ services. ./build_images_for_compose.sh ``` -### 4. Running the core compose file +### 4. Running the core Compose file Stand up the core services. @@ -65,7 +65,7 @@ configuration before hand by running. docker compose -f compose_core.yml config ``` -WARNING - docker compose will prioritize env variables in the following priority +WARNING - docker Compose will prioritize env variables in the following priority 1. From you shell env 2. From the .env file 3. Internally from winthin the image @@ -77,9 +77,9 @@ At this point you should be able to navigate in your browser to NOTE we are using a self signed certificate so you will have to force your browser to allow you to see the page. -### 5. Bringing down the core compose file +### 5. Bringing down the core Compose file -To completely remove the compose instance and all state the following should +To completely remove the Compose instance and all state the following should be run. ```bash @@ -87,12 +87,12 @@ docker compose -f ./compose_core.yml down --volumes ``` NOTE the volumes will remove all cached state. If the '--volumes' flag is -not added then on a subsequent "compose up" the database will not be a clean +not added then on a subsequent "Compose up" the database will not be a clean install but contain state from previous runs. ## Repo Compose Services -The following steps are used to stand up the repo compose file. NOTE, that +The following steps are used to stand up the repo Compose file. NOTE, that because of how Globus is configured, there is an additional configuration and teardown step. @@ -100,8 +100,8 @@ and teardown step. 2. Opening the .env file and entering your configuration 3. Running the generate_globus_files.sh script 4. Building the images -5. Running the compose file -6. Bringing down the compose file. +5. Running the Compose file +6. Bringing down the Compose file. 7. Running the cleanup_globus_files.sh if you want to remove the deployment key and start completely from scratch. @@ -115,18 +115,18 @@ Create the .env file fill in the missing components that are required. ### 2. Enter the .env varaibles for the Repo Services -The .env file will be created in the DataFed/compose folder and will be hidden. +The .env file will be created in the DataFed/Compose folder and will be hidden. The .env file variables can be changed at this point to your configuration. -NOTE the .env file will be read verbatim by compose including any spaces or +NOTE the .env file will be read verbatim by Compose including any spaces or "#" comments so do not include anything but the exact text that needs to be included in the variables. ### 3. Globus configuration This step is only required once, after which the necessary files should exist -in DataFed/compose/globus. These files will contain the Globus configuration -needed for additional cycles of "docker compose up" and "docker compose down". +in DataFed/Compose/globus. These files will contain the Globus configuration +needed for additional cycles of "docker Compose up" and "docker Compose down". ### 4. Building Repo Services @@ -137,7 +137,7 @@ services. ./build_repo_images_for_compose.sh ``` -### 5. Running the Repo compose file +### 5. Running the Repo Compose file Stand up the repo services. @@ -157,13 +157,13 @@ configuration before hand by running. docker compose -f compose_repo.yml config ``` -WARNING - docker compose will prioritize env variables in the following priority +WARNING - docker Compose will prioritize env variables in the following priority 1. From you shell env 2. From the .env file 3. Internally from winthin the image Be sure that you are not accidentally overwriting .env variables. -### 6. Bringing down the compose file +### 6. Bringing down the Compose file ## Cleaning up @@ -204,7 +204,7 @@ entrypoint file can be overwritten by including '--entrypoint /bin/bash' ## Common Errors -### Errors during compose up +### Errors during Compose up Make sure all the ports that are needed are open on the local host. These include, ports From b536f1e319a51131ccd3efbbc82166b220fc6b3c Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 23:22:26 -0400 Subject: [PATCH 80/84] Fix formatting --- scripts/globus/create_guest_collection.py | 15 +++++++++------ scripts/globus/globus_cleanup.py | 3 ++- scripts/globus/initialize_globus_endpoint.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index 400379992..f9a97a766 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -1,4 +1,6 @@ import globus_sdk +from globus_sdk import scopes + import utils import os @@ -60,13 +62,14 @@ client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) -scopes = "openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" +scopes = ("openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities") + authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) gcs_client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) -from globus_sdk import scopes - # The scope the client will need, note that primary scope is for the endpoint, # but it has a dependency on the mapped collection's data_access scope @@ -112,7 +115,7 @@ mapped_collection_id = item["id"] break -if mapped_collection_found == False: +if mapped_collection_found is False: raise Exception("Missing required mapped collection") storage_gateway_found = False @@ -138,7 +141,7 @@ storage_gateway_id = item["id"] break -if storage_gateway_found == False: +if storage_gateway_found is False: raise Exception("Missing required storage gateway") @@ -157,7 +160,7 @@ break # https://github.com/globus/globus-sdk-python/blob/main/docs/examples/guest_collection_creation.rst -if guest_collection_found == False: +if guest_collection_found is False: credential_document = globus_sdk.UserCredentialDocument( storage_gateway_id=storage_gateway_id, identity_id=client_id, diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index aed189272..3ca6a6965 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -165,7 +165,8 @@ ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) - bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" ' + bash_command = f'GCS_CLI_CLIENT_SECRET="{client_secret}" ' bash_command += "globus-connect-server endpoint cleanup " bash_command += f' --deployment-key "{DEPLOYMENT_KEY_PATH}" ' bash_command += " --agree-to-delete-endpoint" diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index db45be7a1..2aa989cae 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -105,7 +105,7 @@ organization = userinfo["identity_provider_display_name"] # Need to determine the project uuid -if utils.projectExists(ac_rt, PROJECT_NAME) == False: +if utils.projectExists(ac_rt, PROJECT_NAME) is False: project_id = utils.createProject(ac_rt, PROJECT_NAME, userinfo) else: project_id = utils.getProjectId(ac_rt, PROJECT_NAME, userinfo) From e384cb3587e69465b36e9cf5fddb6869491519ec Mon Sep 17 00:00:00 2001 From: par-hermes Date: Wed, 10 Apr 2024 03:24:03 +0000 Subject: [PATCH 81/84] cpp-py-formatter --- scripts/globus/create_guest_collection.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index f9a97a766..d047702c7 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -62,9 +62,11 @@ client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) -scopes = ("openid profile email " - "urn:globus:auth:scope:auth.globus.org:manage_projects " - "urn:globus:auth:scope:auth.globus.org:view_identities") +scopes = ( + "openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities" +) authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) From 197546b42ae9cedf805c9af06e954b5436f23dc8 Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 23:24:56 -0400 Subject: [PATCH 82/84] Fix spelling --- compose/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/README.md b/compose/README.md index cf4edc6a3..0a291bd10 100644 --- a/compose/README.md +++ b/compose/README.md @@ -65,7 +65,7 @@ configuration before hand by running. docker compose -f compose_core.yml config ``` -WARNING - docker Compose will prioritize env variables in the following priority +WARNING - Docker Compose will prioritize env variables in the following priority 1. From you shell env 2. From the .env file 3. Internally from winthin the image @@ -126,7 +126,7 @@ included in the variables. This step is only required once, after which the necessary files should exist in DataFed/Compose/globus. These files will contain the Globus configuration -needed for additional cycles of "docker Compose up" and "docker Compose down". +needed for additional cycles of "docker compose up" and "docker compose down". ### 4. Building Repo Services @@ -157,7 +157,7 @@ configuration before hand by running. docker compose -f compose_repo.yml config ``` -WARNING - docker Compose will prioritize env variables in the following priority +WARNING - Docker Compose will prioritize env variables in the following priority 1. From you shell env 2. From the .env file 3. Internally from winthin the image From b2847cd87f7a2d62f6fe4d5355b092378669881e Mon Sep 17 00:00:00 2001 From: Joshua Brown Date: Tue, 9 Apr 2024 23:29:33 -0400 Subject: [PATCH 83/84] Fix scopes keyword --- scripts/globus/create_guest_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index d047702c7..25299f10f 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -62,13 +62,13 @@ client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) -scopes = ( +auth_scopes = ( "openid profile email " "urn:globus:auth:scope:auth.globus.org:manage_projects " "urn:globus:auth:scope:auth.globus.org:view_identities" ) -authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) +authorizer = globus_sdk.ClientCredentialsAuthorizer(client, auth_scopes) gcs_client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) From 655752dd30f7553cfa9b470b6ddddf33530ab011 Mon Sep 17 00:00:00 2001 From: par-hermes Date: Wed, 10 Apr 2024 03:52:36 +0000 Subject: [PATCH 84/84] cpp-py-formatter --- scripts/globus/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 36e3661df..424afd7b4 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -233,6 +233,7 @@ def createClient(auth_client, client_name, project_id, cred_name, cred_file): ) return client_id, client_secret + def getGCSClientIDFromDeploymentFile(deployment_key_file): deployment_key_exists, deployment_key_empty = validFile(deployment_key_file)