From dae37d9a4d0b9d235e82fe0a4a78cd4ec4aac0a1 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Thu, 4 Apr 2024 08:35:08 -0400 Subject: [PATCH 1/6] Create tests for functions that will check ftp:// prefix --- .../gridftp/globus5/authz/CMakeLists.txt | 1 + repository/gridftp/globus5/authz/source/URL.c | 79 +++++++++++ repository/gridftp/globus5/authz/source/URL.h | 30 +++++ .../globus5/authz/tests/CMakeLists.txt | 3 + .../globus5/authz/tests/unit/CMakeLists.txt | 17 +++ .../globus5/authz/tests/unit/test_URL.cpp | 125 ++++++++++++++++++ 6 files changed, 255 insertions(+) create mode 100644 repository/gridftp/globus5/authz/source/URL.c create mode 100644 repository/gridftp/globus5/authz/source/URL.h create mode 100644 repository/gridftp/globus5/authz/tests/CMakeLists.txt create mode 100644 repository/gridftp/globus5/authz/tests/unit/CMakeLists.txt create mode 100644 repository/gridftp/globus5/authz/tests/unit/test_URL.cpp diff --git a/repository/gridftp/globus5/authz/CMakeLists.txt b/repository/gridftp/globus5/authz/CMakeLists.txt index 668592821..584987560 100644 --- a/repository/gridftp/globus5/authz/CMakeLists.txt +++ b/repository/gridftp/globus5/authz/CMakeLists.txt @@ -1 +1,2 @@ add_subdirectory( source ) +add_subdirectory( tests ) diff --git a/repository/gridftp/globus5/authz/source/URL.c b/repository/gridftp/globus5/authz/source/URL.c new file mode 100644 index 000000000..ad898dd44 --- /dev/null +++ b/repository/gridftp/globus5/authz/source/URL.c @@ -0,0 +1,79 @@ +#include +#include + +#include "URL.h" + +// Function to extract the relative path from an FTP URL +// Returns 1 on success, 0 on failure +int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength) { + size_t len_of_prefix = strlen("ftp://"); + size_t len_of_url = strlen(url); + + // Step 1. Check that the URL starts with "ftp://" + if (strncmp(url, "ftp://", len_of_prefix) != 0) { + fprintf(stderr, "Error: URL must start with 'ftp:// but you have provided %s'\n", url); + return 0; + } + + if( len_of_url == len_of_prefix ) { + // This means we have ftp:// but with no relative path and missing + // the final / separating the domain from the relative path. + fprintf(stderr, "Error: Invalid URL format expected ftp://domain/ instead received %s\n", url); + return 0; + } else if ( url[len_of_prefix] == '/' ) { + // If they are not equal the url must be greater because we already + // compared the prefix. Let's make sure we don't have + // ftp:/// where no domain is given this is invalid as well + // + // NOTE the third / will appear at index 6 not 7 + fprintf(stderr, "Error: Invalid URL format missing domain name expected ftp://domain/ instead received %s\n", url); + return 0; + } + // Find the position of the third slash ('/') after "ftp://" + const char *slashPtr = strchr(url + len_of_prefix, '/'); + if (slashPtr == NULL) { + if( len_of_url == len_of_prefix ) { + // This means we have ftp:// but with no relative path and missing + // the final / separating the domain from the relative path. + fprintf(stderr, "Error: Invalid URL format expected ftp://domain/ instead received %s\n", url); + return 0; + } else { + // This means we have ftp://domain but with no relative path and missing + // the final / separating the domain from the relative path. We will + // report this as a success and return a slash + relativePath[0] = '/'; + relativePath[1] = '\0'; + return 1; + } + } + + printf("slashPtr is %s\n", slashPtr); + // Calculate the length of the relative path + size_t pathLength = strlen(slashPtr); + + // Check if the provided buffer is sufficient + if (pathLength >= maxLength) { + fprintf(stderr, "Error: Insufficient buffer size max size is %ld actual size is %ld\n", maxLength, pathLength); + return 0; + } + + // Copy the relative path to the output buffer + strcpy(relativePath, slashPtr); + + return 1; // Success +} + +int comparePrefix(const char *str1, const char *str2, size_t prefix_length) { + size_t len1 = strlen(str1); + size_t len2 = strlen(str2); + + // Ensure the prefix length is not longer than the shortest string length + if (prefix_length > len1 || prefix_length > len2) { + return -1; // Prefix length is longer than one or both of the strings + } + + // Compare the prefixes + return strncmp(str1, str2, prefix_length); +} + + diff --git a/repository/gridftp/globus5/authz/source/URL.h b/repository/gridftp/globus5/authz/source/URL.h new file mode 100644 index 000000000..f9e007406 --- /dev/null +++ b/repository/gridftp/globus5/authz/source/URL.h @@ -0,0 +1,30 @@ +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Function to extract the relative path from an FTP URL + * + * Will take a uri of the form ftp://domain/path/to/file + * + * If the URI has the prefix + * + * ftp://domain Returns 1 for success + * else it will return 0 for failure + **/ +int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength); + +/** + * Will compare two strings and ensure that prefixes are equivalent + * + * On success will return the results of strncmp which will be 0 if they + * match. + **/ +int comparePrefix(const char *str1, const char *str2, size_t prefix_length); + +#ifdef __cplusplus +} +#endif diff --git a/repository/gridftp/globus5/authz/tests/CMakeLists.txt b/repository/gridftp/globus5/authz/tests/CMakeLists.txt new file mode 100644 index 000000000..257856d25 --- /dev/null +++ b/repository/gridftp/globus5/authz/tests/CMakeLists.txt @@ -0,0 +1,3 @@ +if( ENABLE_UNIT_TESTS ) + add_subdirectory(unit) +endif( ENABLE_UNIT_TESTS ) diff --git a/repository/gridftp/globus5/authz/tests/unit/CMakeLists.txt b/repository/gridftp/globus5/authz/tests/unit/CMakeLists.txt new file mode 100644 index 000000000..e72fad5a4 --- /dev/null +++ b/repository/gridftp/globus5/authz/tests/unit/CMakeLists.txt @@ -0,0 +1,17 @@ +# Each test listed in Alphabetical order + + include_directories(${PROJECT_SOURCE_DIR}/repository/gridftp/globus5/authz/source) + add_executable(unit_test_URL test_URL.cpp ../../source/URL.c ) + + target_compile_options(unit_test_URL PRIVATE -fPIC) + + add_dependencies(unit_test_URL common) + if(BUILD_SHARED_LIBS) + target_link_libraries(unit_test_URL PRIVATE ${DATAFED_BOOST_LIBRARIES}) + target_compile_definitions(unit_test_URL PRIVATE BOOST_TEST_DYN_LINK) + else() + target_link_libraries(unit_test_URL PRIVATE ${DATAFED_BOOST_LIBRARIES}) + endif() + # Only want this if using shared boost libraries + add_test(unit_test_URL unit_test_URL) + diff --git a/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp b/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp new file mode 100644 index 000000000..9c371a2cc --- /dev/null +++ b/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp @@ -0,0 +1,125 @@ +#include +#include + +#define BOOST_TEST_MAIN + +#define BOOST_TEST_MODULE buffer +#include +#include + +#include "URL.h" + +BOOST_AUTO_TEST_SUITE(BufferTest) + +BOOST_AUTO_TEST_CASE(test_1_ftpExtractRelativePath) { + char relative_path[100]; // Adjust the buffer size as needed + const char *url1 = "ftp://domain/relative_path"; + + int rv = ftpExtractRelativePath(url1, relative_path, sizeof(relative_path)); + // Should pass + BOOST_CHECK(strcmp(relative_path, "/relative_path") == 0); + BOOST_CHECK(rv == 1); +} + +BOOST_AUTO_TEST_CASE(test_2_ftpExtractRelativePath) { + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp://domain"; + + int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); + // Should not throw an error + BOOST_CHECK( rv == 1); + BOOST_CHECK( strcmp(relative_path, "/") == 0); +} + + +BOOST_AUTO_TEST_CASE(test_3_ftpExtractRelativePath) { + char relative_path[100]; // Adjust the buffer size as needed + const char *url = ""; + + int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); + // Should throw an error + BOOST_CHECK( rv == 0); +} + + +BOOST_AUTO_TEST_CASE(test_4_ftpExtractRelativePath) { + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp:///"; + + int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); + // Should throw an error + BOOST_CHECK( rv == 0); +} + + +BOOST_AUTO_TEST_CASE(test_5_ftpExtractRelativePath) { + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp:/domain///path"; + + int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); + // Should throw an error because prefix is incorrect + BOOST_CHECK( rv == 0); +} + +BOOST_AUTO_TEST_CASE(test_6_ftpExtractRelativePath) { + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp://domain///path"; + + int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); + // Should not throw an error + BOOST_CHECK( rv == 1); + printf("Relative path is %s\n", relative_path); + BOOST_CHECK(strcmp(relative_path, "///path") == 0); +} + +BOOST_AUTO_TEST_CASE(test_1_comparePrefix){ + const char * allowed_prefix="/mnt/storage/globus"; + size_t prefix_len = strlen(allowed_prefix); + const char * relative_path=""; + int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); + + // Should fail + BOOST_CHECK(rv != 0); +} + +BOOST_AUTO_TEST_CASE(test_2_comparePrefix){ + const char * allowed_prefix="/mnt/storage/globus"; + size_t prefix_len = strlen(allowed_prefix); + const char * relative_path="/"; + int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); + + // Should fail + BOOST_CHECK(rv != 0); +} + +BOOST_AUTO_TEST_CASE(test_3_comparePrefix){ + const char * allowed_prefix="/mnt/storage/globus"; + size_t prefix_len = strlen(allowed_prefix); + const char * relative_path="/mnt/storage/globus/"; + int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); + + // Should pass + BOOST_CHECK(rv == 0); +} + +BOOST_AUTO_TEST_CASE(test_4_comparePrefix){ + const char * allowed_prefix="/mnt/storage/globus"; + size_t prefix_len = strlen(allowed_prefix); + const char * relative_path="/mnt/storage/globus"; + int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); + + // Should pass + BOOST_CHECK(rv == 0); +} + +BOOST_AUTO_TEST_CASE(test_5_comparePrefix){ + const char * allowed_prefix="/mnt/storage/globus"; + size_t prefix_len = strlen(allowed_prefix); + const char * relative_path="/mnt/storage/globu"; + int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); + + // Should fail + BOOST_CHECK(rv != 0); +} + +BOOST_AUTO_TEST_SUITE_END() From b5caec4ea7da5f6cda782736e2a394483c4ceb80 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Sat, 6 Apr 2024 04:44:01 +0000 Subject: [PATCH 2/6] Add changelog comment --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e7e96143..7fdf5a62d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ smaller images 4. [912] - Adding working compose instance for core metadata services. 5. [937] - Working metadata services running together as part of CI +6. [945] - Added C functions for comparing POSIX path in Authz module ## PATCH Bug fixes/Technical Debt/Documentation From 71e39e2eb360b9042406a42e5b9b696d37274a9f Mon Sep 17 00:00:00 2001 From: par-hermes Date: Sat, 6 Apr 2024 04:44:45 +0000 Subject: [PATCH 3/6] cpp-py-formatter --- repository/gridftp/globus5/authz/source/URL.c | 105 +++++---- repository/gridftp/globus5/authz/source/URL.h | 5 +- .../globus5/authz/tests/unit/test_URL.cpp | 69 +++--- scripts/globus/globus_cleanup.py | 125 +++++----- scripts/globus/initialize_globus_endpoint.py | 75 +++--- scripts/globus/utils.py | 220 ++++++++++-------- 6 files changed, 331 insertions(+), 268 deletions(-) diff --git a/repository/gridftp/globus5/authz/source/URL.c b/repository/gridftp/globus5/authz/source/URL.c index ad898dd44..7d2d93cd5 100644 --- a/repository/gridftp/globus5/authz/source/URL.c +++ b/repository/gridftp/globus5/authz/source/URL.c @@ -5,41 +5,53 @@ // Function to extract the relative path from an FTP URL // Returns 1 on success, 0 on failure -int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength) { +int ftpExtractRelativePath(const char *url, char *relativePath, + size_t maxLength) { size_t len_of_prefix = strlen("ftp://"); size_t len_of_url = strlen(url); - + // Step 1. Check that the URL starts with "ftp://" - if (strncmp(url, "ftp://", len_of_prefix) != 0) { - fprintf(stderr, "Error: URL must start with 'ftp:// but you have provided %s'\n", url); - return 0; - } + if (strncmp(url, "ftp://", len_of_prefix) != 0) { + fprintf(stderr, + "Error: URL must start with 'ftp:// but you have provided %s'\n", + url); + return 0; + } - if( len_of_url == len_of_prefix ) { - // This means we have ftp:// but with no relative path and missing - // the final / separating the domain from the relative path. - fprintf(stderr, "Error: Invalid URL format expected ftp://domain/ instead received %s\n", url); - return 0; - } else if ( url[len_of_prefix] == '/' ) { - // If they are not equal the url must be greater because we already - // compared the prefix. Let's make sure we don't have - // ftp:/// where no domain is given this is invalid as well - // - // NOTE the third / will appear at index 6 not 7 - fprintf(stderr, "Error: Invalid URL format missing domain name expected ftp://domain/ instead received %s\n", url); - return 0; + if (len_of_url == len_of_prefix) { + // This means we have ftp:// but with no relative path and missing + // the final / separating the domain from the relative path. + fprintf(stderr, + "Error: Invalid URL format expected ftp://domain/ instead received " + "%s\n", + url); + return 0; + } else if (url[len_of_prefix] == '/') { + // If they are not equal the url must be greater because we already + // compared the prefix. Let's make sure we don't have + // ftp:/// where no domain is given this is invalid as well + // + // NOTE the third / will appear at index 6 not 7 + fprintf(stderr, + "Error: Invalid URL format missing domain name expected " + "ftp://domain/ instead received %s\n", + url); + return 0; } - // Find the position of the third slash ('/') after "ftp://" - const char *slashPtr = strchr(url + len_of_prefix, '/'); - if (slashPtr == NULL) { - if( len_of_url == len_of_prefix ) { + // Find the position of the third slash ('/') after "ftp://" + const char *slashPtr = strchr(url + len_of_prefix, '/'); + if (slashPtr == NULL) { + if (len_of_url == len_of_prefix) { // This means we have ftp:// but with no relative path and missing // the final / separating the domain from the relative path. - fprintf(stderr, "Error: Invalid URL format expected ftp://domain/ instead received %s\n", url); + fprintf(stderr, + "Error: Invalid URL format expected ftp://domain/ instead " + "received %s\n", + url); return 0; } else { // This means we have ftp://domain but with no relative path and missing - // the final / separating the domain from the relative path. We will + // the final / separating the domain from the relative path. We will // report this as a success and return a slash relativePath[0] = '/'; relativePath[1] = '\0'; @@ -48,32 +60,33 @@ int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength } printf("slashPtr is %s\n", slashPtr); - // Calculate the length of the relative path - size_t pathLength = strlen(slashPtr); + // Calculate the length of the relative path + size_t pathLength = strlen(slashPtr); - // Check if the provided buffer is sufficient - if (pathLength >= maxLength) { - fprintf(stderr, "Error: Insufficient buffer size max size is %ld actual size is %ld\n", maxLength, pathLength); - return 0; - } + // Check if the provided buffer is sufficient + if (pathLength >= maxLength) { + fprintf( + stderr, + "Error: Insufficient buffer size max size is %ld actual size is %ld\n", + maxLength, pathLength); + return 0; + } - // Copy the relative path to the output buffer - strcpy(relativePath, slashPtr); + // Copy the relative path to the output buffer + strcpy(relativePath, slashPtr); - return 1; // Success + return 1; // Success } int comparePrefix(const char *str1, const char *str2, size_t prefix_length) { - size_t len1 = strlen(str1); - size_t len2 = strlen(str2); - - // Ensure the prefix length is not longer than the shortest string length - if (prefix_length > len1 || prefix_length > len2) { - return -1; // Prefix length is longer than one or both of the strings - } - - // Compare the prefixes - return strncmp(str1, str2, prefix_length); -} + size_t len1 = strlen(str1); + size_t len2 = strlen(str2); + // Ensure the prefix length is not longer than the shortest string length + if (prefix_length > len1 || prefix_length > len2) { + return -1; // Prefix length is longer than one or both of the strings + } + // Compare the prefixes + return strncmp(str1, str2, prefix_length); +} diff --git a/repository/gridftp/globus5/authz/source/URL.h b/repository/gridftp/globus5/authz/source/URL.h index f9e007406..47dbceb85 100644 --- a/repository/gridftp/globus5/authz/source/URL.h +++ b/repository/gridftp/globus5/authz/source/URL.h @@ -15,12 +15,13 @@ extern "C" { * ftp://domain Returns 1 for success * else it will return 0 for failure **/ -int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength); +int ftpExtractRelativePath(const char *url, char *relativePath, + size_t maxLength); /** * Will compare two strings and ensure that prefixes are equivalent * - * On success will return the results of strncmp which will be 0 if they + * On success will return the results of strncmp which will be 0 if they * match. **/ int comparePrefix(const char *str1, const char *str2, size_t prefix_length); diff --git a/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp b/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp index 9c371a2cc..3861d34da 100644 --- a/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp +++ b/repository/gridftp/globus5/authz/tests/unit/test_URL.cpp @@ -12,8 +12,8 @@ BOOST_AUTO_TEST_SUITE(BufferTest) BOOST_AUTO_TEST_CASE(test_1_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url1 = "ftp://domain/relative_path"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url1 = "ftp://domain/relative_path"; int rv = ftpExtractRelativePath(url1, relative_path, sizeof(relative_path)); // Should pass @@ -22,100 +22,97 @@ BOOST_AUTO_TEST_CASE(test_1_ftpExtractRelativePath) { } BOOST_AUTO_TEST_CASE(test_2_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp://domain"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp://domain"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should not throw an error - BOOST_CHECK( rv == 1); - BOOST_CHECK( strcmp(relative_path, "/") == 0); + BOOST_CHECK(rv == 1); + BOOST_CHECK(strcmp(relative_path, "/") == 0); } - BOOST_AUTO_TEST_CASE(test_3_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = ""; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = ""; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should throw an error - BOOST_CHECK( rv == 0); + BOOST_CHECK(rv == 0); } - BOOST_AUTO_TEST_CASE(test_4_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp:///"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp:///"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should throw an error - BOOST_CHECK( rv == 0); + BOOST_CHECK(rv == 0); } - BOOST_AUTO_TEST_CASE(test_5_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp:/domain///path"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp:/domain///path"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should throw an error because prefix is incorrect - BOOST_CHECK( rv == 0); + BOOST_CHECK(rv == 0); } BOOST_AUTO_TEST_CASE(test_6_ftpExtractRelativePath) { - char relative_path[100]; // Adjust the buffer size as needed - const char *url = "ftp://domain///path"; + char relative_path[100]; // Adjust the buffer size as needed + const char *url = "ftp://domain///path"; int rv = ftpExtractRelativePath(url, relative_path, sizeof(relative_path)); // Should not throw an error - BOOST_CHECK( rv == 1); + BOOST_CHECK(rv == 1); printf("Relative path is %s\n", relative_path); BOOST_CHECK(strcmp(relative_path, "///path") == 0); } -BOOST_AUTO_TEST_CASE(test_1_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_1_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path=""; + const char *relative_path = ""; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should fail BOOST_CHECK(rv != 0); } -BOOST_AUTO_TEST_CASE(test_2_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_2_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/"; + const char *relative_path = "/"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should fail BOOST_CHECK(rv != 0); } -BOOST_AUTO_TEST_CASE(test_3_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_3_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/mnt/storage/globus/"; + const char *relative_path = "/mnt/storage/globus/"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should pass BOOST_CHECK(rv == 0); } -BOOST_AUTO_TEST_CASE(test_4_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_4_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/mnt/storage/globus"; + const char *relative_path = "/mnt/storage/globus"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should pass BOOST_CHECK(rv == 0); } -BOOST_AUTO_TEST_CASE(test_5_comparePrefix){ - const char * allowed_prefix="/mnt/storage/globus"; +BOOST_AUTO_TEST_CASE(test_5_comparePrefix) { + const char *allowed_prefix = "/mnt/storage/globus"; size_t prefix_len = strlen(allowed_prefix); - const char * relative_path="/mnt/storage/globu"; + const char *relative_path = "/mnt/storage/globu"; int rv = comparePrefix(allowed_prefix, relative_path, prefix_len); // Should fail diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 764bdc36c..e74f69a81 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,4 +1,3 @@ - import globus_sdk from globus_sdk import AuthClient, AccessTokenAuthorizer import subprocess @@ -8,18 +7,18 @@ import utils # Define your client ID and client secret -CLIENT_ID = 'f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8' # NATIVE +CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" # NATIVE # The Globus project the GCS endpoint will be created in if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") else: - DATAFED_GCS_ROOT_NAME="DataFed Repo" + DATAFED_GCS_ROOT_NAME = "DataFed Repo" if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: - PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") + PROJECT_NAME = os.getenv("DATAFED_GLOBUS_PROJECT_NAME") else: - PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" + PROJECT_NAME = DATAFED_GCS_ROOT_NAME + " Project" # This is for confidential client if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: @@ -29,32 +28,35 @@ # Name of the client secret used by the confidential client if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: - CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") + CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME") else: - CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" + CRED_NAME = DATAFED_GCS_ROOT_NAME + " Cred" # Name of the file where we will store confidential client credentials if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: - CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") + CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") else: - CRED_FILE_PATH="./client_cred.json" + CRED_FILE_PATH = "./client_cred.json" # Name to give to endpoint if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") else: - ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" + ENDPOINT_NAME = DATAFED_GCS_ROOT_NAME + " Endpoint" # Path to deployment key if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: - DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") + DEPLOYMENT_KEY_PATH = os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") else: - DEPLOYMENT_KEY_PATH="./deployment-key.json" + DEPLOYMENT_KEY_PATH = "./deployment-key.json" client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project -client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -66,7 +68,9 @@ print(token_response) -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) ac_rt = AuthClient(authorizer=rt_authorizer) @@ -92,68 +96,84 @@ clients_in_project = utils.getClientsInProject(ac_rt, project_id) if len(clients_in_project) == 0: - print("No clients were detected in the project we can just delete the" - "project and be done.") + print( + "No clients were detected in the project we can just delete the" + "project and be done." + ) else: -# Check if the deployment key exists if it does read it and verify that the -# client exists for the globus connect server if it does not then we will -# call the setup command + # Check if the deployment key exists if it does read it and verify that the + # client exists for the globus connect server if it does not then we will + # call the setup command - gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile(DEPLOYMENT_KEY_PATH) + gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile( + DEPLOYMENT_KEY_PATH + ) - valid_key = utils.isGCSDeploymentKeyValid(ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key) + valid_key = utils.isGCSDeploymentKeyValid( + ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key + ) all_gcs_client_ids = utils.getAllGCSClientIds(ac_rt, project_id, ENDPOINT_NAME) if valid_key is False and len(all_gcs_client_ids) > 0: - print("Looks like gcs client does not exist in the cloud" - f" for the project: {project_id}." - "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like gcs client does not exist in the cloud" + f" for the project: {project_id}." + "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if gcs_id_from_deployment_key is None and len(all_gcs_client_ids) > 0: - print("Looks like deployment key does not exist, please either " - "add the correct deployment." - f" cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like deployment key does not exist, please either " + "add the correct deployment." + f" cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if len(all_gcs_client_ids) > 0: if utils.command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - print("Now that we know a GCS instance exists we have to make sure" - "we have valid credentials to run the globus-connect-server command" - "non interatively, this means we have to create credentials and a" - "client if they don't exist and when we are done with everything" - "delete them.") + print( + "Now that we know a GCS instance exists we have to make sure" + "we have valid credentials to run the globus-connect-server command" + "non interatively, this means we have to create credentials and a" + "client if they don't exist and when we are done with everything" + "delete them." + ) client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) - + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH + ) - ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) + ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+="globus-connect-server endpoint cleanup " - bash_command+=f" --deployment-key \"{DEPLOYMENT_KEY_PATH}\" " - bash_command+=" --agree-to-delete-endpoint" + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += "globus-connect-server endpoint cleanup " + bash_command += f' --deployment-key "{DEPLOYMENT_KEY_PATH}" ' + bash_command += " --agree-to-delete-endpoint" print("Bash command to run") print(bash_command) - - proc = subprocess.Popen(bash_command, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, shell=True, text=True) + + proc = subprocess.Popen( + bash_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True, + text=True, + ) output, error = proc.communicate(input="yes\n") @@ -161,12 +181,11 @@ print("Output:", output) print("Error:", error) - # Now we can try to delete the remaining clients that are in the project # Get all of the clients that are not gcs clients and delete them utils.deleteAllNonGCSClients(ac_rt, project_id) - + # CLOSE - if len(clients_in_project) == 0: @@ -175,5 +194,3 @@ print(f"Attempting to remove project {project_id}") project_remove = ac_rt.delete_project(project_id) print(project_remove) - - diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index a45684fbb..b9e540202 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess import utils @@ -9,18 +8,18 @@ # Hard coded Native Client ID -CLIENT_ID = 'f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8' +CLIENT_ID = "f8d0afca-7ac4-4a3c-ac05-f94f5d9afce8" # The Globus project the GCS endpoint will be created in if os.getenv("DATAFED_GCS_ROOT_NAME") is not None: DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME") else: - DATAFED_GCS_ROOT_NAME="DataFed Repo" + DATAFED_GCS_ROOT_NAME = "DataFed Repo" if os.getenv("DATAFED_GLOBUS_PROJECT_NAME") is not None: - PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME") + PROJECT_NAME = os.getenv("DATAFED_GLOBUS_PROJECT_NAME") else: - PROJECT_NAME=DATAFED_GCS_ROOT_NAME + " Project" + PROJECT_NAME = DATAFED_GCS_ROOT_NAME + " Project" # This is for confidential client if os.getenv("DATAFED_GLOBUS_CLIENT_NAME") is not None: @@ -30,44 +29,47 @@ # Name of the client secret used by the confidential client if os.getenv("DATAFED_GLOBUS_CRED_NAME") is not None: - CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME") + CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME") else: - CRED_NAME= DATAFED_GCS_ROOT_NAME + " Cred" + CRED_NAME = DATAFED_GCS_ROOT_NAME + " Cred" # Name of the file where we will store confidential client credentials if os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") is not None: - CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") + CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH") else: - CRED_FILE_PATH="./client_cred.json" + CRED_FILE_PATH = "./client_cred.json" # Name to give to endpoint if os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") is not None: ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME") else: - ENDPOINT_NAME= DATAFED_GCS_ROOT_NAME + " Endpoint" + ENDPOINT_NAME = DATAFED_GCS_ROOT_NAME + " Endpoint" # Path to deployment key if os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") is not None: - DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") + DEPLOYMENT_KEY_PATH = os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH") else: - DEPLOYMENT_KEY_PATH="./deployment-key.json" + DEPLOYMENT_KEY_PATH = "./deployment-key.json" # Path to deployment key if os.getenv("DATAFED_GLOBUS_CONTROL_PORT") is not None: - DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT") + DATAFED_GLOBUS_CONTROL_PORT = os.getenv("DATAFED_GLOBUS_CONTROL_PORT") else: - DATAFED_GLOBUS_CONTROL_PORT="443" + DATAFED_GLOBUS_CONTROL_PORT = "443" if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: - DATAFED_GLOBUS_SUBSCRIPTION=os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") + DATAFED_GLOBUS_SUBSCRIPTION = os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") else: - DATAFED_GLOBUS_SUBSCRIPTION="" + DATAFED_GLOBUS_SUBSCRIPTION = "" client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project # view_identities to user information for creating GCS server -client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -75,7 +77,9 @@ token_response = client.oauth2_exchange_code_for_tokens(auth_code) # Extract the token -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) # auth_client_refresh_token ac_rt = AuthClient(authorizer=rt_authorizer) @@ -93,20 +97,19 @@ count = utils.countProjects(ac_rt, PROJECT_NAME) if count != 1: - print("Something is wrong there should be at least one project with name" - f" {PROJECT_NAME} instead there are {count} with that name") + print( + "Something is wrong there should be at least one project with name" + f" {PROJECT_NAME} instead there are {count} with that name" + ) sys.exit(1) print(f"Project id is {project_id}") client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH +) # Add the globus client as an admin to the project -ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) +ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) # Get clients in project clients_in_project = utils.getClientsInProject(ac_rt, project_id) @@ -115,13 +118,13 @@ # client exists for the globus connect server if it does not then we will # call the setup command utils.createGCSEndpoint( - ac_rt, - client_id, - client_secret, - project_id, - DEPLOYMENT_KEY_PATH, - ENDPOINT_NAME, - DATAFED_GLOBUS_CONTROL_PORT, - DATAFED_GLOBUS_SUBSCRIPTION, - userinfo) - + ac_rt, + client_id, + client_secret, + project_id, + DEPLOYMENT_KEY_PATH, + ENDPOINT_NAME, + DATAFED_GLOBUS_CONTROL_PORT, + DATAFED_GLOBUS_SUBSCRIPTION, + userinfo, +) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 84194498a..b42b17411 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess from globus_sdk import AuthClient, AccessTokenAuthorizer @@ -6,12 +5,14 @@ import os import sys + def getProjectId(projects, project_name): for project in projects: - if project['display_name'] == project_name: - return project['id'] + if project["display_name"] == project_name: + return project["id"] return None + def projectExists(auth_client, project_name): projects = auth_client.get_projects() project_id = getProjectId(projects, project_name) @@ -21,6 +22,7 @@ def projectExists(auth_client, project_name): project_exists = False return project_exists + def createProject(auth_client, project_name, userinfo): identity_id = userinfo["sub"] @@ -30,35 +32,40 @@ def createProject(auth_client, project_name, userinfo): if project_exists is False: project_create_result = auth_client.create_project( - project_name, - contact_email=email, - admin_ids=[identity_id]) - return project_create_result['project']['id'] + project_name, contact_email=email, admin_ids=[identity_id] + ) + return project_create_result["project"]["id"] projects = auth_client.get_projects() return getProjectId(projects, project_name) + def countProjects(auth_client, project_name): projects = auth_client.get_projects() count = 0 for project in projects: - if project['display_name'] == project_name: + if project["display_name"] == project_name: count += 1 return count + def getClientId(auth_client, client_name, project_id): get_client_result = auth_client.get_clients() - for client in get_client_result['clients']: - if client['name'] == client_name and client['project'] == project_id: - return client['id'] + for client in get_client_result["clients"]: + if client["name"] == client_name and client["project"] == project_id: + return client["id"] return None + def getAllGCSClientIds(auth_client, project_id, endpoint_name): clients_in_project = getClientsInProject(auth_client, project_id) all_gcs_client_ids = [] for client in clients_in_project: - if client['client_type'] == "globus_connect_server" and client['name'] == endpoint_name: - all_gcs_client_ids.append(client['id']) + if ( + client["client_type"] == "globus_connect_server" + and client["name"] == endpoint_name + ): + all_gcs_client_ids.append(client["id"]) return all_gcs_client_ids @@ -66,8 +73,8 @@ def getClientsInProject(auth_client, project_id): # Get clients in project get_client_result = auth_client.get_clients() clients_in_project = [] - for client in get_client_result['clients']: - if client['project'] == project_id: + for client in get_client_result["clients"]: + if client["project"] == project_id: clients_in_project.append(client) return clients_in_project @@ -80,18 +87,22 @@ def createNewClient(auth_client, client_name, project_id): client_exists = True if client_exists is False: - result = auth_client.create_client(client_name, project=project_id, public_client=False) + result = auth_client.create_client( + client_name, project=project_id, public_client=False + ) client_id = result["client"]["id"] return client_id + def getCredentialID(auth_client, client_id, cred_name): get_client_cred_result = auth_client.get_client_credentials(client_id) - for cred in get_client_cred_result['credentials']: - if cred['name'] == cred_name: - return cred['id'] + for cred in get_client_cred_result["credentials"]: + if cred["name"] == cred_name: + return cred["id"] return None + def validFile(file_name): file_exists = False file_empty = True @@ -109,59 +120,57 @@ def getCredentialFromFile(cred_file_name, cred_id): # name cred_exists_locally, cred_empty = validFile(cred_file_name) if cred_empty is False: - with open(cred_file_name, 'r') as f: + with open(cred_file_name, "r") as f: loaded_data = json.load(f) - if loaded_data['client'] == cred_id: - return loaded_data['secret'] + if loaded_data["client"] == cred_id: + return loaded_data["secret"] return None + def getClientIdFromCredFile(cred_file_name): # Check to see if the local secret is the same id and not just the same # name cred_exists_locally, cred_empty = validFile(cred_file_name) if cred_empty is False: - with open(cred_file_name, 'r') as f: + with open(cred_file_name, "r") as f: loaded_data = json.load(f) - return loaded_data['client'] + return loaded_data["client"] return None + def getEndpointIdFromFile(deployment_key_file_path): # Check to see if the local secret is the same id and not just the same # name exists_locally, empty = validFile(deployment_key_file_path) if empty is False: - with open(deployment_key_file_path, 'r') as f: + with open(deployment_key_file_path, "r") as f: loaded_data = json.load(f) - return loaded_data['client_id'] + return loaded_data["client_id"] return None def createNewCredential(auth_client, client_id, cred_name, cred_file): get_client_cred_result = auth_client.get_client_credentials(client_id) - for cred in get_client_cred_result['credentials']: + for cred in get_client_cred_result["credentials"]: # Should have stored secret locally - auth_client.delete_client_credential(client_id, cred['id']) + auth_client.delete_client_credential(client_id, cred["id"]) cred_result = auth_client.create_client_credential(client_id, cred_name) - # Have to change this to a dict + # Have to change this to a dict obj = { - 'client': cred_result['credential']['client'], - 'id': cred_result['credential']['id'], - 'name': cred_result['credential']['name'], - 'secret': cred_result['credential']['secret'] - } - with open(cred_file, 'w') as f: + "client": cred_result["credential"]["client"], + "id": cred_result["credential"]["id"], + "name": cred_result["credential"]["name"], + "secret": cred_result["credential"]["secret"], + } + with open(cred_file, "w") as f: json.dump(obj, f) - return cred_result['credential']['secret'] + return cred_result["credential"]["secret"] -def getClientSecret( - auth_client, - client_id, - cred_name, - cred_id, - cred_file): + +def getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file): client_secret = getCredentialFromFile(cred_file, cred_id) @@ -171,7 +180,7 @@ def getClientSecret( if client_secret: create_new_credential = False remove_cached_credential = False - remove_old_credential = True + remove_old_credential = True if remove_old_credential: auth_client.delete_client_credential(client_id, cred_id) @@ -183,11 +192,8 @@ def getClientSecret( if create_new_credential: # Remove credentials from cloud client_secret = createNewCredential( - auth_client, - client_id, - cred_name, - cred_file - ) + auth_client, client_id, cred_name, cred_file + ) return client_secret @@ -203,8 +209,11 @@ def createClient(auth_client, client_name, project_id, cred_name, cred_file): cred_exists_locally, cred_empty = validFile(cred_file) - client_secret = getClientSecret(auth_client, client_id, cred_name, cred_id, cred_file) - return client_id, client_secret + client_secret = getClientSecret( + auth_client, client_id, cred_name, cred_id, cred_file + ) + return client_id, client_secret + def getGCSClientIDFromDeploymentFile(deployment_key_file): deployment_key_exists, deployment_key_empty = validFile(deployment_key_file) @@ -216,9 +225,9 @@ def getGCSClientIDFromDeploymentFile(deployment_key_file): os.remove(deployment_key_file) else: # If it is not empty get the client id - with open(deployment_key_file, 'r') as f: + with open(deployment_key_file, "r") as f: loaded_data = json.load(f) - return loaded_data['client_id'] + return loaded_data["client_id"] return None @@ -231,44 +240,53 @@ def command_exists(command): # 'which' command returns non-zero exit status if the command is not found return False + def isGCSDeploymentKeyValid(auth_client, project_id, endpoint_name, gcs_id): clients_in_project = getClientsInProject(auth_client, project_id) # Check if the deployment key is valid for the project for client in clients_in_project: - if client['client_type'] == "globus_connect_server" and client['name'] == endpoint_name: + if ( + client["client_type"] == "globus_connect_server" + and client["name"] == endpoint_name + ): if gcs_id: # If gcs_id exists see if it is found remotely - if client['id'] == gcs_id: + if client["id"] == gcs_id: print("Deployment key endpoint is still valid found in cloud") return True else: # Found a globus_connect_server but did not find local deployment # key if deployment_key_empty: - print("Found globus_connect_server already registered but did" - " not find deployment key locally.") + print( + "Found globus_connect_server already registered but did" + " not find deployment key locally." + ) return False def deleteAllNonGCSClients(auth_client, project_id): clients = getClientsInProject(auth_client, project_id) for client in clients: - if client['project'] == project_id and client['client_type'] != "globus_connect_server": - auth_client.delete_client(client['id']) - + if ( + client["project"] == project_id + and client["client_type"] != "globus_connect_server" + ): + auth_client.delete_client(client["id"]) def createGCSEndpoint( - auth_client, - client_id, - client_secret, - project_id, - deployment_key_file, - endpoint_name, - control_port, - subscription_id, - userinfo): + auth_client, + client_id, + client_secret, + project_id, + deployment_key_file, + endpoint_name, + control_port, + subscription_id, + userinfo, +): identity_id = userinfo["sub"] email = userinfo["email"] @@ -277,53 +295,69 @@ def createGCSEndpoint( gcs_id_from_deployment_key = getGCSClientIDFromDeploymentFile(deployment_key_file) - valid_key = isGCSDeploymentKeyValid(auth_client, project_id, endpoint_name, gcs_id_from_deployment_key) + valid_key = isGCSDeploymentKeyValid( + auth_client, project_id, endpoint_name, gcs_id_from_deployment_key + ) if valid_key is False and gcs_id_from_deployment_key: - print("Looks like deployment key exists but does not contain credentials " - f"in the cloud for the project: {project_id}, please either " - "add the correct deployment key or remove the gcs instance" - "registered in the project") + print( + "Looks like deployment key exists but does not contain credentials " + f"in the cloud for the project: {project_id}, please either " + "add the correct deployment key or remove the gcs instance" + "registered in the project" + ) sys.exit(1) -# Create gcs_instance + # Create gcs_instance if valid_key is False: if command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" " - bash_command+=f" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+=f" globus-connect-server endpoint setup \"{endpoint_name}\" " - bash_command+=f" --organization \"{organization}\" " - bash_command+=f" --project-id \"{project_id}\" " - bash_command+=" --agree-to-letsencrypt-tos " - bash_command+=f" --project-admin \"{username}\" " - bash_command+=f" --owner \"{client_id}@clients.auth.globus.org\" " - bash_command+=f" --contact-email \"{email}\" " - bash_command+=f" --deployment-key \"{deployment_key_file}\" " + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" ' + bash_command += f' GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += f' globus-connect-server endpoint setup "{endpoint_name}" ' + bash_command += f' --organization "{organization}" ' + bash_command += f' --project-id "{project_id}" ' + bash_command += " --agree-to-letsencrypt-tos " + bash_command += f' --project-admin "{username}" ' + bash_command += f' --owner "{client_id}@clients.auth.globus.org" ' + bash_command += f' --contact-email "{email}" ' + bash_command += f' --deployment-key "{deployment_key_file}" ' print("Bash command to run") print(bash_command) - - process = subprocess.Popen(bash_command, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) + + process = subprocess.Popen( + bash_command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) # Print the output for line in process.stdout: - print(line, end='') + print(line, end="") deployment_key_exists, deployment_key_empty = validFile(deployment_key_file) if deployment_key_exists is False: - print(f"Something is wrong deployment key does not exist {deployment_key_file} ") + print( + f"Something is wrong deployment key does not exist {deployment_key_file} " + ) sys.exit(1) if deployment_key_empty: - print(f"Something is wrong deployment key is empty {deployment_key_file} ") + print( + f"Something is wrong deployment key is empty {deployment_key_file} " + ) sys.exit(1) # WARNING!!!!!! # This will not work if a node does not first exist, I think at least one # node must be running. - #if len(subscription_id) != 0: + # if len(subscription_id) != 0: # if command_exists("globus-connect-server") is False: # print("Cannot create deployment key, we require globus-connect-server to be installed") # sys.exit(1) @@ -336,10 +370,8 @@ def createGCSEndpoint( # bash_command+=" globus-connect-server endpoint update " # bash_command+=f" --subscription-id \"{subscription_id}\" " # print(bash_command) - # + # # process = subprocess.Popen(bash_command, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) # # Print the output # for line in process.stdout: # print(line, end='') - - From c7a3b9f09cca36c45d95fde70008cd788e838f8a Mon Sep 17 00:00:00 2001 From: par-hermes Date: Sat, 6 Apr 2024 04:57:40 +0000 Subject: [PATCH 4/6] cpp-py-formatter --- common/source/servers/Proxy.cpp | 20 ++- core/server/ClientWorker.cpp | 6 +- core/server/TaskWorker.cpp | 5 +- .../globus5/authz/source/AuthzWorker.cpp | 10 +- .../gridftp/globus5/authz/source/libauthz.c | 164 +++++++++++------- repository/server/RepoServer.cpp | 18 +- repository/server/RequestWorker.cpp | 10 +- scripts/globus/create_guest_collection.py | 142 ++++++++------- scripts/globus/globus_cleanup.py | 120 +++++++------ scripts/globus/initialize_globus_endpoint.py | 64 +++---- scripts/globus/utils.py | 21 ++- 11 files changed, 331 insertions(+), 249 deletions(-) diff --git a/common/source/servers/Proxy.cpp b/common/source/servers/Proxy.cpp index c78ceff7f..e88bd3df0 100644 --- a/common/source/servers/Proxy.cpp +++ b/common/source/servers/Proxy.cpp @@ -123,10 +123,12 @@ void Proxy::run() { // public if (resp_from_client_socket.error == false and resp_from_client_socket.time_out == false) { - if(not resp_from_server_socket.message ) { - DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " - << "response is not defined but no timeouts or errors were " - << "triggered, unable to send to server."); + if (not resp_from_server_socket.message) { + DL_ERROR( + m_log_context, + "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to send to server."); } else { m_communicators[SocketRole::SERVER]->send( *resp_from_client_socket.message); @@ -143,10 +145,12 @@ void Proxy::run() { // ... - Serv Sock - Proxy ------ Client Sock - Serv Sock - Inter App if (resp_from_server_socket.error == false and resp_from_server_socket.time_out == false) { - if(not resp_from_server_socket.message ) { - DL_ERROR(m_log_context, "Proxy::run - Something is wrong, message " - << "response is not defined but no timeouts or errors were " - << "triggered, unable to operate and send to client."); + if (not resp_from_server_socket.message) { + DL_ERROR( + m_log_context, + "Proxy::run - Something is wrong, message " + << "response is not defined but no timeouts or errors were " + << "triggered, unable to operate and send to client."); } else { for (auto &in_operator : m_incoming_operators) { in_operator->execute(*resp_from_server_socket.message); diff --git a/core/server/ClientWorker.cpp b/core/server/ClientWorker.cpp index 40e57f00b..2c3084ee0 100644 --- a/core/server/ClientWorker.cpp +++ b/core/server/ClientWorker.cpp @@ -346,8 +346,10 @@ void ClientWorker::workerThread(LogContext log_context) { ICommunicator::Response response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); if (response.time_out == false and response.error == false) { - if ( not response.message ) { - DL_ERROR(message_log_context, "No timeout or error was reported but message is not defined.") + if (not response.message) { + DL_ERROR( + message_log_context, + "No timeout or error was reported but message is not defined.") } IMessage &message = *response.message; diff --git a/core/server/TaskWorker.cpp b/core/server/TaskWorker.cpp index 87dd0f1ad..95c55ead2 100644 --- a/core/server/TaskWorker.cpp +++ b/core/server/TaskWorker.cpp @@ -339,8 +339,9 @@ TaskWorker::cmdRawDataUpdateSize(TaskWorker &me, const Value &a_task_params, if (response.time_out == true) { return response; } else if (response.error == false) { - if( not response.message ) { - DL_ERROR(log_context, "No timeout or error was reported but no message was defined."); + if (not response.message) { + DL_ERROR(log_context, + "No timeout or error was reported but no message was defined."); } auto proto_msg = std::get(response.message->getPayload()); diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index 4bc7261bd..6a6eaa7e0 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -226,7 +226,8 @@ class AuthzWorker { << client->address()); auto response = client->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); - if (response.message ) { // Make sure the message exists before we try to access it + if (response.message) { // Make sure the message exists before we try to + // access it log_context.correlation_id = std::get( response.message->get(MessageAttribute::CORRELATION_ID)); } @@ -252,9 +253,10 @@ class AuthzWorker { "communicating with the core service: " << response.error_msg); } else { - - if( not response.message ) { - DL_ERROR(log_context, "No error was reported and no time out occured but message is not defined."); + + if (not response.message) { + DL_ERROR(log_context, "No error was reported and no time out occured " + "but message is not defined."); } auto payload = diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index 2dd6b1b7b..fd03bb7b2 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -20,40 +20,63 @@ // Define logging macros #if defined(DONT_USE_SYSLOG) - FILE *log_file = NULL; - bool write_to_file = true; - #define AUTHZ_LOG_DEBUG(fmt, ...) \ - do { if (LOG_LEVEL >= 1) fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0); \ - do { if (LOG_LEVEL >= 1 && write_to_file ) fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INFO(fmt, ...) \ - do { if (LOG_LEVEL >= 2) fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); } while (0); \ - do { if (LOG_LEVEL >= 2 && write_to_file ) fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_ERROR(fmt, ...) \ - do { \ - if (LOG_LEVEL >= 3) fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ - if (LOG_LEVEL >= 3 && write_to_file ) fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ - } while(0) - #define AUTHZ_LOG_INIT(file_path) \ - log_file = fopen(file_path, "a"); \ - if (log_file != NULL) { write_to_file = true; } - #define AUTHZ_LOG_CLOSE() \ - if (log_file != NULL) { fclose(log_file); } +FILE *log_file = NULL; +bool write_to_file = true; +#define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 1) \ + fprintf(stderr, "[DEBUG] " fmt "", ##__VA_ARGS__); \ + } while (0); \ + do { \ + if (LOG_LEVEL >= 1 && write_to_file) \ + fprintf(log_file, "[DEBUG] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INFO(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 2) \ + fprintf(stderr, "[INFO] " fmt "", ##__VA_ARGS__); \ + } while (0); \ + do { \ + if (LOG_LEVEL >= 2 && write_to_file) \ + fprintf(log_file, "[INFO] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_ERROR(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 3) \ + fprintf(stderr, "[ERROR] " fmt "", ##__VA_ARGS__); \ + if (LOG_LEVEL >= 3 && write_to_file) \ + fprintf(log_file, "[ERROR] " fmt "", ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INIT(file_path) \ + log_file = fopen(file_path, "a"); \ + if (log_file != NULL) { \ + write_to_file = true; \ + } +#define AUTHZ_LOG_CLOSE() \ + if (log_file != NULL) { \ + fclose(log_file); \ + } #else - #include - #define AUTHZ_LOG_DEBUG(fmt, ...) \ - do { if (LOG_LEVEL >= 1) syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INFO(fmt, ...) \ - do { if (LOG_LEVEL >= 2) syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_ERROR(fmt, ...) \ - do { if (LOG_LEVEL >= 3) syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); } while (0) - #define AUTHZ_LOG_INIT(file_path) \ - openlog("gsi_authz", 0, LOG_AUTH); - #define AUTHZ_LOG_CLOSE() \ - closelog(); +#include +#define AUTHZ_LOG_DEBUG(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 1) \ + syslog(LOG_DEBUG, "[DEBUG] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INFO(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 2) \ + syslog(LOG_INFO, "[INFO] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_ERROR(fmt, ...) \ + do { \ + if (LOG_LEVEL >= 3) \ + syslog(LOG_ERR, "[ERROR] " fmt, ##__VA_ARGS__); \ + } while (0) +#define AUTHZ_LOG_INIT(file_path) openlog("gsi_authz", 0, LOG_AUTH); +#define AUTHZ_LOG_CLOSE() closelog(); #endif - - typedef void *globus_gsi_authz_handle_t; typedef void (*globus_gsi_authz_cb_t)(void *callback_arg, globus_gsi_authz_handle_t handle, @@ -189,13 +212,14 @@ bool setConfigVal(const char *a_label, char *a_dest, char *a_src, size_t len = strlen(a_src); if (len == 0) { - AUTHZ_LOG_ERROR("DataFed - '%s' value not set.\n", a_label); + AUTHZ_LOG_ERROR("DataFed - '%s' value not set.\n", a_label); return true; } if (len > a_max_len) { - AUTHZ_LOG_ERROR("DataFed - '%s' value too long in authz config file (max %lu).\n", - a_label, a_max_len); + AUTHZ_LOG_ERROR( + "DataFed - '%s' value too long in authz config file (max %lu).\n", + a_label, a_max_len); return true; } @@ -269,7 +293,7 @@ bool loadConfig() { val = strchr(buf, '='); if (!val) { AUTHZ_LOG_ERROR( - "DataFed - Syntax error in authz config file at line %i.\n", lc); + "DataFed - Syntax error in authz config file at line %i.\n", lc); return true; } else { *val = 0; @@ -305,8 +329,8 @@ bool loadConfig() { else { err = true; AUTHZ_LOG_ERROR( - "DataFed - Invalid key, '%s', in authz config file at line %i.\n", - buf, lc); + "DataFed - Invalid key, '%s', in authz config file at line %i.\n", + buf, lc); } if (err) { @@ -336,20 +360,23 @@ bool loadConfig() { strcat(miss, " server_key"); AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); - AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", + getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); if (miss[0] != 0) { - AUTHZ_LOG_ERROR("DataFed - Missing required authz config items:%s\n", miss); + AUTHZ_LOG_ERROR("DataFed - Missing required authz config items:%s\n", + miss); return true; } } else { AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); - AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); + AUTHZ_LOG_INFO(" API, version %s\n", + getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); AUTHZ_LOG_ERROR("DataFed - Could not open authz config file.\n"); return true; @@ -358,14 +385,14 @@ bool loadConfig() { AUTHZ_LOG_INFO("DataFed Authz module started, version %s\n", getVersion()); AUTHZ_LOG_INFO(" API, version %s\n", getAPIVersion()); AUTHZ_LOG_INFO(" Release, version %s\n", - getReleaseVersion()); + getReleaseVersion()); return false; } // The same globus_result_t gsi_authz_init() { - //openlog("gsi_authz", 0, LOG_AUTH); + // openlog("gsi_authz", 0, LOG_AUTH); memset(g_active_contexts, 0, sizeof(g_active_contexts)); // This line is different @@ -402,7 +429,8 @@ globus_result_t gsi_authz_handle_init(va_list ap) { else AUTHZ_LOG_ERROR("gsi_authz_handle_init out of handle context space\n"); } else { - AUTHZ_LOG_DEBUG("gsi_authz_handle_init context handle already initialized\n"); + AUTHZ_LOG_DEBUG( + "gsi_authz_handle_init context handle already initialized\n"); } callback(callback_arg, callback_arg, result); @@ -449,21 +477,21 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { char *callout_id_mapped1 = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", - callout_ids1); + callout_ids1); AUTHZ_LOG_DEBUG("libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", - callout_id_mapped1); + callout_id_mapped1); AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", - g_config.globus_collection_path, action, object); + g_config.globus_collection_path, action, object); if (strcmp(action, "lookup") == 0 || strcmp(action, "chdir") == 0) { AUTHZ_LOG_INFO("Allowed collection path: %s, action: %s, object is %s\n", - g_config.globus_collection_path, action, object); + g_config.globus_collection_path, action, object); result = GLOBUS_SUCCESS; callback(callback_arg, handle, result); return result; } AUTHZ_LOG_ERROR("gsi_authz_authorize_async, handle: %p, act: %s, obj: %s\n", - handle, action, object); + handle, action, object); OM_uint32 min_stat; gss_name_t client = GSS_C_NO_NAME; @@ -486,7 +514,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { gss_display_name(&min_stat, target, &target_buf, &target_type); if (maj_stat == GSS_S_COMPLETE) { AUTHZ_LOG_INFO("Auth client: %s, file: %s, action: %s\n", - (char *)client_buf.value, object, action); + (char *)client_buf.value, object, action); // Testing hack #if 0 @@ -503,7 +531,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { if (strncmp((char *)client_buf.value, "/C=US/O=Globus Consortium/OU=Globus", 35) != 0) { AUTHZ_LOG_ERROR("Invalid certificate subject prefix: %s\n", - (char *)client_buf.value); + (char *)client_buf.value); } else { /* Note: For some reason, globus will provide the CN as either a * UUID that is linked to the client's account and encoded in @@ -523,7 +551,7 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { if (!decodeUUID((char *)client_buf.value + 54, client_id)) { AUTHZ_LOG_ERROR("Failed to decode subject UUID: %s\n", - (char *)client_buf.value + 54); + (char *)client_buf.value + 54); free(client_id); client_id = 0; } @@ -539,23 +567,25 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { } char *callout_ids = getenv("GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS"); - char *callout_username_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); - char *callout_id_mapped = getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); + char *callout_username_mapped = + getenv("GLOBUS_GRIDFTP_MAPPED_USERNAME"); + char *callout_id_mapped = + getenv("GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID"); if (callout_ids != NULL) { AUTHZ_LOG_DEBUG( - "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", - callout_ids); + "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS: %s\n", + callout_ids); client_id = strdup(callout_ids); AUTHZ_LOG_INFO("libauthz.c client_id(s): %s\n", client_id); - } else if ( callout_id_mapped != NULL ) { + } else if (callout_id_mapped != NULL) { AUTHZ_LOG_DEBUG( - "libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", - callout_id_mapped); + "libauthz.c GLOBUS_GRIDFTP_MAPPED_IDENTITY_ID: %s\n", + callout_id_mapped); client_id = strdup(callout_id_mapped); } else { AUTHZ_LOG_ERROR( - "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS.\n"); + "libauthz.c GLOBUS_GRIDFTP_GUEST_IDENTITY_IDS.\n"); } } @@ -566,8 +596,8 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { } else { AUTHZ_LOG_INFO( - "libauthz.c Auth client_id: %s, file: %s, action: %s\n", - client_id, object, action); + "libauthz.c Auth client_id: %s, file: %s, action: %s\n", + client_id, object, action); AUTHZ_LOG_INFO("libauthz.c checkAuthorization FAIL.\n"); } @@ -578,17 +608,17 @@ globus_result_t gsi_authz_authorize_async(va_list ap) { gss_release_buffer(&min_stat, &target_buf); } else { AUTHZ_LOG_ERROR("gss_display_name target FAILED, maj: %d, min: %d\n", - maj_stat, min_stat); + maj_stat, min_stat); } gss_release_buffer(&min_stat, &client_buf); } else { AUTHZ_LOG_ERROR("gss_display_name source FAILED, maj: %d, min: %d\n", - maj_stat, min_stat); + maj_stat, min_stat); } } else { - AUTHZ_LOG_ERROR("gss_inquire_context FAILED, maj: %d, min: %d\n", maj_stat, - min_stat); + AUTHZ_LOG_ERROR("gss_inquire_context FAILED, maj: %d, min: %d\n", + maj_stat, min_stat); } } else { AUTHZ_LOG_ERROR("context handle lookup FAILED\n"); diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index d851eb0b7..e3051e3cc 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -101,7 +101,8 @@ void Server::run() { } void Server::checkServerVersion() { - DL_INFO(m_log_context, "Checking core server connection and version at " << m_config.core_server); + DL_INFO(m_log_context, "Checking core server connection and version at " + << m_config.core_server); // Generate random security keys for anon version request to core server KeyGenerator generator; @@ -148,7 +149,8 @@ void Server::checkServerVersion() { for (int i = 0; i < 10; i++) { DL_INFO(m_log_context, "Attempt " << i << " to initialize communication " - << " with core server at " << m_config.core_server); + << " with core server at " + << m_config.core_server); auto msg = std::make_unique(); auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); message->setPayload(std::move(msg)); @@ -166,14 +168,14 @@ void Server::checkServerVersion() { DL_ERROR(msg_log_context, "Timeout waiting for response from core server: " << m_config.core_server); - } else if(response.error) { - DL_ERROR(msg_log_context, - "Error encountered waiting for core server: " - << m_config.core_server << " msg " << response.error_msg); + } else if (response.error) { + DL_ERROR(msg_log_context, "Error encountered waiting for core server: " + << m_config.core_server << " msg " + << response.error_msg); } else { - + msg_log_context.correlation_id = std::get( - response.message->get(MessageAttribute::CORRELATION_ID)); + response.message->get(MessageAttribute::CORRELATION_ID)); auto payload = std::get(response.message->getPayload()); VersionReply *ver_reply = dynamic_cast(payload); diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index be08293fd..43b0652b5 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -146,9 +146,9 @@ void RequestWorker::workerThread(LogContext log_context) { DL_TRACE(message_log_context, "Checking timeouts: " << response.time_out); if (response.time_out == false and response.error == false) { - if( not response.message) { + if (not response.message) { DL_ERROR(log_context, "Error: No error or timeout occurred but the" - << " message does not exist."); + << " message does not exist."); } else { // May not have a correlation id if the message timed out DL_TRACE(log_context, "Getting correlation_id."); @@ -165,18 +165,18 @@ void RequestWorker::workerThread(LogContext log_context) { if (m_msg_handlers.count(msg_type)) { map::iterator handler = - m_msg_handlers.find(msg_type); + m_msg_handlers.find(msg_type); DL_TRACE(message_log_context, "Calling handler"); auto send_message = - (this->*handler->second)(std::move(response.message)); + (this->*handler->second)(std::move(response.message)); client->send(*(send_message)); DL_TRACE(message_log_context, "Reply sent."); } else { DL_ERROR(message_log_context, - "Received unregistered msg type: " << msg_type); + "Received unregistered msg type: " << msg_type); } } } else if (response.error) { diff --git a/scripts/globus/create_guest_collection.py b/scripts/globus/create_guest_collection.py index 59f6131fe..0099f34fa 100644 --- a/scripts/globus/create_guest_collection.py +++ b/scripts/globus/create_guest_collection.py @@ -1,4 +1,3 @@ - import globus_sdk import subprocess import utils @@ -10,61 +9,76 @@ # The Globus project the GCS endpoint will be created in DATAFED_GCS_ROOT_NAME = os.getenv("DATAFED_GCS_ROOT_NAME", "DataFed Repo") -PROJECT_NAME=os.getenv("DATAFED_GLOBUS_PROJECT_NAME", DATAFED_GCS_ROOT_NAME + " Project") +PROJECT_NAME = os.getenv( + "DATAFED_GLOBUS_PROJECT_NAME", DATAFED_GCS_ROOT_NAME + " Project" +) # This is for confidential client -CLIENT_NAME = os.getenv("DATAFED_GLOBUS_CLIENT_NAME", DATAFED_GCS_ROOT_NAME + " Setup Client") +CLIENT_NAME = os.getenv( + "DATAFED_GLOBUS_CLIENT_NAME", DATAFED_GCS_ROOT_NAME + " Setup Client" +) # Name of the client secret used by the confidential client -CRED_NAME=os.getenv("DATAFED_GLOBUS_CRED_NAME",DATAFED_GCS_ROOT_NAME + " Cred") +CRED_NAME = os.getenv("DATAFED_GLOBUS_CRED_NAME", DATAFED_GCS_ROOT_NAME + " Cred") # Name of the file where we will store confidential client credentials -CRED_FILE_PATH=os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH","./client_cred.json") +CRED_FILE_PATH = os.getenv("DATAFED_GLOBUS_CRED_FILE_PATH", "./client_cred.json") ENDPOINT_ID = os.getenv("GCS_CLI_ENDPOINT_ID") -ENDPOINT_NAME = os.getenv("DATAFED_GLOBUS_ENDPOINT_NAME",DATAFED_GCS_ROOT_NAME + " Endpoint") +ENDPOINT_NAME = os.getenv( + "DATAFED_GLOBUS_ENDPOINT_NAME", DATAFED_GCS_ROOT_NAME + " Endpoint" +) # Path to deployment key -DEPLOYMENT_KEY_PATH=os.getenv("DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH","./deployment-key.json") +DEPLOYMENT_KEY_PATH = os.getenv( + "DATAFED_GLOBUS_DEPLOYMENT_KEY_PATH", "./deployment-key.json" +) # Path to deployment key -DATAFED_GLOBUS_CONTROL_PORT=os.getenv("DATAFED_GLOBUS_CONTROL_PORT", "443") -DATAFED_GCS_URL=os.getenv("DATAFED_GCS_URL") +DATAFED_GLOBUS_CONTROL_PORT = os.getenv("DATAFED_GLOBUS_CONTROL_PORT", "443") +DATAFED_GCS_URL = os.getenv("DATAFED_GCS_URL") client_id = os.getenv("GCS_CLI_CLIENT_ID") client_secret = os.getenv("GCS_CLI_CLIENT_SECRET") mapped_collection_id = os.getenv("MAPPED_COLLECTION_ID") -mapped_collection_name = os.getenv("DATAFED_GCS_COLLECTION_MAPPED", f"{DATAFED_GCS_ROOT_NAME} Collection Mapped") -guest_collection_name = os.getenv("DATAFED_GCS_COLLECTION_GUEST",f"{DATAFED_GCS_ROOT_NAME} Collection Guest") +mapped_collection_name = os.getenv( + "DATAFED_GCS_COLLECTION_MAPPED", f"{DATAFED_GCS_ROOT_NAME} Collection Mapped" +) +guest_collection_name = os.getenv( + "DATAFED_GCS_COLLECTION_GUEST", f"{DATAFED_GCS_ROOT_NAME} Collection Guest" +) storage_gateway_id = os.getenv("STORAGE_GATEWAY_ID") -storage_gateway_name = os.getenv("DATAFED_GCS_STORAGE_GATEWAY",f"{DATAFED_GCS_ROOT_NAME} Storage Gateway") +storage_gateway_name = os.getenv( + "DATAFED_GCS_STORAGE_GATEWAY", f"{DATAFED_GCS_ROOT_NAME} Storage Gateway" +) local_username = os.getenv("DATAFED_REPO_USER") if ENDPOINT_ID is None: raise Exception("GCS_CLI_ENDPOINT_ID must be defined as an env varaible") if DATAFED_GCS_URL is None: - raise Exception("Unable to create guest collection, DATAFED_GCS_URL is not" - " defined.") + raise Exception( + "Unable to create guest collection, DATAFED_GCS_URL is not" " defined." + ) if local_username is None: raise Exception("DATAFED_REPO_USER is not defined.") -#client = globus_sdk.NativeAppAuthClient(CLIENT_ID) +# client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project # view_identities to user information for creating GCS server -#client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) +# client.oauth2_start_flow(requested_scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities", refresh_tokens=True) # -#authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) -#print("Please go to this URL and login: \n", authorize_url) -#auth_code = input("Please enter the authorization code: ") +# authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) +# print("Please go to this URL and login: \n", authorize_url) +# auth_code = input("Please enter the authorization code: ") # -#token_response = client.oauth2_exchange_code_for_tokens(auth_code) +# token_response = client.oauth2_exchange_code_for_tokens(auth_code) ## Extract the token -#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) ## auth_client_refresh_token -#ac_rt = AuthClient(authorizer=rt_authorizer) +# ac_rt = AuthClient(authorizer=rt_authorizer) # -#userinfo = ac_rt.oauth2_userinfo() +# userinfo = ac_rt.oauth2_userinfo() ## Will get the primary email and id -#identity_id = userinfo["sub"] -#email = userinfo["email"] -#username = userinfo["preferred_username"] -#organization = userinfo["identity_provider_display_name"] +# identity_id = userinfo["sub"] +# email = userinfo["email"] +# username = userinfo["preferred_username"] +# organization = userinfo["identity_provider_display_name"] if client_id is None: client_id = getClientIdFromCredFile(CRED_FILE_PATH) @@ -74,21 +88,21 @@ client = globus_sdk.ConfidentialAppAuthClient(client_id, client_secret) -scopes="openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" +scopes = "openid profile email urn:globus:auth:scope:auth.globus.org:manage_projects urn:globus:auth:scope:auth.globus.org:view_identities" authorizer = globus_sdk.ClientCredentialsAuthorizer(client, scopes) -#cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, +# cc_authorizer = globus_sdk.ClientCredentialsAuthorizer(confidential_client, # scopes) -#token_response = client.oauth2_client_credentials_tokens() +# token_response = client.oauth2_client_credentials_tokens() -#refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -#rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) +# refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] +# rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) # the useful values that you want at the end of this -#globus_auth_data = token_response.by_resource_server["auth.globus.org"] -#globus_transfer_data = -#token_response.by_resource_server["transfer.api.globus.org"] -#globus_auth_token = globus_auth_data["access_token"] -#globus_transfer_token = globus_transfer_data["access_token"] +# globus_auth_data = token_response.by_resource_server["auth.globus.org"] +# globus_transfer_data = +# token_response.by_resource_server["transfer.api.globus.org"] +# globus_auth_token = globus_auth_data["access_token"] +# globus_transfer_token = globus_transfer_data["access_token"] gcs_client = globus_sdk.GCSClient(DATAFED_GCS_URL, authorizer=authorizer) @@ -97,10 +111,10 @@ from globus_sdk import scopes # constants -#endpoint_hostname = "https://ecf8ed.08cc.data.globus.org" -#endpoint_id = "769c7ed0-744a-41b5-b4a8-db37b10b1ac9" -#mapped_collection_id = "580ecb92-de56-42ee-a5ec-3d3886767b94" -#storage_gateway_id = "3fdd7f41-4a05-4856-8fcd-2fb50066c590" +# endpoint_hostname = "https://ecf8ed.08cc.data.globus.org" +# endpoint_id = "769c7ed0-744a-41b5-b4a8-db37b10b1ac9" +# mapped_collection_id = "580ecb92-de56-42ee-a5ec-3d3886767b94" +# storage_gateway_id = "3fdd7f41-4a05-4856-8fcd-2fb50066c590" # client credentials # This client identity must have the needed permissions to create a guest @@ -108,9 +122,9 @@ # on the storage gateway that matches the local_username # If using user tokens, the user must be the one with the correct permissions # and identity mapping. -#client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" -#client_secret = "*redacted*" -#local_username = "datafed" +# client_id = "4de65cd7-4363-4510-b652-f8d15a43a0af" +# client_secret = "*redacted*" +# local_username = "datafed" # The scope the client will need, note that primary scope is for the endpoint, # but it has a dependency on the mapped collection's data_access scope @@ -135,13 +149,15 @@ if item["id"] == mapped_collection_id: mapped_collection_found = True if item["display_name"] != mapped_collection_name: - raise Exception("Expected display name is different from what " - "is expected for mapped collection " - f"{mapped_collection_id}, if using non standard" - " display name for mapped collection " - f"{mapped_collection_name} then the " - "MAPPED_COLLECTION_NAME env variable must be " - "set.") + raise Exception( + "Expected display name is different from what " + "is expected for mapped collection " + f"{mapped_collection_id}, if using non standard" + " display name for mapped collection " + f"{mapped_collection_name} then the " + "MAPPED_COLLECTION_NAME env variable must be " + "set." + ) break elif item["display_name"] == mapped_collection_name: mapped_collection_found = True @@ -159,19 +175,21 @@ if item["id"] == storage_gateway_id: storage_gateway_found = True if item["display_name"] != storage_gateway_name: - raise Exception("Expected display name is different from what " - "is expected for storage gateway " - f"{storage_gateway_id}, if using non standard" - " display name for storage gateway " - f"{storage_gateway_name} then the " - "DATAFED_GCS_STORAGE_GATEWAY env variable must be " - "set.") + raise Exception( + "Expected display name is different from what " + "is expected for storage gateway " + f"{storage_gateway_id}, if using non standard" + " display name for storage gateway " + f"{storage_gateway_name} then the " + "DATAFED_GCS_STORAGE_GATEWAY env variable must be " + "set." + ) break elif item["display_name"] == storage_gateway_name: storage_gateway_found = True storage_gateway_id = item["id"] break - + if storage_gateway_found == False: raise Exception("Missing required storage gateway") @@ -189,7 +207,7 @@ guest_collection_found = True guest_collection_id = item["id"] break - + # https://github.com/globus/globus-sdk-python/blob/main/docs/examples/guest_collection_creation.rst if guest_collection_found == False: credential_document = globus_sdk.UserCredentialDocument( @@ -199,7 +217,7 @@ ) client.create_user_credential(credential_document) -# Create the collection + # Create the collection collection_document = globus_sdk.GuestCollectionDocument( public="True", collection_base_path="/", diff --git a/scripts/globus/globus_cleanup.py b/scripts/globus/globus_cleanup.py index 133ee9d3b..502c57733 100644 --- a/scripts/globus/globus_cleanup.py +++ b/scripts/globus/globus_cleanup.py @@ -1,5 +1,5 @@ import globus_sdk -from globus_sdk import AuthClient,GroupsClient, AccessTokenAuthorizer +from globus_sdk import AuthClient, GroupsClient, AccessTokenAuthorizer from globus_sdk.scopes import GroupsScopes import subprocess import json @@ -55,11 +55,12 @@ client = globus_sdk.NativeAppAuthClient(CLIENT_ID) # manage_projects scope to create a project group_scope = GroupsScopes.make_mutable("all") -client.oauth2_start_flow(requested_scopes="openid profile email " - "urn:globus:auth:scope:auth.globus.org:manage_projects " - "urn:globus:auth:scope:auth.globus.org:view_identities " + - str(group_scope), - refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + str(group_scope), + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -71,13 +72,16 @@ print(token_response) -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] +refresh_token_groups = token_response.by_resource_server["groups.api.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, - client) +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, client) ac_rt = AuthClient(authorizer=rt_authorizer) gr_rt = GroupsClient(authorizer=rt_authorizer_groups) @@ -100,68 +104,84 @@ clients_in_project = utils.getClientsInProject(ac_rt, project_id) if len(clients_in_project) == 0: - print("No clients were detected in the project we can just delete the" - "project and be done.") + print( + "No clients were detected in the project we can just delete the" + "project and be done." + ) else: - # Check if the deployment key exists if it does read it and verify that the - # client exists for the globus connect server if it does not then we will - # call the setup command + # Check if the deployment key exists if it does read it and verify that the + # client exists for the globus connect server if it does not then we will + # call the setup command - gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile(DEPLOYMENT_KEY_PATH) + gcs_id_from_deployment_key = utils.getGCSClientIDFromDeploymentFile( + DEPLOYMENT_KEY_PATH + ) - valid_key = utils.isGCSDeploymentKeyValid(ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key) + valid_key = utils.isGCSDeploymentKeyValid( + ac_rt, project_id, ENDPOINT_NAME, gcs_id_from_deployment_key + ) all_gcs_client_ids = utils.getAllGCSClientIds(ac_rt, project_id, ENDPOINT_NAME) if valid_key is False and len(all_gcs_client_ids) > 0: - print("Looks like gcs client does not exist in the cloud" - f" for the project: {project_id}." - "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like gcs client does not exist in the cloud" + f" for the project: {project_id}." + "Maybe you have the wrong deployment key cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if gcs_id_from_deployment_key is None and len(all_gcs_client_ids) > 0: - print("Looks like deployment key does not exist, please either " - "add the correct deployment." - f" cloud_ids {all_gcs_client_ids}" - f"deployment key id {gcs_id_from_deployment_key}") + print( + "Looks like deployment key does not exist, please either " + "add the correct deployment." + f" cloud_ids {all_gcs_client_ids}" + f"deployment key id {gcs_id_from_deployment_key}" + ) sys.exit(1) if len(all_gcs_client_ids) > 0: if utils.command_exists("globus-connect-server") is False: - print("Cannot create deployment key, we require globus-connect-server to be installed") + print( + "Cannot create deployment key, we require globus-connect-server to be installed" + ) sys.exit(1) else: - print("Now that we know a GCS instance exists we have to make sure" - "we have valid credentials to run the globus-connect-server command" - "non interatively, this means we have to create credentials and a" - "client if they don't exist and when we are done with everything" - "delete them.") + print( + "Now that we know a GCS instance exists we have to make sure" + "we have valid credentials to run the globus-connect-server command" + "non interatively, this means we have to create credentials and a" + "client if they don't exist and when we are done with everything" + "delete them." + ) client_id, client_secret = utils.createClient( - ac_rt, - CLIENT_NAME, - project_id, - CRED_NAME, - CRED_FILE_PATH) + ac_rt, CLIENT_NAME, project_id, CRED_NAME, CRED_FILE_PATH + ) + ac_rt.update_project(project_id, admin_ids=[identity_id, client_id]) - ac_rt.update_project(project_id,admin_ids=[identity_id, client_id]) - - bash_command=f"GCS_CLI_CLIENT_ID=\"{client_id}\" GCS_CLI_CLIENT_SECRET=\"{client_secret}\" " - bash_command+="globus-connect-server endpoint cleanup " - bash_command+=f" --deployment-key \"{DEPLOYMENT_KEY_PATH}\" " - bash_command+=" --agree-to-delete-endpoint" + bash_command = f'GCS_CLI_CLIENT_ID="{client_id}" GCS_CLI_CLIENT_SECRET="{client_secret}" ' + bash_command += "globus-connect-server endpoint cleanup " + bash_command += f' --deployment-key "{DEPLOYMENT_KEY_PATH}" ' + bash_command += " --agree-to-delete-endpoint" print("Bash command to run") print(bash_command) - - proc = subprocess.Popen(bash_command, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, shell=True, text=True) + + proc = subprocess.Popen( + bash_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True, + text=True, + ) output, error = proc.communicate(input="yes\n") @@ -169,17 +189,15 @@ print(output) print(error) - # Now we can try to delete the remaining clients that are in the project # Get all of the clients that are not gcs clients and delete them utils.deleteAllNonGCSClients(ac_rt, project_id) - -# CLOSE - if len(clients_in_project) == 0: + # CLOSE - if len(clients_in_project) == 0: -# Try to remove project this will only work if there are no other clients in -# the project + # Try to remove project this will only work if there are no other clients in + # the project print(f"Attempting to remove project {project_id}") project_remove = ac_rt.delete_project(project_id) print(project_remove) diff --git a/scripts/globus/initialize_globus_endpoint.py b/scripts/globus/initialize_globus_endpoint.py index 16afaa519..0f0e79334 100644 --- a/scripts/globus/initialize_globus_endpoint.py +++ b/scripts/globus/initialize_globus_endpoint.py @@ -62,7 +62,7 @@ if os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") is not None: DATAFED_GLOBUS_SUBSCRIPTION = os.getenv("DATAFED_GLOBUS_SUBSCRIPTION") else: - DATAFED_GLOBUS_SUBSCRIPTION=None + DATAFED_GLOBUS_SUBSCRIPTION = None client = globus_sdk.NativeAppAuthClient(CLIENT_ID) @@ -70,11 +70,12 @@ # view_identities to user information for creating GCS server group_scope = GroupsScopes.make_mutable("all") -client.oauth2_start_flow(requested_scopes="openid profile email " - "urn:globus:auth:scope:auth.globus.org:manage_projects " - "urn:globus:auth:scope:auth.globus.org:view_identities " + - str(group_scope), - refresh_tokens=True) +client.oauth2_start_flow( + requested_scopes="openid profile email " + "urn:globus:auth:scope:auth.globus.org:manage_projects " + "urn:globus:auth:scope:auth.globus.org:view_identities " + str(group_scope), + refresh_tokens=True, +) authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print("Please go to this URL and login: \n", authorize_url) @@ -82,12 +83,15 @@ token_response = client.oauth2_exchange_code_for_tokens(auth_code) # Extract the token -refresh_token_auth = token_response.by_resource_server['auth.globus.org']['refresh_token'] -refresh_token_groups = token_response.by_resource_server['groups.api.globus.org']['refresh_token'] +refresh_token_auth = token_response.by_resource_server["auth.globus.org"][ + "refresh_token" +] +refresh_token_groups = token_response.by_resource_server["groups.api.globus.org"][ + "refresh_token" +] rt_authorizer = globus_sdk.RefreshTokenAuthorizer(refresh_token_auth, client) -rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, - client) +rt_authorizer_groups = globus_sdk.RefreshTokenAuthorizer(refresh_token_groups, client) # auth_client_refresh_token ac_rt = AuthClient(authorizer=rt_authorizer) gr_rt = GroupsClient(authorizer=rt_authorizer_groups) @@ -133,20 +137,21 @@ # client exists for the globus connect server if it does not then we will # call the setup command utils.createGCSEndpoint( - ac_rt, - client_id, - client_secret, - project_id, - DEPLOYMENT_KEY_PATH, - ENDPOINT_NAME, - DATAFED_GLOBUS_CONTROL_PORT, - userinfo) + ac_rt, + client_id, + client_secret, + project_id, + DEPLOYMENT_KEY_PATH, + ENDPOINT_NAME, + DATAFED_GLOBUS_CONTROL_PORT, + userinfo, +) if DATAFED_GLOBUS_SUBSCRIPTION is not None: -# Create subscription subgroup + # Create subscription subgroup results = gr_rt.get_group_by_subscription_id(DATAFED_GLOBUS_SUBSCRIPTION) - parent_group_id=results["group_id"] + parent_group_id = results["group_id"] print("Groups by sub") print(results) group_name = f"{DATAFED_GCS_ROOT_NAME} Group" @@ -157,16 +162,15 @@ else: print(f"Group does not exist {group_name}") package = { - "name": group_name, - "description": "DataFed Repository Subscription Group, used for" - "granting access to the application client to setup the repository in " - "Globus", - "parent_id": str(parent_group_id) - } - + "name": group_name, + "description": "DataFed Repository Subscription Group, used for" + "granting access to the application client to setup the repository in " + "Globus", + "parent_id": str(parent_group_id), + } result = gr_rt.create_group(package) - group_id = result['id'] + group_id = result["id"] print("group id") print(group_id) @@ -177,9 +181,7 @@ print("membership_action") print(result) - package = { - "subscription_id": DATAFED_GLOBUS_SUBSCRIPTION - } + package = {"subscription_id": DATAFED_GLOBUS_SUBSCRIPTION} result = gr_rt.update_group(group_id, package) print("update group") print(result) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 226679c53..d4588b51c 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -102,6 +102,7 @@ def getCredentialID(auth_client, client_id, cred_name): return cred["id"] return None + def groupExists(client, group_name): my_groups = client.get_my_groups() print("My groups") @@ -111,6 +112,7 @@ def groupExists(client, group_name): return True return False + def getGroupId(client, group_name): my_groups = client.get_my_groups() for group in my_groups: @@ -119,7 +121,6 @@ def getGroupId(client, group_name): return None - def deleteGroup(client, group_name): my_groups = client.get_my_groups() for group in my_groups: @@ -128,6 +129,7 @@ def deleteGroup(client, group_name): print(f"Removing group: {group_name} with id: {group['id']}") print(result) + def validFile(file_name): file_exists = False file_empty = True @@ -302,14 +304,15 @@ def deleteAllNonGCSClients(auth_client, project_id): def createGCSEndpoint( - auth_client, - client_id, - client_secret, - project_id, - deployment_key_file, - endpoint_name, - control_port, - userinfo): + auth_client, + client_id, + client_secret, + project_id, + deployment_key_file, + endpoint_name, + control_port, + userinfo, +): identity_id = userinfo["sub"] email = userinfo["email"] From 655752dd30f7553cfa9b470b6ddddf33530ab011 Mon Sep 17 00:00:00 2001 From: par-hermes Date: Wed, 10 Apr 2024 03:52:36 +0000 Subject: [PATCH 5/6] cpp-py-formatter --- scripts/globus/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/globus/utils.py b/scripts/globus/utils.py index 36e3661df..424afd7b4 100644 --- a/scripts/globus/utils.py +++ b/scripts/globus/utils.py @@ -233,6 +233,7 @@ def createClient(auth_client, client_name, project_id, cred_name, cred_file): ) return client_id, client_secret + def getGCSClientIDFromDeploymentFile(deployment_key_file): deployment_key_exists, deployment_key_empty = validFile(deployment_key_file) From 41a0ee1dabdc172e31d333f8d31460ae779e2402 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Thu, 11 Apr 2024 14:56:28 -0400 Subject: [PATCH 6/6] A few small fixes for null termination --- repository/gridftp/globus5/authz/source/CMakeLists.txt | 2 ++ repository/gridftp/globus5/authz/source/URL.c | 4 ++-- repository/gridftp/globus5/authz/source/libauthz.c | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/repository/gridftp/globus5/authz/source/CMakeLists.txt b/repository/gridftp/globus5/authz/source/CMakeLists.txt index 0daec480c..20e0c8e06 100644 --- a/repository/gridftp/globus5/authz/source/CMakeLists.txt +++ b/repository/gridftp/globus5/authz/source/CMakeLists.txt @@ -4,6 +4,8 @@ configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/Version.hpp.in" "${CMAKE_CURRENT_SOURCE_DIR}/Version.hpp" @ONLY) + +file( GLOB AUTHZ_Sources "URL.c" ) file( GLOB Sources "*.cpp" "*.c" ) add_library( datafed-authz SHARED ${Sources} ) diff --git a/repository/gridftp/globus5/authz/source/URL.c b/repository/gridftp/globus5/authz/source/URL.c index ad898dd44..ee24d196b 100644 --- a/repository/gridftp/globus5/authz/source/URL.c +++ b/repository/gridftp/globus5/authz/source/URL.c @@ -6,11 +6,11 @@ // Function to extract the relative path from an FTP URL // Returns 1 on success, 0 on failure int ftpExtractRelativePath(const char *url, char *relativePath, size_t maxLength) { - size_t len_of_prefix = strlen("ftp://"); + size_t len_of_prefix = strlen("ftp://\0"); size_t len_of_url = strlen(url); // Step 1. Check that the URL starts with "ftp://" - if (strncmp(url, "ftp://", len_of_prefix) != 0) { + if (strncmp(url, "ftp://\0", len_of_prefix) != 0) { fprintf(stderr, "Error: URL must start with 'ftp:// but you have provided %s'\n", url); return 0; } diff --git a/repository/gridftp/globus5/authz/source/libauthz.c b/repository/gridftp/globus5/authz/source/libauthz.c index 8bea6e112..943f887bc 100644 --- a/repository/gridftp/globus5/authz/source/libauthz.c +++ b/repository/gridftp/globus5/authz/source/libauthz.c @@ -1,6 +1,7 @@ // Local private includes #include "AuthzWorker.h" +#include "URL.h" // Globus third party includes #include