Skip to content

Commit

Permalink
fixed merged conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
nedvedba committed Jan 2, 2024
2 parents d678e9a + a7b8439 commit 308f440
Show file tree
Hide file tree
Showing 47 changed files with 1,769 additions and 1,150 deletions.
12 changes: 12 additions & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# PR Description


# Tasks

* [ ] - A description of the PR has been provided, and a diagram included if it is a new feature.
* [ ] - Formatter has been run
* [ ] - CHANGELOG comment has been added
* [ ] - Labels have been assigned to the pr
* [ ] - A reviwer has been added
* [ ] - A user has been assigned to work on the pr
* [ ] - If new feature a unit test has been added
4 changes: 2 additions & 2 deletions .github/workflows/format-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,5 @@ jobs:
run: black --check .
- name: Lint with flake8
run: |
flake8 --count --select=E9,F63,F7,F82 --show-source --statistics .
flake8 --count --statistics .
flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --max-line-length 100 .
flake8 --count --statistics --max-line-length 100 .
13 changes: 13 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,25 @@ config/datafed-ws.cfg
core/database/foxx/api/version_router.js
core/database/foxx/manifest.json
core/server/Version.hpp
docs/.buildinfo
docs/.doctrees/
docs/.nojekyll
python/datafed_pkg/datafed/VERSION.py
python/datafed_pkg/datafed.egg-info/
python/datafed_pkg/datafed/SDMS_Anon_pb2.py
python/datafed_pkg/datafed/SDMS_Auth_pb2.py
python/datafed_pkg/datafed/SDMS_pb2.py
python/datafed_pkg/datafed/Version_pb2.py
python/datafed_pkg/datafed/__pycache__/
python/datafed_pkg/dist/
web/package.json
repository/gridftp/globus5/authz/source/Version.hpp
repository/server/Version.hpp
scripts/globus/datafed-home-repo.sh
scripts/globus/datafed-home-repo-form.json
scripts/globus/mapping.json
scripts/admin_datafed_backup.sh
scripts/admin_refresh_certs.sh
services/
web/SDMS.proto
web/SDMS_Anon.proto
Expand Down
25 changes: 24 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,26 @@
# Pre-release

## MAJOR Breaking changes

## MINOR Feature
1. [909] - Added Support for Google Analytics

## PATCH Bug fixes/Technical Debt/Documentation
1. [914] - Improve GitHub template
2. [910] - Static code analysis and status checks fixed, improvements to CI
3. [923] - Fixed missing flag in certificate refresh script
4. [917] - Add additional files to .gitignore
5. [915] - Refactor CI to use pipelines Gitlab feature along with pipelines

# v2023.10.23.15.50

## MINOR Feature
1. [906] - Added backup and cert refresh scripts.

## PATCH Bug Fixes/Technical Debt/Documentation
1. [911] - Add GitHub template
2. [913] - Fixed bug, when endpoint info returns an empty array check to seee if array is empty before accessing elements

# v2023.8.21.10.40

## MAJOR Breaking changes
Expand All @@ -13,7 +36,7 @@
7. [879] - Added correlation ids to messages for tracking
8. [879] - Split log output into server specific files.

## PATCH Bug fixes/Technical Debt
## PATCH Bug Fixes/Technical Debt/Documentation
1. [879] - Fixed thread safety of repo list calls in core server which where causing
seg faults
2. [879] - Added better error reporting when attempting to delete repo with running tasks.
Expand Down
30 changes: 20 additions & 10 deletions common/include/common/libjson.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -674,16 +674,20 @@ class Value {
}

Object &asObject() {
if (m_type != VT_OBJECT)
EXCEPT(1, "Value is not an object");

if (m_type != VT_OBJECT) {
std::string error_msg = "Value is not an object, it is instead of type: ";
error_msg += getTypeString();
EXCEPT(1, error_msg);
}
return *m_value.o;
}

const Object &asObject() const {
if (m_type != VT_OBJECT)
EXCEPT(1, "Value is not an object");

if (m_type != VT_OBJECT) {
std::string error_msg = "Value is not an object, it is instead of type: ";
error_msg += getTypeString();
EXCEPT(1, error_msg);
}
return *m_value.o;
}

Expand All @@ -698,15 +702,21 @@ class Value {
}

Array &asArray() {
if (m_type != VT_ARRAY)
EXCEPT(1, "Value is not an array");
if (m_type != VT_ARRAY) {
std::string error_msg = "Value is not an array, it is instead of type: ";
error_msg += getTypeString();
EXCEPT(1, error_msg);
}

return *m_value.a;
}

const Array &asArray() const {
if (m_type != VT_ARRAY)
EXCEPT(1, "Value is not an array");
if (m_type != VT_ARRAY) {
std::string error_msg = "Value is not an array, it is instead of type: ";
error_msg += getTypeString();
EXCEPT(1, error_msg);
}

return *m_value.a;
}
Expand Down
1 change: 1 addition & 0 deletions common/tests/unit/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ foreach(PROG
test_CommunicatorFactory
test_Frame
test_DynaLog
test_Value
test_MessageFactory
test_OperatorFactory
test_ProtoBufFactory
Expand Down
3 changes: 2 additions & 1 deletion common/tests/unit/test_SocketOptions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ BOOST_AUTO_TEST_CASE(testing_AddressSplitterINPROC) {
BOOST_AUTO_TEST_CASE(testing_AddressSplitterNoPort) {

// Still contains ':'
BOOST_CHECK_THROW(AddressSplitter splitter("inproc://www.datafed.com:"), TraceException);
BOOST_CHECK_THROW(AddressSplitter splitter("inproc://www.datafed.com:"),
TraceException);
}

BOOST_AUTO_TEST_CASE(testing_AddressSplitterNoPort2) {
Expand Down
134 changes: 134 additions & 0 deletions common/tests/unit/test_Value.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#define BOOST_TEST_MAIN

#define BOOST_TEST_MODULE libjson
#include <boost/filesystem.hpp>
#include <boost/test/unit_test.hpp>

// Local public includes
#include "common/DynaLog.hpp"
#include "common/TraceException.hpp"
#include "common/libjson.hpp"

// Standard includes
#include <iostream>
#include <string>

// using namespace SDMS;

BOOST_AUTO_TEST_SUITE(LibJSONTest)

BOOST_AUTO_TEST_CASE(testing_object) {

std::string raw_result = "{\n";
raw_result += " \"DATA\": [],\n";
raw_result += " \"DATA_TYPE\": \"endpoint\",\n";
raw_result += " \"acl_available\": true,\n";
raw_result += " \"acl_editable\": false,\n";
raw_result += " \"activated\": false,\n";
raw_result += " \"authentication_assurance_timeout\": null,\n";
raw_result += " \"authentication_policy_id\": null,\n";
raw_result += " \"authentication_timeout_mins\": null,\n";
raw_result += " \"canonical_name\": "
"\"u_t2uyxwqjgvapzmwlfahbi5l4mq#11bc8bd6-6b22-11eb-8287-"
"0275e0cda761\",\n";
raw_result += " \"contact_email\": \"[email protected]\",\n";
raw_result += " \"contact_info\": null,\n";
raw_result += " \"default_directory\": \"/{server_default}/\",\n";
raw_result += " \"department\": \"Research Computing\",\n";
raw_result += " \"description\": \"Library Published Data\",\n";
raw_result += " \"disable_anonymous_writes\": false,\n";
raw_result += " \"disable_verify\": false,\n";
raw_result += " \"display_name\": \"New Anonymous Endpoint\",\n";
raw_result += " \"entity_type\": \"GCSv5_guest_collection\",\n";
raw_result += " \"expire_time\": null,\n";
raw_result += " \"expires_in\": -1,\n";
raw_result += " \"force_encryption\": false,\n";
raw_result += " \"force_verify\": false,\n";
raw_result += " \"french_english_bilingual\": false,\n";
raw_result += " \"gcp_connected\": null,\n";
raw_result += " \"gcp_paused\": null,\n";
raw_result +=
" \"gcs_manager_url\": \"https://e878b.bd7c.data.globus.org\",\n";
raw_result += " \"gcs_version\": \"5.4.67\",\n";
raw_result += " \"globus_connect_setup_key\": null,\n";
raw_result += " \"high_assurance\": false,\n";
raw_result += " \"host_endpoint\": "
"\"u_t2uyxwqjglapzmwbfahbi5l4mq#3b3f5c6c-5b6a-11eb-87bf-"
"02187389bd35\",\n";
raw_result += " \"host_endpoint_display_name\": \"net01\",\n";
raw_result +=
" \"host_endpoint_id\": \"9ea98bda-0135-40fc-b2c1-280e14757c64\",\n";
raw_result += " \"host_path\": null,\n";
raw_result +=
" \"https_server\": \"https://g-f09197.e778b.bd7c.data.globus.org\",\n";
raw_result += " \"id\": \"e1d2afd2-ce11-4482-b127-b4cceef666f6\",\n";
raw_result += " \"in_use\": false,\n";
raw_result += " \"info_link\": null,\n";
raw_result += " \"is_globus_connect\": false,\n";
raw_result += " \"is_go_storage\": false,\n";
raw_result += " \"keywords\": \"Uni,State,Top\",\n";
raw_result += " \"last_accessed_time\": \"2023-10-23T00:00:00+00:00\",\n";
raw_result += " \"local_user_info_available\": true,\n";
raw_result += " \"location\": null,\n";
raw_result +=
" \"mapped_collection_display_name\": \"New Published Data\",\n";
raw_result +=
" \"mapped_collection_id\": \"5b52029b-9a3e-4490-abcf-649dd2f4fd6c\",\n";
raw_result += " \"max_concurrency\": null,\n";
raw_result += " \"max_parallelism\": null,\n";
raw_result += " \"mfa_required\": false,\n";
raw_result += " \"my_effective_roles\": [],\n";
raw_result += " \"myproxy_dn\": null,\n";
raw_result += " \"myproxy_server\": \"myproxy.globusonline.org\",\n";
raw_result += " \"name\": \"11bc8bd6-6b22-11eb-8287-0275e0cda761\",\n";
raw_result += " \"network_use\": null,\n";
raw_result += " \"non_functional\": false,\n";
raw_result += " \"non_functional_endpoint_display_name\": \"net01\",\n";
raw_result += " \"non_functional_endpoint_id\": "
"\"9ea98bda-0935-40fc-b2c1-280e14757c64\",\n";
raw_result += " \"oauth_server\": null,\n";
raw_result += " \"organization\": \"The new University\",\n";
raw_result += " \"owner_id\": \"9ea98bda-0935-40fc-b2c1-280e14757c64\",\n";
raw_result +=
" \"owner_string\": "
"\"[email protected]\",\n";
raw_result += " \"preferred_concurrency\": null,\n";
raw_result += " \"preferred_parallelism\": null,\n";
raw_result += " \"public\": true,\n";
raw_result += " \"requester_pays\": false,\n";
raw_result += " \"s3_owner_activated\": false,\n";
raw_result += " \"s3_url\": null,\n";
raw_result += " \"shareable\": false,\n";
raw_result += " \"sharing_target_endpoint\": "
"\"u_t2uyxw0jgkapzmwbfahbi5l4mq#3b3f5c6c-5b6a-11eb-87bf-"
"02187389bd35\",\n";
raw_result += " \"sharing_target_root_path\": null,\n";
raw_result += " \"storage_type\": null,\n";
raw_result +=
" \"subscription_id\": \"3ba26681-e247-11e6-9d43-22000a1e3b52\",\n";
raw_result += " \"tlsftp_server\": "
"\"tlsftp://g-f09277.e778b.bd7c.data.globus.org:443\",\n";
raw_result += " \"user_message\": null,\n";
raw_result += " \"user_message_link\": null,\n";
raw_result += " \"username\": \"u_t2uyxwqjgvapzmwbfbhbi5l4mq\"\n";
raw_result += "}";

std::cout << "raw result" << std::endl;
// std::cout << raw_result << std::endl;
SDMS::global_logger.setSysLog(false);
SDMS::global_logger.addStream(std::cerr);
SDMS::global_logger.setLevel(SDMS::LogLevel::DEBUG);
SDMS::LogContext log_context;
DL_DEBUG(log_context, raw_result);

libjson::Value result;
result.fromString(raw_result);
std::cout << "Type of value: " << result.getTypeString() << std::endl;
libjson::Value::Object &resp_obj = result.asObject();

libjson::Value::Array &data = resp_obj.getArray("DATA");

BOOST_CHECK(data.size() == 0);
}

BOOST_AUTO_TEST_SUITE_END()
18 changes: 16 additions & 2 deletions core/database/tests/test_foxx.sh
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,24 @@ export NVM_DIR="$HOME/.nvm"

nvm use $NODE_VERSION

FOXX_PREFIX=""
{
# Determine if exists globally first
which foxx
} || {
FOXX_PREFIX="~/bin/"
}

PATH_TO_PASSWD_FILE=${SOURCE}/database_temp.password
if [ "$TEST_TO_RUN" == "all" ]
then
foxx test -u ${local_DATABASE_USER} -p ${PATH_TO_PASSWD_FILE} --database ${local_DATABASE_NAME} /api/${local_FOXX_MAJOR_API_VERSION} --reporter spec
"${FOXX_PREFIX}foxx" test -u "${local_DATABASE_USER}" \
-p "${PATH_TO_PASSWD_FILE}" \
--database "${local_DATABASE_NAME}" \
"/api/${local_FOXX_MAJOR_API_VERSION}" --reporter spec
else
foxx test -u ${local_DATABASE_USER} -p ${PATH_TO_PASSWD_FILE} --database ${local_DATABASE_NAME} /api/${local_FOXX_MAJOR_API_VERSION} "$TEST_TO_RUN" --reporter spec
"${FOXX_PREFIX}foxx" test -u "${local_DATABASE_USER}" \
-p "${PATH_TO_PASSWD_FILE}" \
--database "${local_DATABASE_NAME}" \
"/api/${local_FOXX_MAJOR_API_VERSION}" "$TEST_TO_RUN" --reporter spec
fi
21 changes: 12 additions & 9 deletions core/server/GlobusAPI.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -542,12 +542,11 @@ void GlobusAPI::getEndpointInfo(const std::string &a_ep_id,
long code = get(m_curl_xfr, m_config.glob_xfr_url + "endpoint/", a_ep_id,
a_acc_token, {}, raw_result);

Value result;
try {
if (!raw_result.size())
EXCEPT_PARAM(ID_SERVICE_ERROR, "Empty response. Code: " << code);

Value result;

result.fromString(raw_result);

Value::Object &resp_obj = result.asObject();
Expand Down Expand Up @@ -575,22 +574,26 @@ void GlobusAPI::getEndpointInfo(const std::string &a_ep_id,

// Look at DATA[0].scheme to see if it's gsiftp
Value::Array &data = resp_obj.getArray("DATA");
Value::Object &server_obj = data[0].asObject();

Value &scheme = server_obj.getValue("scheme");
if (data.size() > 0) {
Value::Object &server_obj = data[0].asObject();

if (scheme.isNull())
a_ep_info.supports_encryption = true;
else if (scheme.isString())
a_ep_info.supports_encryption =
(scheme.asString().compare("gsiftp") == 0);
Value &scheme = server_obj.getValue("scheme");

if (scheme.isNull())
a_ep_info.supports_encryption = true;
else if (scheme.isString())
a_ep_info.supports_encryption =
(scheme.asString().compare("gsiftp") == 0);
}
}
} catch (libjson::ParseError &e) {
DL_ERROR(m_log_context, "PARSE FAILED! " << raw_result);
EXCEPT_PARAM(ID_SERVICE_ERROR,
"Globus endpoint API call returned invalid JSON.");
} catch (TraceException &e) {
DL_ERROR(m_log_context, raw_result);
DL_ERROR(m_log_context, "Result type: " << result.getTypeString());
e.addContext("Globus endpoint API call failed.");
throw;
} catch (exception &e) {
Expand Down
2 changes: 1 addition & 1 deletion core/server/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ namespace po = boost::program_options;
int main(int a_argc, char **a_argv) {
global_logger.setSysLog(false);
global_logger.addStream(std::cerr);
global_logger.setLevel(LogLevel::DEBUG);
global_logger.setLevel(LogLevel::INFO);
LogContext log_context;
log_context.thread_name = "core_server";
log_context.thread_id = 0;
Expand Down
2 changes: 0 additions & 2 deletions doc_source/source/conf.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import sphinx_rtd_theme

# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
Expand Down
Loading

0 comments on commit 308f440

Please sign in to comment.