From b92ed8c7bfd262894f7871244f6fcd95521e4925 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Mon, 16 Oct 2023 15:55:56 -0400 Subject: [PATCH 01/10] Added support for Google Analytics --- scripts/generate_datafed.sh | 9 +++++++++ scripts/generate_ws_config.sh | 21 +++++++++++++++++++-- web/datafed-ws.js | 15 ++++++++++----- web/views/head.ect | 11 +++++++++++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index ccb3138ef..07c5021ad 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -142,6 +142,13 @@ else local_DATAFED_CORE_ADDRESS_PORT_INTERNAL=$(printenv DATAFED_CORE_ADDRESS_PORT_INTERNAL) fi +if [ -z "${DATAFED_GOOGLE_ANALYTICS_TAG}" ] +then + local_DATAFED_GOOGLE_ANALYTICS_TAG="" +else + local_DATAFED_GOOGLE_ANALYTICS_TAG=$(printenv DATAFED_GOOGLE_ANALYTICS_TAG) +fi + if [ ! -d "$PATH_TO_CONFIG_DIR" ] then mkdir -p "$PATH_TO_CONFIG_DIR" @@ -206,6 +213,8 @@ export DATAFED_WEB_CERT_PATH="$local_DATAFED_WEB_CERT_PATH" export DATAFED_WEB_USER="" # How the web server communicates with the core server, assumes an internal network export DATAFED_CORE_ADDRESS_PORT_INTERNAL="$local_DATAFED_CORE_ADDRESS_PORT_INTERNAL" +# The id for the associated Google Analytics tag, if left empty, Google Analytics will be disabled +export DATAFED_GOOGLE_ANALYTICS_TAG="$local_DATAFED_GOOGLE_ANALYTICS_TAG" # ************************************************ # Env Variables for Globus Connect Server # ************************************************ diff --git a/scripts/generate_ws_config.sh b/scripts/generate_ws_config.sh index 17d8a40ee..181901054 100755 --- a/scripts/generate_ws_config.sh +++ b/scripts/generate_ws_config.sh @@ -14,7 +14,7 @@ Help() { echo "$(basename $0) Will set up a configuration file for the core server" echo - echo "Syntax: $(basename $0) [-h|s|i|z|y|w|k]" + echo "Syntax: $(basename $0) [-h|s|i|z|y|w|k|t]" echo "options:" echo "-h, --help Print this help message." echo "-s, --globus-secret Globus App secret used by DataFed to authenticate" @@ -38,6 +38,7 @@ Help() echo "-y, --zeromq-system-secret ZeroMQ system secret" echo "-w, --web-cert-path Path to web server certificate file." echo "-k, --web-key-path Path to web server key file." + echo "-t, --google-analytics-tag The tag associated with a Google Analytics stream" } # Set defaults use environment variables by default @@ -97,8 +98,15 @@ else local_DATAFED_CORE_ADDRESS_PORT_INTERNAL=$(printenv DATAFED_CORE_ADDRESS_PORT_INTERNAL) fi +if [ -z "${DATAFED_GOOGLE_ANALYTICS_TAG}" ] +then + local_DATAFED_GOOGLE_ANALYTICS_TAG="" +else + local_DATAFED_GOOGLE_ANALYTICS_TAG=$(printenv DATAFED_GOOGLE_ANALYTICS_TAG) +fi + -VALID_ARGS=$(getopt -o hs:i:z:y:w:k:c: --long 'help',globus-secret:,globus-id:,zeromq-session-secret:,zeromq-system-secret:,web-cert-path:,web-key-path:,core-address-port: -- "$@") +VALID_ARGS=$(getopt -o hs:i:z:y:w:k:c:t: --long 'help',globus-secret:,globus-id:,zeromq-session-secret:,zeromq-system-secret:,web-cert-path:,web-key-path:,core-address-port:google-analytics-tag -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi @@ -145,6 +153,11 @@ while [ : ]; do local_DATAFED_CORE_ADDRESS_PORT_INTERNAL=$2 shift 2 ;; + -t | --google-analytics-tag) + echo "Processing 'DataFed Google Analytics tag' option. Input argument is '$2'" + local_DATAFED_GOOGLE_ANALYTICS_TAG=$2 + shift 2 + ;; --) shift; break ;; @@ -227,6 +240,10 @@ client_secret=${local_DATAFED_GLOBUS_APP_SECRET} # This is the address to talk with the core server which is listening on # port 7513, assuming internal network. server_address=tcp://${local_DATAFED_CORE_ADDRESS_PORT_INTERNAL} + +[operations] +# This is the tag associated with a Google Analytics installation that metrics will be sent to. +google_analytics_tag=${local_DATAFED_GOOGLE_ANALYTICS_TAG} EOF echo diff --git a/web/datafed-ws.js b/web/datafed-ws.js index 32e5a7120..e1b539917 100755 --- a/web/datafed-ws.js +++ b/web/datafed-ws.js @@ -66,7 +66,8 @@ var g_host, g_ver_api_major, g_ver_api_minor, g_ver_api_patch, - g_tls; + g_tls, + g_ga; const nullfr = Buffer.from([]); @@ -294,7 +295,7 @@ app.get('/ui/welcome', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}' auth.globus.org`); - a_resp.render('index',{nonce:a_resp.locals.nonce, theme:theme,version:g_version,test_mode:g_test}); + a_resp.render('index',{nonce:a_resp.locals.nonce, theme:theme,version:g_version,test_mode:g_test,...g_ga}); } }); @@ -306,7 +307,7 @@ app.get('/ui/main', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}'`); - a_resp.render('main',{nonce:a_resp.locals.nonce,user_uid:a_req.session.uid,theme:theme,version:g_version,test_mode:g_test}); + a_resp.render('main',{nonce:a_resp.locals.nonce,user_uid:a_req.session.uid,theme:theme,version:g_version,test_mode:g_test,...g_ga}); }else{ // datafed-user cookie not set, so clear datafed-id before redirect //a_resp.clearCookie( 'datafed-id' ); @@ -333,7 +334,7 @@ app.get('/ui/register', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}' auth.globus.org`); - a_resp.render('register', {nonce:a_resp.locals.nonce, uid: a_req.session.uid, uname: clean, theme: theme, version: g_version, test_mode: g_test }); + a_resp.render('register', {nonce:a_resp.locals.nonce, uid: a_req.session.uid, uname: clean, theme: theme, version: g_version, test_mode: g_test, ...g_ga }); } }); @@ -368,7 +369,7 @@ app.get('/ui/error', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}'`); - a_resp.render('error',{nonce:a_resp.locals.nonce,theme:"light",version:g_version,test_mode:g_test}); + a_resp.render('error',{nonce:a_resp.locals.nonce,theme:"light",version:g_version,test_mode:g_test,...g_ga}); }); /* This is the OAuth redirect URL after a user authenticates with Globus @@ -1824,6 +1825,10 @@ function loadSettings(){ if ( !g_extern_url ){ g_extern_url = "http"+(g_tls?'s':'')+"://" + g_host + ":" + g_port; } + + if ( config.operations ){ + g_ga = { enableGoogleAnalytics: config.operations.google_analytics_tag !== '', googleAnalyticsTag: config.operations.google_analytics_tag }; + } }catch( e ){ logger.error(loadSettings.name, getCurrentLineNumber(), "Could not open/parse configuration file: " + process.argv[2] ); logger.error(loadSettings.name, getCurrentLineNumber(), e.message ); diff --git a/web/views/head.ect b/web/views/head.ect index 25d7c2f15..6017f3b6b 100644 --- a/web/views/head.ect +++ b/web/views/head.ect @@ -1,3 +1,14 @@ +<% if @enableGoogleAnalytics : %> + + + +<% end %> From 00f4d7172e5f073cdad76d4ad0aa0d6d1633c2f6 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 17 Oct 2023 09:21:22 -0400 Subject: [PATCH 02/10] fixed invalid flag --- scripts/generate_ws_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_ws_config.sh b/scripts/generate_ws_config.sh index 181901054..f14a00e4c 100755 --- a/scripts/generate_ws_config.sh +++ b/scripts/generate_ws_config.sh @@ -106,7 +106,7 @@ else fi -VALID_ARGS=$(getopt -o hs:i:z:y:w:k:c:t: --long 'help',globus-secret:,globus-id:,zeromq-session-secret:,zeromq-system-secret:,web-cert-path:,web-key-path:,core-address-port:google-analytics-tag -- "$@") +VALID_ARGS=$(getopt -o hs:i:z:y:w:k:c:t: --long 'help',globus-secret:,globus-id:,zeromq-session-secret:,zeromq-system-secret:,web-cert-path:,web-key-path:,core-address-port:,google-analytics-tag: -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi From 15aefa252632f4d3d176e725b40aefbc8579519c Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 17 Oct 2023 16:30:13 -0400 Subject: [PATCH 03/10] renamed g_ga variable --- web/datafed-ws.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/datafed-ws.js b/web/datafed-ws.js index e1b539917..7293e111d 100755 --- a/web/datafed-ws.js +++ b/web/datafed-ws.js @@ -67,7 +67,7 @@ var g_host, g_ver_api_minor, g_ver_api_patch, g_tls, - g_ga; + g_google_analytics; const nullfr = Buffer.from([]); @@ -295,7 +295,7 @@ app.get('/ui/welcome', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}' auth.globus.org`); - a_resp.render('index',{nonce:a_resp.locals.nonce, theme:theme,version:g_version,test_mode:g_test,...g_ga}); + a_resp.render('index',{nonce:a_resp.locals.nonce, theme:theme,version:g_version,test_mode:g_test,...g_google_analytics}); } }); @@ -307,7 +307,7 @@ app.get('/ui/main', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}'`); - a_resp.render('main',{nonce:a_resp.locals.nonce,user_uid:a_req.session.uid,theme:theme,version:g_version,test_mode:g_test,...g_ga}); + a_resp.render('main',{nonce:a_resp.locals.nonce,user_uid:a_req.session.uid,theme:theme,version:g_version,test_mode:g_test,...g_google_analytics}); }else{ // datafed-user cookie not set, so clear datafed-id before redirect //a_resp.clearCookie( 'datafed-id' ); @@ -334,7 +334,7 @@ app.get('/ui/register', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}' auth.globus.org`); - a_resp.render('register', {nonce:a_resp.locals.nonce, uid: a_req.session.uid, uname: clean, theme: theme, version: g_version, test_mode: g_test, ...g_ga }); + a_resp.render('register', {nonce:a_resp.locals.nonce, uid: a_req.session.uid, uname: clean, theme: theme, version: g_version, test_mode: g_test, ...g_google_analytics }); } }); @@ -369,7 +369,7 @@ app.get('/ui/error', (a_req, a_resp) => { const nonce = crypto.randomBytes(16).toString('base64'); a_resp.locals.nonce = nonce; a_resp.setHeader('Content-Security-Policy', `script-src 'nonce-${nonce}'`); - a_resp.render('error',{nonce:a_resp.locals.nonce,theme:"light",version:g_version,test_mode:g_test,...g_ga}); + a_resp.render('error',{nonce:a_resp.locals.nonce,theme:"light",version:g_version,test_mode:g_test,...g_google_analytics}); }); /* This is the OAuth redirect URL after a user authenticates with Globus @@ -1827,7 +1827,7 @@ function loadSettings(){ } if ( config.operations ){ - g_ga = { enableGoogleAnalytics: config.operations.google_analytics_tag !== '', googleAnalyticsTag: config.operations.google_analytics_tag }; + g_google_analytics = { enableGoogleAnalytics: config.operations.google_analytics_tag !== '', googleAnalyticsTag: config.operations.google_analytics_tag }; } }catch( e ){ logger.error(loadSettings.name, getCurrentLineNumber(), "Could not open/parse configuration file: " + process.argv[2] ); From 691a8faa7efa0518b7f564a106ed50f88c82f57d Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 17 Oct 2023 16:30:38 -0400 Subject: [PATCH 04/10] added a better description for the google analytics tag --- scripts/generate_datafed.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index 07c5021ad..ef8ae013a 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -214,6 +214,8 @@ export DATAFED_WEB_USER="" # How the web server communicates with the core server, assumes an internal network export DATAFED_CORE_ADDRESS_PORT_INTERNAL="$local_DATAFED_CORE_ADDRESS_PORT_INTERNAL" # The id for the associated Google Analytics tag, if left empty, Google Analytics will be disabled +# You can find your tag id by going to the stream details page and it is the field marked as "Measurement ID" +# It will be in the form of "G-XXXXXXXXXX" export DATAFED_GOOGLE_ANALYTICS_TAG="$local_DATAFED_GOOGLE_ANALYTICS_TAG" # ************************************************ # Env Variables for Globus Connect Server From 8068c12d9d7587f460e9481969300d49d7a07ac5 Mon Sep 17 00:00:00 2001 From: nedvedba <145805866+nedvedba@users.noreply.github.com> Date: Mon, 23 Oct 2023 09:37:28 -0400 Subject: [PATCH 05/10] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94a282930..148d1b5a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## MINOR Feature +1. [909] - Added Support for Google Analytics + # v2023.8.21.10.40 ## MAJOR Breaking changes From da9d9b01aa8c16bcde51f9499a3a56a37edf62be Mon Sep 17 00:00:00 2001 From: par-hermes Date: Tue, 24 Oct 2023 16:41:33 +0000 Subject: [PATCH 06/10] cpp-py-formatter --- common/tests/unit/test_SocketOptions.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/tests/unit/test_SocketOptions.cpp b/common/tests/unit/test_SocketOptions.cpp index ba76be426..b9040c5cb 100644 --- a/common/tests/unit/test_SocketOptions.cpp +++ b/common/tests/unit/test_SocketOptions.cpp @@ -36,7 +36,8 @@ BOOST_AUTO_TEST_CASE(testing_AddressSplitterINPROC) { BOOST_AUTO_TEST_CASE(testing_AddressSplitterNoPort) { // Still contains ':' - BOOST_CHECK_THROW(AddressSplitter splitter("inproc://www.datafed.com:"), TraceException); + BOOST_CHECK_THROW(AddressSplitter splitter("inproc://www.datafed.com:"), + TraceException); } BOOST_AUTO_TEST_CASE(testing_AddressSplitterNoPort2) { From 98a515be61efed4d2b96f0f549d6d71b441b41ec Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 30 Oct 2023 12:00:07 -0400 Subject: [PATCH 07/10] Add files to gitignore --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index ea288ff1b..c7df59ef0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,18 @@ config/datafed-ws.cfg core/database/foxx/api/version_router.js core/database/foxx/manifest.json core/server/Version.hpp +docs/.buildinfo +docs/.doctrees/ +docs/.nojekyll python/datafed_pkg/datafed/VERSION.py +python/datafed_pkg/datafed.egg-info/ +python/datafed_pkg/datafed/SDMS_Anon_pb2.py +python/datafed_pkg/datafed/SDMS_Auth_pb2.py +python/datafed_pkg/datafed/SDMS_pb2.py +python/datafed_pkg/datafed/Version_pb2.py +python/datafed_pkg/datafed/__pycache__/ +python/datafed_pkg/dist/ +web/package.json repository/gridftp/globus5/authz/source/Version.hpp repository/server/Version.hpp scripts/globus/datafed-home-repo.sh From 4399d47b92c323dae99fe9eef4df3056002eaeb7 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Thu, 2 Nov 2023 20:48:15 +0000 Subject: [PATCH 08/10] Add line item to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71a3d5a5..12af6ebb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ## PATCH Bug fixes/Technical Debt/Documentation 1. [914] - Improve GitHub template 2. [910] - Static code analysis and status checks fixed, improvements to CI +3. [917] - Add additional files to .gitignore # v2023.10.23.15.50 From 2203ade7c1cd7356fa46195fb9284e74401788f5 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 13 Nov 2023 00:44:31 -0500 Subject: [PATCH 09/10] Add fix to admin script, fix the need for interactive prompt which would hang --- scripts/generate_certificate_refresh_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_certificate_refresh_script.sh b/scripts/generate_certificate_refresh_script.sh index 3725441b8..2ef521140 100755 --- a/scripts/generate_certificate_refresh_script.sh +++ b/scripts/generate_certificate_refresh_script.sh @@ -9,7 +9,7 @@ SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/..") source "${PROJECT_ROOT}/config/datafed.sh" -VERSION="1.0.0" +VERSION="1.0.1" echo "$FILE_NAME $VERSION" ERROR_DETECTED=0 @@ -54,7 +54,7 @@ cat << OUTER_EOF > "$PROJECT_ROOT/scripts/admin_refresh_certs.sh" # and root root DOMAIN="${DATAFED_DOMAIN}" systemctl stop datafed-ws.service -lego --email="${DATAFED_LEGO_EMAIL}" --domains="${DATAFED_DOMAIN}" --tls run +lego --accept-tos --email="${DATAFED_LEGO_EMAIL}" --domains="${DATAFED_DOMAIN}" --tls run DIR_NAME=\$(date +%m-%Y) mkdir -p "\${DIR_NAME}" # Create a copy so we can always go back and check when the last time was From 88109ef8c920341ce758dbb8daaa790820505079 Mon Sep 17 00:00:00 2001 From: "Brown, Joshua" Date: Mon, 13 Nov 2023 07:50:33 -0500 Subject: [PATCH 10/10] Add changelog flag --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71a3d5a5..da31df8af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ## PATCH Bug fixes/Technical Debt/Documentation 1. [914] - Improve GitHub template 2. [910] - Static code analysis and status checks fixed, improvements to CI +3. [923] - Fixed missing flag in certificate refresh script # v2023.10.23.15.50