Merge pull request #550 from episphere/stage

Stage -> Prod Sync (February 2024 Release)

config/dev/.env.yaml

@@ -7,3 +7,7 @@ SG_FROM_EMAIL: [email protected]
 TWILIO_ACCOUNT_SID: projects/nih-nci-dceg-connect-dev/secrets/twilio-account-sid/versions/1
 TWILIO_AUTH_TOKEN: projects/nih-nci-dceg-connect-dev/secrets/twilio-sms-auth-token/versions/1
 TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-dev/secrets/twilio-messaging-service-sid/versions/1
+APP_REGISTRATION_TENANT_ID: projects/nih-nci-dceg-connect-dev/secrets/app-registration-tenant-id/versions/1
+APP_REGISTRATION_CLIENT_ID: projects/nih-nci-dceg-connect-dev/secrets/app-registration-client-id/versions/1
+APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-dev/secrets/app-registration-client-secret/versions/1
+GITHUB_TOKEN: projects/nih-nci-dceg-connect-dev/secrets/questionnaire-sha-tracking/versions/1

config/dev/cloudbuild2.yaml

@@ -16,9 +16,9 @@ steps:
 - name: 'gcr.io/cloud-builders/gcloud'
   args: ['functions', 'deploy', 'exportNotificationsToBucket', '--trigger-topic=${_TOPIC}', '--runtime=${_RUNTIME}', '--timeout=480s', '--source=${_SOURCE}', '--env-vars-file=config/dev/.env.yaml']
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['functions', 'deploy', 'sendGridEventWebhook', '--trigger-http', '--runtime=${_RUNTIME}', '--timeout=240s', '--memory=1024MB',  '--source=${_SOURCE}', '--env-vars-file=config/dev/.env.yaml']
+  args: ['functions', 'deploy', 'webhook', '--trigger-http', '--runtime=${_RUNTIME}', '--timeout=240s', '--memory=1024MB',  '--source=${_SOURCE}', '--env-vars-file=config/dev/.env.yaml']
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['functions', 'add-iam-policy-binding', 'sendGridEventWebhook', '--member=allUsers', '--role=${_ROLE}']
+  args: ['functions', 'add-iam-policy-binding', 'webhook', '--member=allUsers', '--role=${_ROLE}']
 substitutions:
   _SOURCE: https://source.developers.google.com/projects/nih-nci-dceg-connect-dev/repos/github_episphere_connectfaas/moveable-aliases/dev
   _RUNTIME: nodejs20

config/prod/.env.yaml

@@ -6,4 +6,8 @@ SG_FROM_NAME: Connect for Cancer Prevention Study
 SG_FROM_EMAIL: [email protected]
 TWILIO_ACCOUNT_SID: projects/nih-nci-dceg-connect-prod-6d04/secrets/twilio-account-sid/versions/1
 TWILIO_AUTH_TOKEN: projects/nih-nci-dceg-connect-prod-6d04/secrets/twilio-sms-auth-token/versions/1
-TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-prod-6d04/secrets/twilio-messaging-service-sid/versions/1
+TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-prod-6d04/secrets/twilio-messaging-service-sid/versions/1
+APP_REGISTRATION_TENANT_ID: projects/nih-nci-dceg-connect-prod-6d04/secrets/app-registration-tenant-id/versions/1
+APP_REGISTRATION_CLIENT_ID: projects/nih-nci-dceg-connect-prod-6d04/secrets/app-registration-client-id/versions/1
+APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-prod-6d04/secrets/app-registration-client-secret/versions/1
+GITHUB_TOKEN: projects/nih-nci-dceg-connect-prod-6d04/secrets/questionnaire-sha-tracking/versions/1

config/prod/cloudbuild2.yaml

@@ -16,9 +16,9 @@ steps:
 - name: 'gcr.io/cloud-builders/gcloud'
   args: ['functions', 'deploy', 'exportNotificationsToBucket', '--trigger-topic=${_TOPIC}', '--runtime=${_RUNTIME}', '--timeout=480s', '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml']
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['functions', 'deploy', 'sendGridEventWebhook', '--trigger-http', '--runtime=${_RUNTIME}', '--timeout=240s', '--memory=1024MB',  '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml']
+  args: ['functions', 'deploy', 'webhook', '--trigger-http', '--runtime=${_RUNTIME}', '--timeout=240s', '--memory=1024MB',  '--source=${_SOURCE}', '--env-vars-file=config/prod/.env.yaml']
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['functions', 'add-iam-policy-binding', 'sendGridEventWebhook', '--member=allUsers', '--role=${_ROLE}'] 
+  args: ['functions', 'add-iam-policy-binding', 'webhook', '--member=allUsers', '--role=${_ROLE}'] 
 substitutions:
   _SOURCE: https://source.developers.google.com/projects/nih-nci-dceg-connect-prod-6d04/repos/github_episphere_connectfaas
   _RUNTIME: nodejs20

config/stage/.env.yaml

@@ -6,4 +6,8 @@ SG_FROM_NAME: Connect for Cancer Prevention Study (STAGE)
 SG_FROM_EMAIL: [email protected]
 TWILIO_ACCOUNT_SID: projects/nih-nci-dceg-connect-stg-5519/secrets/twilio-account-sid/versions/1
 TWILIO_AUTH_TOKEN: projects/nih-nci-dceg-connect-stg-5519/secrets/twilio-sms-auth-token/versions/1
-TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-stg-5519/secrets/twilio-messaging-service-sid/versions/1
+TWILIO_MESSAGING_SERVICE_SID: projects/nih-nci-dceg-connect-stg-5519/secrets/twilio-messaging-service-sid/versions/1
+APP_REGISTRATION_TENANT_ID: projects/nih-nci-dceg-connect-stg-5519/secrets/app-registration-tenant-id/versions/1
+APP_REGISTRATION_CLIENT_ID: projects/nih-nci-dceg-connect-stg-5519/secrets/app-registration-client-id/versions/1
+APP_REGISTRATION_CLIENT_SECRET: projects/nih-nci-dceg-connect-stg-5519/secrets/app-registration-client-secret/versions/1
+GITHUB_TOKEN: projects/nih-nci-dceg-connect-stg-5519/secrets/questionnaire-sha-tracking/versions/1

config/stage/cloudbuild2.yaml

@@ -16,9 +16,9 @@ steps:
 - name: 'gcr.io/cloud-builders/gcloud'
   args: ['functions', 'deploy', 'exportNotificationsToBucket', '--trigger-topic=${_TOPIC}', '--runtime=${_RUNTIME}', '--timeout=480s', '--source=${_SOURCE}', '--env-vars-file=config/stage/.env.yaml']
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['functions', 'deploy', 'sendGridEventWebhook', '--trigger-http', '--runtime=${_RUNTIME}', '--timeout=240s', '--memory=1024MB',  '--source=${_SOURCE}', '--env-vars-file=config/stage/.env.yaml']
+  args: ['functions', 'deploy', 'webhook', '--trigger-http', '--runtime=${_RUNTIME}', '--timeout=240s', '--memory=1024MB',  '--source=${_SOURCE}', '--env-vars-file=config/stage/.env.yaml']
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['functions', 'add-iam-policy-binding', 'sendGridEventWebhook', '--member=allUsers', '--role=${_ROLE}']
+  args: ['functions', 'add-iam-policy-binding', 'webhook', '--member=allUsers', '--role=${_ROLE}']
 substitutions:
   _SOURCE: https://source.developers.google.com/projects/nih-nci-dceg-connect-stg-5519/repos/github_episphere_connectfaas/moveable-aliases/stage
   _RUNTIME: nodejs20

index.js

@@ -9,7 +9,7 @@ const { dashboard } = require('./utils/dashboard');
 const { getParticipantNotification } = require('./utils/notifications');
 const { importToBigQuery, firestoreExport, exportNotificationsToBucket, importNotificationsToBigquery } = require('./utils/events');
 const { participantDataCleanup } = require('./utils/participantDataCleanup');
-const { sendGridEventWebhook } = require('./utils/sendGridEventWebhook');
+const { webhook } = require('./utils/webhook');
 
 // API End-Points for Sites
 
@@ -66,6 +66,6 @@ exports.importNotificationsToBigquery = importNotificationsToBigquery;
 
 exports.participantDataCleanup = participantDataCleanup;
 
-// End-Points for SendGrid Event Webhook
+// End-Points for Event Webhook
 
-exports.sendGridEventWebhook = sendGridEventWebhook;
+exports.webhook = webhook;

utils/bigquery.js

@@ -36,7 +36,7 @@ async function getParticipantsForNotificationsBQ({
   timeField = "",
   fieldsToFetch = [],
   limit = 100,
-  offset = 0,
+  previousConnectId = 0,
 }) {
   let result = { hasNext: false, fetchedDataArray: [] };
   if (!notificationSpecId || Object.keys(conditions).length === 0) return result;
@@ -70,7 +70,7 @@ async function getParticipantsForNotificationsBQ({
         notificationSpecificationsID = "${notificationSpecId}")
     USING(token)
     WHERE ${bqConditionArray.length === 0 ? "1=1" : bqConditionArray.join(" AND ")}
-    AND isSent IS NOT TRUE LIMIT ${limit} OFFSET ${offset}`;
+    AND isSent IS NOT TRUE AND Connect_ID > ${previousConnectId} ORDER BY Connect_ID LIMIT ${limit}`;
 
   const [rows] = await bigquery.query(queryStr);
   if (rows.length === 0) return result;

utils/connectApp.js

@@ -1,13 +1,15 @@
 const { getResponseJSON, setHeadersDomainRestricted, getUserProfile } = require('./shared');
 const { recruitSubmit, submitSocial, getUserSurveys, getUserCollections } = require('./submission');
-const { retrieveNotifications } = require('./notifications');
+const { retrieveNotifications, sendEmailLink } = require('./notifications');
 const { validateToken, generateToken, updateParticipantFirebaseAuthentication, validateUsersEmailPhone } = require('./validation');
 
 const connectApp = async (req, res) => {
     setHeadersDomainRestricted(req, res);
 
     if(req.method === 'OPTIONS') return res.status(200).json({code: 200});
 
+    if (req.query.api === 'sendEmailLink') return sendEmailLink(req, res);
+
     if(!req.headers.authorization || req.headers.authorization.trim() === ""){
         return res.status(401).json(getResponseJSON('Authorization failed!', 401));
     }
@@ -35,7 +37,7 @@ const connectApp = async (req, res) => {
     console.log(`PWA API: ${api}, called from uid: ${uid}`);
 
   try {
-    if (api === 'submit') return recruitSubmit(req, res, uid);
+    if (api === 'submit') return await recruitSubmit(req, res, uid);
 
     if (api === 'submitSocial') return submitSocial(req, res, uid);
 
@@ -55,6 +57,40 @@ const connectApp = async (req, res) => {
 
     else if (api === 'validateEmailOrPhone') return validateUsersEmailPhone(req, res);
 
+    else if (api === 'getModuleSHA') {
+      if (req.method !== 'GET') {
+        return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405));
+      }
+
+      if (!req.query.path || req.query.path === '') {
+        return res.status(400).json(getResponseJSON('Path parameter is required!', 400));
+      }
+
+      const path = req.query.path;
+
+      const { getModuleSHA } = require('./submission');
+      const shaResult = await getModuleSHA(path);
+
+      return res.status(200).json({data: shaResult, code: 200});
+    }
+    else if (api === 'getSHAFromGitHubCommitData') {
+      if (req.method !== 'GET') {
+        return res.status(405).json(getResponseJSON('Only GET requests are accepted!', 405));
+      }
+
+      if (!req.query.path || req.query.path === '') {
+        return res.status(400).json(getResponseJSON('Path parameter is required!', 400));
+      }
+
+      const surveyStartTimestamp = req.query.surveyStartTimestamp || '';
+      const path = req.query.path;
+
+      const { getSHAFromGitHubCommitData } = require('./submission');
+      const shaResult = await getSHAFromGitHubCommitData(surveyStartTimestamp, path);
+
+      return res.status(200).json({data: shaResult, code: 200});
+    }
+
     else return res.status(400).json(getResponseJSON('Bad request!', 400));
 
   } catch (error) {

utils/events.js

@@ -2,7 +2,7 @@ const firestore = require('@google-cloud/firestore');
 const {BigQuery} = require('@google-cloud/bigquery');
 const {Storage} = require('@google-cloud/storage');
 
-const collectionNameArray = ['participants', 'biospecimen', 'boxes', 'module1_v1', 'module1_v2', 'module2_v1', 'module2_v2', 'module3_v1', 'module4_v1', 'bioSurvey_v1', 'menstrualSurvey_v1', 'clinicalBioSurvey_v1', 'covid19Survey_v1', 'kitAssembly', 'mouthwash_v1', 'sendgridTracking', 'cancerOccurrence'];
+const collectionNameArray = ['participants', 'biospecimen', 'boxes', 'module1_v1', 'module1_v2', 'module2_v1', 'module2_v2', 'module3_v1', 'module4_v1', 'bioSurvey_v1', 'menstrualSurvey_v1', 'clinicalBioSurvey_v1', 'covid19Survey_v1', 'kitAssembly', 'mouthwash_v1', 'sendgridTracking', 'cancerOccurrence', 'promis_v1'];
 
 const firestoreExport = async (eventData, context) => {
   await exportCollectionsToBucket(collectionNameArray);

utils/fieldToConceptIdMapping.js

@@ -251,5 +251,11 @@ module.exports = {
     ssnPartialGiven: 914639140,
     ssnPartialGivenTime: 598680838,
     ssnQcStatus: 605870562,
-    ssnNoCheckRan: 875207881
+    ssnNoCheckRan: 875207881,
+
+    // Survey Status Variables
+    notYetEligible: 789467219,
+    notStarted:     972455046,
+    started:        615768760,
+    submitted:      231311385 
 };