diff --git a/app.js b/app.js index a2863b45..ec060988 100644 --- a/app.js +++ b/app.js @@ -4,7 +4,7 @@ const {join} = require("path"); const cors = require('cors'); const logger = require('morgan'); const createError = require('http-errors'); -const config = require('./config'); +const configuration = require('./config'); const cronJob = require("node-cron"); const createSession = require("./crdc-datahub-database-drivers/session-middleware"); const statusRouter = require("./routers/status-endpoints-router"); @@ -12,7 +12,7 @@ const graphqlRouter = require("./routers/graphql-router"); const {MongoDBCollection} = require("./crdc-datahub-database-drivers/mongodb-collection"); const {DATABASE_NAME, APPLICATION_COLLECTION, USER_COLLECTION, LOG_COLLECTION, APPROVED_STUDIES_COLLECTION, ORGANIZATION_COLLECTION, SUBMISSIONS_COLLECTION, BATCH_COLLECTION, DATA_RECORDS_COLLECTION, VALIDATION_COLLECTION, - CONFIGURATION_COLLECTION, DATA_RECORDS_ARCHIVE_COLLECTION + DATA_RECORDS_ARCHIVE_COLLECTION, QC_RESULTS_COLLECTION } = require("./crdc-datahub-database-drivers/database-constants"); const {Application} = require("./services/application"); const {Submission} = require("./services/submission"); @@ -20,7 +20,7 @@ const {DataRecordService} = require("./services/data-record-service"); const {S3Service} = require("./crdc-datahub-database-drivers/services/s3-service"); const {MongoQueries} = require("./crdc-datahub-database-drivers/mongo-queries"); const {DatabaseConnector} = require("./crdc-datahub-database-drivers/database-connector"); -const {getCurrentTime, subtractDaysFromNow} = require("./crdc-datahub-database-drivers/utility/time-utility"); +const {getCurrentTime} = require("./crdc-datahub-database-drivers/utility/time-utility"); const {EmailService} = require("./services/email"); const {NotifyUser} = require("./services/notify-user"); const {User} = require("./crdc-datahub-database-drivers/services/user"); @@ -32,10 +32,9 @@ const {LOGIN, REACTIVATE_USER} = require("./crdc-datahub-database-drivers/consta const {BatchService} = require("./services/batch-service"); const {AWSService} = require("./services/aws-request"); const {UtilityService} = require("./services/utility"); -const authenticationMiddleware = require("./middleware/authentication-middleware"); -const {ConfigurationService} = require("./services/configurationService"); +const {QcResultService} = require("./services/qc-result-service"); // print environment variables to log -console.info(config); +console.info(configuration); // create logs folder if it does not already exist const LOGS_FOLDER = 'logs'; @@ -56,7 +55,7 @@ app.use(express.static(join(__dirname, 'public'))); app.use("/", statusRouter); // create session -app.use(createSession(config.session_secret, config.session_timeout, config.mongo_db_connection_string)); +app.use(createSession(configuration.session_secret, configuration.session_timeout, configuration.mongo_db_connection_string)); // // authentication middleware // app.use(async (req, res, next) => { @@ -70,22 +69,18 @@ app.use(createSession(config.session_secret, config.session_timeout, config.mong // add graphql endpoint app.use("/api/graphql", graphqlRouter); -const INACTIVE_SUBMISSION_DAYS = "Inactive_Submission_Notify_Days"; -cronJob.schedule(config.schedule_job, async () => { - const dbConnector = new DatabaseConnector(config.mongo_db_connection_string); - const dbService = new MongoQueries(config.mongo_db_connection_string, DATABASE_NAME); - const emailService = new EmailService(config.email_transport, config.emails_enabled); - const notificationsService = new NotifyUser(emailService); +cronJob.schedule(configuration.schedule_job, async () => { + const dbConnector = new DatabaseConnector(configuration.mongo_db_connection_string); + const dbService = new MongoQueries(configuration.mongo_db_connection_string, DATABASE_NAME); dbConnector.connect().then( async () => { + const config = await configuration.updateConfig(dbConnector); + const emailService = new EmailService(config.email_transport, config.emails_enabled); + const notificationsService = new NotifyUser(emailService, config.committee_emails); const applicationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, APPLICATION_COLLECTION); const userCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, USER_COLLECTION); - const configurationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, CONFIGURATION_COLLECTION); - const configurationService = new ConfigurationService(configurationCollection) - const inactiveSubmissionConf = await configurationService.findByType(INACTIVE_SUBMISSION_DAYS); - const inactiveSubmissionsTimeout = Array.isArray(inactiveSubmissionConf?.timeout) && inactiveSubmissionConf?.timeout?.length > 0 ? inactiveSubmissionConf?.timeout : [7, 30, 60]; const emailParams = {url: config.emails_url, officialEmail: config.official_email, inactiveDays: config.inactive_application_days, remindDay: config.remind_application_days, - submissionSystemPortal: config.submission_system_portal, submissionHelpdesk: config.submission_helpdesk, remindSubmissionDay: inactiveSubmissionsTimeout, + submissionSystemPortal: config.submission_system_portal, submissionHelpdesk: config.submission_helpdesk, remindSubmissionDay: config.inactiveSubmissionNotifyDays, finalRemindSubmissionDay: config.inactive_submission_days || 120}; const logCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, LOG_COLLECTION); const approvedStudiesCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, APPROVED_STUDIES_COLLECTION); @@ -94,16 +89,19 @@ cronJob.schedule(config.schedule_job, async () => { const organizationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, ORGANIZATION_COLLECTION); const organizationService = new Organization(organizationCollection); const submissionCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, SUBMISSIONS_COLLECTION); - const userService = new User(userCollection, logCollection, organizationCollection, notificationsService, submissionCollection, applicationCollection, config.official_email, config.tier); + const userService = new User(userCollection, logCollection, organizationCollection, notificationsService, submissionCollection, applicationCollection, config.official_email, config.tier, config.inactive_user_days); const s3Service = new S3Service(); const batchCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, BATCH_COLLECTION); - const awsService = new AWSService(submissionCollection, userService); + const awsService = new AWSService(submissionCollection, userService, config.role_arn, config.presign_expiration); const batchService = new BatchService(s3Service, batchCollection, config.sqs_loader_queue, awsService); + const qcResultCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, QC_RESULTS_COLLECTION); + const qcResultsService = new QcResultService(qcResultCollection, submissionCollection); + const dataRecordCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, DATA_RECORDS_COLLECTION); const dataRecordArchiveCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, DATA_RECORDS_ARCHIVE_COLLECTION); - const dataRecordService = new DataRecordService(dataRecordCollection, dataRecordArchiveCollection, config.file_queue, config.metadata_queue, awsService); + const dataRecordService = new DataRecordService(dataRecordCollection, dataRecordArchiveCollection, config.file_queue, config.metadata_queue, awsService, s3Service, qcResultsService, config.export_queue); const utilityService = new UtilityService(); const dataModelInfo = await utilityService.fetchJsonFromUrl(config.model_url); diff --git a/config.js b/config.js index 74a0e77f..87a6cde3 100644 --- a/config.js +++ b/config.js @@ -1,13 +1,48 @@ require('dotenv').config(); const {readFile2Text} = require("./utility/io-util") +const {ConfigurationService} = require("./services/configurationService"); +const {MongoDBCollection} = require("./crdc-datahub-database-drivers/mongodb-collection"); +const {DATABASE_NAME, CONFIGURATION_COLLECTION} = require("./crdc-datahub-database-drivers/database-constants"); +const EMAIL_SMTP_HOST = "EMAIL_SMTP_HOST"; +const EMAIL_SMTP_PORT = "EMAIL_SMTP_PORT"; +const EMAIL_USER = "EMAIL_USER"; +const EMAIL_PASSWORD = "EMAIL_PASSWORD"; +const EMAIL_URL = "EMAIL_URL"; +const OFFICIAL_EMAIL = "OFFICIAL_EMAIL"; +const INACTIVE_APPLICATION_DAYS= "INACTIVE_APPLICATION_DAYS"; +const REMIND_APPLICATION_DAYS = "REMIND_APPLICATION_DAYS"; +const SUBMISSION_SYSTEM_PORTAL = "SUBMISSION_SYSTEM_PORTAL"; +const SUBMISSION_HELPDESK = "SUBMISSION_HELPDESK"; +const TECH_SUPPORT_EMAIL = "TECH_SUPPORT_EMAIL"; +const INACTIVE_USER_DAYS = "INACTIVE_USER_DAYS"; +const SUBMISSION_DOC_URL = "SUBMISSION_DOC_URL"; +const PROD_URL = "PROD_URL"; +const ROLE_TIMEOUT = "ROLE_TIMEOUT"; +const PRESIGN_EXPIRATION = "PRESIGN_EXPIRATION"; +const TIER = "TIER"; +const LOADER_QUEUE = "LOADER_QUEUE"; +const METADATA_QUEUE = "METADATA_QUEUE"; +const FILE_QUEUE = "FILE_QUEUE"; +const EXPORTER_QUEUE = "EXPORTER_QUEUE"; +const REVIEW_COMMITTEE_EMAIL = "REVIEW_COMMITTEE_EMAIL"; +const MODEL_URL = "MODEL_URL"; +const DATA_COMMONS_LIST = "DATA_COMMONS_LIST"; +const HIDDEN_MODELS = "HIDDEN_MODELS"; +const COMPLETED_RETENTION_DAYS = "COMPLETED_RETENTION_DAYS"; +const INACTIVE_SUBMISSION_DAYS_DELETE = "INACTIVE_SUBMISSION_DAYS_DELETE"; +const DASHBOARD_SESSION_TIMEOUT = "DASHBOARD_SESSION_TIMEOUT"; +const INACTIVE_SUBMISSION_NOTIFY_DAYS = "INACTIVE_SUBMISSION_NOTIFY_DAYS"; +const EMAIL_SMTP = "EMAIL_SMTP"; +const SCHEDULED_JOBS = "SCHEDULED_JOBS"; +const LIST_OF_EMAIL_ADDRESS = "LIST_OF_EMAIL_ADDRESS"; +const LIST_OF_URLS = "LIST_OF_URLS"; +const AWS_SQS_QUEUE = "AWS_SQS_QUEUE"; +const TIMEOUT = "TIMEOUT"; let config = { //info variables version: process.env.VERSION || 'Version not set', date: process.env.DATE || new Date(), - inactive_user_days : process.env.INACTIVE_USER_DAYS || 60, - remind_application_days: process.env.REMIND_APPLICATION_DAYS || 30, - inactive_application_days : process.env.INACTIVE_APPLICATION_DAYS || 45, //Mongo DB mongo_db_user: process.env.MONGO_DB_USER, mongo_db_password: process.env.MONGO_DB_PASSWORD, @@ -19,41 +54,94 @@ let config = { session_timeout: parseInt(process.env.SESSION_TIMEOUT_SECONDS) * 1000 || 30 * 60 * 1000, token_secret: process.env.SESSION_SECRET, token_timeout: parseInt(process.env.TOKEN_TIMEOUT) * 1000 || 30 * 24 * 60 * 60 * 1000, - // Email settings - email_transport: getTransportConfig(), - emails_enabled: process.env.EMAILS_ENABLED ? process.env.EMAILS_ENABLED.toLowerCase() === 'true' : true, - emails_url: process.env.EMAIL_URL ? process.env.EMAIL_URL : 'http://localhost:4010', - official_email: process.env.OFFICIAL_EMAIL || 'CRDCHelpDesk@nih.gov', - // Scheduled cronjob once a day (1am) eastern time at default schedule_job: process.env.SCHEDULE_JOB || "1 0 1 * * *", - // temp url for email - submission_doc_url: process.env.SUBMISSION_DOC_URL || "", - submission_helpdesk: "CRDCSubmissions@nih.gov", - techSupportEmail: process.env.TECH_SUPPORT_EMAIL || "NCICRDCTechSupport@mail.nih.gov", - submission_system_portal: "https://datacommons.cancer.gov/", - prod_url: process.env.PROD_URL || "https://hub.datacommons.cancer.gov/", - submission_bucket: process.env.SUBMISSION_BUCKET, //aws sts assume role role_arn: process.env.ROLE_ARN, - role_timeout: parseInt(process.env.ROLE_TIMEOUT) || 12*3600, - presign_expiration: parseInt(process.env.PRESIGN_EXPIRATION) || 3600, - tier: getTier(), - // aws SQS names - sqs_loader_queue: process.env.LOADER_QUEUE || "crdcdh-queue", - metadata_queue: process.env.METADATA_QUEUE, - file_queue: process.env.FILE_QUEUE, - export_queue: process.env.EXPORTER_QUEUE, - //CRDC Review Committee Emails, separated by "," - committee_emails: process.env.REVIEW_COMMITTEE_EMAIL ? process.env.REVIEW_COMMITTEE_EMAIL.split(',') : ["CRDCSubmisison@nih.gov"], - model_url: getModelUrl(), - //uploader configuration file template - uploaderCLIConfigs: readUploaderCLIConfigTemplate(), - dataCommonsList: process.env.DATA_COMMONS_LIST ? JSON.parse(process.env.DATA_COMMONS_LIST) : ["CDS", "ICDC", "CTDC", "CCDI", "Test MDF", "Hidden Model"], - hiddenModels: process.env.HIDDEN_MODELS ? parseHiddenModels(process.env.HIDDEN_MODELS) : [], - inactive_submission_days: process.env.INACTIVE_SUBMISSION_DAYS_DELETE || 120, - completed_submission_days: process.env.COMPLETED_RETENTION_DAYS || 30, - dashboardSessionTimeout: process.env.DASHBOARD_SESSION_TIMEOUT || 3600, // 60 minutes by default -}; + updateConfig: async (dbConnector)=> { + const configurationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, CONFIGURATION_COLLECTION); + const configurationService = new ConfigurationService(configurationCollection); + // SCHEDULED_JOBS + const scheduledJobsConf = await configurationService.findByType(SCHEDULED_JOBS); + const inactiveUserDaysConf = scheduledJobsConf?.[INACTIVE_USER_DAYS]; + const inactiveApplicationDaysConf = scheduledJobsConf?.[INACTIVE_APPLICATION_DAYS]; + const remindApplicationDaysConf = scheduledJobsConf?.[REMIND_APPLICATION_DAYS]; + const inactiveSubmissionDaysConf = scheduledJobsConf?.[INACTIVE_SUBMISSION_DAYS_DELETE]; + const completedSubmissionDaysConf = scheduledJobsConf?.[COMPLETED_RETENTION_DAYS]; + const inactiveSubmissionNotifyDaysConf = scheduledJobsConf?.[INACTIVE_SUBMISSION_NOTIFY_DAYS]; + // EMAIL_SMTP + const emailSmtpConf = await configurationService.findByType(EMAIL_SMTP); + const emailSmtpHostConf = emailSmtpConf?.[EMAIL_SMTP_HOST]; + const emailSmtpPortConf = emailSmtpConf?.[EMAIL_SMTP_PORT]; + const emailSmtpUserConf = emailSmtpConf?.[EMAIL_USER]; + const emailSmtpPasswordConf = emailSmtpConf?.[EMAIL_PASSWORD]; + // LIST_OF_EMAIL_ADDRESS + const listEmailsConf = await configurationService.findByType(LIST_OF_EMAIL_ADDRESS); + const officialEmailConf = listEmailsConf?.[OFFICIAL_EMAIL]; + const reviewCommitteeEmailConf = listEmailsConf?.[REVIEW_COMMITTEE_EMAIL]; + const techSupportEmailConf = listEmailsConf?.[TECH_SUPPORT_EMAIL]; + const submissionHelpdeskConf = listEmailsConf?.[SUBMISSION_HELPDESK]; + // LIST_OF_URLS + const listURLsConf = await configurationService.findByType(LIST_OF_URLS); + const emailURLConf = listURLsConf?.[EMAIL_URL]; + const submissionDocUrlConf = listURLsConf?.[SUBMISSION_DOC_URL]; + const submissionSystemPortalConf = listURLsConf?.[SUBMISSION_SYSTEM_PORTAL]; + const prodUrlConf = listURLsConf?.[PROD_URL]; + const modelURLConf = listURLsConf?.[MODEL_URL]; + // TIMEOUT + const timeoutConf = await configurationService.findByType(TIMEOUT); + const roleTimeoutConf = timeoutConf?.[ROLE_TIMEOUT]; + const preSignExpirationConf = timeoutConf?.[PRESIGN_EXPIRATION]; + const dashboardSessionTimeoutConf = timeoutConf?.[DASHBOARD_SESSION_TIMEOUT]; + // TIER + const tierConf = await configurationService.findByType(TIER); + // AWS_SQS_QUEUE + const awsQueuesConf = await configurationService.findByType(AWS_SQS_QUEUE); + const loaderQueueConf = awsQueuesConf?.[LOADER_QUEUE]; + const metadataQueueConf = awsQueuesConf?.[METADATA_QUEUE]; + const fileQueueConf = awsQueuesConf?.[FILE_QUEUE]; + const exporterQueueConf = awsQueuesConf?.[EXPORTER_QUEUE]; + // SUBMISSION + const dataCommonsListConf = await configurationService.findByType(DATA_COMMONS_LIST); + const hiddenModelsConf = await configurationService.findByType(HIDDEN_MODELS); + return { + ...config, + inactive_user_days : inactiveUserDaysConf || (process.env.INACTIVE_USER_DAYS || 60), + remind_application_days: remindApplicationDaysConf || (process.env.REMIND_APPLICATION_DAYS || 30), + inactive_application_days : inactiveApplicationDaysConf || (process.env.INACTIVE_APPLICATION_DAYS || 45), + // Email settings + email_transport: getTransportConfig(emailSmtpHostConf, emailSmtpPortConf, emailSmtpUserConf, emailSmtpPasswordConf), + emails_enabled: process.env.EMAILS_ENABLED ? process.env.EMAILS_ENABLED.toLowerCase() === 'true' : true, + emails_url: emailURLConf || (process.env.EMAIL_URL ? process.env.EMAIL_URL : 'http://localhost:4010'), + official_email: officialEmailConf || (process.env.OFFICIAL_EMAIL || 'CRDCHelpDesk@nih.gov'), + // temp url for email + submission_doc_url: submissionDocUrlConf || (process.env.SUBMISSION_DOC_URL || ""), + submission_helpdesk: submissionHelpdeskConf || "CRDCSubmissions@nih.gov", + techSupportEmail: techSupportEmailConf || (process.env.TECH_SUPPORT_EMAIL || "NCICRDCTechSupport@mail.nih.gov"), + submission_system_portal: submissionSystemPortalConf || "https://datacommons.cancer.gov/", + prod_url: prodUrlConf || (process.env.PROD_URL || "https://hub.datacommons.cancer.gov/"), + submission_bucket: process.env.SUBMISSION_BUCKET, + role_timeout: roleTimeoutConf || (parseInt(process.env.ROLE_TIMEOUT) || 12*3600), + presign_expiration: preSignExpirationConf || (parseInt(process.env.PRESIGN_EXPIRATION) || 3600), + tier: getTier(tierConf?.key), + // aws SQS names + sqs_loader_queue: loaderQueueConf || (process.env.LOADER_QUEUE || "crdcdh-queue"), + metadata_queue: metadataQueueConf || process.env.METADATA_QUEUE, + file_queue: fileQueueConf || process.env.FILE_QUEUE, + export_queue: exporterQueueConf || process.env.EXPORTER_QUEUE, + //CRDC Review Committee Emails, separated by "," + committee_emails: reviewCommitteeEmailConf || (process.env.REVIEW_COMMITTEE_EMAIL ? (reviewCommitteeEmailConf || process.env.REVIEW_COMMITTEE_EMAIL)?.split(',') : ["CRDCSubmisison@nih.gov"]), + model_url: modelURLConf || getModelUrl(tierConf?.key), + //uploader configuration file template + uploaderCLIConfigs: readUploaderCLIConfigTemplate(), + dataCommonsList: dataCommonsListConf?.key || (process.env.DATA_COMMONS_LIST ? JSON.parse(process.env.DATA_COMMONS_LIST) : ["CDS", "ICDC", "CTDC", "CCDI", "Test MDF", "Hidden Model"]), + hiddenModels: hiddenModelsConf?.key || (process.env.HIDDEN_MODELS ? parseHiddenModels(process.env.HIDDEN_MODELS) : []), + inactive_submission_days: inactiveSubmissionDaysConf || (process.env.INACTIVE_SUBMISSION_DAYS_DELETE || 120), + completed_submission_days: completedSubmissionDaysConf || (process.env.COMPLETED_RETENTION_DAYS || 30), + dashboardSessionTimeout: dashboardSessionTimeoutConf || (process.env.DASHBOARD_SESSION_TIMEOUT || 3600), // 60 minutes by default + inactiveSubmissionNotifyDays: inactiveSubmissionNotifyDaysConf || [7, 30, 60] // 7, 30, 60 days by default + }; + } +} config.mongo_db_connection_string = `mongodb://${config.mongo_db_user}:${config.mongo_db_password}@${config.mongo_db_host}:${process.env.MONGO_DB_PORT}`; function parseHiddenModels(hiddenModels) { @@ -62,17 +150,17 @@ function parseHiddenModels(hiddenModels) { .map(item => item?.trim()); } -function getTransportConfig() { +function getTransportConfig(host, port, emailUser, emailPassword) { return { - host: process.env.EMAIL_SMTP_HOST, - port: process.env.EMAIL_SMTP_PORT, + host: host || process.env.EMAIL_SMTP_HOST, + port: port || process.env.EMAIL_SMTP_PORT, secure: false, // Optional AWS Email Identity - ...(process.env.EMAIL_USER && { + ...(emailUser || process.env.EMAIL_USER && { secure: false, // true for 465, false for other ports auth: { - user: process.env.EMAIL_USER, // generated ethereal user - pass: process.env.EMAIL_PASSWORD, // generated ethereal password + user: emailUser || process.env.EMAIL_USER, // generated ethereal user + pass: emailPassword || process.env.EMAIL_PASSWORD, // generated ethereal password } } ) @@ -87,12 +175,12 @@ function readUploaderCLIConfigTemplate(){ } return configString; } -function getModelUrl() { +function getModelUrl(dbTier) { // if MODEL_URL exists, it overrides if (process.env.MODEL_URL) { return process.env.MODEL_URL; } - const tier = extractTierName(); + const tier = extractTierName(dbTier); // By default url const modelUrl = ['https://raw.githubusercontent.com/CBIIT/crdc-datahub-models/', tier || 'master', '/cache/content.json'] if (tier?.length > 0) { @@ -101,12 +189,13 @@ function getModelUrl() { return modelUrl.join(""); } -function extractTierName() { - return process.env.TIER?.replace(/prod(uction)?/gi, '')?.replace(/[^a-zA-Z\d]/g, '')?.trim(); +function extractTierName(dbTier) { + const tier = dbTier || process.env.TIER; + return tier?.replace(/prod(uction)?/gi, '')?.replace(/[^a-zA-Z\d]/g, '')?.trim(); } -function getTier() { - const tier = extractTierName(); +function getTier(dbTier) { + const tier = extractTierName(dbTier); return tier?.length > 0 ? `[${tier.toUpperCase()}]` : ''; } diff --git a/crdc-datahub-database-drivers b/crdc-datahub-database-drivers index d972634c..4cb267e9 160000 --- a/crdc-datahub-database-drivers +++ b/crdc-datahub-database-drivers @@ -1 +1 @@ -Subproject commit d972634c1f3fd07d748ab4f7b861f95dc581516f +Subproject commit 4cb267e9c3dd69407cfe37e3448ab7536d0960f1 diff --git a/routers/graphql-router.js b/routers/graphql-router.js index b1a9b831..6c846e14 100644 --- a/routers/graphql-router.js +++ b/routers/graphql-router.js @@ -1,6 +1,7 @@ const {buildSchema} = require('graphql'); const {createHandler} = require("graphql-http/lib/use/express"); -const config = require("../config"); +const configuration = require("../config"); + const {Application} = require("../services/application"); const {Submission} = require("../services/submission"); const {AWSService} = require("../services/aws-request"); @@ -32,8 +33,8 @@ const {DashboardService} = require("../services/dashboardService"); const UserInitializationService = require("../services/user-initialization-service"); const {ConfigurationService} = require("../services/configurationService"); const schema = buildSchema(require("fs").readFileSync("resources/graphql/crdc-datahub.graphql", "utf8")); -const dbService = new MongoQueries(config.mongo_db_connection_string, DATABASE_NAME); -const dbConnector = new DatabaseConnector(config.mongo_db_connection_string); +const dbService = new MongoQueries(configuration.mongo_db_connection_string, DATABASE_NAME); +const dbConnector = new DatabaseConnector(configuration.mongo_db_connection_string); const AuthenticationService = require("../services/authentication-service"); const {apiAuthorization, extractAPINames, PUBLIC} = require("./api-authorization"); const {QcResultService} = require("../services/qc-result-service"); @@ -43,11 +44,12 @@ const INACTIVE_SUBMISSION_DAYS = "Inactive_Submission_Notify_Days"; let root; let authenticationService, userInitializationService; dbConnector.connect().then(async () => { + const config = await configuration.updateConfig(dbConnector); const applicationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, APPLICATION_COLLECTION); const submissionCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, SUBMISSIONS_COLLECTION); const userCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, USER_COLLECTION); const emailService = new EmailService(config.email_transport, config.emails_enabled); - const notificationsService = new NotifyUser(emailService); + const notificationsService = new NotifyUser(emailService, config.committee_emails); const logCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, LOG_COLLECTION); const organizationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, ORGANIZATION_COLLECTION); @@ -55,12 +57,12 @@ dbConnector.connect().then(async () => { const organizationService = new Organization(organizationCollection, userCollection, submissionCollection, applicationCollection, approvedStudiesCollection); const approvedStudiesService = new ApprovedStudiesService(approvedStudiesCollection, organizationService); - const userService = new User(userCollection, logCollection, organizationCollection, notificationsService, submissionCollection, applicationCollection, config.official_email, config.emails_url, config.tier, approvedStudiesCollection); + const userService = new User(userCollection, logCollection, organizationCollection, notificationsService, submissionCollection, applicationCollection, config.official_email, config.emails_url, config.tier, approvedStudiesCollection, config.inactive_user_days); // TODO move userService const userBEService = new UserService(userCollection, logCollection, organizationCollection, organizationService, notificationsService, submissionCollection, applicationCollection, config.official_email, config.emails_url, config.tier); const s3Service = new S3Service(); const batchCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, BATCH_COLLECTION); - const awsService = new AWSService(submissionCollection, userService); + const awsService = new AWSService(submissionCollection, userService, config.role_arn, config.presign_expiration); const utilityService = new UtilityService(); const fetchDataModelInfo = async () => { @@ -76,16 +78,14 @@ dbConnector.connect().then(async () => { const dataRecordCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, DATA_RECORDS_COLLECTION); const dataRecordArchiveCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, DATA_RECORDS_ARCHIVE_COLLECTION); - const dataRecordService = new DataRecordService(dataRecordCollection, dataRecordArchiveCollection, config.file_queue, config.metadata_queue, awsService, s3Service, qcResultsService); + const dataRecordService = new DataRecordService(dataRecordCollection, dataRecordArchiveCollection, config.file_queue, config.metadata_queue, awsService, s3Service, qcResultsService, config.export_queue); const validationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, VALIDATION_COLLECTION); const configurationCollection = new MongoDBCollection(dbConnector.client, DATABASE_NAME, CONFIGURATION_COLLECTION); const configurationService = new ConfigurationService(configurationCollection) - const inactiveSubmissionConf = await configurationService.findByType(INACTIVE_SUBMISSION_DAYS); - const inactiveSubmissionsTimeout = Array.isArray(inactiveSubmissionConf?.timeout) && inactiveSubmissionConf?.timeout?.length > 0 ? inactiveSubmissionConf?.timeout : [7, 30, 60]; const emailParams = {url: config.emails_url, officialEmail: config.official_email, inactiveDays: config.inactive_application_days, remindDay: config.remind_application_days, - submissionSystemPortal: config.submission_system_portal, submissionHelpdesk: config.submission_helpdesk, remindSubmissionDay: inactiveSubmissionsTimeout, techSupportEmail: config.techSupportEmail}; + submissionSystemPortal: config.submission_system_portal, submissionHelpdesk: config.submission_helpdesk, remindSubmissionDay: config.inactiveSubmissionNotifyDays, techSupportEmail: config.techSupportEmail}; const submissionService = new Submission(logCollection, submissionCollection, batchService, userService, organizationService, notificationsService, dataRecordService, config.tier, fetchDataModelInfo, awsService, config.export_queue, s3Service, emailParams, config.dataCommonsList, config.hiddenModels, validationCollection, config.sqs_loader_queue, qcResultsService); const dataInterface = new Application(logCollection, applicationCollection, approvedStudiesService, userService, dbService, notificationsService, emailParams, organizationService, config.tier, institutionService); diff --git a/services/data-record-service.js b/services/data-record-service.js index 688e5f16..35c5b797 100644 --- a/services/data-record-service.js +++ b/services/data-record-service.js @@ -3,7 +3,6 @@ const {VALIDATION} = require("../constants/submission-constants"); const ERRORS = require("../constants/error-constants"); const {ValidationHandler} = require("../utility/validation-handler"); const {getSortDirection} = require("../crdc-datahub-database-drivers/utility/mongodb-utility"); -const config = require("../config"); const {BATCH} = require("../crdc-datahub-database-drivers/constants/batch-constants.js"); const {getCurrentTime} = require("../crdc-datahub-database-drivers/utility/time-utility"); const BATCH_SIZE = 300; @@ -32,7 +31,7 @@ const NODE_RELATION_TYPES = [NODE_RELATION_TYPE_PARENT, NODE_RELATION_TYPE_CHILD const FILE = "file"; class DataRecordService { - constructor(dataRecordsCollection, dataRecordArchiveCollection, fileQueueName, metadataQueueName, awsService, s3Service, qcResultsService) { + constructor(dataRecordsCollection, dataRecordArchiveCollection, fileQueueName, metadataQueueName, awsService, s3Service, qcResultsService, exportQueue) { this.dataRecordsCollection = dataRecordsCollection; this.fileQueueName = fileQueueName; this.metadataQueueName = metadataQueueName; @@ -40,6 +39,7 @@ class DataRecordService { this.s3Service = s3Service; this.dataRecordArchiveCollection = dataRecordArchiveCollection; this.qcResultsService = qcResultsService; + this.exportQueue = exportQueue } async submissionStats(aSubmission) { @@ -235,7 +235,7 @@ class DataRecordService { async exportMetadata(submissionID) { const msg = Message.createFileSubmissionMessage("Export Metadata", submissionID); - return await sendSQSMessageWrapper(this.awsService, msg, submissionID, config.export_queue, submissionID); + return await sendSQSMessageWrapper(this.awsService, msg, submissionID, this.exportQueue, submissionID); } async submissionCrossValidationResults(submissionID, nodeTypes, batchIDs, severities, first, offset, orderBy, sortDirection){ diff --git a/services/notify-user.js b/services/notify-user.js index 5146e443..ac600041 100644 --- a/services/notify-user.js +++ b/services/notify-user.js @@ -3,7 +3,6 @@ const fs = require('fs'); const {createEmailTemplate} = require("../lib/create-email-template"); const sanitizeHtml = require('sanitize-html'); const {replaceMessageVariables} = require("../utility/string-util"); -const config = require("../config"); const NOTIFICATION_USER_HTML_TEMPLATE = "notification-template-user.html"; const ROLE = "Role"; const DATA_COMMONS = "Data Commons"; @@ -19,7 +18,7 @@ const AFFILIATED_ORGANIZATION = "Affiliated Organization"; const CRDC_PORTAL_ADMIN = "CRDC Submission Portal Admins"; class NotifyUser { - constructor(emailService) { + constructor(emailService, committeeEmails) { this.emailService = emailService; this.email_constants = undefined try { @@ -27,6 +26,7 @@ class NotifyUser { } catch (e) { console.error(e) } + this.committeeEmails = committeeEmails; } async send(fn){ @@ -43,8 +43,7 @@ class NotifyUser { await createEmailTemplate("notification-template.html", { message, firstName: this.email_constants.APPLICATION_COMMITTEE_NAME }), - //this.email_constants.APPLICATION_COMMITTEE_EMAIL - config.committee_emails + this.committeeEmails ); }); }