From 180299f4c62faf5c7bf033d3e42b60e87e156f65 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Fri, 19 May 2023 10:17:35 -0400 Subject: [PATCH] fix/large file upload failure * switched to streaming approach for file uploads * removed body size check for Nginx to remove filesize cap when uploading * added 'Name' to WorkflowReport to assist with identification and file generation Ticket(s): DPO3DPKRT-746 --- conf/nginx/nginx-dev.conf | 3 ++- conf/nginx/nginx-prod.conf | 2 +- server/db/api/WorkflowReport.ts | 9 ++++++--- server/db/prisma/schema.prisma | 1 + server/db/sql/scripts/Packrat.SCHEMA.sql | 1 + .../asset/resolvers/mutations/uploadAsset.ts | 6 +++++- server/report/interface/ReportFactory.ts | 3 ++- server/tests/db/dbcreation.test.ts | 9 +++++---- server/utils/zipFile.ts | 1 - .../workflow/impl/Packrat/WorkflowUpload.ts | 20 ++++++++++++++----- 10 files changed, 38 insertions(+), 17 deletions(-) diff --git a/conf/nginx/nginx-dev.conf b/conf/nginx/nginx-dev.conf index 92c0e45c6..34e034c76 100644 --- a/conf/nginx/nginx-dev.conf +++ b/conf/nginx/nginx-dev.conf @@ -30,7 +30,8 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; - client_max_body_size 10240m; + client_max_body_size 0; # disable checking the body size. was: 10240m; + upstream server { server packrat-server:4000; diff --git a/conf/nginx/nginx-prod.conf b/conf/nginx/nginx-prod.conf index b0de99efe..4f5f21b0f 100644 --- a/conf/nginx/nginx-prod.conf +++ b/conf/nginx/nginx-prod.conf @@ -38,7 +38,7 @@ http { # for more information. include /etc/nginx/conf.d/*.conf; - client_max_body_size 10240m; + client_max_body_size 0; # disable checking the body size. was: 10240m; upstream server-prod { server 127.0.0.1:4001; diff --git a/server/db/api/WorkflowReport.ts b/server/db/api/WorkflowReport.ts index 6dc093952..0d5c8f89d 100644 --- a/server/db/api/WorkflowReport.ts +++ b/server/db/api/WorkflowReport.ts @@ -8,6 +8,7 @@ export class WorkflowReport extends DBC.DBObject implements idWorkflow!: number; MimeType!: string; Data!: string; + Name!: string; constructor(input: WorkflowReportBase) { super(input); @@ -18,14 +19,15 @@ export class WorkflowReport extends DBC.DBObject implements protected async createWorker(): Promise { try { - const { idWorkflow, MimeType, Data } = this; + const { idWorkflow, MimeType, Data, Name } = this; ({ idWorkflowReport: this.idWorkflowReport, idWorkflow: this.idWorkflow, MimeType: this.MimeType, - Data: this.Data } = + Data: this.Data, Name: this.Name } = await DBC.DBConnection.prisma.workflowReport.create({ data: { Workflow: { connect: { idWorkflow }, }, MimeType, Data, + Name, }, })); return true; @@ -36,13 +38,14 @@ export class WorkflowReport extends DBC.DBObject implements protected async updateWorker(): Promise { try { - const { idWorkflowReport, idWorkflow, MimeType, Data } = this; + const { idWorkflowReport, idWorkflow, MimeType, Data, Name } = this; return await DBC.DBConnection.prisma.workflowReport.update({ where: { idWorkflowReport, }, data: { Workflow: { connect: { idWorkflow }, }, MimeType, Data, + Name, }, }) ? true : /* istanbul ignore next */ false; } catch (error) /* istanbul ignore next */ { diff --git a/server/db/prisma/schema.prisma b/server/db/prisma/schema.prisma index 5551f4a13..2cd3c91e5 100644 --- a/server/db/prisma/schema.prisma +++ b/server/db/prisma/schema.prisma @@ -875,6 +875,7 @@ model WorkflowReport { idWorkflow Int MimeType String @mariasql.VarChar(256) Data String @mariasql.LongText + Name String @mariasql.VarChar(512) Workflow Workflow @relation(fields: [idWorkflow], references: [idWorkflow], onDelete: NoAction, onUpdate: NoAction, map: "fk_workflowreport_workflow1") @@index([idWorkflow], map: "fk_workflowreport_workflow1") diff --git a/server/db/sql/scripts/Packrat.SCHEMA.sql b/server/db/sql/scripts/Packrat.SCHEMA.sql index 990d48c09..7a2a36baf 100644 --- a/server/db/sql/scripts/Packrat.SCHEMA.sql +++ b/server/db/sql/scripts/Packrat.SCHEMA.sql @@ -626,6 +626,7 @@ CREATE TABLE IF NOT EXISTS `WorkflowReport` ( `idWorkflow` int(11) NOT NULL, `MimeType` varchar(256) NOT NULL, `Data` longtext NOT NULL, + `Name` varchar(512) NOT NULL, PRIMARY KEY (`idWorkflowReport`) ) ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4; diff --git a/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts b/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts index f305455ec..195804ab7 100644 --- a/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts +++ b/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts @@ -47,8 +47,8 @@ class UploadAssetWorker extends ResolverBase { } async upload(): Promise { + // entry point for file upload requests coming from the client this.LS = await ASL.getOrCreateStore(); - const UAR: UploadAssetResult = await this.uploadWorker(); const success: boolean = (UAR.status === UploadStatus.Complete); @@ -63,6 +63,8 @@ class UploadAssetWorker extends ResolverBase { } private async uploadWorker(): Promise { + // creates a WorkflowReport for the upload request allowing for asynchronous handling + const { filename, createReadStream } = this.apolloFile; AuditFactory.audit({ url: `/ingestion/uploads/${filename}`, auth: (this.user !== undefined) }, { eObjectType: COMMON.eSystemObjectType.eAsset, idObject: this.idAsset ?? 0 }, eEventKey.eHTTPUpload); @@ -83,6 +85,7 @@ class UploadAssetWorker extends ResolverBase { return { status: UploadStatus.Failed, error: 'Storage unavailable' }; } + // get a write stream for us to store the incoming stream const WSResult: STORE.WriteStreamResult = await storage.writeStream(filename); if (!WSResult.success || !WSResult.writeStream || !WSResult.storageKey) { LOG.error(`uploadAsset unable to retrieve IStorage.writeStream(): ${WSResult.error}`, LOG.LS.eGQL); @@ -96,6 +99,7 @@ class UploadAssetWorker extends ResolverBase { } try { + // write our incoming stream of bytes to a file in local storage (staging) const fileStream = createReadStream(); const stream = fileStream.pipe(writeStream); diff --git a/server/report/interface/ReportFactory.ts b/server/report/interface/ReportFactory.ts index 82bbdb4ee..ec30b71f6 100644 --- a/server/report/interface/ReportFactory.ts +++ b/server/report/interface/ReportFactory.ts @@ -28,7 +28,8 @@ export class ReportFactory { idWorkflow, MimeType: 'text/html', Data: '', - idWorkflowReport: 0 + idWorkflowReport: 0, + Name: '' }); if (!await workflowReport.create()) { LOG.error(`ReportFactory.getReport() unable to create WorkflowReport for workflow with ID ${JSON.stringify(idWorkflow)}`, LOG.LS.eRPT); diff --git a/server/tests/db/dbcreation.test.ts b/server/tests/db/dbcreation.test.ts index cc391e093..7501d27e4 100644 --- a/server/tests/db/dbcreation.test.ts +++ b/server/tests/db/dbcreation.test.ts @@ -1695,7 +1695,8 @@ describe('DB Creation Test Suite', () => { idWorkflow: workflow.idWorkflow, MimeType: 'test/mimetype', Data: 'WorkflowReport test', - idWorkflowReport: 0 + idWorkflowReport: 0, + Name: 'test_report' }); expect(workflowReport).toBeTruthy(); }); @@ -4274,7 +4275,7 @@ describe('DB Fetch SystemObject Fetch Pair Test Suite', () => { }); test('DB Fetch SystemObject: COMMON.LicenseEnumToString', async () => { - expect(COMMON.LicenseEnumToString(-1)).toEqual('Restricted, Not Publishable'); + expect(COMMON.LicenseEnumToString(4)).toEqual('Restricted, Not Publishable'); // testing int input, -1 not compiling expect(COMMON.LicenseEnumToString(COMMON.eLicense.eViewDownloadCC0)).toEqual('CC0, Publishable w/ Downloads'); expect(COMMON.LicenseEnumToString(COMMON.eLicense.eViewDownloadRestriction)).toEqual('SI ToU, Publishable w/ Downloads'); expect(COMMON.LicenseEnumToString(COMMON.eLicense.eViewOnly)).toEqual('SI ToU, Publishable Only'); @@ -4282,7 +4283,7 @@ describe('DB Fetch SystemObject Fetch Pair Test Suite', () => { }); test('DB Fetch SystemObject: COMMON.PublishedStateEnumToString', async () => { - expect(COMMON.PublishedStateEnumToString(-1)).toEqual('Not Published'); + expect(COMMON.PublishedStateEnumToString(0)).toEqual('Not Published'); // testing int input. -1 not compiling since enums are 0+ expect(COMMON.PublishedStateEnumToString(COMMON.ePublishedState.eNotPublished)).toEqual('Not Published'); expect(COMMON.PublishedStateEnumToString(COMMON.ePublishedState.eAPIOnly)).toEqual('API Only'); expect(COMMON.PublishedStateEnumToString(COMMON.ePublishedState.ePublished)).toEqual('Published'); @@ -4970,7 +4971,7 @@ describe('DB Fetch Special Test Suite', () => { }); test('DB Fetch Special: convertWorkflowJobRunStatusEnumToString', async () => { - expect(DBAPI.convertWorkflowJobRunStatusEnumToString(-1)).toEqual('Uninitialized'); + expect(DBAPI.convertWorkflowJobRunStatusEnumToString(0)).toEqual('Uninitialized'); // testing int input. -1 not compiling expect(DBAPI.convertWorkflowJobRunStatusEnumToString(COMMON.eWorkflowJobRunStatus.eUnitialized)).toEqual('Uninitialized'); expect(DBAPI.convertWorkflowJobRunStatusEnumToString(COMMON.eWorkflowJobRunStatus.eCreated)).toEqual('Created'); expect(DBAPI.convertWorkflowJobRunStatusEnumToString(COMMON.eWorkflowJobRunStatus.eRunning)).toEqual('Running'); diff --git a/server/utils/zipFile.ts b/server/utils/zipFile.ts index 5288781ce..732c0cad9 100644 --- a/server/utils/zipFile.ts +++ b/server/utils/zipFile.ts @@ -28,7 +28,6 @@ export class ZipFile implements IZip { async load(): Promise { try { this._zip = new StreamZip({ file: this._fileName, storeEntries: true }); - return new Promise((resolve) => { /* istanbul ignore else */ if (this._zip) { diff --git a/server/workflow/impl/Packrat/WorkflowUpload.ts b/server/workflow/impl/Packrat/WorkflowUpload.ts index fec00ece8..469ea7cc9 100644 --- a/server/workflow/impl/Packrat/WorkflowUpload.ts +++ b/server/workflow/impl/Packrat/WorkflowUpload.ts @@ -7,7 +7,8 @@ import * as STORE from '../../../storage/interface'; import * as REP from '../../../report/interface'; import * as LOG from '../../../utils/logger'; import * as H from '../../../utils/helpers'; -import { ZipStream } from '../../../utils/zipStream'; +import { Config } from '../../../config'; +import { ZipFile } from '../../../utils'; import { SvxReader } from '../../../utils/parser'; // import * as sharp from 'sharp'; @@ -95,6 +96,7 @@ export class WorkflowUpload implements WF.IWorkflow { if (!asset) return this.handleError(`WorkflowUpload.validateFiles unable to load asset for ${assetVersion.idAsset}`); + // see if the passed in file is a model const isModel: boolean = await this.testIfModel(assetVersion.FileName, asset); const RSR: STORE.ReadStreamResult = await STORE.AssetStorageAdapter.readAssetVersionByID(idAssetVersion); @@ -103,12 +105,18 @@ export class WorkflowUpload implements WF.IWorkflow { this.appendToWFReport(`Upload validation of ${RSR.fileName}`); let fileRes: H.IOResults = { success: true }; - if (isModel) // if we're a model, zipped or not, validate the entire file/collection as is: + if (isModel) { + // if we're a model, zipped or not, validate the entire file/collection as is: fileRes = await this.validateFileModel(RSR.fileName, RSR.readStream, false, idSystemObject); - else if (path.extname(RSR.fileName).toLowerCase() !== '.zip') // not a zip + } else if (path.extname(RSR.fileName).toLowerCase() !== '.zip') { // not a zip + // we are not a zip fileRes = await this.validateFile(RSR.fileName, RSR.readStream, false, idSystemObject, asset); - else { - const ZS: ZipStream = new ZipStream(RSR.readStream); + } else { + + // it's not a model (e.g. Capture Data) + // use ZipFile so we don't need to load it all into memory + const filePath: string = Config.storage.rootStaging+'/'+assetVersion.StorageKeyStaging; + const ZS: ZipFile = new ZipFile(filePath,true); const zipRes: H.IOResults = await ZS.load(); if (!zipRes.success) return this.handleError(`WorkflowUpload.validateFiles unable to unzip asset version ${RSR.fileName}: ${zipRes.error}`); @@ -206,6 +214,8 @@ export class WorkflowUpload implements WF.IWorkflow { } private async testIfModel(fileName: string, asset: DBAPI.Asset): Promise { + // compare our model extension to defined vocabulary for models and geometry files. + // if it doesn't resolve to either type then fail the check. if (await CACHE.VocabularyCache.mapModelFileByExtension(fileName) !== undefined) return true; // might be zipped; check asset