From cd9fcd043efecacad3f38d0d178b42cd9f659a69 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Wed, 7 Feb 2024 12:32:07 -0500 Subject: [PATCH 01/14] (fix) updated all environment variables to point to new EDAN endpoints (#573) dev: PACKRAT_EDAN_3D_API=https://api-internal.edan.si.edu/3d-api-dev/ prod: PACKRAT_EDAN_3D_API=https://api-internal.edan.si.edu/3d-api/ From d251564e2dcb8c47476ea21006dc5e5c08700534 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:49:43 -0500 Subject: [PATCH 02/14] Update Voyager schema for changes made on 1/25/2024 (#577) DPO3DPKRT-787/update-voyager-schema-2024-01-25 --- server/types/voyager/json/meta.schema.json | 12 ++++++ server/types/voyager/json/model.schema.json | 5 +++ server/types/voyager/json/setup.schema.json | 43 +++++++++++++++++++++ server/types/voyager/meta.ts | 6 ++- server/types/voyager/model.ts | 5 ++- server/types/voyager/setup.ts | 6 +-- 6 files changed, 70 insertions(+), 7 deletions(-) diff --git a/server/types/voyager/json/meta.schema.json b/server/types/voyager/json/meta.schema.json index 3d70ee8a8..dedc54f56 100644 --- a/server/types/voyager/json/meta.schema.json +++ b/server/types/voyager/json/meta.schema.json @@ -31,6 +31,10 @@ "height": { "type": "integer", "minimum": 1 + }, + "usage": { + "type": "string", + "enum": [ "Render", "ARCode" ] } }, "required": [ @@ -113,6 +117,14 @@ "uris": { "description": "Location of the audio resource, absolute URL or path relative to this document with language key", "type": "object" + }, + "captionUris": { + "description": "Location of the caption resource, absolute URL or path relative to this document with language key", + "type": "object" + }, + "durations": { + "description": "Length of the audio resource with language key", + "type": "object" } }, "required": [ diff --git a/server/types/voyager/json/model.schema.json b/server/types/voyager/json/model.schema.json index 0af3f951e..55fda8de2 100644 --- a/server/types/voyager/json/model.schema.json +++ b/server/types/voyager/json/model.schema.json @@ -65,6 +65,11 @@ "type": "string", "minLength": 1 }, + "viewId": { + "description": "Id of a snapshot resource with camera view information for this annotation.", + "type": "string", + "minLength": 1 + }, "style": { "type": "string" }, diff --git a/server/types/voyager/json/setup.schema.json b/server/types/voyager/json/setup.schema.json index 414ba900e..38761cbf3 100644 --- a/server/types/voyager/json/setup.schema.json +++ b/server/types/voyager/json/setup.schema.json @@ -15,12 +15,21 @@ "exposure": { "type": "number" }, + "toneMapping": { + "type": "boolean" + }, "gamma": { "type": "number" }, "annotationsVisible": { "type": "boolean" }, + "isWallMountAR": { + "type": "boolean" + }, + "arScale": { + "type": "number" + }, "activeTags": { "type": "string" }, @@ -85,6 +94,12 @@ "autoZoom": { "type": "boolean" }, + "autoRotation": { + "type": "boolean" + }, + "lightsFollowCamera": { + "type": "boolean" + }, "orbit": { "$comment": "TODO: Implement", @@ -188,6 +203,28 @@ }, "position": { "type": "number" + }, + "color": { + "$ref": "./common.schema.json#/definitions/vector3" + } + } + }, + "audio": { + "type": "object", + "properties": { + "narrationId": { + "type": "string" + } + } + }, + "language": { + "type": "object", + "properties": { + "language": { + "type": "string", + "enum": [ + "EN", "ES", "DE", "NL", "JA", "FR", "HAW" + ] } } }, @@ -303,6 +340,12 @@ }, "tours": { "$ref": "#/definitions/tours" + }, + "language": { + "$ref": "#/definitions/language" + }, + "audio": { + "$ref": "#/definitions/audio" } } } diff --git a/server/types/voyager/meta.ts b/server/types/voyager/meta.ts index b41d2c9ad..212860837 100644 --- a/server/types/voyager/meta.ts +++ b/server/types/voyager/meta.ts @@ -1,7 +1,7 @@ -/* eslint-disable quotes, @typescript-eslint/brace-style */ +/* eslint-disable quotes, @typescript-eslint/brace-style, @typescript-eslint/no-explicit-any */ /** * NOTE: this file is part of the definition of a Voyager scene, found in a .svx.json file. - * This was imported from Voyager's source/client/schema on 1/24/2024. It was then modified, + * This was imported from Voyager's source/client/schema on 3/5/2024. It was then modified, * minimally, to allow for use by Packrat. Ideally, in the future, we will extract out the * definition of this shared file format for use by both projects. */ @@ -49,9 +49,11 @@ export interface IImage byteSize: number; width: number; height: number; + usage?: TImageUsage; } export type TImageQuality = "Thumb" | "Low" | "Medium" | "High"; +export type TImageUsage = "Render" | "ARCode"; /** * Refers to an external document or a media file (audio, video, image). diff --git a/server/types/voyager/model.ts b/server/types/voyager/model.ts index 7941041c7..97a27f9d6 100644 --- a/server/types/voyager/model.ts +++ b/server/types/voyager/model.ts @@ -1,7 +1,7 @@ /* eslint-disable quotes, @typescript-eslint/brace-style */ /** * NOTE: this file is part of the definition of a Voyager scene, found in a .svx.json file. - * This was imported from Voyager's source/client/schema on 1/24/2024. It was then modified, + * This was imported from Voyager's source/client/schema on 3/5/2024. It was then modified, * minimally, to allow for use by Packrat. Ideally, in the future, we will extract out the * definition of this shared file format for use by both projects. */ @@ -25,7 +25,7 @@ // import { Dictionary } from "@ff/core/types"; // import { ColorRGBA, EUnitType, TUnitType, Vector3, Vector4 } from "./common"; type Dictionary = Record; -import { ColorRGBA, TUnitType, Vector3, Vector4 } from "./common"; +import { ColorRGBA, /*EUnitType,*/ TUnitType, Vector3, Vector4 } from "./common"; //////////////////////////////////////////////////////////////////////////////// @@ -86,6 +86,7 @@ export interface IAnnotation imageCredit?: Dictionary; imageAltText?: Dictionary; audioId?: string; + viewId?: string; style?: string; visible?: boolean; diff --git a/server/types/voyager/setup.ts b/server/types/voyager/setup.ts index 73a460032..4b3d9fe70 100644 --- a/server/types/voyager/setup.ts +++ b/server/types/voyager/setup.ts @@ -1,7 +1,7 @@ -/* eslint-disable quotes, @typescript-eslint/brace-style */ +/* eslint-disable quotes, @typescript-eslint/brace-style, @typescript-eslint/no-explicit-any */ /** * NOTE: this file is part of the definition of a Voyager scene, found in a .svx.json file. - * This was imported from Voyager's source/client/schema on 1/24/2024. It was then modified, + * This was imported from Voyager's source/client/schema on 3/5/2024. It was then modified, * minimally, to allow for use by Packrat. Ideally, in the future, we will extract out the * definition of this shared file format for use by both projects. */ @@ -26,7 +26,6 @@ import { ELanguageType, TLanguageType } from "./common"; * limitations under the License. */ - export type TShaderMode = "Default" | "Clay" | "XRay" | "Normals" | "Wireframe"; export enum EShaderMode { Default, Clay, XRay, Normals, Wireframe } @@ -57,6 +56,7 @@ export interface ISetup slicer?: ISlicer; tours?: ITours; snapshots?: ISnapshots; + audio?: IAudio; } export interface IInterface From b0e65778b999e8d4628d6af8e8717d7d46ac73de Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Thu, 11 Apr 2024 10:34:50 -0400 Subject: [PATCH 03/14] [DPO3DPKRT-788] Updating assets from Cook with diff names fails (#579) (new) centralizing error reporting for JobCook (new) Asset.fetchFromScene to get all assets attached to a Scene (new) convenience functions for getting Workflow(Step) from a Job (new) generate downloads verifies incoming Cook data for name mismatch (fix) state of Cook Job logged when finished after polling (fix) improved error handling of generate downloads --- server/db/api/Asset.ts | 18 +++ server/db/api/JobRun.ts | 17 ++- server/db/api/Scene.ts | 4 + server/db/api/Workflow.ts | 31 ++++ server/db/api/WorkflowReport.ts | 32 ++++ .../asset/resolvers/mutations/uploadAsset.ts | 7 + server/job/impl/Cook/JobCook.ts | 45 ++++++ .../impl/Cook/JobCookSIGenerateDownloads.ts | 141 +++++++++++++----- server/job/impl/Cook/JobCookSIVoyagerScene.ts | 10 +- server/job/impl/NS/JobPackrat.ts | 11 ++ server/workflow/impl/Packrat/WorkflowJob.ts | 25 +++- 11 files changed, 292 insertions(+), 49 deletions(-) diff --git a/server/db/api/Asset.ts b/server/db/api/Asset.ts index ce3dc7b42..691f7c861 100644 --- a/server/db/api/Asset.ts +++ b/server/db/api/Asset.ts @@ -11,6 +11,8 @@ export class Asset extends DBC.DBObject implements AssetBase, SystemO FileName!: string; idAssetGroup!: number | null; idVAssetType!: number; + // is idVAssetType is 135 (Model) then idSystemObject = the Packrat Model object + // if idVAssetType is 137 (Scene) then idSystemObject = the Packrat Scene object (only for SVX files) idSystemObject!: number | null; StorageKey!: string | null; @@ -142,6 +144,22 @@ export class Asset extends DBC.DBObject implements AssetBase, SystemO } } + /** Fetch assets that are connected to a specific Scene via the scene's id. **/ + static async fetchFromScene(idScene: number): Promise { + + // when grabbing assets we need to grab those that are referenced by SystemObjectXref where + // our Packrat scene is the parent (e.g. models), but we also need to return those assets + // that don't use the xrefs and explicitly link to the Packrat Scene via it's idSystemObject field. + return DBC.CopyArray( + await DBC.DBConnection.prisma.$queryRaw` + SELECT DISTINCT a.* FROM Scene AS scn + JOIN SystemObject AS scnSO ON scn.idScene = scnSO.idScene + JOIN SystemObjectXref AS scnSOX ON scnSO.idSystemObject = scnSOX.idSystemObjectMaster + JOIN Asset AS a ON (a.idSystemObject = scnSOX.idSystemObjectDerived OR a.idSystemObject = scnSO.idSystemObject) + WHERE scn.idScene = ${idScene}; + `,Asset); + } + /** Fetches assets that are connected to the specified idSystemObject (via that object's last SystemObjectVersion, * and that SystemObjectVersionAssetVersionXref's records). For those assets, we look for a match on FileName, idVAssetType */ static async fetchMatching(idSystemObject: number, FileName: string, idVAssetType: number): Promise { diff --git a/server/db/api/JobRun.ts b/server/db/api/JobRun.ts index 8e81c9154..f6347ca9e 100644 --- a/server/db/api/JobRun.ts +++ b/server/db/api/JobRun.ts @@ -9,7 +9,7 @@ import * as H from '../../utils/helpers'; export class JobRun extends DBC.DBObject implements JobRunBase { idJobRun!: number; idJob!: number; - Status!: number; + Status!: number; // defined in common/constantss.ts (ln. 403) Result!: boolean | null; DateStart!: Date | null; DateEnd!: Date | null; @@ -179,4 +179,19 @@ export class JobRun extends DBC.DBObject implements JobRunBase { return null; } } + + static async fetchFromWorkflow(idWorkflow: number): Promise { + // direct get of JubRun(s) from a specific workflow + try { + return DBC.CopyArray ( + await DBC.DBConnection.prisma.$queryRaw` + SELECT jRun.* FROM Workflow AS w + JOIN WorkflowStep AS wStep ON wStep.idWorkflow = w.idWorkflow + JOIN JobRun AS jRun ON jRun.idJobRun = wStep.idJobRun + WHERE w.idWorkflow = ${idWorkflow};`,JobRun); + } catch (error) { + LOG.error('DBAPI.JobRun.fetchFromWorkflow', LOG.LS.eDB, error); + return null; + } + } } diff --git a/server/db/api/Scene.ts b/server/db/api/Scene.ts index 56f9bc610..534c8e32d 100644 --- a/server/db/api/Scene.ts +++ b/server/db/api/Scene.ts @@ -34,6 +34,9 @@ export class Scene extends DBC.DBObject implements SceneBase, SystemO public fetchTableName(): string { return 'Scene'; } public fetchID(): number { return this.idScene; } + public fetchLogInfo(): string { + return `scene: ${this.Name}[id:${this.idScene}] | EDAN: ${this.EdanUUID}`; + } protected async createWorker(): Promise { try { @@ -208,4 +211,5 @@ export class Scene extends DBC.DBObject implements SceneBase, SystemO return null; } } + } diff --git a/server/db/api/Workflow.ts b/server/db/api/Workflow.ts index 8bcd8e812..6fb3ebd83 100644 --- a/server/db/api/Workflow.ts +++ b/server/db/api/Workflow.ts @@ -173,4 +173,35 @@ export class Workflow extends DBC.DBObject implements WorkflowBase return null; } } + + static async fetchFromJobRun(idJobRun: number): Promise { + try { + return DBC.CopyArray ( + await DBC.DBConnection.prisma.$queryRaw` + SELECT w.* FROM JobRun AS jRun + JOIN WorkflowStep AS wStep ON wStep.idJobRun = jRun.idJobRun + JOIN Workflow AS w ON w.idWorkflow = wStep.idWorkflow + WHERE jRun.idJobRun = ${idJobRun};`,Workflow); + } catch (error) { + LOG.error('DBAPI.Workflow.fetchFromJobRun', LOG.LS.eDB, error); + return null; + } + } + + static async fetchAllWithError(includeCancelled: boolean = false, includeUninitialized: boolean = false): Promise { + // return all workflows that contain a step/job that has an error. + // optionally include those cancelled or unitialized + try { + return DBC.CopyArray ( + await DBC.DBConnection.prisma.$queryRaw` + SELECT w.* FROM WorkflowStep AS wStep + JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow + JOIN JobRun AS jRun ON wStep.idJobRun = jRun.idJobRun + JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow + WHERE (wStep.State = 5 ${(includeCancelled?'OR wStep.State = 6 ':'')}${includeUninitialized?'OR wStep.State = 0':''});`,Workflow); + } catch (error) { + LOG.error('DBAPI.Workflow.fetchAllWithError', LOG.LS.eDB, error); + return null; + } + } } diff --git a/server/db/api/WorkflowReport.ts b/server/db/api/WorkflowReport.ts index 0d5c8f89d..0b7934aa0 100644 --- a/server/db/api/WorkflowReport.ts +++ b/server/db/api/WorkflowReport.ts @@ -93,4 +93,36 @@ export class WorkflowReport extends DBC.DBObject implements return null; } } + + static async fetchFromJobRun(idJobRun: number): Promise { + try { + return DBC.CopyArray ( + await DBC.DBConnection.prisma.$queryRaw` + SELECT w.* FROM JobRun AS jRun + JOIN WorkflowStep AS wStep ON wStep.idJobRun = jRun.idJobRun + JOIN WorkflowReport AS wReport ON wReport.idWorkflow = wStep.idWorkflow + WHERE jRun.idJobRun = ${idJobRun};`,WorkflowReport); + } catch (error) { + LOG.error('DBAPI.WorkflowReport.fetchFromJobRun', LOG.LS.eDB, error); + return null; + } + } + + static async fetchAllWithError(includeCancelled: boolean = false, includeUninitialized: boolean = false): Promise { + // return all reports that contain a step/job that has an error. + // optionally include those cancelled or uninitialized. + // TODO: check against JobRun.Result for additional possible errors + try { + return DBC.CopyArray ( + await DBC.DBConnection.prisma.$queryRaw` + SELECT wReport.* FROM WorkflowStep AS wStep + JOIN WorkflowReport AS wReport ON wStep.idWorkflow = wReport.idWorkflow + JOIN JobRun AS jRun ON wStep.idJobRun = jRun.idJobRun + JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow + WHERE (wStep.State = 5 ${(includeCancelled?'OR wStep.State = 6 ':'')}${includeUninitialized?'OR wStep.State = 0':''});`,WorkflowReport); + } catch (error) { + LOG.error('DBAPI.WorkflowReport.fetchAllWithError', LOG.LS.eDB, error); + return null; + } + } } diff --git a/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts b/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts index 995acd45a..90b8af4ff 100644 --- a/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts +++ b/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts @@ -72,6 +72,10 @@ class UploadAssetWorker extends ResolverBase { LOG.error('uploadAsset unable to retrieve user context', LOG.LS.eGQL); return { status: UploadStatus.Noauth, error: 'User not authenticated' }; } + + // if an idAsset was provided then we are trying to update an existing asset + // else if an 'attachment' is specified (this is a child) then we will try to attach + // otherwise, we are adding a new asset to the system if (this.idAsset) await this.appendToWFReport(`Upload starting: UPDATE ${filename}`, true); else if (this.idSOAttachment) @@ -128,10 +132,13 @@ class UploadAssetWorker extends ResolverBase { } private async uploadWorkerOnFinish(storageKey: string, filename: string, idVocabulary: number): Promise { + + // grab our local storage const LSLocal: LocalStore | undefined = ASL.getStore(); if (LSLocal) return await this.uploadWorkerOnFinishWorker(storageKey, filename, idVocabulary); + // if we can't get the local storage system we will use the cache if (this.LS) { LOG.info('uploadAsset missing LocalStore, using cached value', LOG.LS.eGQL); return ASL.run(this.LS, async () => { diff --git a/server/job/impl/Cook/JobCook.ts b/server/job/impl/Cook/JobCook.ts index 089bb8729..e096e0667 100644 --- a/server/job/impl/Cook/JobCook.ts +++ b/server/job/impl/Cook/JobCook.ts @@ -396,9 +396,16 @@ export abstract class JobCook extends JobPackrat { } // look for completion in 'state' member, via value of 'done', 'error', or 'cancelled'; update eJobRunStatus and terminate polling job + // write to the log for the first 10 polling cycles, then every 5th one after that const cookJobReport = axiosResponse.data; if (pollNumber <= 10 || ((pollNumber % 5) == 0)) LOG.info(`JobCook [${this.name()}] polling [${pollNumber}], state: ${cookJobReport['state']}: ${requestUrl}`, LOG.LS.eJOB); + + // if we finished (i.e. not running or waiting) then we push out an additional log statement + // to ensure it's caught + if(cookJobReport['state']!=='waiting' && cookJobReport['state']!=='running') + LOG.info(`JobCook [${this.name()}] polling [exited], state: ${cookJobReport['state']}: ${requestUrl}`, LOG.LS.eJOB); + switch (cookJobReport['state']) { case 'created': await this.recordCreated(); break; case 'waiting': await this.recordWaiting(); break; @@ -666,4 +673,42 @@ export abstract class JobCook extends JobPackrat { LOG.info(error, LOG.LS.eJOB); return res; } + + protected async verifyIncomingCookData(_sceneSource: DBAPI.Scene, _fileMap: Map): Promise { + return { success: true }; + } + + protected extractBaseName(filenames: string[]): string | null { + // extract the base name from the list of incoming filenames and make sure they all share + // the same values. input (currently) requires an SVX file in the list + // TODO: broader support for other 'groups' of filenames that may not have an SVX + const svxFilename: string | undefined = filenames.find(filename => filename.includes('.svx.json')); + if(!svxFilename || svxFilename.length == 0) { + this.logError('JobCookSIGenerateDownloads cannot extract basename. SVX file not found'); + return null; + } + + // get the baseName from the SVX file + const baseName: string = svxFilename.replace(/\.svx\.json$/, ''); + + // compare with others in the list to make sure they match + const errorNames: string[] = filenames.filter(filename => !filename.startsWith(baseName)); + if(errorNames.length>0) { + this.logError(`JobCookSIGenerateDownloads filenames don't share base name. (${errorNames.join(' | ')})`); + return null; + } + + // return success + return baseName; + } + + // private getSceneFilenamesFromMap(fileMap: Map): string[] { + // const result: string[] = []; + // fileMap.forEach((value, _key) => { + // if (value.includes('svx.json')) { + // result.push(value); + // } + // }); + // return result; + // } } diff --git a/server/job/impl/Cook/JobCookSIGenerateDownloads.ts b/server/job/impl/Cook/JobCookSIGenerateDownloads.ts index 94ca69d6b..b4cf539ec 100644 --- a/server/job/impl/Cook/JobCookSIGenerateDownloads.ts +++ b/server/job/impl/Cook/JobCookSIGenerateDownloads.ts @@ -169,7 +169,10 @@ export class JobCookSIGenerateDownloads extends JobCook download filename @@ -212,6 +215,12 @@ export class JobCookSIGenerateDownloads extends JobCook obj.success === false).map(obj => obj.error).join('","') }"]`; - return await this.appendToReportAndLog(`JobCookSIGenerateDownloads failed processing of returned download model files (name: ${sceneSource.Name} | idScene: ${sceneSource.idScene} | errors: ${errors})`,true); + return this.logError(`JobCookSIGenerateDownloads failed processing of returned download model files (name: ${sceneSource.Name} | idScene: ${sceneSource.idScene} | errors: ${errors})`); } // update all ModelSceneXrefs transforms for generated models in the svx scene with what's in the @@ -352,7 +359,7 @@ export class JobCookSIGenerateDownloads extends JobCook 0 ? MSXSources[0] : null; @@ -466,7 +467,7 @@ export class JobCookSIGenerateDownloads extends JobCook 1) - this.logError(`JobCookSIGenerateDownloads.processModelFile created multiple asset versions, unexpectedly, ingesting ${fileItem.fileName}`); + await this.logError(`JobCookSIGenerateDownloads.processModelFile created multiple asset versions, unexpectedly, ingesting ${fileItem.fileName}`); // if no SysObj exists for this model then we check our cache for one let idSystemObjectModel: number | null = modelSO ? modelSO.idSystemObject : null; @@ -623,7 +624,7 @@ export class JobCookSIGenerateDownloads extends JobCook { if (!this.sceneParameterHelper) - return this.logError('JobCookSIGenerateDownloads.processSceneFile called without needed parameters'); + return await this.logError('JobCookSIGenerateDownloads.processSceneFile called without needed parameters'); const svxFile: string = fileItem.fileName; //this.parameters.svxFile ?? 'scene.svx.json'; const svxData = fileItem.data; const vScene: DBAPI.Vocabulary | undefined = await this.computeVocabAssetTypeScene(); const vModel: DBAPI.Vocabulary | undefined = await this.computeVocabAssetTypeModelGeometryFile(); if (!vScene || !vModel) - return this.logError(`JobCookSIGenerateDownloads.processSceneFile unable to calculate vocabulary needed to ingest scene file ${svxFile}`); + return await this.logError(`JobCookSIGenerateDownloads.processSceneFile unable to calculate vocabulary needed to ingest scene file ${svxFile}`); LOG.info(`JobCookSIGenerateDownloads.processSceneFile[${svxFile}] parse scene`, LOG.LS.eJOB); // LOG.info(`JobCookSIGenerateDownloads.processSceneFile fetched scene:${H.Helpers.JSONStringify(svxData)}`, LOG.LS.eJOB); @@ -762,7 +763,7 @@ export class JobCookSIGenerateDownloads extends JobCook1) - return this.logError(`multiple valid scenes found (${scenes.length}). cannot find asset to update (idScene: ${fileItem.fileName})`); + return await this.logError(`multiple valid scenes found (${scenes.length}). cannot find asset to update (idScene: ${fileItem.fileName})`); // If needed, create a new scene (if we have no scenes, or if we have multiple scenes, then create a new one); // If we have just one scene, before reusing it, see if the model names all match up @@ -829,19 +830,19 @@ export class JobCookSIGenerateDownloads extends JobCook 0) { const SOX2: DBAPI.SystemObjectXref | null = await DBAPI.SystemObjectXref.wireObjectsIfNeeded(OG.item[0], scene); if (!SOX2) - return this.logError(`JobCookSIGenerateDownloads.processSceneFile unable to wire item ${H.Helpers.JSONStringify(OG.item[0])} to Scene ${H.Helpers.JSONStringify(scene)}: database error`); + return await this.logError(`JobCookSIGenerateDownloads.processSceneFile unable to wire item ${H.Helpers.JSONStringify(OG.item[0])} to Scene ${H.Helpers.JSONStringify(scene)}: database error`); } // LOG.info(`JobCookSIGenerateDownloads.processSceneFile[${svxFile}] wire ModelSource to Scene: ${H.Helpers.JSONStringify(SOX)}`, LOG.LS.eJOB); } else { @@ -868,7 +869,7 @@ export class JobCookSIGenerateDownloads extends JobCook 1) - LOG.error(`JobCookSIGenerateDownloads.processSceneFile created multiple asset versions, unexpectedly, ingesting ${svxFile}`, LOG.LS.eJOB); + await this.logError(`JobCookSIGenerateDownloads.processSceneFile created multiple asset versions, unexpectedly, ingesting ${svxFile}`); const SOI: DBAPI.SystemObjectInfo | undefined = await CACHE.SystemObjectCache.getSystemFromScene(scene); const assetVersion: DBAPI.AssetVersion | null = (IAR.assetVersions && IAR.assetVersions.length > 0) ? IAR.assetVersions[0] : null; @@ -999,6 +1000,68 @@ export class JobCookSIGenerateDownloads extends JobCook): Promise { + + const result: H.IOResults = await super.verifyIncomingCookData(sceneSource, fileMap); + if(result.success == false) + return this.logError('verifyIncomingCookData base failed'); + + // get assets in Packrat Scene + const sceneAssets: DBAPI.Asset[] | null = await DBAPI.Asset.fetchFromScene(sceneSource.idScene); + if(!sceneAssets || sceneAssets.length == 0) + return this.logError(`JobCookSIGenerateDownloads.verifyIncomingCookData cannot find any assets for the Packrat scene. (${sceneSource.fetchLogInfo()})`); + const sceneAssetFilenames: string[] = sceneAssets.map(asset => asset.FileName); + LOG.info(`JobCookSIGenerateDownloads.verifyIncomingCookData\n\t${H.Helpers.JSONStringify(sceneAssetFilenames)}\n\t${H.Helpers.JSONStringify(fileMap)}`,LOG.LS.eDEBUG); + + // determine baseName from existing assets in the Packrat scene (error on diff) + const sceneBaseName: string | null = this.extractBaseName(sceneAssets.map(asset => asset.FileName)); + if(!sceneBaseName) + return this.logError(`JobCookSIGenerateDownloads.verifyIncomingCookData cannot extract base name. (${sceneSource.fetchLogInfo()})`); + + // get list of filenames from the incoming fileMap + const incomingAssetFilenames: string[] = [...fileMap.values()]; + + // define expected suffixes to look for in the system + // NOTE: need to update this list if si-generate-downloads returns new/different assets + const suffixes: string[] = ['-150k-4096_std.glb','-100k-2048_std_draco.glb','.svx.json','-100k-2048_std.usdz','-full_resolution-obj_std.zip','-150k-4096-gltf_std.zip','-150k-4096-obj_std.zip']; + const mismatches: string[] = []; + + // cycle through Scene assets comparing with incoming. + // if we have a file with the provided suffix in the Scene then check for a full name match with the incoming map + for(let i=0; i filename.endsWith(suffixes[i])); + if(found) + return this.logError(`CRITICAL: JobCookSIGenerateDownloads.verifyIncomingCookData found an incoming asset (${found}) that doesn't share the scene (id: ${sceneSource.idScene}) base name (${sceneBaseName}).`); + else { + LOG.info(`JobCookSIGenerateDownloads.verifyIncomingCookData didn't find ${filename}. Assuming it's new...`,LOG.LS.eDEBUG); + } + } + } + + // if we have mismatches throw an error and dump out specifics + if(mismatches.length > 0) + return this.logError(`JobCookSIGenerateDownloads.verifyIncomingCookData couldn't find matching assets with '${sceneBaseName}' base name. (${mismatches.length}: ${incomingAssetFilenames.join('|')}`); + + LOG.info(`JobCookSIGenerateDownloads incoming Cook data verified. (${sceneSource.fetchLogInfo()} | baseName: ${sceneBaseName})`,LOG.LS.eJOB); + return { success: true }; + } + // private async getModelDetailsFromVoyagerScene(svxFile): Promise { // // Retrieve svx.json data // const RSR: STORE.ReadStreamResult = await this.fetchFile(svxFile); diff --git a/server/job/impl/Cook/JobCookSIVoyagerScene.ts b/server/job/impl/Cook/JobCookSIVoyagerScene.ts index 86a6ae66d..ac9ec0d83 100644 --- a/server/job/impl/Cook/JobCookSIVoyagerScene.ts +++ b/server/job/impl/Cook/JobCookSIVoyagerScene.ts @@ -531,10 +531,10 @@ export class JobCookSIVoyagerScene extends JobCook { - const error: string = `${this.name()} ${errorBase}`; - await this.appendToReportAndLog(error, true); - return { success: false, error }; - } + // private async logError(errorBase: string): Promise { + // const error: string = `${this.name()} ${errorBase}`; + // await this.appendToReportAndLog(error, true); + // return { success: false, error }; + // } } diff --git a/server/job/impl/NS/JobPackrat.ts b/server/job/impl/NS/JobPackrat.ts index dcb67a206..5eed8beaf 100644 --- a/server/job/impl/NS/JobPackrat.ts +++ b/server/job/impl/NS/JobPackrat.ts @@ -50,6 +50,8 @@ export abstract class JobPackrat implements JOB.IJob { for (let attempt: number = 0; attempt < JOB_RETRY_COUNT; attempt++) { await this.recordCreated(); this._results = await this.startJobWorker(fireDate); + LOG.info(`JobPackrat.executeJob start job worker results (${H.Helpers.JSONStringify(this._results)})`,LOG.LS.eDEBUG); + if (!this._results.success) await this.recordFailure(null, this._results.error); @@ -204,5 +206,14 @@ export abstract class JobPackrat implements JOB.IJob { this.updateEngines(true, true); // don't block } } + + protected async logError(errorMessage: string, addToReport: boolean = true): Promise { + // const error: string = `JobCookSIGenerateDownloads.${errorMessage}`; + // TODO: prepend with recipe type/info/jobId. overriden by each derived class + const error = `[${this.name()}] ${errorMessage}`; + if(addToReport == true) + await this.appendToReportAndLog(error, true); + return { success: false, error: errorMessage }; + } // #endregion } diff --git a/server/workflow/impl/Packrat/WorkflowJob.ts b/server/workflow/impl/Packrat/WorkflowJob.ts index 94671b020..214fd61ab 100644 --- a/server/workflow/impl/Packrat/WorkflowJob.ts +++ b/server/workflow/impl/Packrat/WorkflowJob.ts @@ -81,6 +81,8 @@ export class WorkflowJob implements WF.IWorkflow { frequency: null // null means create but don't run }; + // create our job, but don't start it so we can hook it up to the WorkflowStep first + // this is done to ensure the Job can reference the associated WorkflowStep. const job: JOB.IJob | null = await jobEngine.create(jobCreationParameters); if (!job) { const error: string = `WorkflowJob.start unable to start job ${jobCreationParameters.eJobType @@ -127,6 +129,7 @@ export class WorkflowJob implements WF.IWorkflow { let updateWFSNeeded: boolean = false; let workflowComplete: boolean = false; + // see if the Job stopped and handle any errors const eWorkflowStepState: COMMON.eWorkflowJobRunStatus = jobRun.getStatus(); switch (eWorkflowStepState) { case COMMON.eWorkflowJobRunStatus.eDone: @@ -140,11 +143,13 @@ export class WorkflowJob implements WF.IWorkflow { break; } + // if our state is different then we update the WorkflowStep, which holds the job if (eWorkflowStepState != eWorkflowStepStateOrig) { workflowStep.setState(eWorkflowStepState); updateWFSNeeded = true; } + // update when we finished this job (if applicable) if (dateCompleted) { workflowStep.DateCompleted = dateCompleted; updateWFSNeeded = true; @@ -157,11 +162,13 @@ export class WorkflowJob implements WF.IWorkflow { let dbUpdateResult: boolean = true; + // if we need to update our WorkflowStep (DB) due to change in state, do so if (updateWFSNeeded) dbUpdateResult = await workflowStep.update() && dbUpdateResult; if (workflowComplete && this.workflowData.workflow) dbUpdateResult = await this.workflowData.workflow.update() && dbUpdateResult; + // if the workflow finished then we tell it to cleanup any mutex's created if (workflowComplete) { LOG.info(`WorkflowJob.update releasing ${this.completionMutexes.length} waiter(s) ${JSON.stringify(this.workflowJobParameters)}: ${jobRun.idJobRun} ${COMMON.eWorkflowJobRunStatus[jobRun.getStatus()]} -> ${COMMON.eWorkflowJobRunStatus[eWorkflowStepState]}`, LOG.LS.eWF); this.signalCompletion(); @@ -188,30 +195,40 @@ export class WorkflowJob implements WF.IWorkflow { } signalCompletion() { + // set our flag for completion and cycle through all pending mutex to cancel them + // BUG: in rare situations this is not called on fast/successful Cook jobs and thus goes + // until the full timeout (10hr). this.complete = true; for (const mutex of this.completionMutexes) mutex.cancel(); } async waitForCompletion(timeout: number): Promise { + if (this.complete) return this.results; + + // create a new mutex with the provided timeout value. + // 'acquire()' returns a promise that resolves when the mutex is 'released'. + // this mutex is stored in the array of mutext (i.e. actions) needed for job completion + // a mutex will lock until it's timeout is satisfied or receives E_CANCEL, which is triggered + // when the Job finishes and calls signalCompletion(). const waitMutex: MutexInterface = withTimeout(new Mutex(), timeout); this.completionMutexes.push(waitMutex); const releaseOuter = await waitMutex.acquire(); // first acquire should succeed try { const releaseInner = await waitMutex.acquire(); // second acquire should wait - releaseInner(); + releaseInner(); // releases the lock } catch (error) { - if (error === E_CANCELED) // we're done -- cancel comes from signalCompletion() + if (error === E_CANCELED) // we're done -- cancel comes from signalCompletion() return this.results; - else if (error === E_TIMEOUT) // we timed out + else if (error === E_TIMEOUT) // we timed out return { success: false, error: `WorkflowJob.waitForCompletion timed out after ${timeout}ms` }; else return { success: false, error: `WorkflowJob.waitForCompletion failure: ${JSON.stringify(error)}` }; } finally { - releaseOuter(); + releaseOuter(); // releases the lock } return this.results; } From 841b082b37bb42f1c037b5e3850f720110c61d3b Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:20:09 -0400 Subject: [PATCH 04/14] [DPO3DPKRT-752] decouple explicit storage path in workflow (#580) now using storage interface to get paths for capture data uploads --- server/workflow/impl/Packrat/WorkflowUpload.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/workflow/impl/Packrat/WorkflowUpload.ts b/server/workflow/impl/Packrat/WorkflowUpload.ts index 5ccc9ef8c..ea8b7a354 100644 --- a/server/workflow/impl/Packrat/WorkflowUpload.ts +++ b/server/workflow/impl/Packrat/WorkflowUpload.ts @@ -7,7 +7,6 @@ import * as STORE from '../../../storage/interface'; import * as REP from '../../../report/interface'; import * as LOG from '../../../utils/logger'; import * as H from '../../../utils/helpers'; -import { Config } from '../../../config'; import { ZipFile } from '../../../utils'; import { SvxReader } from '../../../utils/parser'; @@ -112,10 +111,18 @@ export class WorkflowUpload implements WF.IWorkflow { // we are not a zip fileRes = await this.validateFile(RSR.fileName, RSR.readStream, false, idSystemObject, asset); } else { - // it's not a model (e.g. Capture Data) + // grab our storage instance + const storage: STORE.IStorage | null = await STORE.StorageFactory.getInstance(); /* istanbul ignore next */ + if (!storage) { + const error: string = 'WorkflowUpload.validateFiles: Unable to retrieve Storage Implementation from StorageFactory.getInstace()'; + return this.handleError(error); + } + + // figure out our path + const filePath: string = await storage.stagingFileName(assetVersion.StorageKeyStaging); + // use ZipFile so we don't need to load it all into memory - const filePath: string = Config.storage.rootStaging+'/'+assetVersion.StorageKeyStaging; const ZS: ZipFile = new ZipFile(filePath); const zipRes: H.IOResults = await ZS.load(); if (!zipRes.success) From 59f14e892485c236c339d9541c25e25fe6a3624a Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Thu, 11 Apr 2024 12:08:46 -0400 Subject: [PATCH 05/14] Update to Voyager Schema for v0.39.0 (#581) (new) support for splash screens and auto near/far plane calculations --- server/types/voyager/document.ts | 1 + server/types/voyager/json/document.schema.json | 6 +++++- server/types/voyager/json/meta.schema.json | 4 ++++ server/types/voyager/meta.ts | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/types/voyager/document.ts b/server/types/voyager/document.ts index 5ce89adc4..a64a56ed9 100644 --- a/server/types/voyager/document.ts +++ b/server/types/voyager/document.ts @@ -101,6 +101,7 @@ export interface ICamera type: TCameraType; perspective?: IPerspectiveCameraProps; orthographic?: IOrthographicCameraProps; + autoNearFar?: boolean; } /** diff --git a/server/types/voyager/json/document.schema.json b/server/types/voyager/json/document.schema.json index 70114ba58..b1a2a1f09 100644 --- a/server/types/voyager/json/document.schema.json +++ b/server/types/voyager/json/document.schema.json @@ -188,13 +188,17 @@ "znear", "zfar" ] + }, + "autoNearFar": { + "type": "boolean", + "default": true } }, "required": [ "type" ], "not": { - "required": [ "perspective", "orthographic" ] + "required": [ "perspective", "orthographic", "autoNearFar" ] } }, diff --git a/server/types/voyager/json/meta.schema.json b/server/types/voyager/json/meta.schema.json index dedc54f56..7d3b1417a 100644 --- a/server/types/voyager/json/meta.schema.json +++ b/server/types/voyager/json/meta.schema.json @@ -78,6 +78,10 @@ "description": "Array of tags, categorizing the annotation with language key.", "type": "object" }, + "intros": { + "description": "Introductory splash screen text with language key.", + "type": "object" + }, "uri": { "description": "Location of the article resource, absolute URL or path relative to this document", "type": "string", diff --git a/server/types/voyager/meta.ts b/server/types/voyager/meta.ts index 212860837..be7844536 100644 --- a/server/types/voyager/meta.ts +++ b/server/types/voyager/meta.ts @@ -70,6 +70,7 @@ export interface IArticle leads?: Dictionary; tags?: string[]; taglist?: Dictionary; + intros?: Dictionary; mimeType?: string; thumbnailUri?: string; From 5ebfd780b3e6cd43663b0860a639033c70320e98 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Thu, 11 Apr 2024 14:53:45 -0400 Subject: [PATCH 06/14] Improved report tracking during Cook Jobs (#583) (new) appended CookJobId to WorkflowReports when job is started (new) attached Cook job report to WorkflowReport on 'cancel' or 'failure' (new) Cook job report available when job starts and updated each polling cycle --- server/job/impl/Cook/JobCook.ts | 23 +++++++++--------- server/job/impl/NS/JobPackrat.ts | 40 +++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/server/job/impl/Cook/JobCook.ts b/server/job/impl/Cook/JobCook.ts index e096e0667..cc3f2426b 100644 --- a/server/job/impl/Cook/JobCook.ts +++ b/server/job/impl/Cook/JobCook.ts @@ -406,14 +406,23 @@ export abstract class JobCook extends JobPackrat { if(cookJobReport['state']!=='waiting' && cookJobReport['state']!=='running') LOG.info(`JobCook [${this.name()}] polling [exited], state: ${cookJobReport['state']}: ${requestUrl}`, LOG.LS.eJOB); + // extract our Cook JobID + const cookJobID: string = cookJobReport['id']; + + // depending on our state we handle our state changes switch (cookJobReport['state']) { - case 'created': await this.recordCreated(); break; + case 'created': await this.recordCreated(); break; case 'waiting': await this.recordWaiting(); break; - case 'running': await this.recordStart(); break; + case 'running': await this.recordStart(cookJobID); break; case 'done': await this.recordSuccess(JSON.stringify(cookJobReport)); return { success: true, allowRetry: false, connectFailure: false, otherCookError: false }; case 'error': await this.recordFailure(JSON.stringify(cookJobReport), cookJobReport['error']); return { success: false, allowRetry: false, connectFailure: false, otherCookError: false, error: cookJobReport['error'] }; case 'cancelled': await this.recordCancel(JSON.stringify(cookJobReport), cookJobReport['error']); return { success: false, allowRetry: false, connectFailure: false, otherCookError: false, error: cookJobReport['error'] }; } + + // we always update our output so it represents the latest from Cook + // TODO: measure performance and wrap into staggered updates if needed + await this.updateJobOutput(JSON.stringify(cookJobReport)); + } catch (err) { return this.handleRequestException(err, requestUrl, 'get', undefined); } @@ -701,14 +710,4 @@ export abstract class JobCook extends JobPackrat { // return success return baseName; } - - // private getSceneFilenamesFromMap(fileMap: Map): string[] { - // const result: string[] = []; - // fileMap.forEach((value, _key) => { - // if (value.includes('svx.json')) { - // result.push(value); - // } - // }); - // return result; - // } } diff --git a/server/job/impl/NS/JobPackrat.ts b/server/job/impl/NS/JobPackrat.ts index 5eed8beaf..8f198110a 100644 --- a/server/job/impl/NS/JobPackrat.ts +++ b/server/job/impl/NS/JobPackrat.ts @@ -120,6 +120,14 @@ export abstract class JobPackrat implements JOB.IJob { return await this._report.append(content); } + protected async updateJobOutput(output: string, updateEngines: boolean=false): Promise { + //this._dbJobRun.Result = true; + this._dbJobRun.Output = output; + await this._dbJobRun.update(); + if(updateEngines) + this.updateEngines(true); // don't block + } + protected async recordCreated(): Promise { const updated: boolean = (this._dbJobRun.getStatus() == COMMON.eWorkflowJobRunStatus.eUnitialized); if (updated) { @@ -141,10 +149,10 @@ export abstract class JobPackrat implements JOB.IJob { } } - protected async recordStart(): Promise { + protected async recordStart(idJob: string): Promise { const updated: boolean = (this._dbJobRun.getStatus() != COMMON.eWorkflowJobRunStatus.eRunning); if (updated) { - this.appendToReportAndLog(`JobPackrat [${this.name()}] Starting`); + this.appendToReportAndLog(`JobPackrat [${this.name()}] Starting (CookJobId: ${idJob})`); this._dbJobRun.DateStart = new Date(); this._dbJobRun.setStatus(COMMON.eWorkflowJobRunStatus.eRunning); await this._dbJobRun.update(); @@ -159,8 +167,8 @@ export abstract class JobPackrat implements JOB.IJob { this._results = { success: true }; // do this before we await this._dbJobRun.update() this._dbJobRun.DateEnd = new Date(); this._dbJobRun.Result = true; + this._dbJobRun.Output = output ?? this._dbJobRun.Output; // if we don't have output, keep what we've got. this._dbJobRun.setStatus(COMMON.eWorkflowJobRunStatus.eDone); - this._dbJobRun.Output = output; await this._dbJobRun.update(); if (this._report) { @@ -179,11 +187,19 @@ export abstract class JobPackrat implements JOB.IJob { this.appendToReportAndLog(`JobPackrat [${this.name()}] Failure: ${errorMsg}`, true); this._results = { success: false, error: errorMsg }; // do this before we await this._dbJobRun.update() this._dbJobRun.DateEnd = new Date(); - this._dbJobRun.Result = false; + this._dbJobRun.Result = true; this._dbJobRun.setStatus(COMMON.eWorkflowJobRunStatus.eError); - this._dbJobRun.Output = output; + this._dbJobRun.Output = output ?? this._dbJobRun.Output; // if we don't have output, keep what we've got. this._dbJobRun.Error = errorMsg ?? ''; await this._dbJobRun.update(); + + // make sure we attach our report if there is one + if (this._report) { + const pathDownload: string = RouteBuilder.DownloadJobRun(this._dbJobRun.idJobRun , eHrefMode.ePrependServerURL); + const hrefDownload: string = H.Helpers.computeHref(pathDownload, 'Cook Job Output'); + await this._report.append(`${hrefDownload}
\n`); + } + this.updateEngines(true, true); // don't block } } @@ -197,18 +213,26 @@ export abstract class JobPackrat implements JOB.IJob { } else this.appendToReportAndLog(`JobPackrat [${this.name()}] Cancel`, true); + console.log(`>>> cancel: ${output}`); this._results = { success: false, error: 'Job Cancelled' + (errorMsg ? ` ${errorMsg}` : '') }; // do this before we await this._dbJobRun.update() this._dbJobRun.DateEnd = new Date(); - this._dbJobRun.Result = false; + this._dbJobRun.Result = true; this._dbJobRun.setStatus(COMMON.eWorkflowJobRunStatus.eCancelled); - this._dbJobRun.Output = output; + this._dbJobRun.Output = output ?? this._dbJobRun.Output; // if we don't have output, keep what we've got. await this._dbJobRun.update(); + + // make sure we attach our report if there is one + if (this._report) { + const pathDownload: string = RouteBuilder.DownloadJobRun(this._dbJobRun.idJobRun , eHrefMode.ePrependServerURL); + const hrefDownload: string = H.Helpers.computeHref(pathDownload, 'Cook Job Output'); + await this._report.append(`${hrefDownload}
\n`); + } + this.updateEngines(true, true); // don't block } } protected async logError(errorMessage: string, addToReport: boolean = true): Promise { - // const error: string = `JobCookSIGenerateDownloads.${errorMessage}`; // TODO: prepend with recipe type/info/jobId. overriden by each derived class const error = `[${this.name()}] ${errorMessage}`; if(addToReport == true) From 28a9a8ccb63fac624ca657500342e28efc85e274 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:11:17 -0400 Subject: [PATCH 07/14] Improved UX for selecting scene generation during Ingestion (#584) (new) dropdown for more intuitive selection of voyager scene options (new) tooltip describing conditions for auto-generation of scenes (fix) front-end always generating scenes for master models --- .../Control/SceneGenerateWorkflowControl.tsx | 2 +- .../components/Metadata/Model/index.tsx | 32 +++++++++++++++---- client/src/pages/Ingestion/hooks/useIngest.ts | 13 +++++--- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx b/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx index d512f729e..e1500cbe6 100644 --- a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx +++ b/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx @@ -23,7 +23,7 @@ function SceneGenerateWorkflowControl(props: SceneGenerateWorkflowControlProps): /> Generate Voyager Scene - To enable, Units must be set to mm, cm, m, in, ft, or yd, Purpose must be set to Master, and Model File Type must be set to obj, ply, stl, x3d, wrl, dae, or fbx + (X) To enable, Units must be set to mm, cm, m, in, ft, or yd, Purpose must be set to Master, and Model File Type must be set to obj, ply, stl, x3d, wrl, dae, or fbx ); diff --git a/client/src/pages/Ingestion/components/Metadata/Model/index.tsx b/client/src/pages/Ingestion/components/Metadata/Model/index.tsx index 80da68d23..ef383dcc8 100644 --- a/client/src/pages/Ingestion/components/Metadata/Model/index.tsx +++ b/client/src/pages/Ingestion/components/Metadata/Model/index.tsx @@ -7,7 +7,7 @@ * * This component renders the metadata fields specific to model asset. */ -import { Box, makeStyles, Typography, Table, TableBody, TableCell, TableContainer, TableRow, Paper, Select, MenuItem } from '@material-ui/core'; +import { Box, makeStyles, Typography, Table, TableBody, TableCell, TableContainer, TableRow, Paper, Select, MenuItem, Tooltip } from '@material-ui/core'; import React, { useState, useEffect } from 'react'; import { AssetIdentifiers, DateInputField, ReadOnlyRow, TextArea } from '../../../../../components'; import { StateIdentifier, StateRelatedObject, useSubjectStore, useMetadataStore, useVocabularyStore, useRepositoryStore, FieldErrors } from '../../../../../store'; @@ -23,7 +23,7 @@ import { apolloClient } from '../../../../../graphql/index'; import { useStyles as useTableStyles } from '../../../../Repository/components/DetailsView/DetailsTab/CaptureDataDetails'; import { errorFieldStyling } from '../Photogrammetry'; import SubtitleControl from '../Control/SubtitleControl'; -import SceneGenerateWorkflowControl from '../Control/SceneGenerateWorkflowControl'; +// import SceneGenerateWorkflowControl from '../Control/SceneGenerateWorkflowControl'; import { enableSceneGenerateCheck } from '../../../../../store/utils'; import clsx from 'clsx'; import lodash from 'lodash'; @@ -222,8 +222,9 @@ function Model(props: ModelProps): React.ReactElement { }; const setSceneGenerate = ({ target }): void => { - const { name, checked } = target; - updateMetadataField(metadataIndex, name, !checked, MetadataType.model); + const { name, value } = target; + console.log(`skipSceneGenerate: ${value} | eval:${(value==='false')} | model:${model.skipSceneGenerate}`); + updateMetadataField(metadataIndex, name, (value==='false'), MetadataType.model); }; const setIdField = ({ target }): void => { @@ -374,13 +375,13 @@ function Model(props: ModelProps): React.ReactElement { hasError={fieldErrors?.model.subtitles ?? false} /> - + {/* - + */} )} @@ -469,6 +470,25 @@ function Model(props: ModelProps): React.ReactElement { + + Generate Voyager Scene + + To enable, Units must be set to mm, cm, m, in, ft, or yd, Purpose must be set to Master, and Model File Type must be set to obj, ply, stl, x3d, wrl, dae, or fbx}> + + + + diff --git a/client/src/pages/Ingestion/hooks/useIngest.ts b/client/src/pages/Ingestion/hooks/useIngest.ts index f5ef3055b..1de0884db 100644 --- a/client/src/pages/Ingestion/hooks/useIngest.ts +++ b/client/src/pages/Ingestion/hooks/useIngest.ts @@ -78,6 +78,7 @@ function useIngest(): UseIngest { })); const item: StateItem = getSelectedItem() || { id: '', entireSubject: false, selected: false, subtitle: '', idProject: -1, projectName: '' }; + // console.log('item',item); const isDefaultItem = item.id === defaultItem.id; @@ -107,7 +108,7 @@ function useIngest(): UseIngest { const metadatasList = metadatas.length === 0 ? getMetadatas() : metadatas; lodash.forEach(metadatasList, metadata => { - // console.log('ingestionStart metadata', metadata); + // console.log('ingestionStart metadata',metadata); const { file, photogrammetry, model, scene, other, sceneAttachment } = metadata; const { photogrammetry: isPhotogrammetry, model: isModel, scene: isScene, attachment: isAttachment, other: isOther } = getAssetType(file.type); @@ -186,7 +187,8 @@ function useIngest(): UseIngest { sourceObjects, derivedObjects, updateNotes, - subtitles + subtitles, + skipSceneGenerate } = model; let { @@ -198,6 +200,7 @@ function useIngest(): UseIngest { } else if (typeof dateCreated === 'object') { dateCreated = nonNullValue('dateCreated', dateCreated.toISOString()); } + // console.log('model',model); const ingestIdentifiers: IngestIdentifierInput[] = getIngestIdentifiers(identifiers); const selectedSubtitle = subtitles.find(subtitle => subtitle.selected); @@ -214,8 +217,10 @@ function useIngest(): UseIngest { directory, systemCreated, sourceObjects, - derivedObjects + derivedObjects, + skipSceneGenerate }; + // console.log('modelData', modelData); const idAsset: number | undefined = idToIdAssetMap.get(file.id); if (idAsset) { @@ -306,7 +311,7 @@ function useIngest(): UseIngest { other: ingestOther, sceneAttachment: ingestSceneAttachment }; - // console.log(`** IngestDataInput`, input); + console.log('** IngestDataInput',input); const ingestDataMutation: FetchResult = await apolloClient.mutate({ mutation: IngestDataDocument, From 752df8c555132e91bc5d7f23478a51f14888ffe6 Mon Sep 17 00:00:00 2001 From: Eric Maslowski Date: Sun, 14 Apr 2024 21:52:37 -0400 Subject: [PATCH 08/14] Cleanup component for metadata scene generation flag --- .../Control/SceneGenerateWorkflowControl.tsx | 32 ------------------- .../components/Metadata/Model/index.tsx | 8 ----- 2 files changed, 40 deletions(-) delete mode 100644 client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx diff --git a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx b/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx deleted file mode 100644 index e1500cbe6..000000000 --- a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import React from 'react'; -import { Box, Checkbox, Typography } from '@material-ui/core'; - - -interface SceneGenerateWorkflowControlProps { - disabled: boolean; - selected: boolean; - setCheckboxField: ({ target }: { target: EventTarget }) => void; -} - -function SceneGenerateWorkflowControl(props: SceneGenerateWorkflowControlProps): React.ReactElement { - const { disabled, selected, setCheckboxField } = props; - return ( - - - - Generate Voyager Scene - (X) To enable, Units must be set to mm, cm, m, in, ft, or yd, Purpose must be set to Master, and Model File Type must be set to obj, ply, stl, x3d, wrl, dae, or fbx - - - ); -} - -export default SceneGenerateWorkflowControl; \ No newline at end of file diff --git a/client/src/pages/Ingestion/components/Metadata/Model/index.tsx b/client/src/pages/Ingestion/components/Metadata/Model/index.tsx index ef383dcc8..e7cc45405 100644 --- a/client/src/pages/Ingestion/components/Metadata/Model/index.tsx +++ b/client/src/pages/Ingestion/components/Metadata/Model/index.tsx @@ -23,7 +23,6 @@ import { apolloClient } from '../../../../../graphql/index'; import { useStyles as useTableStyles } from '../../../../Repository/components/DetailsView/DetailsTab/CaptureDataDetails'; import { errorFieldStyling } from '../Photogrammetry'; import SubtitleControl from '../Control/SubtitleControl'; -// import SceneGenerateWorkflowControl from '../Control/SceneGenerateWorkflowControl'; import { enableSceneGenerateCheck } from '../../../../../store/utils'; import clsx from 'clsx'; import lodash from 'lodash'; @@ -375,13 +374,6 @@ function Model(props: ModelProps): React.ReactElement { hasError={fieldErrors?.model.subtitles ?? false} /> - {/* - - */} )} From 754fe64fd08c0c52361492f840710a8bc2801518 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Wed, 7 Feb 2024 12:32:07 -0500 Subject: [PATCH 09/14] (fix) updated all environment variables to point to new EDAN endpoints (#573) dev: PACKRAT_EDAN_3D_API=https://api-internal.edan.si.edu/3d-api-dev/ prod: PACKRAT_EDAN_3D_API=https://api-internal.edan.si.edu/3d-api/ From 611c671264a04ca7d9fdc95f6c9a26477de28822 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:11:17 -0400 Subject: [PATCH 10/14] Improved UX for selecting scene generation during Ingestion (#584) (new) dropdown for more intuitive selection of voyager scene options (new) tooltip describing conditions for auto-generation of scenes (fix) front-end always generating scenes for master models --- .../Control/SceneGenerateWorkflowControl.tsx | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx diff --git a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx b/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx new file mode 100644 index 000000000..e1500cbe6 --- /dev/null +++ b/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx @@ -0,0 +1,32 @@ +import React from 'react'; +import { Box, Checkbox, Typography } from '@material-ui/core'; + + +interface SceneGenerateWorkflowControlProps { + disabled: boolean; + selected: boolean; + setCheckboxField: ({ target }: { target: EventTarget }) => void; +} + +function SceneGenerateWorkflowControl(props: SceneGenerateWorkflowControlProps): React.ReactElement { + const { disabled, selected, setCheckboxField } = props; + return ( + + + + Generate Voyager Scene + (X) To enable, Units must be set to mm, cm, m, in, ft, or yd, Purpose must be set to Master, and Model File Type must be set to obj, ply, stl, x3d, wrl, dae, or fbx + + + ); +} + +export default SceneGenerateWorkflowControl; \ No newline at end of file From de3fc0631661363890ee8a0c8192459919e4877f Mon Sep 17 00:00:00 2001 From: Eric Maslowski Date: Sun, 14 Apr 2024 21:52:37 -0400 Subject: [PATCH 11/14] Cleanup component for metadata scene generation flag --- .../Control/SceneGenerateWorkflowControl.tsx | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx diff --git a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx b/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx deleted file mode 100644 index e1500cbe6..000000000 --- a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import React from 'react'; -import { Box, Checkbox, Typography } from '@material-ui/core'; - - -interface SceneGenerateWorkflowControlProps { - disabled: boolean; - selected: boolean; - setCheckboxField: ({ target }: { target: EventTarget }) => void; -} - -function SceneGenerateWorkflowControl(props: SceneGenerateWorkflowControlProps): React.ReactElement { - const { disabled, selected, setCheckboxField } = props; - return ( - - - - Generate Voyager Scene - (X) To enable, Units must be set to mm, cm, m, in, ft, or yd, Purpose must be set to Master, and Model File Type must be set to obj, ply, stl, x3d, wrl, dae, or fbx - - - ); -} - -export default SceneGenerateWorkflowControl; \ No newline at end of file From 3a8e24e9f9affa19eef96859629f479d1cdce20f Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Tue, 7 May 2024 13:13:37 -0400 Subject: [PATCH 12/14] DPO3DPKRT-798/manual execution generate downloads (#586) (new) generate downloads button on Scene details page (no more automatic generation) (new) endpoints for initiating generate downloads + getting it's status (new) fetchBySystemObject added to DBAPI.Scene (new) Helpers.getStackTrace() to return stack trace as string (new) JobRun.fetchByJobType for getting all jobs of a type & status (new) JobRun.usesScene/Model to see if it references a scene/model (new) DB.JobRun.fetchActiveByScene grabs all active jobs by scene id (new) DB.Project.fetchFromScene to get the project associated with a specific scene (new) DB.Asset.fetchFromModel/Scene to get the asset for a specific model/scene (new) DB.Model.fetchMasterFromScene get our master model given a Scene id (new) IWorkflow.getWorkflowObject to access DB.Workflow data (new) added the master model 'units' to computeSceneInfo (new) return workflow and workflowReport if job already running (new) AsyncLocalStore.clone for extending the current LocalStore. helps to extend context (new) optional output of LocalStore usage to logs (new) Exposing AsyncResource for LocalStore use (fix) API.request now catches errors and bad 'ok' status (fix) updated error status codes to 200 so request succeeds but w/ error (client requirement) (fix) catch additional places idScene can exist in parameters (fix) verify button state when data is updated client-side (fix) preventing publishing or license change from adding/removing downloads (fix) better logging for HTTP/API requests and LocalStore creation/use (fix) reordered routes to avoid loss of context with LocalStore (fix) LocalStore was not always receiving a valid idUser (fix) removed idRequestMiddleware2. all requests now have a LocalStore (fix) maxWait/timeout in Prisma.transaction for failed concurrent ops (fix) cleanup WebDEV server initialization and assignment in Express (fix) new LocalStore forcing idRequest to 0 if nextID = false. now pass desired ID for duplicating (fix) LocalStore losing context inside WebDAV and _openWriteStream() (fix) AsyncLocalStore is undefined during Jest testing. --- client/src/api/index.ts | 36 +++- .../components/DetailsView/index.tsx | 113 +++++++++- server/collections/impl/PublishScene.ts | 15 +- server/db/api/Asset.ts | 25 +++ server/db/api/JobRun.ts | 86 ++++++++ server/db/api/Model.ts | 18 ++ server/db/api/Project.ts | 15 ++ server/db/api/Scene.ts | 21 ++ server/db/api/SystemObjectVersion.ts | 10 +- server/event/interface/EventEnums.ts | 2 + server/http/index.ts | 126 ++++++++---- server/http/routes/WebDAVServer.ts | 24 ++- server/http/routes/api/generateDownloads.ts | 160 +++++++++++++++ .../impl/Cook/JobCookSIGenerateDownloads.ts | 1 - server/report/impl/Report.ts | 11 +- server/tests/cache/VocabularyCache.test.ts | 1 + server/utils/helpers.ts | 8 + server/utils/localStore.ts | 63 +++++- server/utils/logger.ts | 4 +- .../workflow/impl/Packrat/WorkflowEngine.ts | 193 +++++++++++++++++- .../impl/Packrat/WorkflowIngestion.ts | 13 ++ server/workflow/impl/Packrat/WorkflowJob.ts | 12 ++ .../workflow/impl/Packrat/WorkflowUpload.ts | 12 ++ server/workflow/interface/IWorkflow.ts | 1 + server/workflow/interface/IWorkflowEngine.ts | 8 + 25 files changed, 902 insertions(+), 76 deletions(-) create mode 100644 server/http/routes/api/generateDownloads.ts diff --git a/client/src/api/index.ts b/client/src/api/index.ts index 231011f14..5ba288c8a 100644 --- a/client/src/api/index.ts +++ b/client/src/api/index.ts @@ -7,7 +7,8 @@ */ enum API_ROUTES { LOGIN = 'auth/login', - LOGOUT = 'auth/logout' + LOGOUT = 'auth/logout', + GEN_DOWNLOADS = 'api/scene/gen-downloads' } export type AuthResponseType = { @@ -16,6 +17,13 @@ export type AuthResponseType = { originalUrl?: string; }; +export type RequestResponse = { + success: boolean; + message?: string; + originalUrl?: string; + data?: any; +}; + export default class API { static async login(email: string, password: string): Promise { const body = JSON.stringify({ email, password }); @@ -31,6 +39,20 @@ export default class API { return this.request(API_ROUTES.LOGOUT); } + static async generateDownloads(idSystemObject: number, statusOnly: boolean = false): Promise { + // initiates the generate downloads routine and either runs the recipe with the given id or returns its status. + // idSystemObject = the SystemObject id for the Packrat Scene making this request + const body = JSON.stringify({ idSystemObject }); + + let options; + if(statusOnly) + options = { method: 'GET' }; + else + options = { method: 'POST', body }; + + return this.request(`${API_ROUTES.GEN_DOWNLOADS}?id=${idSystemObject}`, options); + } + static async request(route: string, options: RequestInit = {}): Promise { const serverEndpoint = API.serverEndpoint(); const defaultOptions: RequestInit = { @@ -41,7 +63,17 @@ export default class API { ...options }; - return fetch(`${serverEndpoint}/${route}`, defaultOptions).then(response => response.json()); + // TODO: return an error response + return fetch(`${serverEndpoint}/${route}`, defaultOptions) + .then(response => { + // Check if the response returned a successful status code + if (!response.ok) + return { success: false, message: response.statusText }; + return response.json(); // Assuming the server responds with JSON + }) + .catch(error => { + console.error(`[Packrat] could not complete request (${route}) due to error: ${error}`); + }); } static serverEndpoint(): string { diff --git a/client/src/pages/Repository/components/DetailsView/index.tsx b/client/src/pages/Repository/components/DetailsView/index.tsx index 604b30e67..639b226ec 100644 --- a/client/src/pages/Repository/components/DetailsView/index.tsx +++ b/client/src/pages/Repository/components/DetailsView/index.tsx @@ -6,6 +6,8 @@ * * This component renders repository details view for the Repository UI. */ +import API, { RequestResponse } from '../../../../api'; + import { Box } from '@material-ui/core'; import { makeStyles } from '@material-ui/core/styles'; import React, { useEffect, useState } from 'react'; @@ -101,6 +103,9 @@ function DetailsView(): React.ReactElement { // eslint-disable-next-line @typescript-eslint/no-explicit-any const [detailQuery, setDetailQuery] = useState({}); const [isUpdatingData, setIsUpdatingData] = useState(false); + const [isGeneratingDownloads, setIsGeneratingDownloads] = useState(false); + const [canGenerateDownloads, setCanGenerateDownloads] = useState(true); + const [objectRelationship, setObjectRelationship] = useState(RelatedObjectType.Source); const [loadingIdentifiers, setLoadingIdentifiers] = useState(true); const idSystemObject: number = Number.parseInt(params.idSystemObject as string, 10); @@ -167,6 +172,30 @@ function DetailsView(): React.ReactElement { }; + const verifyGenerateDownloads = async (): Promise => { + // check whether we can actually generate downloads + // TODO: check QC checkbox status, (ideally) if a generate downloads is already running, ... + console.log('Verifying Generate Downloads...'); + + // make a call to our generate downloads endpoint with the current scene id + const response: RequestResponse = await API.generateDownloads(idSystemObject, true); + console.log(response); + if(response.success === false) { + console.log(`[Packrat - ERROR] cannot verify if generate downloads is available. (${response.message})`); + setCanGenerateDownloads(false); + return false; + } + + // see if we can actually run based on if a job isn't already running + // and our scene meets the core requirements + const canRun: boolean = (response.data.isJobRunning === false) && (response.data.isSceneValid === true); + + // we have success so enable it + console.log(`[PACKRAT - DEBUG] can generate downloads: ${canRun}`); + setCanGenerateDownloads(canRun); + return canRun; + }; + useEffect(() => { if (data) { const handleDetailTab = async () => { @@ -211,6 +240,10 @@ function DetailsView(): React.ReactElement { return () => onUploaderReset(); }, []); + useEffect(() => { + verifyGenerateDownloads(); + }, []); + if (!data || !params.idSystemObject) { return ; } @@ -524,7 +557,66 @@ function DetailsView(): React.ReactElement { return false; } finally { setIsUpdatingData(false); + + // check our generate downloads button + await verifyGenerateDownloads(); + } + }; + + const generateDownloads = async (): Promise => { + // fire off download generation for the scene. (UI element, ln. 593) + // TODO: move into 'Assets' tab (currently lacking context/details on if we're a scene) + console.log('Generating Downloads...'); + if(isGeneratingDownloads === true || canGenerateDownloads === false) { + console.error('[Packrat] cannot generate downloads. not sure how you got here...'); + return false; } + setIsGeneratingDownloads(true); + setCanGenerateDownloads(false); + + // get our current time so we can make sure we exeecute for a certain amount of time + const startTime: number = Date.now(); + + // make a call to our generate downloads endpoint with the current scene id + // return sucess when the job is started or if one is already running + const response: RequestResponse = await API.generateDownloads(idSystemObject); + if(response.success === false) { + + // if the job is running then handle differently + if(response.message && response.message.includes('already running')) { + console.log(`[Packrat - WARN] cannot generate downloads. (${response.message})`); + toast.warn('Not generating downloads. Job already running. Please wait for it to finish.'); + } else { + console.log(`[Packrat - ERROR] cannot generate downloads. (${response.message})`); + toast.error('Cannot generate downloads. Check the report.'); + } + + // update our button state + setCanGenerateDownloads(true); + + // set to false to stop 'loading' animation on button. doesn't (currently) represent state of job on server + setIsGeneratingDownloads(false); + return false; + } + + console.log(params); + console.log(data); + + // wait for a period of time before resetting our buttons + // setting a minimal time improves UX and shows spinning logo + const diffTime = Math.max(1,2000-(Date.now()-startTime)); + setTimeout(() => { + // TODO: keep polling to set our button state back + // if our message is already running then we notify the user + toast.success('Generating Downloads started. This may take awhile...'); + + // cleanup our button state + setCanGenerateDownloads(true); + setIsGeneratingDownloads(false); + }, diffTime); + + //console.log(`waiting ${diffTime/1000}s before cleaning up button`); + return true; }; const immutableNameTypes = new Set([eSystemObjectType.eItem, eSystemObjectType.eModel, eSystemObjectType.eScene]); @@ -583,8 +675,25 @@ function DetailsView(): React.ReactElement { Update - {(updatedIdentifiers || updatedMetadata) &&
Update needed to save your data
} - {(objectType === eSystemObjectType.eScene || objectType === eSystemObjectType.eModel) && document.getElementById('Voyager-Explorer')?.scrollIntoView()} style={{ marginLeft: 5 }}>View} + {(updatedIdentifiers || updatedMetadata) && +
Update needed to save your data
} + + {(objectType === eSystemObjectType.eScene || objectType === eSystemObjectType.eModel) && + document.getElementById('Voyager-Explorer')?.scrollIntoView()} + style={{ marginLeft: 5 }} + >View} + + {(objectType === eSystemObjectType.eScene) && + Generate Downloads} diff --git a/server/collections/impl/PublishScene.ts b/server/collections/impl/PublishScene.ts index c770fb1c7..d4cc1d152 100644 --- a/server/collections/impl/PublishScene.ts +++ b/server/collections/impl/PublishScene.ts @@ -301,7 +301,7 @@ export class PublishScene { return DownloadMSXMap; } - static async handleSceneUpdates(idScene: number, idSystemObject: number, _idUser: number | undefined, + static async handleSceneUpdates(idScene: number, _idSystemObject: number, _idUser: number | undefined, oldPosedAndQCd: boolean, newPosedAndQCd: boolean, LicenseOld: DBAPI.License | undefined, LicenseNew: DBAPI.License | undefined): Promise { // if we've changed Posed and QC'd, and/or we've updated our license, create or remove downloads @@ -319,8 +319,9 @@ export class PublishScene { return PublishScene.sendResult(false, `Unable to fetch workflow engine for download generation for scene ${idScene}`); // trigger the workflow/recipe - workflowEngine.generateSceneDownloads(idScene, { idUserInitiator: _idUser }); // don't await - return { success: true, downloadsGenerated: true, downloadsRemoved: false }; + // HACK: disable automatic download generation for the moment + // workflowEngine.generateSceneDownloads(idScene, { idUserInitiator: _idUser }); // don't await + return { success: true, downloadsGenerated: false, downloadsRemoved: false }; } else { // Remove downloads LOG.info(`PublishScene.handleSceneUpdates removing downloads for scene ${idScene}`, LOG.LS.eGQL); // Compute downloads @@ -338,9 +339,11 @@ export class PublishScene { assetVersionOverrideMap.set(asset.idAsset, 0); } - const SOV: DBAPI.SystemObjectVersion | null = await DBAPI.SystemObjectVersion.cloneObjectAndXrefs(idSystemObject, null, 'Removing Downloads', assetVersionOverrideMap); - if (!SOV) - return PublishScene.sendResult(false, `Unable to clone system object version for idSystemObject ${idSystemObject} for scene ${idScene}`); + // HACK: preventing modifying downloads in the Scene. + // Publishing should not add/remove assets but select what is included when sent to EDAN + // const SOV: DBAPI.SystemObjectVersion | null = await DBAPI.SystemObjectVersion.cloneObjectAndXrefs(idSystemObject, null, 'Removing Downloads', assetVersionOverrideMap); + // if (!SOV) + // return PublishScene.sendResult(false, `Unable to clone system object version for idSystemObject ${idSystemObject} for scene ${idScene}`); return { success: true, downloadsGenerated: false, downloadsRemoved: true }; } } diff --git a/server/db/api/Asset.ts b/server/db/api/Asset.ts index 691f7c861..8436504de 100644 --- a/server/db/api/Asset.ts +++ b/server/db/api/Asset.ts @@ -160,6 +160,31 @@ export class Asset extends DBC.DBObject implements AssetBase, SystemO `,Asset); } + static async fetchFromModel(idModel: number): Promise { + // get any assets associated with the given Model. Can include meshes, textures, and materials + return DBC.CopyArray( + await DBC.DBConnection.prisma.$queryRaw` + SELECT * FROM Model AS mdl + JOIN SystemObject AS mdlSO ON mdl.idModel = mdlSO.idModel + JOIN Asset AS a ON a.idSystemObject = mdlSO.idSystemObject + WHERE mdl.idModel = ${idModel}; + `,Asset); + } + + static async fetchVoyagerSceneFromScene(idScene: number): Promise { + // get the voyager scene asset associated with the Packrat scene (if any) + // TODO: get asset type id from VocabularyID + const idvAssetType: number = 137; + + return DBC.CopyArray( + await DBC.DBConnection.prisma.$queryRaw` + SELECT a.* FROM Scene AS scn + JOIN SystemObject AS scnSO ON scn.idScene = scnSO.idScene + JOIN Asset AS a ON a.idSystemObject = scnSO.idSystemObject + WHERE scn.idScene = ${idScene} AND a.idVAssetType = ${idvAssetType}; + `,Asset); + } + /** Fetches assets that are connected to the specified idSystemObject (via that object's last SystemObjectVersion, * and that SystemObjectVersionAssetVersionXref's records). For those assets, we look for a match on FileName, idVAssetType */ static async fetchMatching(idSystemObject: number, FileName: string, idVAssetType: number): Promise { diff --git a/server/db/api/JobRun.ts b/server/db/api/JobRun.ts index f6347ca9e..855a72812 100644 --- a/server/db/api/JobRun.ts +++ b/server/db/api/JobRun.ts @@ -43,6 +43,49 @@ export class JobRun extends DBC.DBObject implements JobRunBase { getStatus(): COMMON.eWorkflowJobRunStatus { return convertWorkflowJobRunStatusToEnum(this.Status); } setStatus(eStatus: COMMON.eWorkflowJobRunStatus): void { this.Status = eStatus; } + usesScene(idScene: number): boolean { + // checks to see if this job was associated with a specific Packrat Scene + if(!this.Parameters || this.Parameters.length===0) + return false; + + // grab our parameters, parse them and see if we have the scene + try { + const json = JSON.parse(this.Parameters); + + if(json.idScene) + return (json.idScene == idScene); + + if(!json.model && !json.model.idScene) + return (json.model.idScene == idScene); + + return false; + } catch(error) { + LOG.error(`JobRun.usesScene failed. (${error})`,LOG.LS.eDB); + return false; + } + } + usesModel(idModel: number): boolean { + // checks to see if this job was associated with a specific model + if(!this.Parameters || this.Parameters.length===0) + return false; + + // grab our parameters, parse them and see if we have the model + try { + const json = JSON.parse(this.Parameters); + + if(json.idModel) + return (json.idModel == idModel); + + if(!json.model && !json.model.idModel) + return (json.model.idModel == idModel); + + return false; + } catch(error) { + LOG.error(`JobRun.usesModel failed. (${error})`,LOG.LS.eDB); + return false; + } + } + protected async createWorker(): Promise { try { const { idJob, Status, Result, DateStart, DateEnd, Configuration, Parameters, Output, Error } = this; @@ -180,6 +223,49 @@ export class JobRun extends DBC.DBObject implements JobRunBase { } } + static async fetchByJobFiltered(idJob: number, active: boolean = true): Promise { + // get the JobRuns that are of a specific type and (optionally) those active/inactive + // TODO: pass in array of status codes want to consider. can't find accepted way to include array values to queryRaw + try { + if(active===true) { + // return those uninitialized(0), created(1), running(2), waiting(3) + return DBC.CopyArray ( + await DBC.DBConnection.prisma.$queryRaw` + SELECT * FROM JobRun AS jr + JOIN WorkflowStep AS wfStep ON jr.idJobRun = wfStep.idJobRun + WHERE (jr.idJob = ${idJob}) AND (jr.Status IN (${0},${1},${2},${3}));`,JobRun); + } else { + // return those done(4), error(5), or cancelled(6) + return DBC.CopyArray ( + await DBC.DBConnection.prisma.$queryRaw` + SELECT * FROM JobRun AS jr + JOIN WorkflowStep AS wfStep ON jr.idJobRun = wfStep.idJobRun + WHERE (jr.idJob = ${idJob}) AND (jr.Status IN (${4},${5},${6}));`,JobRun); + } + } catch (error) { + LOG.error('DBAPI.JobRun.fetchFromWorkflow', LOG.LS.eDB, error); + return null; + } + } + + static async fetchActiveByScene(idJob: number, idScene: number): Promise { + const jobs: JobRun[] = []; + const activeJobs: JobRun[] | null = await JobRun.fetchByJobFiltered(idJob,true); + if(!activeJobs) { + LOG.error('DBAPI.JobRun.fetchActiveByScene failed to get filtered jobs',LOG.LS.eDB); + return null; + } + + // see if it's ours and if so, store the job's id + for(let i=0; i< activeJobs.length; i++){ + if(activeJobs[i].usesScene(idScene)) { + jobs.push(activeJobs[i]); + } + } + + return jobs; + } + static async fetchFromWorkflow(idWorkflow: number): Promise { // direct get of JubRun(s) from a specific workflow try { diff --git a/server/db/api/Model.ts b/server/db/api/Model.ts index 5eac2e9ab..973ca945a 100644 --- a/server/db/api/Model.ts +++ b/server/db/api/Model.ts @@ -161,6 +161,7 @@ export class Model extends DBC.DBObject implements ModelBase, SystemO } static async fetchFromXref(idScene: number): Promise { + // get's the assets using the ModelSceneXref table, which includes derivative models if (!idScene) return null; try { @@ -172,6 +173,23 @@ export class Model extends DBC.DBObject implements ModelBase, SystemO } } + static async fetchMasterFromScene(idScene: number): Promise { + // get the master Model associated with a given Scene + // TODO: get 'Master' model type id from VocabularyID + const idvMasterModelType: number = 45; + + return DBC.CopyArray( + await DBC.DBConnection.prisma.$queryRaw` + SELECT mdl.* FROM Scene AS scn + JOIN SystemObject AS scnSO ON scn.idScene = scnSO.idScene + JOIN SystemObjectXref AS scnSOX ON scnSO.idSystemObject = scnSOX.idSystemObjectDerived + JOIN SystemObject AS masterSO ON (scnSOX.idSystemObjectMaster = masterSO.idSystemObject AND masterSO.idModel IS NOT NULL) + JOIN Model AS mdl ON (masterSO.idModel = mdl.idModel AND mdl.idVPurpose = ${idvMasterModelType}) + JOIN SystemObject AS mdlSO ON mdl.idModel = mdlSO.idModel + WHERE scn.idScene = ${idScene}; + `,Model); + } + /** * Computes the array of Models that are connected to any of the specified items. * Models are connected to system objects; we examine those system objects which are in a *derived* relationship diff --git a/server/db/api/Project.ts b/server/db/api/Project.ts index 5dd643beb..2e90ebe33 100644 --- a/server/db/api/Project.ts +++ b/server/db/api/Project.ts @@ -161,4 +161,19 @@ export class Project extends DBC.DBObject implements ProjectBase, S return null; } } + + static async fetchFromScene(idScene: number): Promise { + // return the project(s) associated with a particular scene + return DBC.CopyArray( + await DBC.DBConnection.prisma.$queryRaw` + SELECT proj.* FROM Scene AS scn + JOIN SystemObject AS scnSO ON scn.idScene = scnSO.idScene + JOIN SystemObjectXref AS scnSOX ON scnSO.idSystemObject = scnSOX.idSystemObjectDerived + JOIN SystemObject AS itemSO ON scnSOX.idSystemObjectMaster = itemSO.idSystemObject AND itemSO.idItem IS NOT NULL + JOIN SystemObjectXref AS itemSOX ON itemSO.idSystemObject = itemSOX.idSystemObjectDerived + JOIN SystemObject AS projSO ON itemSOX.idSystemObjectMaster = projSO.idSystemObject AND projSO.idProject IS NOT NULL + JOIN Project AS proj ON projSO.idProject = proj.idProject + WHERE scn.idScene = ${idScene}; + `,Project); + } } diff --git a/server/db/api/Scene.ts b/server/db/api/Scene.ts index 534c8e32d..a2f8ce9cc 100644 --- a/server/db/api/Scene.ts +++ b/server/db/api/Scene.ts @@ -149,6 +149,27 @@ export class Scene extends DBC.DBObject implements SceneBase, SystemO } } + static async fetchBySystemObject(idSystemObject: number): Promise { + if (!idSystemObject) + return null; + try { + const scenes: Scene[] | null = DBC.CopyArray( + await DBC.DBConnection.prisma.$queryRaw` + SELECT * FROM Scene AS scn + JOIN SystemObject AS so ON (scn.idScene = so.idScene) + WHERE so.idSystemObject = ${idSystemObject};`, Scene); + + // if we have scenes just return the first one, else null + if(scenes) + return scenes[0]; + + return null; + } catch (error) /* istanbul ignore next */ { + LOG.error('DBAPI.Scene.fetchBySystemObject', LOG.LS.eDB, error); + return null; + } + } + /** * Computes the array of Scenes that are connected to any of the specified items. * Scenes are connected to system objects; we examine those system objects which are in a *derived* relationship diff --git a/server/db/api/SystemObjectVersion.ts b/server/db/api/SystemObjectVersion.ts index fd9524025..b825108ca 100644 --- a/server/db/api/SystemObjectVersion.ts +++ b/server/db/api/SystemObjectVersion.ts @@ -143,6 +143,14 @@ export class SystemObjectVersion extends DBC.DBObject i if (!DBC.DBConnection.isFullPrismaClient(prismaClient)) return SystemObjectVersion.cloneObjectAndXrefsTrans(idSystemObject, idSystemObjectVersion, Comment, assetVersionOverrideMap, assetsUnzipped); + // set our options for all transactions. these timeout values help with situations + // where concurrent requests to the DB take longer than expected. (in ms) + const transactionOptions = { + maxWait: 10000, // how long Prisma will wait to get a connection (default: 2000) + timeout: 20000 // how long an interactive transaction can take (default: 5000) + // isolationLevel: Prisma.TransactionIsolationLevel.Serializable, // how much concurrency is allowed 'serializable' is most strict (default: RepeatableRead) + }; + // otherwise, start a new transaction: // LOG.info('DBAPI.SystemObjectVersion.cloneObjectAndXrefs starting a new DB transaction', LOG.LS.eDB); return await prismaClient.$transaction(async (prisma) => { @@ -150,7 +158,7 @@ export class SystemObjectVersion extends DBC.DBObject i const retValue: SystemObjectVersion | null = await SystemObjectVersion.cloneObjectAndXrefsTrans(idSystemObject, idSystemObjectVersion, Comment, assetVersionOverrideMap, assetsUnzipped); DBC.DBConnection.clearPrismaTransaction(transactionNumber); return retValue; - }); + },transactionOptions); } catch (error) /* istanbul ignore next */ { LOG.error('DBAPI.SystemObjectVersion.cloneObjectAndXrefs', LOG.LS.eDB, error); return null; diff --git a/server/event/interface/EventEnums.ts b/server/event/interface/EventEnums.ts index 367659547..4dbdfeae1 100644 --- a/server/event/interface/EventEnums.ts +++ b/server/event/interface/EventEnums.ts @@ -3,6 +3,7 @@ export enum eEventTopic { eDB = 2, ePublish = 3, eHTTP = 4, + eScene = 5 } export enum eEventKey { @@ -14,4 +15,5 @@ export enum eEventKey { eHTTPDownload = 6, eHTTPUpload = 7, eAuthFailed = 8, + eGenDownloads = 9 } diff --git a/server/http/index.ts b/server/http/index.ts index c5726903c..534e18815 100644 --- a/server/http/index.ts +++ b/server/http/index.ts @@ -4,6 +4,7 @@ import { EventFactory } from '../event/interface/EventFactory'; import { ASL, LocalStore } from '../utils/localStore'; import { Config } from '../config'; import * as LOG from '../utils/logger'; +import * as H from '../utils/helpers'; import { UsageMonitor } from '../utils/osStats'; import { logtest } from './routes/logtest'; @@ -14,6 +15,7 @@ import { Downloader, download } from './routes/download'; import { errorhandler } from './routes/errorhandler'; import { WebDAVServer } from './routes/WebDAVServer'; import { getCookResource } from './routes/resources'; +import { generateDownloads } from './routes/api/generateDownloads'; import express, { Request, Express, RequestHandler } from 'express'; import cors from 'cors'; @@ -34,6 +36,7 @@ const monitorVerboseSamples: number = 300; */ export class HttpServer { public app: Express = express(); + private WDSV: WebDAVServer | null = null; private static _singleton: HttpServer | null = null; static async getInstance(): Promise { @@ -47,6 +50,17 @@ export class HttpServer { private async initializeServer(): Promise { LOG.info('**************************', LOG.LS.eSYS); LOG.info('Packrat Server Initialized', LOG.LS.eSYS); + + // get our webDAV server + this.WDSV = await WebDAVServer.server(); + if (this.WDSV) { + LOG.info('initialized WebDAV Server', LOG.LS.eSYS); + } else { + LOG.error('Failed to initialize WebDAV server', LOG.LS.eSYS); + return false; + } + + // configure our routes/endpoints const res: boolean = await this.configureMiddlewareAndRoutes(); // call to initalize the EventFactory, which in turn will initialize the AuditEventGenerator, supplying the IEventEngine @@ -60,85 +74,125 @@ export class HttpServer { static bodyProcessorExclusions: RegExp = /^\/(?!webdav).*$/; private async configureMiddlewareAndRoutes(): Promise { - this.app.use(HttpServer.idRequestMiddleware); - this.app.use(cors(authCorsConfig)); + // First step is to modify the request body as needed. We do this first + // because the express.json() 3rd party library breaks any context created + // by the AsyncLocalStore as it waits for the request/body to arrive. this.app.use(HttpServer.bodyProcessorExclusions, express.json() as RequestHandler); // do not extract webdav PUT bodies into request.body element this.app.use(HttpServer.bodyProcessorExclusions, express.urlencoded({ extended: true }) as RequestHandler); + + // get our cookie and auth system rolling. We do this here so we can extract + // our user information (if present) and have it for creating the LocalStore. + this.app.use(cors(authCorsConfig)); this.app.use(cookieParser()); this.app.use(authSession); this.app.use(passport.initialize()); this.app.use(passport.session()); - this.app.use('/auth', HttpServer.idRequestMiddleware2); + // create our LocalStore for all future interactions + this.app.use(HttpServer.logRequest); + this.app.use(HttpServer.assignLocalStore); + + // if we have a WebDAV server (always), attach it to express + if(this.WDSV) + this.app.use(webdav.extensions.express(WebDAVServer.httpRoute, this.WDSV.webdav())); + + // authentication and graphQL endpoints this.app.use('/auth', AuthRouter); - this.app.use('/graphql', HttpServer.idRequestMiddleware2); - this.app.use('/graphql', HttpServer.graphqlLoggingMiddleware); this.app.use('/graphql', graphqlUploadExpress()); + // start our ApolloServer const server = new ApolloServer(ApolloServerOptions); await server.start(); server.applyMiddleware({ app: this.app, cors: false }); + // utility endpoints this.app.get('/logtest', logtest); this.app.get('/heartbeat', heartbeat); this.app.get('/solrindex', solrindex); this.app.get('/solrindexprofiled', solrindexprofiled); this.app.get('/migrate', migrate); this.app.get('/migrate/*', migrate); - this.app.get(`${Downloader.httpRoute}*`, HttpServer.idRequestMiddleware2); this.app.get(`${Downloader.httpRoute}*`, download); + // Packrat API endpoints (WIP) this.app.get('/resources/cook', getCookResource); + this.app.get('/api/scene/gen-downloads', generateDownloads); + this.app.post('/api/scene/gen-downloads', generateDownloads); - const WDSV: WebDAVServer | null = await WebDAVServer.server(); - if (WDSV) { - this.app.use(WebDAVServer.httpRoute, HttpServer.idRequestMiddleware2); - this.app.use(webdav.extensions.express(WebDAVServer.httpRoute, WDSV.webdav())); - } else - LOG.error('HttpServer.configureMiddlewareAndRoutes failed to initialize WebDAV server', LOG.LS.eHTTP); - + // if we're here then we handle any errors that may have surfaced this.app.use(errorhandler); // keep last + // if we're not testing then open up server on the correct port if (process.env.NODE_ENV !== 'test') { this.app.listen(Config.http.port, () => { LOG.info(`Server is running on port ${Config.http.port}`, LOG.LS.eSYS); }); } + + // only gets here if no other route is satisfied return true; } // creates a LocalStore populated with the next requestID - private static idRequestMiddleware(req: Request, _res, next): void { - if (!req.originalUrl.startsWith('/auth/') && - !req.originalUrl.startsWith('/graphql') && - !req.originalUrl.startsWith(Downloader.httpRoute) && - !req.originalUrl.startsWith(WebDAVServer.httpRoute)) { - const user = req['user']; - const idUser = user ? user['idUser'] : undefined; - ASL.run(new LocalStore(true, idUser), () => { - LOG.info(req.originalUrl, LOG.LS.eHTTP); - next(); - }); - } else - next(); - } - - private static idRequestMiddleware2(req: Request, _res, next): void { + private static assignLocalStore(req: Request, _res, next): void { const user = req['user']; const idUser = user ? user['idUser'] : undefined; ASL.run(new LocalStore(true, idUser), () => { - if (!req.originalUrl.startsWith('/graphql')) - LOG.info(`HTTP request for: ${req.originalUrl}`, LOG.LS.eHTTP); + LOG.info(`HTTP.idRequestMiddleware creating new LocalStore (url: ${req.originalUrl} | idUser: ${idUser})`,LOG.LS.eHTTP); next(); }); } - private static graphqlLoggingMiddleware(req: Request, _res, next): void { - const query: string | null = computeGQLQuery(req); - if (query && query !== '__schema') // silence __schema logging, issued by GraphQL playground - LOG.info(`${query} ${JSON.stringify(req.body.variables)}`, LOG.LS.eGQL); - return next(); + // utility routines and middleware + private static logRequest(req: Request, _res, next): void { + // TODO: more detailed information about the request + // figure out who is calling this + const user = req['user']; + const idUser = user ? user['idUser'] : undefined; + + // our method (GET, POST, ...) + let method = req.method.toUpperCase(); + + // get our query + let query = req.originalUrl; + let queryParams = H.Helpers.JSONStringify(req.query); + if(req.originalUrl.includes('/graphql')) { + method = 'GQL'; + const queryGQL = computeGQLQuery(req); + if(queryGQL && queryGQL !== '__schema') { + query = queryGQL; + queryParams = H.Helpers.JSONStringify(req.body.variables); + } else + query = `Unknown GraphQL: ${query}|${req.path}`; + } + LOG.info(`New ${method} request [${query}] made by user ${idUser}. (${queryParams})`,LOG.LS.eHTTP); + next(); } + // private static checkLocalStore(label: string) { + // return function (_req, _res, next) { + // //LOG.info(`HTTP.checkLocalStore [${label}]. (url: ${req.originalUrl} | ${H.Helpers.JSONStringify(ASL.getStore())})`,LOG.LS.eDEBUG); + // ASL.checkLocalStore(label); + // next(); + // }; + // } + // private static printRequest(req: Request, label: string = 'Request'): void { + // if(!req) { + // console.log('nothing'); + // } + // LOG.info(`${label}: ${H.Helpers.JSONStringify({ + // // headers: req.headers, + // // body: req.body, + // query: req.query, + // params: req.params, + // url: req.url, + // method: req.method, + // ip: req.ip, + // path: req.path, + // sid: req.cookies?.connect?.sid ?? undefined, + // // cookies: req.cookies, + // user: req.user + // })}`,LOG.LS.eDEBUG); + // } } process.on('uncaughtException', (err) => { diff --git a/server/http/routes/WebDAVServer.ts b/server/http/routes/WebDAVServer.ts index 57ea8c76b..1bbbfde27 100644 --- a/server/http/routes/WebDAVServer.ts +++ b/server/http/routes/WebDAVServer.ts @@ -4,11 +4,10 @@ import * as STORE from '../../storage/interface'; import * as DBAPI from '../../db'; import * as CACHE from '../../cache'; import * as COMMON from '@dpo-packrat/common'; -import * as H from '../../utils/helpers'; import { BufferStream } from '../../utils/bufferStream'; import { AuditFactory } from '../../audit/interface/AuditFactory'; import { eEventKey } from '../../event/interface/EventEnums'; -import { ASL, LocalStore } from '../../utils/localStore'; +import { ASL, ASR, LocalStore } from '../../utils/localStore'; import { isAuthenticated } from '../auth'; import { DownloaderParser, DownloaderParserResults } from './DownloaderParser'; @@ -417,6 +416,7 @@ class WebDAVFileSystem extends webdav.FileSystem { async _openWriteStream(pathWD: webdav.Path, _info: webdav.OpenWriteStreamInfo, callback: webdav.ReturnCallback, callbackComplete: webdav.SimpleCallback): Promise { + try { /* const lockUUID: string | undefined = await this.setLock(pathWD, _info.context, callback); @@ -470,8 +470,7 @@ class WebDAVFileSystem extends webdav.FileSystem { // await this.removeLock(pathWD, info.context, lockUUID); return; } - - LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}), FileName ${FileName}, FilePath ${FilePath}, asset type ${COMMON.eVocabularyID[eVocab]}, SOBased ${JSON.stringify(SOBased, H.Helpers.saferStringify)}`, LOG.LS.eHTTP); + // LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}), FileName ${FileName}, FilePath ${FilePath}, asset type ${COMMON.eVocabularyID[eVocab]}, SOBased ${JSON.stringify(SOBased, H.Helpers.saferStringify)}`, LOG.LS.eHTTP); const LS: LocalStore = await ASL.getOrCreateStore(); const idUserCreator: number = LS?.idUser ?? 0; @@ -484,9 +483,12 @@ class WebDAVFileSystem extends webdav.FileSystem { // BS.on('unpipe', async () => { LOG.info(`WebDAVFileSystem._openWriteStream: (W) onUnPipe for ${asset ? JSON.stringify(asset, H.Helpers.saferStringify) : 'new asset'}`, LOG.LS.eHTTP); }); // BS.on('drain', async () => { LOG.info(`WebDAVFileSystem._openWriteStream: (W) onDrain for ${asset ? JSON.stringify(asset, H.Helpers.saferStringify) : 'new asset'}`, LOG.LS.eHTTP); }); // BS.on('close', async () => { LOG.info(`WebDAVFileSystem._openWriteStream: (W) onClose for ${asset ? JSON.stringify(asset, H.Helpers.saferStringify) : 'new asset'}`, LOG.LS.eHTTP); }); - BS.on('finish', async () => { + + // we wrap our function in AsyncResource to preserve the context since the callback may be called outside + // the original context and thus breaking LocalStore. + BS.on('finish', ASR.bind(async () => { try { - LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}): (W) onFinish for ${asset ? JSON.stringify(asset, H.Helpers.saferStringify) : 'new asset'}`, LOG.LS.eHTTP); + // LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}): (W) onFinish for ${asset ? JSON.stringify(asset, H.Helpers.saferStringify) : 'new asset'}`, LOG.LS.eDEBUG); const ISI: STORE.IngestStreamOrFileInput = { readStream: BS, localFilePath: null, @@ -503,17 +505,16 @@ class WebDAVFileSystem extends webdav.FileSystem { // Serialize access per DP.idSystemObjectV via Semaphore, allowing only 1 ingestion at a time per system object const writeLock: SemaphoreInterface | undefined = await this.computeWriteLock(DP.idSystemObjectV, pathS); - if (writeLock) { for (let lockAttempt = 1; lockAttempt <= 5; lockAttempt++) { try { await writeLock.runExclusive(async (_value) => { try { - LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}): (W) onFinish ingestStream START`, LOG.LS.eHTTP); + // LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}): (W) onFinish ingestStream START`, LOG.LS.DEBUG); return await this.ingestStream(ISI, pathS); } finally { await this.releaseWriteLock(DP.idSystemObjectV, pathS); - LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}): (W) onFinish ingestStream END`, LOG.LS.eHTTP); + // LOG.info(`WebDAVFileSystem._openWriteStream(${pathS}): (W) onFinish ingestStream END`, LOG.LS.eDEBUG); } }); return; @@ -529,14 +530,15 @@ class WebDAVFileSystem extends webdav.FileSystem { LOG.error(`WebDAVFileSystem._openWriteStream(${pathS}): (W) onFinish ingestStream`, LOG.LS.eHTTP, error); } } - } else + } else { return await this.ingestStream(ISI, pathS); + } } catch (error) { LOG.error(`WebDAVFileSystem._openWriteStream(${pathWD}) (W) onFinish`, LOG.LS.eHTTP, error); } finally { callbackComplete(undefined); } - }); + })); // LOG.info('WebDAVFileSystem._openWriteStream callback()', LOG.LS.eHTTP); callback(undefined, BS); diff --git a/server/http/routes/api/generateDownloads.ts b/server/http/routes/api/generateDownloads.ts new file mode 100644 index 000000000..484ddb3fe --- /dev/null +++ b/server/http/routes/api/generateDownloads.ts @@ -0,0 +1,160 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import * as LOG from '../../../utils/logger'; +import * as DBAPI from '../../../db'; +import * as H from '../../../utils/helpers'; +import { ASL, LocalStore } from '../../../utils/localStore'; + +import { eEventKey } from '../../../event/interface/EventEnums'; +import { AuditFactory } from '../../../audit/interface/AuditFactory'; +import { isAuthenticated } from '../../auth'; + +import { Request, Response } from 'express'; +import { WorkflowFactory, IWorkflowEngine, WorkflowCreateResult, WorkflowParameters } from '../../../workflow/interface'; + +type GenDownloadsStatus = { + idWorkflow?: number, // do we have a workflow/report so we can (later) jump to its status page + idWorkflowReport?: number, // do we have a report for this workflow to help with future polling + isSceneValid: boolean, // if the referenced scene is QC'd and has basic requirements met + isJobRunning: boolean, // is there a job already running +}; + +type GenDownloadsResponse = { + success: boolean, // was the request successful + message?: string, // errors from the request|workflow to put in console or display to user + data?: GenDownloadsStatus +}; + +const generateResponse = (success: boolean, message?: string | undefined, data?: GenDownloadsStatus | undefined): GenDownloadsResponse => { + return { + success, + message, + data + }; +}; + +export async function generateDownloads(req: Request, res: Response): Promise { + // LOG.info('Generating Downloads from API request...', LOG.LS.eHTTP); + + // make sure we're authenticated (i.e. see if request has a 'user' object) + if (!isAuthenticated(req)) { + AuditFactory.audit({ url: req.path, auth: false }, { eObjectType: 0, idObject: 0 }, eEventKey.eGenDownloads); + LOG.error('API.generateDownloads failed. not authenticated.', LOG.LS.eHTTP); + res.status(403).send('no authenticated'); + return; + } + + // get our method to see what we should do + let statusOnly: boolean = true; + switch(req.method.toLocaleLowerCase()) { + case 'get': statusOnly = true; break; + case 'post': statusOnly = false; break; + default: { + LOG.error('API.generateDownloads failed. unsupported HTTP method',LOG.LS.eHTTP); + res.status(400).send(JSON.stringify(generateResponse(false,'invalid HTTP method'))); + return; + } + } + + // get our LocalStore. If we don't have one then bail. it is needed for the user id, auditing, and workflows + const LS: LocalStore | undefined = ASL.getStore(); + if(!LS || !LS.idUser){ + LOG.error('API.generateDownloads failed. cannot get LocalStore or idUser',LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,'missing store/user'))); + return; + } + + // extract query param for idSystemObject + const idSystemObject: number = parseInt((req.query.id as string) ?? '0'); + if(idSystemObject === 0) { + LOG.error(`API.generateDownloads failed. invalid id: ${idSystemObject}`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,`invalid id parameter: ${req.query.id ?? -1}`))); + return; + } + + // grab it and make sure it's a scene + const scene: DBAPI.Scene | null = await DBAPI.Scene.fetchBySystemObject(idSystemObject); + if(!scene) { + LOG.error(`API.generateDownloads failed. cannot find Scene. (id:${idSystemObject})`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,`cannot find scene: ${idSystemObject}`))); + return; + } + + // get any generate downloads workflows/jobs running + // HACK: hardcoding the job id since vocabulary is returning different values for looking up the job + // enum should provide 149, but is returning 125. The actual idJob is 8 (see above) + const idJob: number = 8; + + // if we only want the status then we need to do some quick checks ourself instead of WorkflowEngine + if(statusOnly === true) { + // see if scene is valid + const isSceneValid: boolean = (scene.PosedAndQCd)?true:false; + if(isSceneValid === false) { + LOG.error(`API.generateDownloads failed. scene is not QC'd. (id:${idSystemObject} | scene:${scene.idScene})`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,'scene has not be QC\'d.'))); + return; + } + + // get any active jobs + const activeJobs: DBAPI.JobRun[] | null = await DBAPI.JobRun.fetchActiveByScene(idJob,scene.idScene); + if(!activeJobs) { + LOG.error(`API.generateDownloads failed. cannot determine if job is running. (id:${idSystemObject} | idScene: ${scene.idScene})`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,'failed to get active jobs from DB', { isSceneValid, isJobRunning: false }))); + return; + } + + // if we're running, we don't duplicate our efforts + const idActiveJobRun: number[] = activeJobs.map(job => job.idJobRun); + if(activeJobs.length > 0) { + // get our workflow & report from the first active job id + let idWorkflow: number | undefined = undefined; + let idWorkflowReport: number | undefined = undefined; + const workflowReport: DBAPI.WorkflowReport[] | null = await DBAPI.WorkflowReport.fetchFromJobRun(activeJobs[0].idJobRun); + if(workflowReport && workflowReport.length>0) { + idWorkflowReport = workflowReport[0].idWorkflowReport; + idWorkflow = workflowReport[0].idWorkflow; + } else + LOG.info(`API.generateDownloads unable to get workflowReport (idScene: ${scene.idScene} | idJobRun: ${activeJobs[0].idJobRun}}).`,LOG.LS.eHTTP); + + // return our response and log it + LOG.info(`API.generateDownloads job already running (idScene: ${scene.idScene} | idJobRun: ${idActiveJobRun.join(',')}}).`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,'job already running',{ isSceneValid, isJobRunning: (activeJobs.length>0), idWorkflow, idWorkflowReport }))); + return; + } + + // send our info back to the client + LOG.info(`API.generateDownloads job is not running but valid. (id: ${scene.idScene} | scene: ${scene.Name})`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(true,'scene is valid and no job is running',{ isSceneValid, isJobRunning: (activeJobs.length>0) }))); + return; + } + + // if we're here then we want to try and initiate the workflow + const wfEngine: IWorkflowEngine | null = await WorkflowFactory.getInstance(); + if(!wfEngine) { + LOG.error(`API.generateDownloads failed to get WorkflowEngine. (id: ${scene.idScene} | scene: ${scene.Name})`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,'failed to get WorkflowEngine'))); + return; + } + + // build our parameters for the workflow + const workflowParams: WorkflowParameters = { + idUserInitiator: LS.idUser + }; + + // create our workflow for generating downloads + const result: WorkflowCreateResult = await wfEngine.generateDownloads(scene.idScene, workflowParams); + LOG.info(`API.generateDownloads post creation. (result: ${H.Helpers.JSONStringify(result)})`,LOG.LS.eDEBUG); + const isSceneValid: boolean = result.data.isSceneValid ?? false; + const isJobRunning: boolean = (result.data.activeJobs.length>0) ?? false; + const idWorkflow: number | undefined = (result.data.workflow?.idWorkflow) ?? undefined; + const idWorkflowReport: number | undefined = (result.data.workflowReport?.idWorkflowReport) ?? undefined; + + // make sure we saw success, otherwise bail + if(result.success===false) { + LOG.error(`API.generateDownloads failed to generate downloads: ${result.message}`,LOG.LS.eHTTP); + res.status(200).send(JSON.stringify(generateResponse(false,result.message,{ isSceneValid, isJobRunning, idWorkflow, idWorkflowReport }))); + return; + } + + // return success + res.status(200).send(JSON.stringify(generateResponse(true,`Generating Downloads for: ${scene.Name}`,{ isSceneValid, isJobRunning, idWorkflow, idWorkflowReport }))); +} \ No newline at end of file diff --git a/server/job/impl/Cook/JobCookSIGenerateDownloads.ts b/server/job/impl/Cook/JobCookSIGenerateDownloads.ts index b4cf539ec..d3d1d10e0 100644 --- a/server/job/impl/Cook/JobCookSIGenerateDownloads.ts +++ b/server/job/impl/Cook/JobCookSIGenerateDownloads.ts @@ -181,7 +181,6 @@ export class JobCookSIGenerateDownloads extends JobCook { - // grab our Packrat Scene from the database. idScene is a parameter passed in when creating this object // grab our Packrat Scene from the database. idScene is a parameter passed in when creating this object diff --git a/server/report/impl/Report.ts b/server/report/impl/Report.ts index a4b40ec0f..1fc914e43 100644 --- a/server/report/impl/Report.ts +++ b/server/report/impl/Report.ts @@ -3,15 +3,16 @@ import { WorkflowReport } from '../../db'; import * as H from '../../utils/helpers'; export class Report implements IReport { - private _workflowReport: WorkflowReport; + workflowReport: WorkflowReport; + constructor(workflowReport: WorkflowReport) { - this._workflowReport = workflowReport; + this.workflowReport = workflowReport; } async append(content: string): Promise { - const seperator: string = (this._workflowReport.Data) ? '
\n' : ''; - this._workflowReport.Data += seperator + content; - if (await this._workflowReport.update()) + const seperator: string = (this.workflowReport.Data) ? '
\n' : ''; + this.workflowReport.Data += seperator + content; + if (await this.workflowReport.update()) return { success: true }; return { success: false, error: 'Database error persisting WorkflowReport' }; } diff --git a/server/tests/cache/VocabularyCache.test.ts b/server/tests/cache/VocabularyCache.test.ts index ebee7a091..d6132e81c 100644 --- a/server/tests/cache/VocabularyCache.test.ts +++ b/server/tests/cache/VocabularyCache.test.ts @@ -86,6 +86,7 @@ function vocabularyCacheTestWorker(eMode: eCacheTestMode): void { for (const sVocabID in COMMON.eVocabularyID) { if (!isNaN(Number(sVocabID))) continue; + const eVocabID: COMMON.eVocabularyID = (COMMON.eVocabularyID)[sVocabID]; const vocabulary: DBAPI.Vocabulary | undefined = await VocabularyCache.vocabularyByEnum(eVocabID); // LOG.info(`*** sVocab=${sVocabID}, eVocabID=${COMMON.eVocabularyID[eVocabID]}, vocabulary=${JSON.stringify(vocabulary)}`, LOG.LS.eTEST); diff --git a/server/utils/helpers.ts b/server/utils/helpers.ts index 0de7a18ec..fd9352f9f 100644 --- a/server/utils/helpers.ts +++ b/server/utils/helpers.ts @@ -539,4 +539,12 @@ export class Helpers { if (typeof value === 'number' && value > 0 && value < 2147483648) return true; return false; } + + static getStackTrace(label: string = 'Capture Stack'): string { + try { + throw new Error(label); + } catch (error) { + return (error as Error).stack || `${label}: No stack trace available`; + } + } } \ No newline at end of file diff --git a/server/utils/localStore.ts b/server/utils/localStore.ts index 636e02c1b..3c8baa92b 100644 --- a/server/utils/localStore.ts +++ b/server/utils/localStore.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { AsyncLocalStorage } from 'async_hooks'; -// import * as LOG from './logger'; +import { AsyncLocalStorage, AsyncResource } from 'async_hooks'; +import * as LOG from './logger'; +import * as H from './helpers'; export class LocalStore { idRequest: number; @@ -13,11 +14,12 @@ export class LocalStore { private static idRequestNext: number = 0; private static getIDRequestNext(): number { + // LOG.info(`LocalStore.getIDRequestNext incrementing ID. (${LocalStore.idRequestNext}->${LocalStore.idRequestNext+1})`,LOG.LS.eDEBUG); return ++LocalStore.idRequestNext; } - constructor(getNextID: boolean, idUser: any | undefined) { - this.idRequest = (getNextID) ? LocalStore.getIDRequestNext() : 0; + constructor(getNextID: boolean, idUser: any | undefined, idRequest?: number | undefined) { + this.idRequest = (getNextID) ? LocalStore.getIDRequestNext() : (idRequest ?? 0); this.idUser = (typeof(idUser) === 'number') ? idUser : null; this.idWorkflow = []; } @@ -62,16 +64,63 @@ export class LocalStore { } export class AsyncLocalStore extends AsyncLocalStorage { - async getOrCreateStore(): Promise { + // we shouldn't need this routine as all LocalStore should be created when the request is first made + // so the idRequest and idUser are consistent throughout all related operations. + // TODO: phase out in favor of getStore(). + async getOrCreateStore(idUser: number | undefined = undefined, logUse: boolean = false): Promise { let LS: LocalStore | undefined = this.getStore(); - if (LS) + if (LS) { + if(logUse===true) + LOG.info(`AsyncLocalStore.getOrCreateStore using existing store (idRequest: ${LS.idRequest} | idUser: ${LS.idUser})`,LOG.LS.eDEBUG); + // LOG.info(`\t ${H.Helpers.getStackTrace('AsyncLocalStore.getOrCreateStore')}`,LOG.LS.eDEBUG); + + if(!LS.idUser && idUser) { + LOG.error(`AsyncLocalStore.getOrCreateStore adding missing user id (idRequest: ${LS.idRequest} | idUser: ${LS.idUser})`,LOG.LS.eDEBUG); + LS.idUser = idUser; + } return LS; + } return new Promise((resolve) => { - LS = new LocalStore(true, undefined); + LS = new LocalStore(true, idUser); + if(logUse===true) + LOG.error(`AsyncLocalStore.getOrCreateStore creating a new store. lost context? (idRequest: ${LS.idRequest} | idUser: ${idUser})`,LOG.LS.eSYS); + // LOG.error(`\t${H.Helpers.getStackTrace('AsyncLocalStore.getOrCreateStore')}`,LOG.LS.eDEBUG); this.run(LS, () => { resolve(LS!); }); // eslint-disable-line @typescript-eslint/no-non-null-assertion }); } + + clone(src: LocalStore | undefined, fn: () => unknown): LocalStore | null { + // creates a new LocalStore based on an existing one. This is used for wrappers + // around non-async libraries and preserves the context info entering a new + // context for logging/auditing. + // + // NOTE: any changes DO NOT propagate back out to the previous context. + + // if we don't have a source, get from the current store + src = src ?? this.getStore(); + if(!src) { + LOG.error('AsyncLocalStore.clone no source store found',LOG.LS.eSYS); + return null; + } + + LOG.info(`AsyncLocalStore.clone from existing LocalStore (idRequest: ${src.idRequest} | idUser: ${src.idUser})`,LOG.LS.eSYS); + + // create our new LocalStore and pass in our callback function + const LS: LocalStore = new LocalStore(false, src.idUser, src.idRequest); + this.run(LS,fn); + return LS; + } + + checkLocalStore(label: string = 'LocalStore', logUndefined: boolean = false): void { + label = `LocalStore [check: ${label}]`; + LOG.info(`${label} (${H.Helpers.JSONStringify(this.getStore())})`,LOG.LS.eDEBUG); + + // if we don't have a store then dump the trace so we know where it came from + if(!this.getStore() && logUndefined===true) + LOG.info(`\t ${H.Helpers.getStackTrace(label)}`,LOG.LS.eDEBUG); + } } +export { AsyncResource as ASR }; export const ASL: AsyncLocalStore = new AsyncLocalStore(); diff --git a/server/utils/logger.ts b/server/utils/logger.ts index a57c2e379..35544f21f 100644 --- a/server/utils/logger.ts +++ b/server/utils/logger.ts @@ -95,7 +95,9 @@ function configureLogger(logPath: string | null): void { // winston.format.json() // winston.format.simple(), winston.format.printf((info) => { - const LS: LocalStore | undefined = ASL.getStore(); + // we catch if ASL is undefined (ASL?) as it will be during JEST testing + // otherwise it should never be undefined. + const LS: LocalStore | undefined = ASL?.getStore(); const idRequest: number | undefined = LS?.idRequest; const reqID: string = idRequest ? ('00000' + (idRequest % 100000)).slice(-5) : ' --- '; diff --git a/server/workflow/impl/Packrat/WorkflowEngine.ts b/server/workflow/impl/Packrat/WorkflowEngine.ts index 98c1b3496..b4c9d79f4 100644 --- a/server/workflow/impl/Packrat/WorkflowEngine.ts +++ b/server/workflow/impl/Packrat/WorkflowEngine.ts @@ -9,6 +9,8 @@ import * as LOG from '../../../utils/logger'; import * as CACHE from '../../../cache'; import * as COMMON from '@dpo-packrat/common'; import * as DBAPI from '../../../db'; +// import * as REP from '../../../report/interface'; +// import { Report } from '../../../report/impl/Report'; import { NameHelpers, ModelHierarchy, UNKNOWN_NAME } from '../../../utils/nameHelpers'; import { ASL, LocalStore } from '../../../utils/localStore'; import * as H from '../../../utils/helpers'; @@ -42,6 +44,7 @@ type ComputeSceneInfoResult = { assetVersionMTL?: DBAPI.AssetVersion | undefined; scene?: DBAPI.Scene | undefined; licenseResolver?: DBAPI.LicenseResolver | undefined; + units?: string | undefined; }; export class WorkflowEngine implements WF.IWorkflowEngine { @@ -128,6 +131,7 @@ export class WorkflowEngine implements WF.IWorkflowEngine { } async generateSceneDownloads(idScene: number, workflowParams: WF.WorkflowParameters): Promise { + LOG.info(`WorkflowEngine.generateSceneDownloads working...(idScene:${idScene})`,LOG.LS.eWF); const scene: DBAPI.Scene | null = await DBAPI.Scene.fetch(idScene); if (!scene) { @@ -149,6 +153,173 @@ export class WorkflowEngine implements WF.IWorkflowEngine { return await this.eventIngestionIngestObjectScene(CSIR, workflowParams, true); } + async generateDownloads(idScene: number, workflowParams: WF.WorkflowParameters): Promise { + + // TODO: make this it's own workflow implementation instead of a routine (e.g. WorkflowGenDownloads) + // TODO: move scene verification into Scene class (returns CSIR). we then check everything is available for gen downloads here + + //#region get and verify scene + // grab our scene from the DB + const scene: DBAPI.Scene | null = await DBAPI.Scene.fetch(idScene); + if(!scene) { + LOG.error(`API.generateDownloads failed. cannot find Scene. (idScene:${idScene})`,LOG.LS.eHTTP); + return { success: false, message: 'cannot find scene' }; + } + + // get our system object + const sceneSO: DBAPI.SystemObject | null = await scene.fetchSystemObject(); + if(!sceneSO) { + LOG.error(`WorkflowEngine.generateDownloads failed. Scene is invalid without SystemObject. (idScene: ${scene.idScene})`,LOG.LS.eWF); + return { success: false, message: 'cannot get SystemObject', data: { isSceneValid: false } }; + } + + // get our information about the scene + const CSIR: ComputeSceneInfoResult | null = await this.computeSceneInfo(scene.idScene,sceneSO.idSystemObject); + if(!CSIR || !CSIR.idScene || CSIR.exitEarly==true) { + LOG.error(`WorkflowEngine.generateDownloads failed. Scene is invalid. (idScene: ${scene.idScene})`,LOG.LS.eWF); + return { success: false, message: 'cannot compute scene info', data: { isSceneValid: false } }; + } + LOG.info(`WorkflowEngine.generateDownloads verify scene (idScene:${CSIR.idScene} | sceneFile: ${CSIR.assetSVX?.FileName} | idModel: ${CSIR.idModel} | modelFile: ${CSIR.assetVersionGeometry?.FileName})`,LOG.LS.eDEBUG); + + // make sure we have a voyager scene + if(!CSIR.assetSVX) { + LOG.error(`WorkflowEngine.generateDownloads failed. No voyager scene found (idScene: ${scene.idScene})`,LOG.LS.eWF); + return { success: false, message: 'no voyager scene found', data: { isSceneValid: false } }; + } + + // make sure we have a master model + if(!CSIR.assetVersionGeometry || !CSIR.idModel) { + LOG.error(`WorkflowEngine.generateDownloads failed. No master model found (idScene: ${scene.idScene})`,LOG.LS.eWF); + return { success: false, message: 'no master model found', data: { isSceneValid: false } }; + } + + // make sure we can run the recipe (valid scene, not running, etc) + if(scene.PosedAndQCd === false) { + LOG.error(`WorkflowEngine.generateDownloads failed. Scene is invalid. (idScene: ${scene.idScene})`,LOG.LS.eWF); + return { success: false, message: 'not posed or QC\'d', data: { isSceneValid: false } }; + } + const isSceneValid: boolean = true; + //#endregion + + //#region check for duplicate jobs + // make sure we don't have any jobs running. >0 if a running job was found. + const activeJobs: DBAPI.JobRun[] | null = await DBAPI.JobRun.fetchActiveByScene(8,scene.idScene); + if(!activeJobs) { + LOG.error(`WorkflowEngine.generateDownloads failed. cannot determine if job is running. (idScene: ${scene.idScene})`,LOG.LS.eWF); + return { success: false, message: 'failed to get acti e jobs from DB', data: { isSceneValid: false } }; + } + + // if we're running, we don't duplicate our efforts + // TODO: allow for cancelling/overwritting existing jobs + const idActiveJobRun: number[] = activeJobs.map(job => job.idJobRun); + if(activeJobs.length > 0) { + // get our workflow & report from the first active job id + let idWorkflow: number | undefined = undefined; + let idWorkflowReport: number | undefined = undefined; + const workflowReport: DBAPI.WorkflowReport[] | null = await DBAPI.WorkflowReport.fetchFromJobRun(activeJobs[0].idJobRun); + if(workflowReport && workflowReport.length>0) { + idWorkflowReport = workflowReport[0].idWorkflowReport; + idWorkflow = workflowReport[0].idWorkflow; + } else + LOG.info(`WorkflowEngine.generateDownloads unable to get workflowReport (idScene: ${scene.idScene} | idJobRun: ${activeJobs[0].idJobRun}}).`,LOG.LS.eHTTP); + + LOG.info(`WorkflowEngine.generateDownloads did not start. Job already running (idScene: ${scene.idScene} | activeJobRun: ${idActiveJobRun.join(',')}}).`,LOG.LS.eWF); + return { success: false, message: 'Job already running', data: { isSceneValid: true, activeJobs, idWorkflow, idWorkflowReport } }; + } + //#endregion + + //#region get system objects to act on + const SOGeometry: DBAPI.SystemObject| null = await CSIR.assetVersionGeometry.fetchSystemObject(); + if (!SOGeometry) { + LOG.error(`WorkflowEngine.eventIngestionIngestObjectScene unable to compute geometry file systemobject from ${JSON.stringify(CSIR.assetVersionGeometry, H.Helpers.saferStringify)}`, LOG.LS.eWF); + return { success: false, message: 'cannot get SystemObject for geometry file', data: { isSceneValid: false, activeJobs } }; + } + const idSystemObject: number[] = [SOGeometry.idSystemObject]; + + const SOSVX: DBAPI.SystemObject| null = CSIR.assetSVX ? await CSIR.assetSVX.fetchSystemObject() : null; + if (!SOSVX) { + LOG.error(`WorkflowEngine.eventIngestionIngestObjectScene unable to compute scene file systemobject from ${JSON.stringify(CSIR.assetSVX, H.Helpers.saferStringify)}`, LOG.LS.eWF); + return { success: false, message: 'cannot get SystemObject for voyager scene file', data: { isSceneValid: false, activeJobs } }; + } + idSystemObject.push(SOSVX.idSystemObject); + + const SODiffuse: DBAPI.SystemObject| null = CSIR.assetVersionDiffuse ? await CSIR.assetVersionDiffuse.fetchSystemObject() : null; + if (SODiffuse) + idSystemObject.push(SODiffuse.idSystemObject); + + const SOMTL: DBAPI.SystemObject| null = CSIR.assetVersionMTL ? await CSIR.assetVersionMTL.fetchSystemObject() : null; + if (SOMTL) + idSystemObject.push(SOMTL.idSystemObject); + //#endregion + + // build our base names + const { sceneBaseName, modelBaseName } = await WorkflowEngine.computeSceneAndModelBaseNames(CSIR.idModel, CSIR.assetVersionGeometry.FileName); + LOG.info(`WorkflowEngine.generateDownloads compute names (sceneBaseName: ${sceneBaseName} | modelBaseName: ${modelBaseName})`,LOG.LS.eDEBUG); + + // #region build our scene parameters + const parameterHelper: COOK.JobCookSIVoyagerSceneParameterHelper | null = await COOK.JobCookSIVoyagerSceneParameterHelper.compute(CSIR.idModel); + if(parameterHelper==null) { + LOG.error(`WorkflowEngine.generateDownloads cannot create workflow parameters\n(CSIR:${H.Helpers.JSONStringify(CSIR)})`, LOG.LS.eWF); + return { success: false, message: 'cannot create workflow parameters', data: { isSceneValid, activeJobs } }; + } + + // create our parameters for generating downloads job + const jobParamSIGenerateDownloads: WFP.WorkflowJobParameters = + new WFP.WorkflowJobParameters(COMMON.eVocabularyID.eJobJobTypeCookSIGenerateDownloads, + new COOK.JobCookSIGenerateDownloadsParameters(CSIR.idScene, CSIR.idModel, CSIR.assetVersionGeometry.FileName, + CSIR.assetSVX.FileName, CSIR.assetVersionDiffuse?.FileName, CSIR.assetVersionMTL?.FileName, sceneBaseName, CSIR.units, undefined, parameterHelper )); + + // get our project for this scene. + // if no project found then have integrity issue and should not make things by creating additional assets + const sceneProjects: DBAPI.Project[] | null = await DBAPI.Project.fetchFromScene(scene.idScene); + if(!sceneProjects || sceneProjects.length!=1) { + LOG.error(`WorkflowEngine.generateDownloads cannot find project for scene. (idScene: ${scene.idScene})`, LOG.LS.eWF); + return { success: false, message: `cannot create workflow if scene does not have a Project (${scene.idScene})`, data: { isSceneValid, activeJobs } }; + } + + // create parameters for the workflow based on those created for the job + const wfParamSIGenerateDownloads: WF.WorkflowParameters = { + eWorkflowType: COMMON.eVocabularyID.eWorkflowTypeCookJob, + idSystemObject, + idProject: sceneProjects[0].idProject, + idUserInitiator: workflowParams.idUserInitiator, + parameters: jobParamSIGenerateDownloads, + }; + LOG.info(`WorkflowEngine.generateDownloads generating downloads... (${H.Helpers.JSONStringify(wfParamSIGenerateDownloads)})`, LOG.LS.eDEBUG); + //#endregion + + const doCreate: boolean = true; + if(doCreate) { + // create our workflow + const wf: WF.IWorkflow | null = await this.create(wfParamSIGenerateDownloads); + if (!wf) { + LOG.error(`WorkflowEngine.generateDownloads unable to create Cook si-generate-downloads workflow: ${H.Helpers.JSONStringify(wfParamSIGenerateDownloads)}`, LOG.LS.eWF); + return { success: false, message: 'cannot create downloads workflow', data: { isSceneValid, activeJobs } }; + } + + // get our Workflow object from the database + const workflow: DBAPI.Workflow | null = await wf.getWorkflowObject(); + if(!workflow) { + LOG.error(`WorkflowEngine.generateDownloads unable to get DB object for workflow. (${H.Helpers.JSONStringify(wfParamSIGenerateDownloads)})`, LOG.LS.eWF); + return { success: false, message: 'cannot get worfklow object', data: { isSceneValid, activeJobs } }; + } + LOG.info(`WorkflowEngine.generateDownloads retrieved workflow (${workflow.idWorkflow} | ${workflow.idWorkflowSet})`,LOG.LS.eDEBUG); + + // get our workflow report for the new workflow + const workflowReport: DBAPI.WorkflowReport[] | null = await DBAPI.WorkflowReport.fetchFromWorkflow(workflow.idWorkflow); + if(!workflowReport || workflowReport.length <= 0) { + LOG.error(`WorkflowEngine.generateDownloads unable to get workflow report. (${workflow.idWorkflow}))`, LOG.LS.eWF); + return { success: false, message: 'cannot get worfklow report object', data: { isSceneValid, activeJobs } }; + } + LOG.info(`WorkflowEngine.generateDownloads retrieved workflow report (${workflowReport[0].idWorkflowReport})`,LOG.LS.eDEBUG); + LOG.info(`\t ${H.Helpers.JSONStringify(workflowReport[0].Data)}`,LOG.LS.eDEBUG); + + // return success + return { success: true, message: 'generating downloads', data: { isSceneValid, activeJobs, workflow, workflowReport } }; + } else + return { success: true, message: 'generating downloads', data: { isSceneValid, activeJobs } }; + } + private async eventIngestionIngestObject(workflowParams: WF.WorkflowParameters | null): Promise { LOG.info(`WorkflowEngine.eventIngestionIngestObject params=${JSON.stringify(workflowParams)}`, LOG.LS.eWF); if (!workflowParams || !workflowParams.idSystemObject) @@ -237,7 +408,7 @@ export class WorkflowEngine implements WF.IWorkflowEngine { return workflows.length > 0 ? workflows : null; } - private async eventIngestionIngestObjectModel(CMIR: ComputeModelInfoResult, workflowParams: WF.WorkflowParameters, assetsIngested: boolean, generateDownloads: boolean = true): Promise { + private async eventIngestionIngestObjectModel(CMIR: ComputeModelInfoResult, workflowParams: WF.WorkflowParameters, assetsIngested: boolean, generateDownloads: boolean = false): Promise { if (!assetsIngested) { LOG.info(`WorkflowEngine.eventIngestionIngestObjectModel skipping post-ingest workflows as no assets were updated for ${JSON.stringify(CMIR, H.Helpers.saferStringify)}`, LOG.LS.eWF); return null; @@ -317,6 +488,13 @@ export class WorkflowEngine implements WF.IWorkflowEngine { // do we want to generate downloads for this ingestion if(generateDownloads===true) { + // TEMP: preventing automatic download generation + if(generateDownloads===true) { + LOG.info(`!!! WorkflowEngine.generateDownloads reached w/ ingest model. (idScene: ${CMIR.idModel})`,LOG.LS.eDEBUG); + console.trace('WorkflowEngine.generateDownloads'); + return null; + } + // does this ingested model have a scene child? If so, initiate WorkflowJob for cook si-generate-downloads const SODerived: DBAPI.SystemObject[] | null = CMIR.idSystemObjectModel ? await DBAPI.SystemObject.fetchDerivedFromXref(CMIR.idSystemObjectModel) : null; if (!SODerived) @@ -379,7 +557,7 @@ export class WorkflowEngine implements WF.IWorkflowEngine { return workflows.length > 0 ? workflows : null; } - private async eventIngestionIngestObjectScene(CSIR: ComputeSceneInfoResult, workflowParams: WF.WorkflowParameters, assetsIngested: boolean, generateDownloads: boolean = true): Promise { + private async eventIngestionIngestObjectScene(CSIR: ComputeSceneInfoResult, workflowParams: WF.WorkflowParameters, assetsIngested: boolean, generateDownloads: boolean = false): Promise { if (!assetsIngested) { LOG.info(`WorkflowEngine.eventIngestionIngestObjectScene skipping post-ingest workflows as no assets were updated for ${JSON.stringify(CSIR, H.Helpers.saferStringify)}`, LOG.LS.eWF); return null; @@ -413,9 +591,15 @@ export class WorkflowEngine implements WF.IWorkflowEngine { idSystemObject.push(SOMTL.idSystemObject); // do we want to generate downloads for this scene - // TODO: currently always true. needs to be fed upstream from business logic if(generateDownloads===true) { + // TEMP: preventing automatic download generation + if(generateDownloads===true) { + LOG.info(`!!! WorkflowEngine.generateDownloads reached w/ ingest scene. (idScene: ${CSIR.idScene})`,LOG.LS.eDEBUG); + console.trace('WorkflowEngine.generateDownloads'); + return null; + } + // initiate WorkflowJob for cook si-generate-download const { sceneBaseName } = await WorkflowEngine.computeSceneAndModelBaseNames(CSIR.idModel, CSIR.assetVersionGeometry.FileName); @@ -754,6 +938,7 @@ export class WorkflowEngine implements WF.IWorkflowEngine { } // Search for master model for this scene, among "source" objects + // assuming Scene has models as source/parent const SOMasters: DBAPI.SystemObject[] | null = await DBAPI.SystemObject.fetchMasterFromXref(idSystemObjectScene); if (!SOMasters) { LOG.error(`WorkflowEngine.computeSceneInfo unable to compute scene's master objects from system object ${JSON.stringify(idSystemObjectScene)}`, LOG.LS.eWF); @@ -779,7 +964,7 @@ export class WorkflowEngine implements WF.IWorkflowEngine { const retValue = { exitEarly: false, idScene, idModel: CMIR.idModel, idSystemObjectScene, assetSVX, assetVersionGeometry: CMIR.assetVersionGeometry, assetVersionDiffuse: CMIR.assetVersionDiffuse, - assetVersionMTL: CMIR.assetVersionMTL, scene, licenseResolver }; + assetVersionMTL: CMIR.assetVersionMTL, scene, licenseResolver, units: CMIR.units }; LOG.info(`WorkflowEngine.computeSceneInfo returning ${JSON.stringify(retValue, H.Helpers.saferStringify)}`, LOG.LS.eWF); return retValue; } diff --git a/server/workflow/impl/Packrat/WorkflowIngestion.ts b/server/workflow/impl/Packrat/WorkflowIngestion.ts index 9c62cb622..8e3983542 100644 --- a/server/workflow/impl/Packrat/WorkflowIngestion.ts +++ b/server/workflow/impl/Packrat/WorkflowIngestion.ts @@ -2,6 +2,7 @@ import * as WF from '../../interface'; import * as DBAPI from '../../../db'; import * as H from '../../../utils/helpers'; import * as COMMON from '@dpo-packrat/common'; +import * as LOG from '../../../utils/logger'; // This Workflow represents an ingestion action, typically initiated by a user. // The workflow itself performs no work (ingestion is performed in the graphQl ingestData routine) @@ -56,4 +57,16 @@ export class WorkflowIngestion implements WF.IWorkflow { async workflowConstellation(): Promise { return this.workflowData; } + + async getWorkflowObject(): Promise { + + // get our constellation + const wfConstellation: DBAPI.WorkflowConstellation | null = await this.workflowConstellation(); + if(!wfConstellation) { + LOG.error('WorkflowIngestion.getWorkflowObject failed. No constellation found. unitialized?',LOG.LS.eWF); + return null; + } + + return wfConstellation.workflow; + } } diff --git a/server/workflow/impl/Packrat/WorkflowJob.ts b/server/workflow/impl/Packrat/WorkflowJob.ts index 214fd61ab..be4570728 100644 --- a/server/workflow/impl/Packrat/WorkflowJob.ts +++ b/server/workflow/impl/Packrat/WorkflowJob.ts @@ -266,4 +266,16 @@ export class WorkflowJob implements WF.IWorkflow { this.idAssetVersions = WFUVersion.idAssetVersions; return { success: true }; } + + async getWorkflowObject(): Promise { + + // get our constellation + const wfConstellation: DBAPI.WorkflowConstellation | null = await this.workflowConstellation(); + if(!wfConstellation) { + LOG.error('WorkflowJob.getWorkflowObject failed. No constellation found. unitialized?',LOG.LS.eWF); + return null; + } + + return wfConstellation.workflow; + } } \ No newline at end of file diff --git a/server/workflow/impl/Packrat/WorkflowUpload.ts b/server/workflow/impl/Packrat/WorkflowUpload.ts index ea8b7a354..8f54df30f 100644 --- a/server/workflow/impl/Packrat/WorkflowUpload.ts +++ b/server/workflow/impl/Packrat/WorkflowUpload.ts @@ -249,4 +249,16 @@ export class WorkflowUpload implements WF.IWorkflow { this.results = { success: false, error: (this.results.error ? this.results.error + '/n' : '') + error }; return this.results; } + + async getWorkflowObject(): Promise { + + // get our constellation + const wfConstellation: DBAPI.WorkflowConstellation | null = await this.workflowConstellation(); + if(!wfConstellation) { + LOG.error('WorkflowUpload.getWorkflowObject failed. No constellation found. unitialized?',LOG.LS.eWF); + return null; + } + + return wfConstellation.workflow; + } } diff --git a/server/workflow/interface/IWorkflow.ts b/server/workflow/interface/IWorkflow.ts index 2d8f689ce..941af8017 100644 --- a/server/workflow/interface/IWorkflow.ts +++ b/server/workflow/interface/IWorkflow.ts @@ -15,4 +15,5 @@ export interface IWorkflow { updateStatus(eStatus: COMMON.eWorkflowJobRunStatus): Promise; waitForCompletion(timeout: number): Promise; workflowConstellation(): Promise; + getWorkflowObject(): Promise; } diff --git a/server/workflow/interface/IWorkflowEngine.ts b/server/workflow/interface/IWorkflowEngine.ts index 363b6a118..6826cbc72 100644 --- a/server/workflow/interface/IWorkflowEngine.ts +++ b/server/workflow/interface/IWorkflowEngine.ts @@ -10,9 +10,17 @@ export interface WorkflowParameters { parameters?: any | undefined; // Additional workflow parameters; each workflow template should define their own parameter interface } +export interface WorkflowCreateResult { + success: boolean; + message?: string | undefined; + workflow?: IWorkflow[] | undefined; + data?: any | undefined; +} + export interface IWorkflowEngine { create(workflowParams: WorkflowParameters): Promise; jobUpdated(idJobRun: number): Promise; event(eWorkflowEvent: COMMON.eVocabularyID, workflowParams: WorkflowParameters | null): Promise; generateSceneDownloads(idScene: number, workflowParams: WorkflowParameters): Promise; + generateDownloads(idScene: number, workflowParams: WorkflowParameters): Promise; } From c3446250d7f3e8603e905954adc9dfc34227af69 Mon Sep 17 00:00:00 2001 From: EMaslowskiQ <118929649+EMaslowskiQ@users.noreply.github.com> Date: Fri, 10 May 2024 12:29:59 -0400 Subject: [PATCH 13/14] DPO3DPKRT-808/cleaner model details during ingest (#587) (new) placed inspection model details inside a collapsible section (fix) layout and border issues for consistency --- .../Metadata/Control/SubtitleControl.tsx | 5 +- .../Metadata/Model/ObjectMeshTable.tsx | 4 +- .../components/Metadata/Model/index.tsx | 65 +++++++++++++++---- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/client/src/pages/Ingestion/components/Metadata/Control/SubtitleControl.tsx b/client/src/pages/Ingestion/components/Metadata/Control/SubtitleControl.tsx index 8bd4242f4..2ee15e764 100644 --- a/client/src/pages/Ingestion/components/Metadata/Control/SubtitleControl.tsx +++ b/client/src/pages/Ingestion/components/Metadata/Control/SubtitleControl.tsx @@ -21,9 +21,10 @@ const useStyles = makeStyles(({ palette, typography }) => ({ display: 'flex', flexDirection: 'column', backgroundColor: ({ hasPrimaryTheme, hasError }: { hasPrimaryTheme: boolean, hasError: boolean }) => hasError ? '#e57373' : hasPrimaryTheme ? palette.primary.light : palette.secondary.light, - width: 'fit-content', + width: '100%', minWidth: 300, - borderRadius: 5 + borderRadius: 5, + paddingBottom: '0.5rem' }, selected: { cursor: 'pointer', diff --git a/client/src/pages/Ingestion/components/Metadata/Model/ObjectMeshTable.tsx b/client/src/pages/Ingestion/components/Metadata/Model/ObjectMeshTable.tsx index 722f5c79b..e8f3d0e09 100644 --- a/client/src/pages/Ingestion/components/Metadata/Model/ObjectMeshTable.tsx +++ b/client/src/pages/Ingestion/components/Metadata/Model/ObjectMeshTable.tsx @@ -17,7 +17,8 @@ const useStyles = makeStyles(({ palette }) => ({ width: 'fit-content', height: 'fit-content', padding: '5px', - outline: '1px solid rgba(141, 171, 196, 0.4)' + outline: '1px solid rgba(141, 171, 196, 0.4)', + marginRight: '1rem' }, caption: { flex: '1 1 0%', @@ -84,7 +85,6 @@ function ObjectMeshTable({ modelObjects }): React.ReactElement { alignItems: 'center', columnGap: 10, }; - return ( <> {modelObjects.map(modelObject => { diff --git a/client/src/pages/Ingestion/components/Metadata/Model/index.tsx b/client/src/pages/Ingestion/components/Metadata/Model/index.tsx index e7cc45405..ee778c646 100644 --- a/client/src/pages/Ingestion/components/Metadata/Model/index.tsx +++ b/client/src/pages/Ingestion/components/Metadata/Model/index.tsx @@ -7,7 +7,7 @@ * * This component renders the metadata fields specific to model asset. */ -import { Box, makeStyles, Typography, Table, TableBody, TableCell, TableContainer, TableRow, Paper, Select, MenuItem, Tooltip } from '@material-ui/core'; +import { Box, makeStyles, Typography, Table, TableBody, TableCell, TableContainer, TableRow, Paper, Select, MenuItem, Tooltip, IconButton, Collapse } from '@material-ui/core'; import React, { useState, useEffect } from 'react'; import { AssetIdentifiers, DateInputField, ReadOnlyRow, TextArea } from '../../../../../components'; import { StateIdentifier, StateRelatedObject, useSubjectStore, useMetadataStore, useVocabularyStore, useRepositoryStore, FieldErrors } from '../../../../../store'; @@ -28,6 +28,10 @@ import clsx from 'clsx'; import lodash from 'lodash'; import { toast } from 'react-toastify'; +// icons +import KeyboardArrowDownIcon from '@material-ui/icons/KeyboardArrowDown'; +import KeyboardArrowUpIcon from '@material-ui/icons/KeyboardArrowUp'; + const useStyles = makeStyles(({ palette }) => ({ container: { marginTop: 20 @@ -40,7 +44,8 @@ const useStyles = makeStyles(({ palette }) => ({ width: 'fit-content', height: 'fit-content', padding: '5px', - outline: '1px solid rgba(141, 171, 196, 0.4)' + outline: '1px solid rgba(141, 171, 196, 0.4)', + marginRight: '1rem' }, dataEntry: { display: 'flex', @@ -147,6 +152,7 @@ function Model(props: ModelProps): React.ReactElement { } ]); const [sceneGenerateDisabled, setSceneGenerateDisabled] = useState(false); + const [showDetails, setShowDetails] = useState(true); const urlParams = new URLSearchParams(window.location.search); const idAssetVersion = urlParams.get('fileId'); @@ -364,7 +370,7 @@ function Model(props: ModelProps): React.ReactElement { {!idAsset && ( <> - + - + Model - - - - - - - + - + setShowDetails(showDetails === true ? false:true )} + > + Inspection Details + {showDetails === true ? ():( )} + + + + + + Model + + + + + + + + + + + + + + + + + + + Date: Fri, 10 May 2024 12:49:34 -0400 Subject: [PATCH 14/14] DPO3DPKRT-755: collection ID added to Subject search and listing results (#588) --- .../components/SubjectItem/SubjectList.tsx | 2 +- .../components/SubjectItem/SubjectListItem.tsx | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/client/src/pages/Ingestion/components/SubjectItem/SubjectList.tsx b/client/src/pages/Ingestion/components/SubjectItem/SubjectList.tsx index fe47e4d89..7120a8538 100644 --- a/client/src/pages/Ingestion/components/SubjectItem/SubjectList.tsx +++ b/client/src/pages/Ingestion/components/SubjectItem/SubjectList.tsx @@ -46,7 +46,7 @@ function SubjectList(props: SubjectListProps): React.ReactElement { const [addSubject, removeSubject] = useSubjectStore(state => [state.addSubject, state.removeSubject]); const classes = useStyles(); - const header: string[] = ['ARK / ID', 'UNIT', 'NAME']; + const header: string[] = ['ARK / ID', 'UNIT', 'NAME', 'ID']; const getSubjectList = ({ id, arkId, unit, name, collectionId }: StateSubject, index: number) => ( {unit} - - {name} - - {selected ? : } - + {name} + + + {collectionId} + + + + {selected ? : }