Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DPO3DPKRT-701/generate downloads support for scenes (scene packages) #568

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
65bd09c
Fix:
EMaslowskiQ Jan 31, 2023
d13240d
[DPO3DPKRT-746] fix/large file upload failure (#543)
EMaslowskiQ May 18, 2023
bdf593c
Updated GLTF dependencies to v2.5.1 (current: 3.5.1)
EMaslowskiQ Aug 30, 2023
c055be0
DPO3DPKRT-760/new edan publishing api (#549)
EMaslowskiQ Oct 6, 2023
785f292
Fixes DPO3DPKRT-764 allowing for easier local Proxy operation
EMaslowskiQ Oct 6, 2023
b474b31
DPO3DPKRT-763/generate download scenes workaround (#552)
EMaslowskiQ Oct 10, 2023
ebbc293
Merge branch 'master' into develop
EMaslowskiQ Oct 11, 2023
f643eba
DPO3DPKRT-768/Update LDAP Environment Vars (#554)
EMaslowskiQ Nov 6, 2023
52e6cc6
DPO3DPKRT-767/Support Multiple Cook Resources (#555)
EMaslowskiQ Nov 7, 2023
9aef1e6
DPO3DPKRT-767/support-multiple-cook-resources : jobs 'created' suppor…
EMaslowskiQ Nov 8, 2023
c85c581
Updated to LDAPS. (#559)
EMaslowskiQ Nov 30, 2023
7ca9f71
DPO3DPKRT-769/encoding webdav uri for voyager edit (#558)
EMaslowskiQ Dec 12, 2023
c61028e
Merge branch 'master' of https://github.com/Smithsonian/dpo-packrat i…
EMaslowskiQ Dec 12, 2023
7a86c47
IngestionSubjectProjectAlgo
EMaslowskiQ Dec 12, 2023
97f21ec
(new) remove all non-ascii characters for filename sanitization (#562)
EMaslowskiQ Jan 3, 2024
184f537
DPO3DPKRT-774/streams not reporting generated errors (#563)
EMaslowskiQ Jan 3, 2024
9721e2c
(fix) returning error message for zipStream
EMaslowskiQ Jan 3, 2024
ae80106
Merge branch 'master' of https://github.com/Smithsonian/dpo-packrat i…
EMaslowskiQ Jan 3, 2024
e7cd3b1
DPO3DPKRT-761/improved cook responses for generate downloads (#565)
EMaslowskiQ Jan 4, 2024
fa16fde
(fix) ignore local batch scripts for environment setup
EMaslowskiQ Jan 8, 2024
88a78f9
Merge branch 'develop' into DPO3DPKRT-701/generate-downloads-support-…
EMaslowskiQ Jan 8, 2024
e571ba6
(new) eDEBUG log type specifically for log outputs during development to
EMaslowskiQ Jan 19, 2024
8aedb69
(fix) commenting out getModelDetailsFromVoyagerScene since unused
EMaslowskiQ Jan 19, 2024
bdc48a8
(fix) support for draco/usdz generated downloads
EMaslowskiQ Jan 19, 2024
62a0d6a
(fix) EDAN resource titles showing undefined model type. modified tags
EMaslowskiQ Jan 22, 2024
65d2f5d
(fix) cleanout or comment debug statements
EMaslowskiQ Jan 23, 2024
1f3a46e
(fix) inclusion of usdz/draco files in scene package
EMaslowskiQ Jan 24, 2024
e655661
merging 'develop' to help with rebasing and remove unnecessary commit…
EMaslowskiQ Jan 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions server/collections/impl/EdanCollection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,9 @@ export class EdanCollection implements COL.ICollection {
}

async publish(idSystemObject: number, ePublishState: number): Promise<boolean> {

LOG.info(`EdanCollection.publish (idSystemObject: ${idSystemObject}`,LOG.LS.eDEBUG);

switch (ePublishState) {
case COMMON.ePublishedState.eNotPublished:
case COMMON.ePublishedState.eAPIOnly:
Expand Down
56 changes: 47 additions & 9 deletions server/collections/impl/PublishScene.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ export class PublishScene {
if (resource)
resourceMap.set(SAC, resource);
}
// LOG.info(`>>> PublishScene.computeResourceMap (scene: ${this.sceneFile} | ${H.Helpers.JSONStringify(resourceMap)})`,LOG.LS.eDEBUG);
return resourceMap;
}

Expand All @@ -86,12 +87,16 @@ export class PublishScene {
return false;
}

if (!this.scene || !this.subject)
if (!this.scene || !this.subject) {
LOG.info(`PublishScene.publish cannot publish. no scene/subject. (scene: ${this.scene?.Name} | subject: ${this.subject?.Name})`,LOG.LS.eCOLL);
return false;
}

// stage scene
if (!await this.stageSceneFiles() || !this.sharedName)
if (!await this.stageSceneFiles() || !this.sharedName) {
LOG.error(`PublishScene.publish cannot stage files. (scene: ${this.scene?.Name} | sharedName: ${this.sharedName})`,LOG.LS.eCOLL);
return false;
}

// create EDAN 3D Package
let edanRecord: COL.EdanRecord | null = await ICol.createEdan3DPackage(this.sharedName, this.sceneFile);
Expand Down Expand Up @@ -356,10 +361,13 @@ export class PublishScene {
return false;
}

LOG.info(`PublishScene.collectAssets downloadMSXMap ${H.Helpers.JSONStringify(this.DownloadMSXMap)}`, LOG.LS.eDEBUG);

// first pass through assets: detect and record the scene file (first file to match *.svx.json); lookup supporting information
// prepare to extract and discard the path to this file, so that the scene zip is "rooted" at the svx.json
let stageRes: H.IOResults = { success: true };
for (const assetVersion of this.assetVersions) {
// grab our system object for this asset version
const asset: DBAPI.Asset | null = await DBAPI.Asset.fetch(assetVersion.idAsset);
LOG.info(`PublishScene.collectAssets considering assetVersion=${JSON.stringify(assetVersion, H.Helpers.saferStringify)} asset=${JSON.stringify(asset, H.Helpers.saferStringify)}`, LOG.LS.eCOLL);
if (!asset) {
Expand All @@ -368,6 +376,9 @@ export class PublishScene {
}

// determine if assetVersion is an attachment by examining metadata
// NOTE: isAttachment metadata is set in computeResourceMap to label an asset as a 'resource' (i.e. downloadable)
// however, usdz/draco files have dual use and are resources, but also connected to a voyager scene. below is a special
// case for this situation until downloadable is decoupled from Usage.
let isAttachment: boolean = false;
const SOAssetVersion: DBAPI.SystemObject | null = await assetVersion.fetchSystemObject();
if (!SOAssetVersion) {
Expand All @@ -383,20 +394,34 @@ export class PublishScene {
}
}
} else
LOG.error(`PublishScene.collectAssets unable to compute metadata for ${JSON.stringify(assetVersion, H.Helpers.saferStringify)}`, LOG.LS.eCOLL);
LOG.error(`PublishScene.collectAssets unable to compute metadata for ${H.Helpers.JSONStringify(assetVersion)}`, LOG.LS.eCOLL);

// if we have our asset and a system object (always the case?) then we see if this asset is in the download/resource
// map...
if (asset.idSystemObject) {
const modelSceneXref: DBAPI.ModelSceneXref | undefined = this.DownloadMSXMap.get(asset.idSystemObject ?? 0);
if (!modelSceneXref)
if (!modelSceneXref) {
// LOG.info(`>>> PublishScene.collectAssets adding to list, no MSX (${asset.FileName}|${isAttachment}|${H.Helpers.JSONStringify(metadataSet)}`, LOG.LS.eDEBUG);
this.SacList.push({ idSystemObject: asset.idSystemObject, asset, assetVersion, metadataSet: isAttachment ? metadataSet : undefined });
else {
} else {
const model: DBAPI.Model | null = await DBAPI.Model.fetch(modelSceneXref.idModel);
if (!model) {
LOG.error(`PublishScene.collectAssets unable to load model from xref ${JSON.stringify(modelSceneXref, H.Helpers.saferStringify)}`, LOG.LS.eCOLL);
LOG.error(`PublishScene.collectAssets unable to load model from xref ${H.Helpers.JSONStringify(modelSceneXref)}`, LOG.LS.eCOLL);
return false;
}
// LOG.info(`>>> PublishScene.collectAssets adding to list with MSX (${asset.FileName}|${isAttachment}|${H.Helpers.JSONStringify(modelSceneXref)}`, LOG.LS.eDEBUG);
this.SacList.push({ idSystemObject: asset.idSystemObject, asset, assetVersion, model, modelSceneXref, metadataSet: isAttachment ? metadataSet : undefined });

// HACK: special case for handling dual-use assets (draco/usdz), which will be added as downloads/attachments by default.
// whe check if we're dealing with that asset and add like other scene referenced assets. (i.e. we omit the MSX and metadataSet)
if(modelSceneXref.Usage === 'App3D' || modelSceneXref.Usage === 'iOSApp3D') {
LOG.info(`>>> PublishScene.collectAssets adding draco/usdz assets again (${asset.FileName})`,LOG.LS.eDEBUG);
this.SacList.push({ idSystemObject: asset.idSystemObject, asset, assetVersion, metadataSet: undefined });
}
}

} else {
LOG.info(`PublishScene.collectAssets not adding. no idSystemObject ${H.Helpers.JSONStringify(asset)}|${H.Helpers.JSONStringify(metadataSet)}`, LOG.LS.eDEBUG);
}

if (!this.sceneFile && assetVersion.FileName.toLowerCase().endsWith('.svx.json')) {
Expand Down Expand Up @@ -427,10 +452,16 @@ export class PublishScene {
}

private async stageSceneFiles(): Promise<boolean> {
if (this.SacList.length <= 0 || !this.scene)
if (this.SacList.length <= 0 || !this.scene) {
LOG.error(`PublishScene.stageSceneFiles casnnot stage files for scene (${this.sceneFile}). No scene or assets list (${this.scene}|${this.SacList.length})`,LOG.LS.eCOLL);
return false;
}
let stageRes: H.IOResults = { success: true };

// log what will be included
const assets: string[] = this.SacList.map(SAC => { return `${SAC.asset.FileName}(${SAC.assetVersion.Version})`; });
LOG.info(`PublishScene.stageSceneFiles collecting assets for scene (${this.sceneFile}): ${assets.join(',')}`,LOG.LS.eCOLL);

// second pass: zip up appropriate assets; prepare to copy downloads
const zip: ZIP.ZipStream = new ZIP.ZipStream();
for (const SAC of this.SacList.values()) {
Expand Down Expand Up @@ -704,11 +735,16 @@ export class PublishScene {
switch (SAC.modelSceneXref.Usage?.replace('Download:', '').toLowerCase()) {
case undefined:
case 'webassetglblowuncompressed': category = 'Low resolution'; MODEL_FILE_TYPE = 'glb'; break;
case 'webassetglbarcompressed': category = 'Low resolution'; MODEL_FILE_TYPE = 'glb'; DRACO_COMPRESSED = true; break;
case 'usdz': category = 'iOS AR model'; MODEL_FILE_TYPE = 'usdz'; break;
case 'objzipfull': category = 'Full resolution'; MODEL_FILE_TYPE = 'obj'; break;
case 'objziplow': category = 'Low resolution'; MODEL_FILE_TYPE = 'obj'; break;
case 'gltfziplow': category = 'Low resolution'; MODEL_FILE_TYPE = 'gltf'; break;

// HACK: special case to account for USDZ/Draco models which have dual purpose 'usage'
// and requires specific Usage to work. fix when decoupling 'downloadable' property.
case 'usdz':
case 'iosapp3d': category = 'iOS AR model'; MODEL_FILE_TYPE = 'usdz'; break;
case 'webassetglbarcompressed':
case 'app3d': category = 'Low resolution'; MODEL_FILE_TYPE = 'glb'; DRACO_COMPRESSED = true; break;
}
}

Expand Down Expand Up @@ -739,6 +775,8 @@ export class PublishScene {
LOG.error(`PublishScene.updatePublishedState unable to update published state for ${JSON.stringify(this.systemObjectVersion, H.Helpers.saferStringify)}`, LOG.LS.eCOLL);
return false;
}
} else {
LOG.info(`PublishScene.updatePublishedState skipping.... ${H.Helpers.JSONStringify(ePublishedStateIntended)}`, LOG.LS.eDEBUG);
}

return true;
Expand Down
5 changes: 3 additions & 2 deletions server/config/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ export type ConfigType = {
};

const oneDay = 24 * 60 * 60;
const fifteenMinutes = 1 * 15 * 60;
// const fifteenMinutes = 1 * 15 * 60;
const threeHours = 3 * 60 * 60;
const debugSessionTimeout = false;

export const Config: ConfigType = {
Expand All @@ -118,7 +119,7 @@ export const Config: ConfigType = {
auth: {
type: process.env.PACKRAT_AUTH_TYPE == 'LDAP' ? AUTH_TYPE.LDAP : AUTH_TYPE.LOCAL,
session: {
maxAge: !debugSessionTimeout ? (((process.env.NODE_ENV === 'production') ? fifteenMinutes : oneDay) * 1000) : 8000, // expiration time 15 minutes, in milliseconds
maxAge: !debugSessionTimeout ? (((process.env.NODE_ENV === 'production') ? threeHours : oneDay) * 1000) : 8000, // expiration time 15 minutes, in milliseconds
checkPeriod: oneDay // prune expired entries every 24 hours
},
ldap: {
Expand Down
Loading
Loading