Release 0.9.16 - Cook Response Validation, Report/Error Handling (#585)

* [DPO3DPKRT-788] Updating assets from Cook with diff names fails (#579) * [DPO3DPKRT-752] Decouple explicit storage path in workflow (#580) * [DPO3DPKRT-800] Update to Voyager Schema for v0.39.0 (#581) * [DPO3DPKRT-799] Improved report tracking during Cook Jobs (#583) * [DPO3DPKRT-793] Improved UX for selecting scene generation during Ingestion (#584)
Smithsonian · Apr 15, 2024 · d084a09 · d084a09
1 parent 512ea73
commit d084a09
Show file tree

Hide file tree

Showing 19 changed files with 377 additions and 109 deletions.
diff --git a/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx b/client/src/pages/Ingestion/components/Metadata/Control/SceneGenerateWorkflowControl.tsx
diff --git a/client/src/pages/Ingestion/components/Metadata/Model/index.tsx b/client/src/pages/Ingestion/components/Metadata/Model/index.tsx
@@ -7,7 +7,7 @@
  *
  * This component renders the metadata fields specific to model asset.
  */
-import { Box, makeStyles, Typography, Table, TableBody, TableCell, TableContainer, TableRow, Paper, Select, MenuItem } from '@material-ui/core';
+import { Box, makeStyles, Typography, Table, TableBody, TableCell, TableContainer, TableRow, Paper, Select, MenuItem, Tooltip } from '@material-ui/core';
 import React, { useState, useEffect } from 'react';
 import { AssetIdentifiers, DateInputField, ReadOnlyRow, TextArea } from '../../../../../components';
 import { StateIdentifier, StateRelatedObject, useSubjectStore, useMetadataStore, useVocabularyStore, useRepositoryStore, FieldErrors } from '../../../../../store';
@@ -23,7 +23,6 @@ import { apolloClient } from '../../../../../graphql/index';
 import { useStyles as useTableStyles } from '../../../../Repository/components/DetailsView/DetailsTab/CaptureDataDetails';
 import { errorFieldStyling } from '../Photogrammetry';
 import SubtitleControl from '../Control/SubtitleControl';
-import SceneGenerateWorkflowControl from '../Control/SceneGenerateWorkflowControl';
 import { enableSceneGenerateCheck } from '../../../../../store/utils';
 import clsx from 'clsx';
 import lodash from 'lodash';
@@ -222,8 +221,9 @@ function Model(props: ModelProps): React.ReactElement {
     };
 
     const setSceneGenerate = ({ target }): void => {
-        const { name, checked } = target;
-        updateMetadataField(metadataIndex, name, !checked, MetadataType.model);
+        const { name, value } = target;
+        console.log(`skipSceneGenerate: ${value} | eval:${(value==='false')} | model:${model.skipSceneGenerate}`);
+        updateMetadataField(metadataIndex, name, (value==='false'), MetadataType.model);
     };
 
     const setIdField = ({ target }): void => {
@@ -374,13 +374,6 @@ function Model(props: ModelProps): React.ReactElement {
                                     hasError={fieldErrors?.model.subtitles ?? false}
                                 />
                             </Box>
-                            <Box style={{ marginBottom: 10 }}>
-                                <SceneGenerateWorkflowControl
-                                    selected={model.skipSceneGenerate}
-                                    disabled={sceneGenerateDisabled}
-                                    setCheckboxField={setSceneGenerate}
-                                />
-                            </Box>
                         </>
                     )}
                     <Box className={classes.modelDetailsContainer}>
@@ -469,6 +462,25 @@ function Model(props: ModelProps): React.ReactElement {
                                                 </Select>
                                             </TableCell>
                                         </TableRow>
+                                        <TableRow className={tableClasses.tableRow}>
+                                            <TableCell className={tableClasses.tableCell}><Typography className={tableClasses.labelText}>Generate Voyager Scene</Typography></TableCell>
+                                            <TableCell className={tableClasses.tableCell}>
+                                                <Tooltip title={<span style={{ fontSize: '0.75rem', fontWeight: 300 }}>To enable, <u><b>Units</b></u> must be set to mm, cm, m, in, ft, or yd, <u><b>Purpose</b></u> must be set to Master, and <u><b>Model File Type</b></u> must be set to obj, ply, stl, x3d, wrl, dae, or fbx</span>}>
+                                                    <Select
+                                                        value={String(!model.skipSceneGenerate)}
+                                                        name='skipSceneGenerate'
+                                                        onChange={setSceneGenerate}
+                                                        disableUnderline
+                                                        disabled={sceneGenerateDisabled}
+                                                        className={clsx(tableClasses.select, tableClasses.datasetFieldSelect)}
+                                                        SelectDisplayProps={{ style: { paddingLeft: '10px', borderRadius: '5px' } }}
+                                                    >
+                                                        <MenuItem value='true'>True</MenuItem>
+                                                        <MenuItem value='false'>False</MenuItem>
+                                                    </Select>
+                                                </Tooltip>
+                                            </TableCell>
+                                        </TableRow>
                                     </TableBody>
                                 </Table>
                             </TableContainer>

diff --git a/client/src/pages/Ingestion/hooks/useIngest.ts b/client/src/pages/Ingestion/hooks/useIngest.ts
@@ -78,6 +78,7 @@ function useIngest(): UseIngest {
             }));
 
             const item: StateItem = getSelectedItem() || { id: '', entireSubject: false, selected: false, subtitle: '', idProject: -1, projectName: '' };
+            // console.log('item',item);
 
             const isDefaultItem = item.id === defaultItem.id;
 
@@ -107,7 +108,7 @@ function useIngest(): UseIngest {
 
             const metadatasList = metadatas.length === 0 ? getMetadatas() : metadatas;
             lodash.forEach(metadatasList, metadata => {
-                // console.log('ingestionStart metadata', metadata);
+                // console.log('ingestionStart metadata',metadata);
                 const { file, photogrammetry, model, scene, other, sceneAttachment } = metadata;
                 const { photogrammetry: isPhotogrammetry, model: isModel, scene: isScene, attachment: isAttachment, other: isOther } = getAssetType(file.type);
 
@@ -186,7 +187,8 @@ function useIngest(): UseIngest {
                         sourceObjects,
                         derivedObjects,
                         updateNotes,
-                        subtitles
+                        subtitles,
+                        skipSceneGenerate
                     } = model;
 
                     let {
@@ -198,6 +200,7 @@ function useIngest(): UseIngest {
                     } else if (typeof dateCreated === 'object') {
                         dateCreated = nonNullValue<string>('dateCreated', dateCreated.toISOString());
                     }
+                    // console.log('model',model);
 
                     const ingestIdentifiers: IngestIdentifierInput[] = getIngestIdentifiers(identifiers);
                     const selectedSubtitle = subtitles.find(subtitle => subtitle.selected);
@@ -214,8 +217,10 @@ function useIngest(): UseIngest {
                         directory,
                         systemCreated,
                         sourceObjects,
-                        derivedObjects
+                        derivedObjects,
+                        skipSceneGenerate
                     };
+                    // console.log('modelData', modelData);
 
                     const idAsset: number | undefined = idToIdAssetMap.get(file.id);
                     if (idAsset) {
@@ -306,7 +311,7 @@ function useIngest(): UseIngest {
                 other: ingestOther,
                 sceneAttachment: ingestSceneAttachment
             };
-            // console.log(`** IngestDataInput`, input);
+            console.log('** IngestDataInput',input);
 
             const ingestDataMutation: FetchResult<IngestDataMutation> = await apolloClient.mutate({
                 mutation: IngestDataDocument,

diff --git a/server/db/api/Asset.ts b/server/db/api/Asset.ts
@@ -11,6 +11,8 @@ export class Asset extends DBC.DBObject<AssetBase> implements AssetBase, SystemO
     FileName!: string;
     idAssetGroup!: number | null;
     idVAssetType!: number;
+    // is idVAssetType is 135 (Model) then idSystemObject = the Packrat Model object
+    // if idVAssetType is 137 (Scene) then idSystemObject = the Packrat Scene object (only for SVX files)
     idSystemObject!: number | null;
     StorageKey!: string | null;
 
@@ -142,6 +144,22 @@ export class Asset extends DBC.DBObject<AssetBase> implements AssetBase, SystemO
         }
     }
 
+    /** Fetch assets that are connected to a specific Scene via the scene's id. **/
+    static async fetchFromScene(idScene: number): Promise<Asset[] | null> {
+
+        // when grabbing assets we need to grab those that are referenced by SystemObjectXref where
+        // our Packrat scene is the parent (e.g. models), but we also need to return those assets
+        // that don't use the xrefs and explicitly link to the Packrat Scene via it's idSystemObject field.
+        return DBC.CopyArray<AssetBase, Asset>(
+            await DBC.DBConnection.prisma.$queryRaw<Asset[]>`
+                SELECT DISTINCT a.* FROM Scene AS scn
+                JOIN SystemObject AS scnSO ON scn.idScene = scnSO.idScene 
+                JOIN SystemObjectXref AS scnSOX ON scnSO.idSystemObject = scnSOX.idSystemObjectMaster
+                JOIN Asset AS a ON (a.idSystemObject = scnSOX.idSystemObjectDerived OR a.idSystemObject = scnSO.idSystemObject)
+                WHERE scn.idScene = ${idScene};
+            `,Asset);
+    }
+
     /** Fetches assets that are connected to the specified idSystemObject (via that object's last SystemObjectVersion,
      * and that SystemObjectVersionAssetVersionXref's records). For those assets, we look for a match on FileName, idVAssetType */
     static async fetchMatching(idSystemObject: number, FileName: string, idVAssetType: number): Promise<Asset | null> {

diff --git a/server/db/api/JobRun.ts b/server/db/api/JobRun.ts
@@ -9,7 +9,7 @@ import * as H from '../../utils/helpers';
 export class JobRun extends DBC.DBObject<JobRunBase> implements JobRunBase {
     idJobRun!: number;
     idJob!: number;
-    Status!: number;
+    Status!: number;    // defined in common/constantss.ts (ln. 403)
     Result!: boolean | null;
     DateStart!: Date | null;
     DateEnd!: Date | null;
@@ -179,4 +179,19 @@ export class JobRun extends DBC.DBObject<JobRunBase> implements JobRunBase {
             return null;
         }
     }
+
+    static async fetchFromWorkflow(idWorkflow: number): Promise<JobRun[] | null> {
+        // direct get of JubRun(s) from a specific workflow
+        try {
+            return DBC.CopyArray<JobRunBase, JobRun> (
+                await DBC.DBConnection.prisma.$queryRaw<JobRun[]>`
+                SELECT jRun.* FROM Workflow AS w
+                JOIN WorkflowStep AS wStep ON wStep.idWorkflow = w.idWorkflow
+                JOIN JobRun AS jRun ON jRun.idJobRun = wStep.idJobRun
+                WHERE w.idWorkflow = ${idWorkflow};`,JobRun);
+        } catch (error) {
+            LOG.error('DBAPI.JobRun.fetchFromWorkflow', LOG.LS.eDB, error);
+            return null;
+        }
+    }
 }
diff --git a/server/db/api/Scene.ts b/server/db/api/Scene.ts
@@ -34,6 +34,9 @@ export class Scene extends DBC.DBObject<SceneBase> implements SceneBase, SystemO
 
     public fetchTableName(): string { return 'Scene'; }
     public fetchID(): number { return this.idScene; }
+    public fetchLogInfo(): string {
+        return `scene: ${this.Name}[id:${this.idScene}] | EDAN: ${this.EdanUUID}`;
+    }
 
     protected async createWorker(): Promise<boolean> {
         try {
@@ -208,4 +211,5 @@ export class Scene extends DBC.DBObject<SceneBase> implements SceneBase, SystemO
             return null;
         }
     }
+
 }
diff --git a/server/db/api/Workflow.ts b/server/db/api/Workflow.ts
@@ -173,4 +173,35 @@ export class Workflow extends DBC.DBObject<WorkflowBase> implements WorkflowBase
             return null;
         }
     }
+
+    static async fetchFromJobRun(idJobRun: number): Promise<Workflow[] | null> {
+        try {
+            return DBC.CopyArray<WorkflowBase, Workflow> (
+                await DBC.DBConnection.prisma.$queryRaw<Workflow[]>`
+                SELECT w.* FROM JobRun AS jRun
+                JOIN WorkflowStep AS wStep ON wStep.idJobRun = jRun.idJobRun
+                JOIN Workflow AS w ON w.idWorkflow = wStep.idWorkflow
+                WHERE jRun.idJobRun = ${idJobRun};`,Workflow);
+        } catch (error) {
+            LOG.error('DBAPI.Workflow.fetchFromJobRun', LOG.LS.eDB, error);
+            return null;
+        }
+    }
+
+    static async fetchAllWithError(includeCancelled: boolean = false, includeUninitialized: boolean = false): Promise<Workflow[] | null> {
+        // return all workflows that contain a step/job that has an error.
+        // optionally include those cancelled or unitialized
+        try {
+            return DBC.CopyArray<WorkflowBase, Workflow> (
+                await DBC.DBConnection.prisma.$queryRaw<Workflow[]>`
+                SELECT w.* FROM WorkflowStep AS wStep
+                JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow
+                JOIN JobRun AS jRun ON wStep.idJobRun = jRun.idJobRun
+                JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow
+                WHERE (wStep.State = 5 ${(includeCancelled?'OR wStep.State = 6 ':'')}${includeUninitialized?'OR wStep.State = 0':''});`,Workflow);
+        } catch (error) {
+            LOG.error('DBAPI.Workflow.fetchAllWithError', LOG.LS.eDB, error);
+            return null;
+        }
+    }
 }
diff --git a/server/db/api/WorkflowReport.ts b/server/db/api/WorkflowReport.ts
@@ -93,4 +93,36 @@ export class WorkflowReport extends DBC.DBObject<WorkflowReportBase> implements
             return null;
         }
     }
+
+    static async fetchFromJobRun(idJobRun: number): Promise<WorkflowReport[] | null> {
+        try {
+            return DBC.CopyArray<WorkflowReportBase, WorkflowReport> (
+                await DBC.DBConnection.prisma.$queryRaw<WorkflowReport[]>`
+                SELECT w.* FROM JobRun AS jRun
+                JOIN WorkflowStep AS wStep ON wStep.idJobRun = jRun.idJobRun
+                JOIN WorkflowReport AS wReport ON wReport.idWorkflow = wStep.idWorkflow
+                WHERE jRun.idJobRun = ${idJobRun};`,WorkflowReport);
+        } catch (error) {
+            LOG.error('DBAPI.WorkflowReport.fetchFromJobRun', LOG.LS.eDB, error);
+            return null;
+        }
+    }
+
+    static async fetchAllWithError(includeCancelled: boolean = false, includeUninitialized: boolean = false): Promise<WorkflowReport[] | null> {
+        // return all reports that contain a step/job that has an error.
+        // optionally include those cancelled or uninitialized.
+        // TODO: check against JobRun.Result for additional possible errors
+        try {
+            return DBC.CopyArray<WorkflowReportBase, WorkflowReport> (
+                await DBC.DBConnection.prisma.$queryRaw<WorkflowReport[]>`
+                SELECT wReport.* FROM WorkflowStep AS wStep
+                JOIN WorkflowReport AS wReport ON wStep.idWorkflow = wReport.idWorkflow
+                JOIN JobRun AS jRun ON wStep.idJobRun = jRun.idJobRun
+                JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow
+                WHERE (wStep.State = 5 ${(includeCancelled?'OR wStep.State = 6 ':'')}${includeUninitialized?'OR wStep.State = 0':''});`,WorkflowReport);
+        } catch (error) {
+            LOG.error('DBAPI.WorkflowReport.fetchAllWithError', LOG.LS.eDB, error);
+            return null;
+        }
+    }
 }
diff --git a/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts b/server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts
@@ -72,6 +72,10 @@ class UploadAssetWorker extends ResolverBase {
             LOG.error('uploadAsset unable to retrieve user context', LOG.LS.eGQL);
             return { status: UploadStatus.Noauth, error: 'User not authenticated' };
         }
+
+        // if an idAsset was provided then we are trying to update an existing asset
+        // else if an 'attachment' is specified (this is a child) then we will try to attach
+        // otherwise, we are adding a new asset to the system
         if (this.idAsset)
             await this.appendToWFReport(`<b>Upload starting</b>: UPDATE ${filename}`, true);
         else if (this.idSOAttachment)
@@ -128,10 +132,13 @@ class UploadAssetWorker extends ResolverBase {
     }
 
     private async uploadWorkerOnFinish(storageKey: string, filename: string, idVocabulary: number): Promise<UploadAssetResult> {
+
+        // grab our local storage
         const LSLocal: LocalStore | undefined = ASL.getStore();
         if (LSLocal)
             return await this.uploadWorkerOnFinishWorker(storageKey, filename, idVocabulary);
 
+        // if we can't get the local storage system we will use the cache
         if (this.LS) {
             LOG.info('uploadAsset missing LocalStore, using cached value', LOG.LS.eGQL);
             return ASL.run(this.LS, async () => {