[DPO3DPKRT-788] Updating assets from Cook with diff names fails (#579)

(new) centralizing error reporting for JobCook
(new) Asset.fetchFromScene to get all assets attached to a Scene
(new) convenience functions for getting Workflow(Step) from a Job
(new) generate downloads verifies incoming Cook data for name mismatch
(fix) state of Cook Job logged when finished after polling
(fix) improved error handling of generate downloads

server/db/api/Asset.ts

@@ -11,6 +11,8 @@ export class Asset extends DBC.DBObject<AssetBase> implements AssetBase, SystemO
     FileName!: string;
     idAssetGroup!: number | null;
     idVAssetType!: number;
+    // is idVAssetType is 135 (Model) then idSystemObject = the Packrat Model object
+    // if idVAssetType is 137 (Scene) then idSystemObject = the Packrat Scene object (only for SVX files)
     idSystemObject!: number | null;
     StorageKey!: string | null;
 
@@ -142,6 +144,22 @@ export class Asset extends DBC.DBObject<AssetBase> implements AssetBase, SystemO
         }
     }
 
+    /** Fetch assets that are connected to a specific Scene via the scene's id. **/
+    static async fetchFromScene(idScene: number): Promise<Asset[] | null> {
+
+        // when grabbing assets we need to grab those that are referenced by SystemObjectXref where
+        // our Packrat scene is the parent (e.g. models), but we also need to return those assets
+        // that don't use the xrefs and explicitly link to the Packrat Scene via it's idSystemObject field.
+        return DBC.CopyArray<AssetBase, Asset>(
+            await DBC.DBConnection.prisma.$queryRaw<Asset[]>`
+                SELECT DISTINCT a.* FROM Scene AS scn
+                JOIN SystemObject AS scnSO ON scn.idScene = scnSO.idScene 
+                JOIN SystemObjectXref AS scnSOX ON scnSO.idSystemObject = scnSOX.idSystemObjectMaster
+                JOIN Asset AS a ON (a.idSystemObject = scnSOX.idSystemObjectDerived OR a.idSystemObject = scnSO.idSystemObject)
+                WHERE scn.idScene = ${idScene};
+            `,Asset);
+    }
+
     /** Fetches assets that are connected to the specified idSystemObject (via that object's last SystemObjectVersion,
      * and that SystemObjectVersionAssetVersionXref's records). For those assets, we look for a match on FileName, idVAssetType */
     static async fetchMatching(idSystemObject: number, FileName: string, idVAssetType: number): Promise<Asset | null> {

server/db/api/JobRun.ts

@@ -9,7 +9,7 @@ import * as H from '../../utils/helpers';
 export class JobRun extends DBC.DBObject<JobRunBase> implements JobRunBase {
     idJobRun!: number;
     idJob!: number;
-    Status!: number;
+    Status!: number;    // defined in common/constantss.ts (ln. 403)
     Result!: boolean | null;
     DateStart!: Date | null;
     DateEnd!: Date | null;
@@ -179,4 +179,19 @@ export class JobRun extends DBC.DBObject<JobRunBase> implements JobRunBase {
             return null;
         }
     }
+
+    static async fetchFromWorkflow(idWorkflow: number): Promise<JobRun[] | null> {
+        // direct get of JubRun(s) from a specific workflow
+        try {
+            return DBC.CopyArray<JobRunBase, JobRun> (
+                await DBC.DBConnection.prisma.$queryRaw<JobRun[]>`
+                SELECT jRun.* FROM Workflow AS w
+                JOIN WorkflowStep AS wStep ON wStep.idWorkflow = w.idWorkflow
+                JOIN JobRun AS jRun ON jRun.idJobRun = wStep.idJobRun
+                WHERE w.idWorkflow = ${idWorkflow};`,JobRun);
+        } catch (error) {
+            LOG.error('DBAPI.JobRun.fetchFromWorkflow', LOG.LS.eDB, error);
+            return null;
+        }
+    }
 }

server/db/api/Scene.ts

@@ -34,6 +34,9 @@ export class Scene extends DBC.DBObject<SceneBase> implements SceneBase, SystemO
 
     public fetchTableName(): string { return 'Scene'; }
     public fetchID(): number { return this.idScene; }
+    public fetchLogInfo(): string {
+        return `scene: ${this.Name}[id:${this.idScene}] | EDAN: ${this.EdanUUID}`;
+    }
 
     protected async createWorker(): Promise<boolean> {
         try {
@@ -208,4 +211,5 @@ export class Scene extends DBC.DBObject<SceneBase> implements SceneBase, SystemO
             return null;
         }
     }
+
 }

server/db/api/Workflow.ts

@@ -173,4 +173,35 @@ export class Workflow extends DBC.DBObject<WorkflowBase> implements WorkflowBase
             return null;
         }
     }
+
+    static async fetchFromJobRun(idJobRun: number): Promise<Workflow[] | null> {
+        try {
+            return DBC.CopyArray<WorkflowBase, Workflow> (
+                await DBC.DBConnection.prisma.$queryRaw<Workflow[]>`
+                SELECT w.* FROM JobRun AS jRun
+                JOIN WorkflowStep AS wStep ON wStep.idJobRun = jRun.idJobRun
+                JOIN Workflow AS w ON w.idWorkflow = wStep.idWorkflow
+                WHERE jRun.idJobRun = ${idJobRun};`,Workflow);
+        } catch (error) {
+            LOG.error('DBAPI.Workflow.fetchFromJobRun', LOG.LS.eDB, error);
+            return null;
+        }
+    }
+
+    static async fetchAllWithError(includeCancelled: boolean = false, includeUninitialized: boolean = false): Promise<Workflow[] | null> {
+        // return all workflows that contain a step/job that has an error.
+        // optionally include those cancelled or unitialized
+        try {
+            return DBC.CopyArray<WorkflowBase, Workflow> (
+                await DBC.DBConnection.prisma.$queryRaw<Workflow[]>`
+                SELECT w.* FROM WorkflowStep AS wStep
+                JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow
+                JOIN JobRun AS jRun ON wStep.idJobRun = jRun.idJobRun
+                JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow
+                WHERE (wStep.State = 5 ${(includeCancelled?'OR wStep.State = 6 ':'')}${includeUninitialized?'OR wStep.State = 0':''});`,Workflow);
+        } catch (error) {
+            LOG.error('DBAPI.Workflow.fetchAllWithError', LOG.LS.eDB, error);
+            return null;
+        }
+    }
 }

server/db/api/WorkflowReport.ts

@@ -93,4 +93,36 @@ export class WorkflowReport extends DBC.DBObject<WorkflowReportBase> implements
             return null;
         }
     }
+
+    static async fetchFromJobRun(idJobRun: number): Promise<WorkflowReport[] | null> {
+        try {
+            return DBC.CopyArray<WorkflowReportBase, WorkflowReport> (
+                await DBC.DBConnection.prisma.$queryRaw<WorkflowReport[]>`
+                SELECT w.* FROM JobRun AS jRun
+                JOIN WorkflowStep AS wStep ON wStep.idJobRun = jRun.idJobRun
+                JOIN WorkflowReport AS wReport ON wReport.idWorkflow = wStep.idWorkflow
+                WHERE jRun.idJobRun = ${idJobRun};`,WorkflowReport);
+        } catch (error) {
+            LOG.error('DBAPI.WorkflowReport.fetchFromJobRun', LOG.LS.eDB, error);
+            return null;
+        }
+    }
+
+    static async fetchAllWithError(includeCancelled: boolean = false, includeUninitialized: boolean = false): Promise<WorkflowReport[] | null> {
+        // return all reports that contain a step/job that has an error.
+        // optionally include those cancelled or uninitialized.
+        // TODO: check against JobRun.Result for additional possible errors
+        try {
+            return DBC.CopyArray<WorkflowReportBase, WorkflowReport> (
+                await DBC.DBConnection.prisma.$queryRaw<WorkflowReport[]>`
+                SELECT wReport.* FROM WorkflowStep AS wStep
+                JOIN WorkflowReport AS wReport ON wStep.idWorkflow = wReport.idWorkflow
+                JOIN JobRun AS jRun ON wStep.idJobRun = jRun.idJobRun
+                JOIN Workflow AS w ON wStep.idWorkflow = w.idWorkflow
+                WHERE (wStep.State = 5 ${(includeCancelled?'OR wStep.State = 6 ':'')}${includeUninitialized?'OR wStep.State = 0':''});`,WorkflowReport);
+        } catch (error) {
+            LOG.error('DBAPI.WorkflowReport.fetchAllWithError', LOG.LS.eDB, error);
+            return null;
+        }
+    }
 }

server/graphql/schema/asset/resolvers/mutations/uploadAsset.ts

@@ -72,6 +72,10 @@ class UploadAssetWorker extends ResolverBase {
             LOG.error('uploadAsset unable to retrieve user context', LOG.LS.eGQL);
             return { status: UploadStatus.Noauth, error: 'User not authenticated' };
         }
+
+        // if an idAsset was provided then we are trying to update an existing asset
+        // else if an 'attachment' is specified (this is a child) then we will try to attach
+        // otherwise, we are adding a new asset to the system
         if (this.idAsset)
             await this.appendToWFReport(`<b>Upload starting</b>: UPDATE ${filename}`, true);
         else if (this.idSOAttachment)
@@ -128,10 +132,13 @@ class UploadAssetWorker extends ResolverBase {
     }
 
     private async uploadWorkerOnFinish(storageKey: string, filename: string, idVocabulary: number): Promise<UploadAssetResult> {
+
+        // grab our local storage
         const LSLocal: LocalStore | undefined = ASL.getStore();
         if (LSLocal)
             return await this.uploadWorkerOnFinishWorker(storageKey, filename, idVocabulary);
 
+        // if we can't get the local storage system we will use the cache
         if (this.LS) {
             LOG.info('uploadAsset missing LocalStore, using cached value', LOG.LS.eGQL);
             return ASL.run(this.LS, async () => {

server/job/impl/Cook/JobCook.ts

@@ -396,9 +396,16 @@ export abstract class JobCook<T> extends JobPackrat {
             }
 
             // look for completion in 'state' member, via value of 'done', 'error', or 'cancelled'; update eJobRunStatus and terminate polling job
+            // write to the log for the first 10 polling cycles, then every 5th one after that
             const cookJobReport = axiosResponse.data;
             if (pollNumber <= 10 || ((pollNumber % 5) == 0))
                 LOG.info(`JobCook [${this.name()}] polling [${pollNumber}], state: ${cookJobReport['state']}: ${requestUrl}`, LOG.LS.eJOB);
+
+            // if we finished (i.e. not running or waiting) then we push out an additional log statement
+            // to ensure it's caught
+            if(cookJobReport['state']!=='waiting' && cookJobReport['state']!=='running')
+                LOG.info(`JobCook [${this.name()}] polling [exited], state: ${cookJobReport['state']}: ${requestUrl}`, LOG.LS.eJOB);
+
             switch (cookJobReport['state']) {
                 case 'created':     await this.recordCreated();                                                         break;
                 case 'waiting':     await this.recordWaiting();                                                         break;
@@ -666,4 +673,42 @@ export abstract class JobCook<T> extends JobPackrat {
             LOG.info(error, LOG.LS.eJOB);
         return res;
     }
+
+    protected async verifyIncomingCookData(_sceneSource: DBAPI.Scene, _fileMap: Map<string,string>): Promise<H.IOResults> {
+        return { success: true };
+    }
+
+    protected extractBaseName(filenames: string[]): string | null {
+        // extract the base name from the list of incoming filenames and make sure they all share
+        // the same values. input (currently) requires an SVX file in the list
+        // TODO: broader support for other 'groups' of filenames that may not have an SVX
+        const svxFilename: string | undefined = filenames.find(filename => filename.includes('.svx.json'));
+        if(!svxFilename || svxFilename.length == 0) {
+            this.logError('JobCookSIGenerateDownloads cannot extract basename. SVX file not found');
+            return null;
+        }
+
+        // get the baseName from the SVX file
+        const baseName: string = svxFilename.replace(/\.svx\.json$/, '');
+
+        // compare with others in the list to make sure they match
+        const errorNames: string[] = filenames.filter(filename => !filename.startsWith(baseName));
+        if(errorNames.length>0) {
+            this.logError(`JobCookSIGenerateDownloads filenames don't share base name. (${errorNames.join(' | ')})`);
+            return null;
+        }
+
+        // return success
+        return baseName;
+    }
+
+    // private getSceneFilenamesFromMap(fileMap: Map<string, string>): string[] {
+    //     const result: string[] = [];
+    //     fileMap.forEach((value, _key) => {
+    //         if (value.includes('svx.json')) {
+    //             result.push(value);
+    //         }
+    //     });
+    //     return result;
+    // }
 }