diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index bfec1cc1..65d7c539 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -263,3 +263,11 @@ model aqi_data_classifications { update_user_id String @db.VarChar(200) update_utc_timestamp DateTime @db.Timestamp(6) } + +model aqi_imported_data { + aqi_imported_data_id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + file_name String? @db.VarChar(200) + original_file_name String? @db.VarChar(200) + imported_guids Json? + create_utc_timestamp DateTime @db.Timestamp(6) +} diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 6e4d73e7..3c393da1 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -24,7 +24,9 @@ export class AqiApiService { async fieldVisits(body: any) { try { const response = await this.axiosInstance.post("/v1/fieldvisits", body); - this.logger.log(`API call to POST Field Visits succeeded: ${response.status}`); + this.logger.log( + `API call to POST Field Visits succeeded: ${response.status}`, + ); return response.data.id; } catch (err) { console.error( @@ -35,11 +37,16 @@ export class AqiApiService { } async putFieldVisits(GUID: string, body: any) { - try{ - const response = await this.axiosInstance.put(`/v1/fieldvisits/${GUID}`, body); - this.logger.log(`API call to PUT Field Visits succeeded: ${response.status}`); + try { + const response = await this.axiosInstance.put( + `/v1/fieldvisits/${GUID}`, + body, + ); + this.logger.log( + `API call to PUT Field Visits succeeded: ${response.status}`, + ); return response.data.id; - }catch (err){ + } catch (err) { console.error( "API CALL TO PUT Field Visits failed: ", err.response.data.message, @@ -50,7 +57,9 @@ export class AqiApiService { async fieldActivities(body: any) { try { const response = await this.axiosInstance.post("/v1/activities", body); - this.logger.log(`API call to POST Activities succeeded: ${response.status}`); + this.logger.log( + `API call to POST Activities succeeded: ${response.status}`, + ); return response.data.id; } catch (err) { console.error( @@ -61,11 +70,16 @@ export class AqiApiService { } async putFieldActivities(GUID: string, body: any) { - try{ - const response = await this.axiosInstance.put(`/v1/activities/${GUID}`, body); - this.logger.log(`API call to PUT Field Activities succeeded: ${response.status}`); + try { + const response = await this.axiosInstance.put( + `/v1/activities/${GUID}`, + body, + ); + this.logger.log( + `API call to PUT Field Activities succeeded: ${response.status}`, + ); return response.data.id; - }catch (err){ + } catch (err) { console.error( "API CALL TO PUT Field Activities failed: ", err.response.data.message, @@ -76,7 +90,9 @@ export class AqiApiService { async fieldSpecimens(body: any) { try { const response = await this.axiosInstance.post("/v1/specimens", body); - this.logger.log(`API call to POST Specimens succeeded: ${response.status}`); + this.logger.log( + `API call to POST Specimens succeeded: ${response.status}`, + ); return response.data.id; } catch (err) { console.error( @@ -87,11 +103,16 @@ export class AqiApiService { } async putSpecimens(GUID: string, body: any) { - try{ - const response = await this.axiosInstance.put(`/v1/specimens/${GUID}`, body); - this.logger.log(`API call to PUT Specimens succeeded: ${response.status}`); + try { + const response = await this.axiosInstance.put( + `/v1/specimens/${GUID}`, + body, + ); + this.logger.log( + `API call to PUT Specimens succeeded: ${response.status}`, + ); return response.data.id; - }catch (err){ + } catch (err) { console.error( "API CALL TO PUT Specimens failed: ", err.response.data.message, @@ -99,6 +120,23 @@ export class AqiApiService { } } + async getObservationsFromFile(fileName: string) { + try { + let observations = ( + await this.axiosInstance.get("/v2/observations?limit=1000") + ).data.domainObjects; + + const relatedData = observations.filter((observation) => + observation.extendedAttributes + .some((attribute) => attribute.text === fileName) + ) + .map((observation) => observation.id); + return relatedData; + } catch (err) { + console.error("API CALL TO GET Observations from File failed: ", err); + } + } + async importObservations(fileName: any, method: string) { const formData = new FormData(); formData.append("file", fs.createReadStream(fileName)); @@ -145,7 +183,6 @@ export class AqiApiService { const obsStatus = await this.getObservationsStatusResult(statusURL); const errorMessages = this.parseObsResultResponse(obsStatus); - return errorMessages; } } catch (err) { @@ -169,7 +206,7 @@ export class AqiApiService { }, }); - await wait(5000); + await wait(7000); const obsResultResponse = await axios.get( `${process.env.AQI_BASE_URL}/v2/observationimports/${response.data.id}/result`, @@ -244,7 +281,7 @@ export class AqiApiService { aqi_field_visit_start_time: queryParam[1], }, }); - if (result.length > 0){ + if (result.length > 0) { return result[0].aqi_field_visits_id; } else { return null; @@ -397,7 +434,7 @@ export class AqiApiService { if (uniqueObservations.length > 0) { try { let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v2/observations?specimentIds=${uniqueSpecimens}`, + `${process.env.AQI_BASE_URL}/v2/observations?specimenIds=${uniqueSpecimens}`, { headers: { Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index a7be7801..4725da30 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -569,7 +569,11 @@ export class FileParseValidateService { return specimenIds; } - async formulateObservationFile(observationData: any, fileName: string) { + async formulateObservationFile( + observationData: any, + fileName: string, + originalFileName: string, + ) { const obsToWrite: ObservationFile[] = []; observationData.map((source) => { @@ -584,6 +588,7 @@ export class FileParseValidateService { newObs[targetKey] = source[sourceKey]; } }); + newObs["EA_FileID"] = originalFileName; obsToWrite.push(newObs); }); @@ -964,6 +969,45 @@ export class FileParseValidateService { return; } + async saveAQIInsertedElements( + fileName: string, + originalFileName: string, + visitInfo: any[], + activityInfo: any[], + specimenInfo: any[], + ) { + let importedGUIDS = {}; + + const visitGUIDS = visitInfo.map((visit) => visit.rec.fieldVisit); + const activityGUIDS = activityInfo.map( + (activity) => activity.rec.activity.id, + ); + const specimenGUIDS = specimenInfo.map( + (specimen) => specimen.rec.specimen.id, + ); + + const observationGUIDS = + await this.aqiService.getObservationsFromFile(originalFileName); + + importedGUIDS["observations"] = observationGUIDS; + importedGUIDS["specimens"] = specimenGUIDS; + importedGUIDS["activities"] = activityGUIDS; + importedGUIDS["visits"] = visitGUIDS; + + const imported_guids_data = { + file_name: fileName, + original_file_name: originalFileName, + imported_guids: importedGUIDS, + create_utc_timestamp: new Date(), + }; + + await this.prisma.$transaction(async (prisma) => { + await prisma.aqi_imported_data.create({ + data: imported_guids_data + }); + }) + } + async parseFile( file: string, fileName: string, @@ -1033,6 +1077,7 @@ export class FileParseValidateService { const ObsFilePath = await this.formulateObservationFile( allObservations, fileName, + originalFileName, ); const uniqueMinistryContacts = Array.from( @@ -1194,13 +1239,13 @@ export class FileParseValidateService { ); // Save the created GUIDs to aqi_inserted_elements - // console.log(visitInfo); - // console.log(activityInfo); - // console.log(specimenInfo); - - // console.log(visitInfo.length); - // console.log(activityInfo.length); - // console.log(specimenInfo.length); + await this.saveAQIInsertedElements( + fileName, + originalFileName, + visitInfo, + activityInfo, + specimenInfo, + ); const file_error_log_data = { file_submission_id: file_submission_id, diff --git a/migrations/sql/V1.0.0__submission_status_code.sql b/migrations/sql/V1.0.0__submission_status_code.sql index eea42528..9884996f 100644 --- a/migrations/sql/V1.0.0__submission_status_code.sql +++ b/migrations/sql/V1.0.0__submission_status_code.sql @@ -69,7 +69,7 @@ values ( (now() at time zone 'utc'), 'VMANAWAT', (now() at time zone 'utc') - ) + ), ( 'DELETED', 'Deleted', diff --git a/migrations/sql/V1.1.1__aqi_imported_data.sql b/migrations/sql/V1.1.1__aqi_imported_data.sql new file mode 100644 index 00000000..becc2cee --- /dev/null +++ b/migrations/sql/V1.1.1__aqi_imported_data.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS enmods.aqi_imported_data ( + aqi_imported_data_id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + file_name varchar(200) NULL, + original_file_name varchar(200) NULL, + imported_guids JSONB NULL, + create_utc_timestamp timestamp NOT NULL +); \ No newline at end of file