From 521731bfe1c86f559aa9d16c61b38038a66e67aa Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 1 Oct 2024 14:27:21 -0700 Subject: [PATCH 01/18] Updated validation and parsing logic + started writing deletion logic --- backend/src/aqi_api/aqi_api.service.ts | 96 +++++++++++++++++++ .../file_parse_and_validation.service.ts | 49 ++++++---- .../file_submissions.controller.ts | 6 +- .../file_submissions.module.ts | 3 +- .../file_submissions.service.ts | 9 +- frontend/src/common/api.ts | 35 +++++++ frontend/src/common/manage-files.ts | 7 ++ frontend/src/pages/Dashboard.tsx | 7 +- 8 files changed, 182 insertions(+), 30 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index b86ecd0f..672a3765 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -3,6 +3,7 @@ import axios, { AxiosError, AxiosInstance } from "axios"; import * as fs from "fs"; import FormData from "form-data"; import { PrismaService } from "nestjs-prisma"; +import path from "path"; @Injectable() export class AqiApiService { @@ -253,4 +254,99 @@ export class AqiApiService { return Array.from(map.values()); } + + getUnique(type: string, data: any[]): any[] { + switch (type) { + case "obs": + let uniqueObservations = []; + data.filter((item) => { + const pairKey = `${item.id}-${item.customId}`; + if (!uniqueObservations.includes(pairKey)) { + uniqueObservations.push(pairKey); + } + }); + return uniqueObservations; + case "specimen": + let uniqueSpecimens =[]; + data.filter((item) => { + const pairKey = `${item.id}`; + if (!uniqueSpecimens.includes(pairKey)) { + uniqueSpecimens.push(pairKey); + } + }); + return uniqueSpecimens; + case "activity": + let uniqueActivities = []; + data.filter((item) => { + const pairKey = `${item.id}-${item.customId}`; + if (!uniqueActivities.includes(pairKey)) { + uniqueActivities.push(pairKey); + } + }); + return uniqueActivities; + case "visit": + let uniqueVisits = []; + data.filter((item) => { + const pairKey = `${item.id}-${item.customId}`; + if (!uniqueVisits.includes(pairKey)) { + uniqueVisits.push(pairKey); + } + }); + return uniqueVisits; + default: + return []; + } + } + + async deleteRelatedData(file_name: string) { + let file_name_to_search = `obs-${path.parse(file_name).name}`; + let allObservations = [], + uniqueObservations = []; + let allSpecimens = [], + uniqueSpecimens = []; + let allActivities = [], + uniqueActivities = []; + let allVisits = [], + uniqueVisits = []; + try { + let observations = ( + await this.axiosInstance.get("/v2/observations?limit=1000") + ).data.domainObjects; + const relatedData = observations.filter((observation) => + observation.activity.loggerFileName.includes(file_name_to_search), + ); + + relatedData.forEach((observation) => { + allObservations.push({ + id: observation.id, + customID: observation.customId, + }); + allSpecimens.push({ + id: observation.specimen.id, + customID: observation.specimen.name, + }); + allActivities.push({ + id: observation.activity.id, + customID: observation.activity.customId, + }); + allVisits.push({ + id: observation.fieldVisit.id, + startTime: observation.fieldVisit.startTime, + }); + }); + + uniqueObservations = this.getUnique("obs", allObservations); + uniqueSpecimens = this.getUnique("specimen", allSpecimens); + uniqueActivities = this.getUnique("activity", allActivities); + uniqueVisits = this.getUnique("visit", allVisits); + + console.log(uniqueObservations.length) + console.log(uniqueSpecimens.length) + console.log(uniqueActivities.length) + console.log(uniqueVisits.length) + + } catch (err) { + console.error(`API call to fetch Observations failed: `, err); + } + } } diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index e5d34ad8..a8366d42 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -369,7 +369,7 @@ export class FileParseValidateService { Object.assign(currentVisitAndLoc, { fieldVisit: await this.aqiService.fieldVisits(postData), }); - visitAndLocId.push({ rec: currentVisitAndLoc, count: row.count }); + visitAndLocId.push({ rec: currentVisitAndLoc, count: row.count, positions: row.positions }); } return visitAndLocId; @@ -460,7 +460,7 @@ export class FileParseValidateService { startTime: row.rec.ObservedDateTime, }, }); - activityId.push({ rec: currentActivity, count: row.count }); + activityId.push({ rec: currentActivity, count: row.count, positions: row.positions }); } return activityId; } @@ -584,19 +584,32 @@ export class FileParseValidateService { } getUniqueWithCounts(data: any[]) { - const map = new Map(); + const map = new Map(); + + data.forEach((obj, index) => { + const key = JSON.stringify(obj); + if (map.has(key)) { + const entry = map.get(key)!; + entry.count++; + entry.positions.push(index); + } else { + map.set(key, { rec: obj, count: 1, positions: [index] }); + } + }) + const dupeCount = Array.from(map.values()); + return dupeCount; + } - data.forEach((visit) => { - const key = JSON.stringify(visit); - map.set(key, (map.get(key) || 0) + 1); - }); + expandList (data: any[]) { + const expandedList: any[] = []; + + data.forEach(({rec, positions}) => { + positions.forEach(position => { + expandedList[position] = rec + }) + }) - const dupeCount = Array.from(map.entries()).map(([key, count]) => ({ - rec: JSON.parse(key), - count, - })); - - return dupeCount; + return expandedList; } async localValidation(allRecords, observaionFilePath) { @@ -1002,9 +1015,7 @@ export class FileParseValidateService { const uniqueVisitsWithCounts = this.getUniqueWithCounts(allFieldVisits); let visitInfo = await this.postFieldVisits(uniqueVisitsWithCounts); - let expandedVisitInfo = visitInfo.flatMap((visit) => - Array(visit.count).fill(visit.rec), - ); + let expandedVisitInfo = this.expandList(visitInfo) /* * Merge the expanded visitInfo with allFieldActivities @@ -1023,10 +1034,8 @@ export class FileParseValidateService { let activityInfo = await this.postFieldActivities( uniqueActivitiesWithCounts, ); - let expandedActivityInfo = activityInfo.flatMap((activity) => - Array(activity.count).fill(activity.rec), - ); - + let expandedActivityInfo = this.expandList(activityInfo) + /* * Merge the expanded activityInfo with allSpecimens * Collapse allSpecimens with a dupe count diff --git a/backend/src/file_submissions/file_submissions.controller.ts b/backend/src/file_submissions/file_submissions.controller.ts index 45559bb0..7cc68de6 100644 --- a/backend/src/file_submissions/file_submissions.controller.ts +++ b/backend/src/file_submissions/file_submissions.controller.ts @@ -79,8 +79,8 @@ export class FileSubmissionsController { return this.fileSubmissionsService.update(+id, updateFileSubmissionDto); } - @Delete(":id") - remove(@Param("id") id: string) { - return this.fileSubmissionsService.remove(+id); + @Delete(":file_name/:id") + remove(@Param("file_name") file_name: string, @Param("id") id: string) { + return this.fileSubmissionsService.remove(file_name, id); } } diff --git a/backend/src/file_submissions/file_submissions.module.ts b/backend/src/file_submissions/file_submissions.module.ts index 177951fd..e07c9079 100644 --- a/backend/src/file_submissions/file_submissions.module.ts +++ b/backend/src/file_submissions/file_submissions.module.ts @@ -3,10 +3,11 @@ import { FileSubmissionsService } from "./file_submissions.service"; import { FileSubmissionsController } from "./file_submissions.controller"; import { SanitizeService } from "src/sanitize/sanitize.service"; import { ObjectStoreModule } from "src/objectStore/objectStore.module"; +import { AqiApiService } from "src/aqi_api/aqi_api.service"; @Module({ controllers: [FileSubmissionsController], - providers: [FileSubmissionsService, SanitizeService], + providers: [FileSubmissionsService, SanitizeService, AqiApiService], exports: [FileSubmissionsService], imports: [ObjectStoreModule], }) diff --git a/backend/src/file_submissions/file_submissions.service.ts b/backend/src/file_submissions/file_submissions.service.ts index 3541dd76..2deae119 100644 --- a/backend/src/file_submissions/file_submissions.service.ts +++ b/backend/src/file_submissions/file_submissions.service.ts @@ -3,16 +3,18 @@ import { CreateFileSubmissionDto } from "./dto/create-file_submission.dto"; import { UpdateFileSubmissionDto } from "./dto/update-file_submission.dto"; import { PrismaService } from "nestjs-prisma"; import { FileResultsWithCount } from "src/interface/fileResultsWithCount"; -import { file_submission, Prisma } from "@prisma/client"; +import { Prisma } from "@prisma/client"; import { FileInfo } from "src/types/types"; import { randomUUID } from "crypto"; import { ObjectStoreService } from "src/objectStore/objectStore.service"; +import { AqiApiService } from "src/aqi_api/aqi_api.service"; @Injectable() export class FileSubmissionsService { constructor( private prisma: PrismaService, private readonly objectStore: ObjectStoreService, + private readonly aqiService: AqiApiService, ) {} async create(body: any, file: Express.Multer.File) { @@ -199,8 +201,9 @@ export class FileSubmissionsService { return `This action updates a #${id} fileSubmission`; } - remove(id: number) { - return `This action removes a #${id} fileSubmission`; + async remove(file_name: string, id: string) { + await this.aqiService.deleteRelatedData(file_name) + return false } async getFromS3(fileName: string) { diff --git a/frontend/src/common/api.ts b/frontend/src/common/api.ts index d68f3508..f6d92faf 100644 --- a/frontend/src/common/api.ts +++ b/frontend/src/common/api.ts @@ -142,6 +142,41 @@ export const put = ( }); }; +export const deleteMethod = ( + parameters: ApiRequestParameters, + headers?: {}, +): Promise => { + let config: AxiosRequestConfig = { headers: headers }; + return new Promise((resolve, reject) => { + const { url, requiresAuthentication, params } = parameters; + + if (requiresAuthentication) { + axios.defaults.headers.common["Authorization"] = + `Bearer ${localStorage.getItem(AUTH_TOKEN)}`; + } + + if (params) { + config.params = params; + } + + axios + .delete(url, config) + .then((response: AxiosResponse) => { + const { data, status } = response; + + if (status === STATUS_CODES.Unauthorized) { + window.location = KEYCLOAK_URL; + } + + resolve(data as T); + }) + .catch((error: AxiosError) => { + console.log(error.message); + reject(error); + }); + }); +}; + export const generateApiParameters = ( url: string, params?: T, diff --git a/frontend/src/common/manage-files.ts b/frontend/src/common/manage-files.ts index 6c2d8087..44c3c6f3 100644 --- a/frontend/src/common/manage-files.ts +++ b/frontend/src/common/manage-files.ts @@ -45,3 +45,10 @@ export const downloadFileLogs = async (fileID: String) => { const response = await api.get(getParameters); return response; }; + +export const deleteFile = async (fileName: string, submissionId: string) => { + const url = `${config.API_BASE_URL}/v1/file_submissions/${fileName}/${submissionId}`; + const getParameters = api.generateApiParameters(url); + const response = await api.deleteMethod(getParameters); + return response; +} diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 0f98c98b..699f19ff 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -26,6 +26,7 @@ import { import { FileStatusCode } from "@/types/types"; import { getFileStatusCodes } from "@/common/manage-dropdowns"; import { + deleteFile, downloadFile, downloadFileLogs, searchFiles, @@ -511,9 +512,9 @@ async function handleMessages( document.body.removeChild(link); // Clean up } -function handleDelete(fileName: string, submission_id: string): void { - console.log(fileName); - console.log(submission_id); +async function handleDelete(fileName: string, submission_id: string): Promise { + let deleted = false; + deleted = await deleteFile(fileName, submission_id); } function getMimeType(fileName: string) { From e60bcdff5cd9f21ee1311d252e265f2696525df6 Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 1 Oct 2024 14:28:28 -0700 Subject: [PATCH 02/18] adding new status change when finished importing --- .../file_parse_and_validation.service.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index a8366d42..20c81143 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -1050,6 +1050,11 @@ export class FileParseValidateService { await this.postFieldSpecimens(uniqueSpecimensWithCounts); await this.aqiService.importObservations(ObsFilePath, ""); + + await this.fileSubmissionsService.updateFileStatus( + file_submission_id, + "SUBMITTED", + ); } } } From fcef4b544618169e33316e92b479a2c2217d18a1 Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 1 Oct 2024 16:12:43 -0700 Subject: [PATCH 03/18] small updates to error log service and delete procedure --- backend/src/aqi_api/aqi_api.service.ts | 11 ++-- .../file_error_logs.service.ts | 64 ++++++++++--------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 672a3765..48fad80e 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -260,7 +260,7 @@ export class AqiApiService { case "obs": let uniqueObservations = []; data.filter((item) => { - const pairKey = `${item.id}-${item.customId}`; + const pairKey = `${item.id}`; if (!uniqueObservations.includes(pairKey)) { uniqueObservations.push(pairKey); } @@ -278,7 +278,7 @@ export class AqiApiService { case "activity": let uniqueActivities = []; data.filter((item) => { - const pairKey = `${item.id}-${item.customId}`; + const pairKey = `${item.id}`; if (!uniqueActivities.includes(pairKey)) { uniqueActivities.push(pairKey); } @@ -287,7 +287,7 @@ export class AqiApiService { case "visit": let uniqueVisits = []; data.filter((item) => { - const pairKey = `${item.id}-${item.customId}`; + const pairKey = `${item.id}`; if (!uniqueVisits.includes(pairKey)) { uniqueVisits.push(pairKey); } @@ -340,10 +340,7 @@ export class AqiApiService { uniqueActivities = this.getUnique("activity", allActivities); uniqueVisits = this.getUnique("visit", allVisits); - console.log(uniqueObservations.length) - console.log(uniqueSpecimens.length) - console.log(uniqueActivities.length) - console.log(uniqueVisits.length) + } catch (err) { console.error(`API call to fetch Observations failed: `, err); diff --git a/backend/src/file_error_logs/file_error_logs.service.ts b/backend/src/file_error_logs/file_error_logs.service.ts index b5590aaf..e5b6cefa 100644 --- a/backend/src/file_error_logs/file_error_logs.service.ts +++ b/backend/src/file_error_logs/file_error_logs.service.ts @@ -36,42 +36,44 @@ export class FileErrorLogsService { } function formulateErrorFile(logs: any) { - let formattedMessages = ""; - const [date, timeWithZ] = new Date(logs[0].create_utc_timestamp) - .toISOString() - .split("T"); - const time = timeWithZ.replace("Z", ""); - let fileOperation = "" + if (logs){ + let formattedMessages = ""; + const [date, timeWithZ] = new Date(logs[0].create_utc_timestamp) + .toISOString() + .split("T"); + const time = timeWithZ.replace("Z", ""); + let fileOperation = "" - if (logs[0].file_operation_code === 'VALIDATE'){ - fileOperation = "True"; - }else{ - fileOperation = "False"; - } + if (logs[0].file_operation_code === 'VALIDATE'){ + fileOperation = "True"; + }else{ + fileOperation = "False"; + } - formattedMessages = - `User's Original File: ${logs[0].original_file_name}\n` + - `${date} ${time}\n\n` + - `QA Only: ${fileOperation}\n\n` + - `The following warnings/errors were found during the validation/import of the data.\n` + - `The data will need to be corrected and uploaded again for validation/import to ENMODS.\n` + - `If you have any questions, please contact the ministry contact(s) listed below.\n\n` + - `-----------------------------------------------------------------------\n` + - `Ministry Contact: ${logs[0].ministry_contact}\n` + - `-----------------------------------------------------------------------\n\n`; + formattedMessages = + `User's Original File: ${logs[0].original_file_name}\n` + + `${date} ${time}\n\n` + + `QA Only: ${fileOperation}\n\n` + + `The following warnings/errors were found during the validation/import of the data.\n` + + `The data will need to be corrected and uploaded again for validation/import to ENMODS.\n` + + `If you have any questions, please contact the ministry contact(s) listed below.\n\n` + + `-----------------------------------------------------------------------\n` + + `Ministry Contact: ${logs[0].ministry_contact}\n` + + `-----------------------------------------------------------------------\n\n`; - logs[0].error_log.forEach((log) => { - const rowNum = log.rowNum; + logs[0].error_log.forEach((log) => { + const rowNum = log.rowNum; - for (const [key, msg] of Object.entries(log.message)) { - formattedMessages += `${log.type}: Row ${rowNum}: ${key} - ${msg}\n`; + for (const [key, msg] of Object.entries(log.message)) { + formattedMessages += `${log.type}: Row ${rowNum}: ${key} - ${msg}\n`; + } + }); + + if (logs[0].error_log.length >= 1) { + formattedMessages += + "\nData was not updated in ENMODS due to errors found in the submission file. Please correct the data and resubmit."; } - }); - if (logs[0].error_log.length >= 1) { - formattedMessages += - "\nData was not updated in ENMODS due to errors found in the submission file. Please correct the data and resubmit."; + return formattedMessages; } - - return formattedMessages; } From 76dc5cad35946174e4527ef68b4c582db83fba86 Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:39:28 -0700 Subject: [PATCH 04/18] Finished first iteration of the delete function --- backend/src/aqi_api/aqi_api.service.ts | 113 +++++++++++++++++- .../file_error_logs.service.ts | 2 +- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 48fad80e..82718b11 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -267,7 +267,7 @@ export class AqiApiService { }); return uniqueObservations; case "specimen": - let uniqueSpecimens =[]; + let uniqueSpecimens = []; data.filter((item) => { const pairKey = `${item.id}`; if (!uniqueSpecimens.includes(pairKey)) { @@ -340,8 +340,117 @@ export class AqiApiService { uniqueActivities = this.getUnique("activity", allActivities); uniqueVisits = this.getUnique("visit", allVisits); + // Delete all the observations for the activities imported from AQI + if (uniqueObservations.length > 0) { + try { + let deletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v2/observations?specimentIds=${uniqueSpecimens}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQI_ACCESS_TOKEN, + }, + }, + ); + console.log("AQI OBS DELETION: " + deletion); + } catch (err) { + console.error(`API call to delete AQI observation failed: `, err); + } + } - + // Delete all the specimens for the activities imported from AQI and the PSQL db + if (uniqueSpecimens.length > 0) { + for (const specimen of uniqueSpecimens) { + try { + let aqiDeletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v1/specimens/${specimen}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQI_ACCESS_TOKEN, + }, + }, + ); + console.log("AQI SPECIMEN DELETION: " + aqiDeletion); + } catch (err) { + console.error(`API call to delete AQI specimen failed: `, err); + } + } + try { + const dbDeletion = await this.prisma.aqi_specimens.deleteMany({ + where: { + aqi_specimens_id: { + in: uniqueSpecimens, + }, + }, + }); + console.log("DB SPECIMEN DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB specimen failed: `, err); + } + } + + // Delete all the activities for the visits imported + if (uniqueActivities.length > 0) { + try { + let deletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v1/activities?ids=${uniqueActivities}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQ, + }, + }, + ); + console.log("AQI ACTIVITY DELETION: " + deletion); + } catch (err) { + console.error(`API call to delete DB activity failed: `, err); + } + + try { + const dbDeletion = await this.prisma.aqi_field_activities.deleteMany({ + where: { + aqi_field_activities_id: { + in: uniqueActivities, + }, + }, + }); + console.log("DB ACTIVITY DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB activities failed: `, err); + } + } + + // Delete all the visits for the visits imported + if (uniqueVisits.length > 0) { + try { + let deletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v1/fieldVisits?ids=${uniqueVisits}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQI_ACCESS_TOKEN, + }, + }, + ); + console.log("AQI VISIT DELETION: " + deletion); + } catch (err) { + console.error(`API call to delete AQI visit failed: `, err); + } + + try { + const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({ + where: { + aqi_field_visits_id: { + in: uniqueVisits, + }, + }, + }); + console.log("DB VISIT DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB visits failed: `, err); + } + } } catch (err) { console.error(`API call to fetch Observations failed: `, err); } diff --git a/backend/src/file_error_logs/file_error_logs.service.ts b/backend/src/file_error_logs/file_error_logs.service.ts index e5b6cefa..2efdc518 100644 --- a/backend/src/file_error_logs/file_error_logs.service.ts +++ b/backend/src/file_error_logs/file_error_logs.service.ts @@ -36,7 +36,7 @@ export class FileErrorLogsService { } function formulateErrorFile(logs: any) { - if (logs){ + if (logs.length > 0){ let formattedMessages = ""; const [date, timeWithZ] = new Date(logs[0].create_utc_timestamp) .toISOString() From 6e4bc3a5b5e5e38f46a3e25fc7086aaf5ba621ee Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:51:47 -0700 Subject: [PATCH 05/18] Changed active ind when file is deleted --- .../file_parse_and_validation.service.ts | 8 ++++---- backend/src/file_submissions/file_submissions.service.ts | 9 +++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index 20c81143..e0eefa18 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -962,10 +962,10 @@ export class FileParseValidateService { * Do the local validation for each section here - if passed then go to the API calls - else create the message/file/email for the errors */ - // await this.fileSubmissionsService.updateFileStatus( - // file_submission_id, - // "INPROGRESS", - // ); + await this.fileSubmissionsService.updateFileStatus( + file_submission_id, + "INPROGRESS", + ); const localValidationResults = await this.localValidation( allRecords, diff --git a/backend/src/file_submissions/file_submissions.service.ts b/backend/src/file_submissions/file_submissions.service.ts index 2deae119..cfaa7b11 100644 --- a/backend/src/file_submissions/file_submissions.service.ts +++ b/backend/src/file_submissions/file_submissions.service.ts @@ -115,6 +115,7 @@ export class FileSubmissionsService { submitter_user_id: {}, submitter_agency_name: {}, submission_status_code: {}, + active_ind: true }; if (body.fileName) { @@ -203,6 +204,14 @@ export class FileSubmissionsService { async remove(file_name: string, id: string) { await this.aqiService.deleteRelatedData(file_name) + await this.prisma.file_submission.update({ + where:{ + submission_id: id, + }, + data:{ + active_ind: false, + } + }) return false } From 411b0a7e61ccfef5abbd4bf1b64c88989f37b377 Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:44:41 -0700 Subject: [PATCH 06/18] Added some logging and waits --- backend/src/aqi_api/aqi_api.service.ts | 4 ++-- backend/src/cron-job/cron-job.service.ts | 8 +++++++- .../file_parse_and_validation.service.ts | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 82718b11..05bc784b 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -125,7 +125,7 @@ export class AqiApiService { }, }); - await this.wait(10); + await this.wait(15); const obsResultResponse = await axios.get( `${process.env.AQI_BASE_URL}/v2/observationimports/${response.data.id}/result`, @@ -425,7 +425,7 @@ export class AqiApiService { if (uniqueVisits.length > 0) { try { let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v1/fieldVisits?ids=${uniqueVisits}`, + `${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${uniqueVisits}`, { headers: { Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, diff --git a/backend/src/cron-job/cron-job.service.ts b/backend/src/cron-job/cron-job.service.ts index ceb9c2db..162cc09e 100644 --- a/backend/src/cron-job/cron-job.service.ts +++ b/backend/src/cron-job/cron-job.service.ts @@ -17,6 +17,9 @@ export class CronJobService { private dataPullDownComplete: boolean = false; + private wait = (seconds: number) => + new Promise((resolve) => setTimeout(resolve, seconds * 1000)); + constructor( private prisma: PrismaService, private readonly fileParser: FileParseValidateService, @@ -451,7 +454,10 @@ export class CronJobService { file.original_file_name, file.submission_id, file.file_operation_code, - ); + ); + this.logger.log(`SENT FILE: ${file.file_name}`); + await this.wait(10) + this.logger.log(`WAITING FOR PREVIOUS FILE`) } this.dataPullDownComplete = false; return; diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index e0eefa18..1f972676 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -845,7 +845,7 @@ export class FileParseValidateService { record.FieldVisitStartTime, ]); if (visitExists) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in AQI Field Visits}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Visit": "Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in AQI Field Visits"}}`; errorLogs.push(JSON.parse(errorLog)); } @@ -855,7 +855,7 @@ export class FileParseValidateService { [record.ActivityName, record.FieldVisitStartTime, record.LocationID], ); if (activityExists) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in AQI Activities}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Activity": "Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in AQI Activities"}}`; errorLogs.push(JSON.parse(errorLog)); } @@ -867,7 +867,7 @@ export class FileParseValidateService { record.LocationID, ]); if (specimenExists) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in AQI Specimen}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Specimen": "Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in AQI Specimen"}}`; errorLogs.push(JSON.parse(errorLog)); } } From d975fd6326ee7150ef36d20ad8fc34df61a0623b Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:09:41 -0700 Subject: [PATCH 07/18] Success file error log download --- .../file_error_logs.service.ts | 29 ++++++++++++++++++- .../file_parse_and_validation.service.ts | 15 ++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/src/file_error_logs/file_error_logs.service.ts b/backend/src/file_error_logs/file_error_logs.service.ts index 2efdc518..841f14a3 100644 --- a/backend/src/file_error_logs/file_error_logs.service.ts +++ b/backend/src/file_error_logs/file_error_logs.service.ts @@ -36,7 +36,7 @@ export class FileErrorLogsService { } function formulateErrorFile(logs: any) { - if (logs.length > 0){ + if (logs[0].error_log.length > 0){ let formattedMessages = ""; const [date, timeWithZ] = new Date(logs[0].create_utc_timestamp) .toISOString() @@ -74,6 +74,33 @@ function formulateErrorFile(logs: any) { "\nData was not updated in ENMODS due to errors found in the submission file. Please correct the data and resubmit."; } + return formattedMessages; + }else{ + const [date, timeWithZ] = new Date(logs[0].create_utc_timestamp) + .toISOString() + .split("T"); + const time = timeWithZ.replace("Z", ""); + let formattedMessages = ""; + let fileOperation = "" + + if (logs[0].file_operation_code === 'VALIDATE'){ + fileOperation = "True"; + }else{ + fileOperation = "False"; + } + formattedMessages = + `User's Original File: ${logs[0].original_file_name}\n` + + `${date} ${time}\n\n` + + `QA Only: ${fileOperation}\n\n` + + `The following warnings/errors were found during the validation/import of the data.\n` + + `The data will need to be corrected and uploaded again for validation/import to ENMODS.\n` + + `If you have any questions, please contact the ministry contact(s) listed below.\n\n` + + `-----------------------------------------------------------------------\n` + + `Ministry Contact: ${logs[0].ministry_contact}\n` + + `-----------------------------------------------------------------------\n\n` + + `No errors were found during the validation/import of the data.\n\n` + + `The file was successfully imported.\n\n`; + return formattedMessages; } } diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index 1f972676..6f93a7ac 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -1055,6 +1055,21 @@ export class FileParseValidateService { file_submission_id, "SUBMITTED", ); + + const file_error_log_data = { + file_submission_id: file_submission_id, + file_name: fileName, + original_file_name: originalFileName, + file_operation_code: file_operation_code, + ministry_contact: uniqueMinistryContacts.join(', '), + error_log: localValidationResults, + create_utc_timestamp: new Date(), + }; + + await this.prisma.file_error_logs.create({ + data: file_error_log_data, + }); + return; } } } From 1232c541371a29439c8503b01d78b55f73ea29a0 Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:54:10 -0700 Subject: [PATCH 08/18] Added delete confirmation and updates to file status when deleted --- .../file_submissions.service.ts | 29 +- frontend/src/pages/Dashboard.tsx | 298 ++++++++++-------- .../sql/V1.0.0__submission_status_code.sql | 10 + 3 files changed, 197 insertions(+), 140 deletions(-) diff --git a/backend/src/file_submissions/file_submissions.service.ts b/backend/src/file_submissions/file_submissions.service.ts index cfaa7b11..6a19a5b7 100644 --- a/backend/src/file_submissions/file_submissions.service.ts +++ b/backend/src/file_submissions/file_submissions.service.ts @@ -115,7 +115,7 @@ export class FileSubmissionsService { submitter_user_id: {}, submitter_agency_name: {}, submission_status_code: {}, - active_ind: true + active_ind: true, }; if (body.fileName) { @@ -203,16 +203,23 @@ export class FileSubmissionsService { } async remove(file_name: string, id: string) { - await this.aqiService.deleteRelatedData(file_name) - await this.prisma.file_submission.update({ - where:{ - submission_id: id, - }, - data:{ - active_ind: false, - } - }) - return false + try { + await this.aqiService.deleteRelatedData(file_name); + await this.prisma.$transaction(async (prisma) => { + const updateFileStatus = await this.prisma.file_submission.update({ + where: { + submission_id: id, + }, + data: { + submission_status_code: "DELETED", + }, + }); + }); + return true; + } catch (err) { + console.error(`Error deleting file: ${err.message}`); + return false; + } } async getFromS3(fileName: string) { diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 699f19ff..f549bfe3 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -32,106 +32,108 @@ import { searchFiles, } from "@/common/manage-files"; -const columns = [ - { - field: "file_name", - headerName: "File Name", - sortable: true, - filterable: true, - flex: 1.5, - renderCell: (params) => ( - - handleDownload(params.row.file_name, params.row.original_file_name) - } - > - {params.row.original_file_name} - - ), - }, - { - field: "submission_date", - headerName: "Submission Date", - sortable: true, - filterable: true, - flex: 2, - }, - { - field: "submitter_user_id", - headerName: "Submitter Username", - sortable: true, - filterable: true, - flex: 2, - }, - { - field: "submitter_agency_name", - headerName: "Submitter Agency", - sortable: true, - filterable: true, - flex: 2, - }, - { - field: "submission_status_code", - headerName: "Status", - sortable: true, - filterable: true, - flex: 1.5, - }, - { - field: "sample_count", - headerName: "# Samples", - sortable: true, - filterable: true, - flex: 1, - }, - { - field: "results_count", - headerName: "# Results", - sortable: true, - filterable: true, - flex: 1, - }, - { - field: "delete", - headerName: "Delete", - flex: 0.75, - renderCell: (params) => ( - - handleDelete(params.row.file_name, params.row.submission_id) - } - > - - - ), - }, - { - field: "messages", - headerName: "Messages", - flex: 1, - renderCell: (params) => ( - - handleMessages( - params.row.submission_id, - params.row.original_file_name, - ) - } - > - - - ), - }, -]; - export default function Dashboard() { + const { open, currentItem, handleOpen, handleClose } = useHandleOpen(); + + const columns = [ + { + field: "file_name", + headerName: "File Name", + sortable: true, + filterable: true, + flex: 1.5, + renderCell: (params) => ( + + handleDownload(params.row.file_name, params.row.original_file_name) + } + > + {params.row.original_file_name} + + ), + }, + { + field: "submission_date", + headerName: "Submission Date", + sortable: true, + filterable: true, + flex: 2, + }, + { + field: "submitter_user_id", + headerName: "Submitter Username", + sortable: true, + filterable: true, + flex: 2, + }, + { + field: "submitter_agency_name", + headerName: "Submitter Agency", + sortable: true, + filterable: true, + flex: 2, + }, + { + field: "submission_status_code", + headerName: "Status", + sortable: true, + filterable: true, + flex: 1.5, + }, + { + field: "sample_count", + headerName: "# Samples", + sortable: true, + filterable: true, + flex: 1, + }, + { + field: "results_count", + headerName: "# Results", + sortable: true, + filterable: true, + flex: 1, + }, + { + field: "delete", + headerName: "Delete", + flex: 0.75, + renderCell: (params) => ( + + handleOpen(params.row.file_name, params.row.submission_id) + } + > + + + ), + }, + { + field: "messages", + headerName: "Messages", + flex: 1, + renderCell: (params) => ( + + handleMessages( + params.row.submission_id, + params.row.original_file_name, + ) + } + > + + + ), + }, + ]; + const [formData, setFormData] = useState({ fileName: "", submissionDateTo: "", @@ -169,6 +171,7 @@ export default function Dashboard() { }; const handleSearch = async (event) => { + console.log('here2') if (event != null) { event.preventDefault(); setPaginationModel({ page: 0, pageSize: 10 }); @@ -217,10 +220,16 @@ export default function Dashboard() { handleFormInputChange("submitterAgency", event); }; + const handleCloseAndSubmit = async () => { + console.log('here') + handleClose(); + await handleSearch(undefined); + } + useEffect(() => { async function fetchFileStatusCodes() { - await getFileStatusCodes().then((response) => { - const newSubmissionCodes = submissionStatusCodes.items; + await getFileStatusCodes().then((response: any) => { + const newSubmissionCodes: any = submissionStatusCodes.items; Object.keys(response).map((key) => { newSubmissionCodes[key] = response[key]; }); @@ -239,12 +248,6 @@ export default function Dashboard() { } }, [paginationModel]); - const [selectedRow, setSelectedRow] = useState(null); - - const handleClose = () => { - setSelectedRow(null); - }; - return ( <>
{submissionStatusCodes - ? submissionStatusCodes.items.map((option) => ( + ? submissionStatusCodes.items.map((option: any) => ( setSelectedRow(params.row)} sx={{ width: "1400px", height: `${paginationModel.pageSize * 100}` }} /> - - Row Details - - - - {selectedRow && - Object.entries(selectedRow).map(([key, value]) => ( - - {key} - {value} - - ))} - -
+
+ +
+ + Delete File + + + Are you sure you want to delete{" "} + {currentItem ? currentItem.file_name : ""} ? + - - + @@ -475,12 +496,28 @@ export default function Dashboard() { ); } +function useHandleOpen() { + const [open, setOpen] = useState(false); + const [currentItem, setCurrentItem] = useState({}); + + const handleOpen = (file_name: string, submission_id: string) => { + setCurrentItem({ file_name, submission_id }); + setOpen(true); + }; + const handleClose = () => { + setOpen(false); + setCurrentItem(null); + }; + + return { open, currentItem, handleOpen, handleClose }; +} + async function handleDownload( fileName: string, originalFileName: string, ): Promise { const fileMimeType = getMimeType(fileName); - await downloadFile(fileName).then((response) => { + await downloadFile(fileName).then((response: any) => { const fileBuffer = new Uint8Array(response.data); const blob = new Blob([fileBuffer], { type: fileMimeType }); const link = document.createElement("a"); @@ -502,7 +539,7 @@ async function handleMessages( ? fileNameParts.slice(0, -1).join(".") : original_file_name; - const errorMessages = await downloadFileLogs(submission_id); + const errorMessages: any = await downloadFileLogs(submission_id); const blob = new Blob([errorMessages], { type: "text/plain" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); @@ -512,9 +549,12 @@ async function handleMessages( document.body.removeChild(link); // Clean up } -async function handleDelete(fileName: string, submission_id: string): Promise { +async function handleDelete( + fileName: string, + submission_id: string, +): Promise { let deleted = false; - deleted = await deleteFile(fileName, submission_id); + await deleteFile(fileName, submission_id); } function getMimeType(fileName: string) { diff --git a/migrations/sql/V1.0.0__submission_status_code.sql b/migrations/sql/V1.0.0__submission_status_code.sql index 7f2918ed..eea42528 100644 --- a/migrations/sql/V1.0.0__submission_status_code.sql +++ b/migrations/sql/V1.0.0__submission_status_code.sql @@ -69,4 +69,14 @@ values ( (now() at time zone 'utc'), 'VMANAWAT', (now() at time zone 'utc') + ) + ( + 'DELETED', + 'Deleted', + 25, + true, + 'VMANAWAT', + (now() at time zone 'utc'), + 'VMANAWAT', + (now() at time zone 'utc') ); \ No newline at end of file From 59e24f2f848b8773bbfa895be1a285d53c0a7f55 Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:17:46 -0700 Subject: [PATCH 09/18] Updates to corn jobs to ensure it waits. Added initial processing in case the data need to be sent for an update instead of an insertion --- backend/src/aqi_api/aqi_api.service.ts | 37 +++- backend/src/cron-job/cron-job.service.ts | 43 ++-- .../file_parse_and_validation.service.ts | 184 ++++++++++++++---- 3 files changed, 195 insertions(+), 69 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 05bc784b..aea0fd07 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -10,9 +10,6 @@ export class AqiApiService { private readonly logger = new Logger(AqiApiService.name); private axiosInstance: AxiosInstance; - private wait = (seconds: number) => - new Promise((resolve) => setTimeout(resolve, seconds * 1000)); - constructor(private prisma: PrismaService) { this.axiosInstance = axios.create({ baseURL: process.env.AQI_BASE_URL, @@ -37,6 +34,20 @@ export class AqiApiService { } } + async putFieldVisits(GUID: string, body: any) { + console.log(body) + try{ + const response = await this.axiosInstance.put(`/v1/fieldvisits/${GUID}`, body); + this.logger.log(`API call to Field Visits succeeded: ${response.status}`); + return response.data.id; + }catch (err){ + console.error( + "API CALL TO PUT Field Visits failed: ", + err.response.data.message, + ); + } + } + async fieldActivities(body: any) { try { const response = await this.axiosInstance.post("/v1/activities", body); @@ -109,6 +120,8 @@ export class AqiApiService { const obsStatus = await this.getObservationsStatusResult(statusURL); const errorMessages = this.parseObsResultResponse(obsStatus); + console.log(obsStatus); + console.log(errorMessages); return errorMessages; } } catch (err) { @@ -117,6 +130,13 @@ export class AqiApiService { } async getObservationsStatusResult(statusURL: string) { + const wait = async (ms: number) => { + const seconds = ms / 1000; + for (let i = 1; i <= seconds; i++) { + await new Promise((resolve) => setTimeout(resolve, ms)); // wait 1 second + } + }; + try { const response = await axios.get(statusURL, { headers: { @@ -125,7 +145,7 @@ export class AqiApiService { }, }); - await this.wait(15); + await wait(5000); const obsResultResponse = await axios.get( `${process.env.AQI_BASE_URL}/v2/observationimports/${response.data.id}/result`, @@ -200,6 +220,7 @@ export class AqiApiService { aqi_field_visit_start_time: queryParam[1], }, }); + return result[0].aqi_field_visits_id; } catch (err) { console.error(`API CALL TO ${dbTable} failed: `, err); } @@ -212,6 +233,7 @@ export class AqiApiService { aqi_location_custom_id: queryParam[2], }, }); + return result[0].aqi_field_activities_id; } catch (err) { console.error(`API CALL TO ${dbTable} failed: `, err); } @@ -225,16 +247,11 @@ export class AqiApiService { aqi_location_custom_id: queryParam[3], }, }); + return result[0].aqi_specimens_id; } catch (err) { console.error(`API CALL TO ${dbTable} failed: `, err); } } - - if (result.length > 0) { - return true; - } else { - return false; - } } mergeErrorMessages(localErrors: any[], remoteErrors: any[]) { diff --git a/backend/src/cron-job/cron-job.service.ts b/backend/src/cron-job/cron-job.service.ts index 162cc09e..bf0052b2 100644 --- a/backend/src/cron-job/cron-job.service.ts +++ b/backend/src/cron-job/cron-job.service.ts @@ -16,10 +16,6 @@ export class CronJobService { private tableModels; private dataPullDownComplete: boolean = false; - - private wait = (seconds: number) => - new Promise((resolve) => setTimeout(resolve, seconds * 1000)); - constructor( private prisma: PrismaService, private readonly fileParser: FileParseValidateService, @@ -445,22 +441,31 @@ export class CronJobService { console.log("************** NO FILES TO VALIDATE **************"); return; } else { - for (const file of filesToValidate) { - const fileBinary = await this.objectStore.getFileData(file.file_name); + this.processFiles(filesToValidate).then(() => { + this.logger.log("All files processed."); + }); + } + } - this.fileParser.parseFile( - fileBinary, - file.file_name, - file.original_file_name, - file.submission_id, - file.file_operation_code, - ); - this.logger.log(`SENT FILE: ${file.file_name}`); - await this.wait(10) - this.logger.log(`WAITING FOR PREVIOUS FILE`) - } - this.dataPullDownComplete = false; - return; + async processFiles(files) { + const wait = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); + for (const file of files) { + const fileBinary = await this.objectStore.getFileData(file.file_name); + this.logger.log(`SENT FILE: ${file.file_name}`); + + await this.fileParser.parseFile( + fileBinary, + file.file_name, + file.original_file_name, + file.submission_id, + file.file_operation_code, + ); + + this.logger.log(`WAITING FOR PREVIOUS FILE`); + this.logger.log("GOING TO NEXT FILE"); } + this.dataPullDownComplete = false; + return; } } diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index 6f93a7ac..53a7fdea 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -369,7 +369,11 @@ export class FileParseValidateService { Object.assign(currentVisitAndLoc, { fieldVisit: await this.aqiService.fieldVisits(postData), }); - visitAndLocId.push({ rec: currentVisitAndLoc, count: row.count, positions: row.positions }); + visitAndLocId.push({ + rec: currentVisitAndLoc, + count: row.count, + positions: row.positions, + }); } return visitAndLocId; @@ -460,12 +464,17 @@ export class FileParseValidateService { startTime: row.rec.ObservedDateTime, }, }); - activityId.push({ rec: currentActivity, count: row.count, positions: row.positions }); + activityId.push({ + rec: currentActivity, + count: row.count, + positions: row.positions, + }); } return activityId; } async postFieldSpecimens(specimenData: any) { + let specimenIds = []; for (const row of specimenData) { let postData = {}; const extendedAttribs = { extendedAttributes: [] }; @@ -529,7 +538,21 @@ export class FileParseValidateService { Object.assign(postData, extendedAttribs); await this.aqiService.fieldSpecimens(postData); + let currentSpecimen = {}; + Object.assign(currentSpecimen, { + specimen: { + id: await this.aqiService.fieldSpecimens(postData), + customId: row.rec.SpecimenName, + startTime: row.rec.ObservedDateTime, + }, + }); + specimenIds.push({ + rec: currentSpecimen, + count: row.count, + positions: row.positions, + }); } + return specimenIds; } async formulateObservationFile(observationData: any, fileName: string) { @@ -584,7 +607,10 @@ export class FileParseValidateService { } getUniqueWithCounts(data: any[]) { - const map = new Map(); + const map = new Map< + string, + { rec: any; count: number; positions: number[] } + >(); data.forEach((obj, index) => { const key = JSON.stringify(obj); @@ -595,26 +621,28 @@ export class FileParseValidateService { } else { map.set(key, { rec: obj, count: 1, positions: [index] }); } - }) + }); const dupeCount = Array.from(map.values()); return dupeCount; } - expandList (data: any[]) { + expandList(data: any[]) { const expandedList: any[] = []; - - data.forEach(({rec, positions}) => { - positions.forEach(position => { - expandedList[position] = rec - }) - }) + + data.forEach(({ rec, positions }) => { + positions.forEach((position) => { + expandedList[position] = rec; + }); + }); return expandedList; } async localValidation(allRecords, observaionFilePath) { let errorLogs = []; + let existingRecords = []; for (const [index, record] of allRecords.entries()) { + let existingGUIDS = {}; const isoDateTimeRegex = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(:(\d{2})(\.\d+)?)?(Z|([+-]\d{2}:\d{2}))?$/; @@ -845,7 +873,8 @@ export class FileParseValidateService { record.FieldVisitStartTime, ]); if (visitExists) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Visit": "Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in AQI Field Visits"}}`; + existingGUIDS["visit"] = visitExists; + let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Visit": "Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in AQI Field Visits"}}`; errorLogs.push(JSON.parse(errorLog)); } @@ -855,7 +884,8 @@ export class FileParseValidateService { [record.ActivityName, record.FieldVisitStartTime, record.LocationID], ); if (activityExists) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Activity": "Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in AQI Activities"}}`; + existingGUIDS["activity"] = activityExists; + let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Activity": "Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in AQI Activities"}}`; errorLogs.push(JSON.parse(errorLog)); } @@ -867,9 +897,11 @@ export class FileParseValidateService { record.LocationID, ]); if (specimenExists) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Specimen": "Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in AQI Specimen"}}`; + existingGUIDS["specimen"] = specimenExists; + let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Specimen": "Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in AQI Specimen"}}`; errorLogs.push(JSON.parse(errorLog)); } + existingRecords.push({ rowNum: index, existingGUIDS: existingGUIDS }); } // Do a dry run of the observations @@ -883,7 +915,36 @@ export class FileParseValidateService { observationsErrors, ); - return finalErrorLog; + return [finalErrorLog, existingRecords]; + } + + async rejectFileAndLogErrors( + file_submission_id: string, + fileName: string, + originalFileName: string, + file_operation_code: string, + ministryContacts: string, + localValidationResults: any[], + ) { + await this.fileSubmissionsService.updateFileStatus( + file_submission_id, + "REJECTED", + ); + + const file_error_log_data = { + file_submission_id: file_submission_id, + file_name: fileName, + original_file_name: originalFileName, + file_operation_code: file_operation_code, + ministry_contact: ministryContacts, + error_log: localValidationResults, + create_utc_timestamp: new Date(), + }; + + await this.prisma.file_error_logs.create({ + data: file_error_log_data, + }); + return; } async parseFile( @@ -957,7 +1018,9 @@ export class FileParseValidateService { fileName, ); - const uniqueMinistryContacts = Array.from(new Set(allRecords.map(rec => rec.MinistryContact))) + const uniqueMinistryContacts = Array.from( + new Set(allRecords.map((rec) => rec.MinistryContact)), + ); /* * Do the local validation for each section here - if passed then go to the API calls - else create the message/file/email for the errors */ @@ -972,32 +1035,62 @@ export class FileParseValidateService { ObsFilePath, ); - if (localValidationResults.some((item) => item.type === "ERROR")) { + if (localValidationResults[0].some((item) => item.type === "ERROR")) { /* * Set the file status to 'REJECTED' * Save the error logs to the database table * Send the an email to the submitter and the ministry contact that is inside the file */ - await this.fileSubmissionsService.updateFileStatus( + + await this.rejectFileAndLogErrors( file_submission_id, - "REJECTED", + fileName, + originalFileName, + file_operation_code, + uniqueMinistryContacts.join(", "), + localValidationResults, + ); + return; + } else if ( + localValidationResults[0].some((item) => item.type === "WARN") + ) { + const { + existingVisitGUIDS, + existingActivityGUIDS, + existingSpecimenGUIDS, + } = localValidationResults[1].reduce( + (acc, { existingGUIDS }) => { + acc.existingVisitGUIDS.push(existingGUIDS.visit); + acc.existingActivityGUIDS.push(existingGUIDS.activity); + acc.existingSpecimenGUIDS.push(existingGUIDS.specimen); + return acc; + }, + { + existingVisitGUIDS: [] as string[], + existingActivityGUIDS: [] as string[], + existingSpecimenGUIDS: [] as string[], + }, ); - const file_error_log_data = { - file_submission_id: file_submission_id, - file_name: fileName, - original_file_name: originalFileName, - file_operation_code: file_operation_code, - ministry_contact: uniqueMinistryContacts.join(', '), - error_log: localValidationResults, - create_utc_timestamp: new Date(), - }; - - await this.prisma.file_error_logs.create({ - data: file_error_log_data, + const allVisitsWithGUIDS = allFieldVisits.map((visit, index) => { + return { + id: existingVisitGUIDS[index], + ...visit, + }; }); - return; - } else if (!(await localValidationResults).includes("ERROR")) { + + const uniqueVisitsWithIDsAndCounts = + this.getUniqueWithCounts(allVisitsWithGUIDS); + + for (const fieldVisit of uniqueVisitsWithIDsAndCounts) { + const GUID = fieldVisit.rec.id + delete fieldVisit.rec.id; + await this.aqiService.putFieldVisits(GUID, fieldVisit.rec) + } + } else if ( + !(await localValidationResults[0]).includes("ERROR") && + !(await localValidationResults[0]).includes("WARN") + ) { await this.fileSubmissionsService.updateFileStatus( file_submission_id, "VALIDATED", @@ -1015,7 +1108,7 @@ export class FileParseValidateService { const uniqueVisitsWithCounts = this.getUniqueWithCounts(allFieldVisits); let visitInfo = await this.postFieldVisits(uniqueVisitsWithCounts); - let expandedVisitInfo = this.expandList(visitInfo) + let expandedVisitInfo = this.expandList(visitInfo); /* * Merge the expanded visitInfo with allFieldActivities @@ -1034,8 +1127,8 @@ export class FileParseValidateService { let activityInfo = await this.postFieldActivities( uniqueActivitiesWithCounts, ); - let expandedActivityInfo = this.expandList(activityInfo) - + let expandedActivityInfo = this.expandList(activityInfo); + /* * Merge the expanded activityInfo with allSpecimens * Collapse allSpecimens with a dupe count @@ -1047,25 +1140,36 @@ export class FileParseValidateService { }); const uniqueSpecimensWithCounts = this.getUniqueWithCounts(allSpecimens); - await this.postFieldSpecimens(uniqueSpecimensWithCounts); + let specimenInfo = await this.postFieldSpecimens( + uniqueSpecimensWithCounts, + ); - await this.aqiService.importObservations(ObsFilePath, ""); + await this.aqiService.importObservations(ObsFilePath, "import"); await this.fileSubmissionsService.updateFileStatus( file_submission_id, "SUBMITTED", ); + // Save the created GUIDs to aqi_inserted_elements + console.log(visitInfo); + console.log(activityInfo); + console.log(specimenInfo); + + console.log(visitInfo.length); + console.log(activityInfo.length); + console.log(specimenInfo.length); + const file_error_log_data = { file_submission_id: file_submission_id, file_name: fileName, original_file_name: originalFileName, file_operation_code: file_operation_code, - ministry_contact: uniqueMinistryContacts.join(', '), + ministry_contact: uniqueMinistryContacts.join(", "), error_log: localValidationResults, create_utc_timestamp: new Date(), }; - + await this.prisma.file_error_logs.create({ data: file_error_log_data, }); From 2aaf0c71e1d587f63187c3f74c49e889b6e49fac Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:47:30 -0700 Subject: [PATCH 10/18] Implementing PUT methods for file components in case of a partial delete/partial upload --- backend/src/aqi_api/aqi_api.service.ts | 62 +++++-- .../file_parse_and_validation.service.ts | 168 +++++++++++------- .../file_submissions.service.ts | 16 +- 3 files changed, 163 insertions(+), 83 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index aea0fd07..6e4d73e7 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -24,21 +24,20 @@ export class AqiApiService { async fieldVisits(body: any) { try { const response = await this.axiosInstance.post("/v1/fieldvisits", body); - this.logger.log(`API call to Field Visits succeeded: ${response.status}`); + this.logger.log(`API call to POST Field Visits succeeded: ${response.status}`); return response.data.id; } catch (err) { console.error( - "API CALL TO Field Visits failed: ", + "API CALL TO POST Field Visits failed: ", err.response.data.message, ); } } async putFieldVisits(GUID: string, body: any) { - console.log(body) try{ const response = await this.axiosInstance.put(`/v1/fieldvisits/${GUID}`, body); - this.logger.log(`API call to Field Visits succeeded: ${response.status}`); + this.logger.log(`API call to PUT Field Visits succeeded: ${response.status}`); return response.data.id; }catch (err){ console.error( @@ -51,11 +50,24 @@ export class AqiApiService { async fieldActivities(body: any) { try { const response = await this.axiosInstance.post("/v1/activities", body); - this.logger.log(`API call to Activities succeeded: ${response.status}`); + this.logger.log(`API call to POST Activities succeeded: ${response.status}`); return response.data.id; } catch (err) { console.error( - "API CALL TO Activities failed: ", + "API CALL TO POST Activities failed: ", + err.response.data.message, + ); + } + } + + async putFieldActivities(GUID: string, body: any) { + try{ + const response = await this.axiosInstance.put(`/v1/activities/${GUID}`, body); + this.logger.log(`API call to PUT Field Activities succeeded: ${response.status}`); + return response.data.id; + }catch (err){ + console.error( + "API CALL TO PUT Field Activities failed: ", err.response.data.message, ); } @@ -64,11 +76,24 @@ export class AqiApiService { async fieldSpecimens(body: any) { try { const response = await this.axiosInstance.post("/v1/specimens", body); - this.logger.log(`API call to Specimens succeeded: ${response.status}`); + this.logger.log(`API call to POST Specimens succeeded: ${response.status}`); return response.data.id; } catch (err) { console.error( - "API CALL TO Specimens failed: ", + "API CALL TO POST Specimens failed: ", + err.response.data.message, + ); + } + } + + async putSpecimens(GUID: string, body: any) { + try{ + const response = await this.axiosInstance.put(`/v1/specimens/${GUID}`, body); + this.logger.log(`API call to PUT Specimens succeeded: ${response.status}`); + return response.data.id; + }catch (err){ + console.error( + "API CALL TO PUT Specimens failed: ", err.response.data.message, ); } @@ -120,8 +145,7 @@ export class AqiApiService { const obsStatus = await this.getObservationsStatusResult(statusURL); const errorMessages = this.parseObsResultResponse(obsStatus); - console.log(obsStatus); - console.log(errorMessages); + return errorMessages; } } catch (err) { @@ -220,7 +244,11 @@ export class AqiApiService { aqi_field_visit_start_time: queryParam[1], }, }); - return result[0].aqi_field_visits_id; + if (result.length > 0){ + return result[0].aqi_field_visits_id; + } else { + return null; + } } catch (err) { console.error(`API CALL TO ${dbTable} failed: `, err); } @@ -233,7 +261,11 @@ export class AqiApiService { aqi_location_custom_id: queryParam[2], }, }); - return result[0].aqi_field_activities_id; + if (result.length > 0) { + return result[0].aqi_field_activities_id; + } else { + return null; + } } catch (err) { console.error(`API CALL TO ${dbTable} failed: `, err); } @@ -247,7 +279,11 @@ export class AqiApiService { aqi_location_custom_id: queryParam[3], }, }); - return result[0].aqi_specimens_id; + if (result.length > 0) { + return result[0].aqi_specimens_id; + } else { + return null; + } } catch (err) { console.error(`API CALL TO ${dbTable} failed: `, err); } diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index 53a7fdea..a7be7801 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -312,7 +312,7 @@ export class FileParseValidateService { } } - async postFieldVisits(visitData: any) { + async fieldVisitJson(visitData: any, apiType: string) { const visitAndLocId = []; for (const row of visitData) { let postData: any = {}; @@ -360,26 +360,31 @@ export class FileParseValidateService { Object.assign(postData, { notes: row.rec.FieldVisitComments }); Object.assign(postData, { planningStatus: row.rec.PlanningStatus }); - let currentVisitAndLoc: any = {}; + if (apiType === "post") { + let currentVisitAndLoc: any = {}; - Object.assign(currentVisitAndLoc, { - samplingLocation: postData.samplingLocation, - }); + Object.assign(currentVisitAndLoc, { + samplingLocation: postData.samplingLocation, + }); - Object.assign(currentVisitAndLoc, { - fieldVisit: await this.aqiService.fieldVisits(postData), - }); - visitAndLocId.push({ - rec: currentVisitAndLoc, - count: row.count, - positions: row.positions, - }); + Object.assign(currentVisitAndLoc, { + fieldVisit: await this.aqiService.fieldVisits(postData), + }); + visitAndLocId.push({ + rec: currentVisitAndLoc, + count: row.count, + positions: row.positions, + }); + } else if (apiType === "put") { + const GUIDtoUpdate = row.rec.id; + await this.aqiService.putFieldVisits(GUIDtoUpdate, postData); + } } return visitAndLocId; } - async postFieldActivities(activityData: any) { + async fieldActivityJson(activityData: any, apiType: string) { let activityId = []; for (const row of activityData) { @@ -456,24 +461,29 @@ export class FileParseValidateService { }); Object.assign(postData, { customId: row.rec.ActivityName }); - let currentActivity = {}; - Object.assign(currentActivity, { - activity: { - id: await this.aqiService.fieldActivities(postData), - customId: row.rec.ActivityName, - startTime: row.rec.ObservedDateTime, - }, - }); - activityId.push({ - rec: currentActivity, - count: row.count, - positions: row.positions, - }); + if (apiType === "post") { + let currentActivity = {}; + Object.assign(currentActivity, { + activity: { + id: await this.aqiService.fieldActivities(postData), + customId: row.rec.ActivityName, + startTime: row.rec.ObservedDateTime, + }, + }); + activityId.push({ + rec: currentActivity, + count: row.count, + positions: row.positions, + }); + } else { + const GUIDtoUpdate = row.rec.id; + await this.aqiService.putFieldActivities(GUIDtoUpdate, postData); + } } return activityId; } - async postFieldSpecimens(specimenData: any) { + async specimensJson(specimenData: any, apiType: string) { let specimenIds = []; for (const row of specimenData) { let postData = {}; @@ -537,20 +547,24 @@ export class FileParseValidateService { Object.assign(postData, { activity: row.rec.activity }); Object.assign(postData, extendedAttribs); - await this.aqiService.fieldSpecimens(postData); - let currentSpecimen = {}; - Object.assign(currentSpecimen, { - specimen: { - id: await this.aqiService.fieldSpecimens(postData), - customId: row.rec.SpecimenName, - startTime: row.rec.ObservedDateTime, - }, - }); - specimenIds.push({ - rec: currentSpecimen, - count: row.count, - positions: row.positions, - }); + if (apiType === "post") { + let currentSpecimen = {}; + Object.assign(currentSpecimen, { + specimen: { + id: await this.aqiService.fieldSpecimens(postData), + customId: row.rec.SpecimenName, + startTime: row.rec.ObservedDateTime, + }, + }); + specimenIds.push({ + rec: currentSpecimen, + count: row.count, + positions: row.positions, + }); + } else if (apiType === "put") { + const GUIDtoUpdate = row.rec.id; + await this.aqiService.putSpecimens(GUIDtoUpdate, postData); + } } return specimenIds; } @@ -872,7 +886,7 @@ export class FileParseValidateService { record.LocationID, record.FieldVisitStartTime, ]); - if (visitExists) { + if (visitExists != null) { existingGUIDS["visit"] = visitExists; let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Visit": "Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in AQI Field Visits"}}`; errorLogs.push(JSON.parse(errorLog)); @@ -883,7 +897,7 @@ export class FileParseValidateService { "aqi_field_activities", [record.ActivityName, record.FieldVisitStartTime, record.LocationID], ); - if (activityExists) { + if (activityExists != null) { existingGUIDS["activity"] = activityExists; let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Activity": "Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in AQI Activities"}}`; errorLogs.push(JSON.parse(errorLog)); @@ -896,12 +910,15 @@ export class FileParseValidateService { record.ActivityName, record.LocationID, ]); - if (specimenExists) { + if (specimenExists != null) { existingGUIDS["specimen"] = specimenExists; let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Specimen": "Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in AQI Specimen"}}`; errorLogs.push(JSON.parse(errorLog)); } - existingRecords.push({ rowNum: index, existingGUIDS: existingGUIDS }); + + if (Object.keys(existingGUIDS).length > 0) { + existingRecords.push({ rowNum: index, existingGUIDS: existingGUIDS }); + } } // Do a dry run of the observations @@ -1041,14 +1058,13 @@ export class FileParseValidateService { * Save the error logs to the database table * Send the an email to the submitter and the ministry contact that is inside the file */ - await this.rejectFileAndLogErrors( file_submission_id, fileName, originalFileName, file_operation_code, uniqueMinistryContacts.join(", "), - localValidationResults, + localValidationResults[0], ); return; } else if ( @@ -1081,12 +1097,33 @@ export class FileParseValidateService { const uniqueVisitsWithIDsAndCounts = this.getUniqueWithCounts(allVisitsWithGUIDS); + await this.fieldVisitJson(uniqueVisitsWithIDsAndCounts, "put"); + + const allActivitiesWithGUIDS = allFieldActivities.map( + (activity, index) => { + return { + id: existingActivityGUIDS[index], + ...activity, + }; + }, + ); - for (const fieldVisit of uniqueVisitsWithIDsAndCounts) { - const GUID = fieldVisit.rec.id - delete fieldVisit.rec.id; - await this.aqiService.putFieldVisits(GUID, fieldVisit.rec) - } + const uniqueActivitiesWithIDsAndCounts = this.getUniqueWithCounts( + allActivitiesWithGUIDS, + ); + await this.fieldActivityJson(uniqueActivitiesWithIDsAndCounts, "put"); + + const allSpecimensWithGUIDS = allSpecimens.map((specimen, index) => { + return { + id: existingSpecimenGUIDS[index], + ...specimen, + }; + }); + + const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( + allSpecimensWithGUIDS, + ); + await this.specimensJson(uniqueSpecimensWithIDsAndCounts, "put"); } else if ( !(await localValidationResults[0]).includes("ERROR") && !(await localValidationResults[0]).includes("WARN") @@ -1107,7 +1144,10 @@ export class FileParseValidateService { */ const uniqueVisitsWithCounts = this.getUniqueWithCounts(allFieldVisits); - let visitInfo = await this.postFieldVisits(uniqueVisitsWithCounts); + let visitInfo = await this.fieldVisitJson( + uniqueVisitsWithCounts, + "post", + ); let expandedVisitInfo = this.expandList(visitInfo); /* @@ -1124,8 +1164,9 @@ export class FileParseValidateService { const uniqueActivitiesWithCounts = this.getUniqueWithCounts(allFieldActivities); - let activityInfo = await this.postFieldActivities( + let activityInfo = await this.fieldActivityJson( uniqueActivitiesWithCounts, + "post", ); let expandedActivityInfo = this.expandList(activityInfo); @@ -1140,8 +1181,9 @@ export class FileParseValidateService { }); const uniqueSpecimensWithCounts = this.getUniqueWithCounts(allSpecimens); - let specimenInfo = await this.postFieldSpecimens( + let specimenInfo = await this.specimensJson( uniqueSpecimensWithCounts, + "post", ); await this.aqiService.importObservations(ObsFilePath, "import"); @@ -1152,13 +1194,13 @@ export class FileParseValidateService { ); // Save the created GUIDs to aqi_inserted_elements - console.log(visitInfo); - console.log(activityInfo); - console.log(specimenInfo); + // console.log(visitInfo); + // console.log(activityInfo); + // console.log(specimenInfo); - console.log(visitInfo.length); - console.log(activityInfo.length); - console.log(specimenInfo.length); + // console.log(visitInfo.length); + // console.log(activityInfo.length); + // console.log(specimenInfo.length); const file_error_log_data = { file_submission_id: file_submission_id, @@ -1166,7 +1208,7 @@ export class FileParseValidateService { original_file_name: originalFileName, file_operation_code: file_operation_code, ministry_contact: uniqueMinistryContacts.join(", "), - error_log: localValidationResults, + error_log: localValidationResults[0], create_utc_timestamp: new Date(), }; diff --git a/backend/src/file_submissions/file_submissions.service.ts b/backend/src/file_submissions/file_submissions.service.ts index 6a19a5b7..2f21e8c7 100644 --- a/backend/src/file_submissions/file_submissions.service.ts +++ b/backend/src/file_submissions/file_submissions.service.ts @@ -188,13 +188,15 @@ export class FileSubmissionsService { } async updateFileStatus(submission_id: string, status: string) { - await this.prisma.file_submission.update({ - where: { - submission_id: submission_id, - }, - data: { - submission_status_code: status, - }, + await this.prisma.$transaction(async (prisma) => { + const updateStatus = await this.prisma.file_submission.update({ + where: { + submission_id: submission_id, + }, + data: { + submission_status_code: status, + }, + }); }); } From 8775294644b3b6e81f3a418854ea5ff6eb5cb80b Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:21:40 -0700 Subject: [PATCH 11/18] conditional rendering of messages and delete button --- frontend/src/pages/Dashboard.tsx | 126 ++++++++++++++++++++++--------- 1 file changed, 90 insertions(+), 36 deletions(-) diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index f549bfe3..f73fd5ca 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -1,15 +1,10 @@ -import apiService from "@/service/api-service"; import Button from "@mui/material/Button"; import Dialog from "@mui/material/Dialog"; import DialogActions from "@mui/material/DialogActions"; import DialogContent from "@mui/material/DialogContent"; import DialogTitle from "@mui/material/DialogTitle"; -import Table from "@mui/material/Table"; -import TableBody from "@mui/material/TableBody"; -import TableCell from "@mui/material/TableCell"; -import TableRow from "@mui/material/TableRow"; import { DataGrid, GridToolbar } from "@mui/x-data-grid"; -import { useEffect, useState } from "react"; +import { ChangeEvent, useEffect, useState } from "react"; import { DeleteRounded, Description } from "@mui/icons-material"; import _kc from "@/keycloak"; import { @@ -23,7 +18,6 @@ import { TextField, Typography, } from "@mui/material"; -import { FileStatusCode } from "@/types/types"; import { getFileStatusCodes } from "@/common/manage-dropdowns"; import { deleteFile, @@ -42,7 +36,9 @@ export default function Dashboard() { sortable: true, filterable: true, flex: 1.5, - renderCell: (params) => ( + renderCell: (params: { + row: { file_name: string; original_file_name: string }; + }) => ( ( - - handleOpen(params.row.file_name, params.row.submission_id) - } - > - - - ), + renderCell: (params: { + row: { + file_name: string; + original_file_name: string; + submission_id: string; + submission_status_code: string; + }; + }) => { + if (params.row.submission_status_code === "SUBMITTED") { + return ( + + handleOpen( + params.row.original_file_name, + params.row.submission_id, + ) + } + > + + + ); + } else { + return ( + + handleOpen( + params.row.original_file_name, + params.row.submission_id, + ) + } + > + + + ); + } + }, }, { field: "messages", headerName: "Messages", flex: 1, - renderCell: (params) => ( - - handleMessages( - params.row.submission_id, - params.row.original_file_name, - ) - } - > - - - ), + renderCell: (params: { + row: { + file_name: string; + original_file_name: string; + submission_id: string; + submission_status_code: string; + }; + }) => { + if ( + params.row.submission_status_code === "VALIDATED" || + params.row.submission_status_code === "REJECTED" || + params.row.submission_status_code === "SUBMITTED" + ) { + return ( + + handleMessages( + params.row.submission_id, + params.row.original_file_name, + ) + } + > + + + ); + } else { + return ( + + handleMessages( + params.row.submission_id, + params.row.original_file_name, + ) + } + > + + + ); + } + }, }, ]; @@ -143,7 +198,7 @@ export default function Dashboard() { fileStatus: "", }); - const handleFormInputChange = (key, event) => { + const handleFormInputChange = (key:string, event: ChangeEvent) => { setFormData({ ...formData, [key]: event.target.value, @@ -160,7 +215,7 @@ export default function Dashboard() { pageSize: 10, }); - const handlePaginationChange = (params) => { + const handlePaginationChange = (params: {pageSize: number, page: number}) => { setTimeout(() => { if (params.pageSize != paginationModel.pageSize) { setPaginationModel({ page: 0, pageSize: params.pageSize }); @@ -171,7 +226,6 @@ export default function Dashboard() { }; const handleSearch = async (event) => { - console.log('here2') if (event != null) { event.preventDefault(); setPaginationModel({ page: 0, pageSize: 10 }); @@ -221,10 +275,10 @@ export default function Dashboard() { }; const handleCloseAndSubmit = async () => { - console.log('here') + console.log("here"); handleClose(); await handleSearch(undefined); - } + }; useEffect(() => { async function fetchFileStatusCodes() { From 49be55392391d59a2ac16183ff02e06031644280 Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:22:43 -0700 Subject: [PATCH 12/18] new db table and procedure to save imported guids in db to use for deletion --- backend/prisma/schema.prisma | 8 ++ backend/src/aqi_api/aqi_api.service.ts | 75 ++++++++++++++----- .../file_parse_and_validation.service.ts | 61 +++++++++++++-- .../sql/V1.0.0__submission_status_code.sql | 2 +- migrations/sql/V1.1.1__aqi_imported_data.sql | 7 ++ 5 files changed, 125 insertions(+), 28 deletions(-) create mode 100644 migrations/sql/V1.1.1__aqi_imported_data.sql diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index bfec1cc1..65d7c539 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -263,3 +263,11 @@ model aqi_data_classifications { update_user_id String @db.VarChar(200) update_utc_timestamp DateTime @db.Timestamp(6) } + +model aqi_imported_data { + aqi_imported_data_id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + file_name String? @db.VarChar(200) + original_file_name String? @db.VarChar(200) + imported_guids Json? + create_utc_timestamp DateTime @db.Timestamp(6) +} diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 6e4d73e7..3c393da1 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -24,7 +24,9 @@ export class AqiApiService { async fieldVisits(body: any) { try { const response = await this.axiosInstance.post("/v1/fieldvisits", body); - this.logger.log(`API call to POST Field Visits succeeded: ${response.status}`); + this.logger.log( + `API call to POST Field Visits succeeded: ${response.status}`, + ); return response.data.id; } catch (err) { console.error( @@ -35,11 +37,16 @@ export class AqiApiService { } async putFieldVisits(GUID: string, body: any) { - try{ - const response = await this.axiosInstance.put(`/v1/fieldvisits/${GUID}`, body); - this.logger.log(`API call to PUT Field Visits succeeded: ${response.status}`); + try { + const response = await this.axiosInstance.put( + `/v1/fieldvisits/${GUID}`, + body, + ); + this.logger.log( + `API call to PUT Field Visits succeeded: ${response.status}`, + ); return response.data.id; - }catch (err){ + } catch (err) { console.error( "API CALL TO PUT Field Visits failed: ", err.response.data.message, @@ -50,7 +57,9 @@ export class AqiApiService { async fieldActivities(body: any) { try { const response = await this.axiosInstance.post("/v1/activities", body); - this.logger.log(`API call to POST Activities succeeded: ${response.status}`); + this.logger.log( + `API call to POST Activities succeeded: ${response.status}`, + ); return response.data.id; } catch (err) { console.error( @@ -61,11 +70,16 @@ export class AqiApiService { } async putFieldActivities(GUID: string, body: any) { - try{ - const response = await this.axiosInstance.put(`/v1/activities/${GUID}`, body); - this.logger.log(`API call to PUT Field Activities succeeded: ${response.status}`); + try { + const response = await this.axiosInstance.put( + `/v1/activities/${GUID}`, + body, + ); + this.logger.log( + `API call to PUT Field Activities succeeded: ${response.status}`, + ); return response.data.id; - }catch (err){ + } catch (err) { console.error( "API CALL TO PUT Field Activities failed: ", err.response.data.message, @@ -76,7 +90,9 @@ export class AqiApiService { async fieldSpecimens(body: any) { try { const response = await this.axiosInstance.post("/v1/specimens", body); - this.logger.log(`API call to POST Specimens succeeded: ${response.status}`); + this.logger.log( + `API call to POST Specimens succeeded: ${response.status}`, + ); return response.data.id; } catch (err) { console.error( @@ -87,11 +103,16 @@ export class AqiApiService { } async putSpecimens(GUID: string, body: any) { - try{ - const response = await this.axiosInstance.put(`/v1/specimens/${GUID}`, body); - this.logger.log(`API call to PUT Specimens succeeded: ${response.status}`); + try { + const response = await this.axiosInstance.put( + `/v1/specimens/${GUID}`, + body, + ); + this.logger.log( + `API call to PUT Specimens succeeded: ${response.status}`, + ); return response.data.id; - }catch (err){ + } catch (err) { console.error( "API CALL TO PUT Specimens failed: ", err.response.data.message, @@ -99,6 +120,23 @@ export class AqiApiService { } } + async getObservationsFromFile(fileName: string) { + try { + let observations = ( + await this.axiosInstance.get("/v2/observations?limit=1000") + ).data.domainObjects; + + const relatedData = observations.filter((observation) => + observation.extendedAttributes + .some((attribute) => attribute.text === fileName) + ) + .map((observation) => observation.id); + return relatedData; + } catch (err) { + console.error("API CALL TO GET Observations from File failed: ", err); + } + } + async importObservations(fileName: any, method: string) { const formData = new FormData(); formData.append("file", fs.createReadStream(fileName)); @@ -145,7 +183,6 @@ export class AqiApiService { const obsStatus = await this.getObservationsStatusResult(statusURL); const errorMessages = this.parseObsResultResponse(obsStatus); - return errorMessages; } } catch (err) { @@ -169,7 +206,7 @@ export class AqiApiService { }, }); - await wait(5000); + await wait(7000); const obsResultResponse = await axios.get( `${process.env.AQI_BASE_URL}/v2/observationimports/${response.data.id}/result`, @@ -244,7 +281,7 @@ export class AqiApiService { aqi_field_visit_start_time: queryParam[1], }, }); - if (result.length > 0){ + if (result.length > 0) { return result[0].aqi_field_visits_id; } else { return null; @@ -397,7 +434,7 @@ export class AqiApiService { if (uniqueObservations.length > 0) { try { let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v2/observations?specimentIds=${uniqueSpecimens}`, + `${process.env.AQI_BASE_URL}/v2/observations?specimenIds=${uniqueSpecimens}`, { headers: { Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index a7be7801..4725da30 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -569,7 +569,11 @@ export class FileParseValidateService { return specimenIds; } - async formulateObservationFile(observationData: any, fileName: string) { + async formulateObservationFile( + observationData: any, + fileName: string, + originalFileName: string, + ) { const obsToWrite: ObservationFile[] = []; observationData.map((source) => { @@ -584,6 +588,7 @@ export class FileParseValidateService { newObs[targetKey] = source[sourceKey]; } }); + newObs["EA_FileID"] = originalFileName; obsToWrite.push(newObs); }); @@ -964,6 +969,45 @@ export class FileParseValidateService { return; } + async saveAQIInsertedElements( + fileName: string, + originalFileName: string, + visitInfo: any[], + activityInfo: any[], + specimenInfo: any[], + ) { + let importedGUIDS = {}; + + const visitGUIDS = visitInfo.map((visit) => visit.rec.fieldVisit); + const activityGUIDS = activityInfo.map( + (activity) => activity.rec.activity.id, + ); + const specimenGUIDS = specimenInfo.map( + (specimen) => specimen.rec.specimen.id, + ); + + const observationGUIDS = + await this.aqiService.getObservationsFromFile(originalFileName); + + importedGUIDS["observations"] = observationGUIDS; + importedGUIDS["specimens"] = specimenGUIDS; + importedGUIDS["activities"] = activityGUIDS; + importedGUIDS["visits"] = visitGUIDS; + + const imported_guids_data = { + file_name: fileName, + original_file_name: originalFileName, + imported_guids: importedGUIDS, + create_utc_timestamp: new Date(), + }; + + await this.prisma.$transaction(async (prisma) => { + await prisma.aqi_imported_data.create({ + data: imported_guids_data + }); + }) + } + async parseFile( file: string, fileName: string, @@ -1033,6 +1077,7 @@ export class FileParseValidateService { const ObsFilePath = await this.formulateObservationFile( allObservations, fileName, + originalFileName, ); const uniqueMinistryContacts = Array.from( @@ -1194,13 +1239,13 @@ export class FileParseValidateService { ); // Save the created GUIDs to aqi_inserted_elements - // console.log(visitInfo); - // console.log(activityInfo); - // console.log(specimenInfo); - - // console.log(visitInfo.length); - // console.log(activityInfo.length); - // console.log(specimenInfo.length); + await this.saveAQIInsertedElements( + fileName, + originalFileName, + visitInfo, + activityInfo, + specimenInfo, + ); const file_error_log_data = { file_submission_id: file_submission_id, diff --git a/migrations/sql/V1.0.0__submission_status_code.sql b/migrations/sql/V1.0.0__submission_status_code.sql index eea42528..9884996f 100644 --- a/migrations/sql/V1.0.0__submission_status_code.sql +++ b/migrations/sql/V1.0.0__submission_status_code.sql @@ -69,7 +69,7 @@ values ( (now() at time zone 'utc'), 'VMANAWAT', (now() at time zone 'utc') - ) + ), ( 'DELETED', 'Deleted', diff --git a/migrations/sql/V1.1.1__aqi_imported_data.sql b/migrations/sql/V1.1.1__aqi_imported_data.sql new file mode 100644 index 00000000..becc2cee --- /dev/null +++ b/migrations/sql/V1.1.1__aqi_imported_data.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS enmods.aqi_imported_data ( + aqi_imported_data_id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + file_name varchar(200) NULL, + original_file_name varchar(200) NULL, + imported_guids JSONB NULL, + create_utc_timestamp timestamp NOT NULL +); \ No newline at end of file From 63c2280d47201eb82dcbfe8535813328ee75b88b Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:16:06 -0700 Subject: [PATCH 13/18] Adding procedure to delete only those guids that were imported from a file --- backend/src/aqi_api/aqi_api.service.ts | 241 +++++++++++-------------- frontend/src/pages/Dashboard.tsx | 9 +- 2 files changed, 111 insertions(+), 139 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 3c393da1..139d2bbc 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -4,6 +4,7 @@ import * as fs from "fs"; import FormData from "form-data"; import { PrismaService } from "nestjs-prisma"; import path from "path"; +import { JsonValue } from "@prisma/client/runtime/library"; @Injectable() export class AqiApiService { @@ -126,11 +127,13 @@ export class AqiApiService { await this.axiosInstance.get("/v2/observations?limit=1000") ).data.domainObjects; - const relatedData = observations.filter((observation) => - observation.extendedAttributes - .some((attribute) => attribute.text === fileName) - ) - .map((observation) => observation.id); + const relatedData = observations + .filter((observation) => + observation.extendedAttributes.some( + (attribute) => attribute.text === fileName, + ), + ) + .map((observation) => observation.id); return relatedData; } catch (err) { console.error("API CALL TO GET Observations from File failed: ", err); @@ -388,53 +391,37 @@ export class AqiApiService { } } - async deleteRelatedData(file_name: string) { - let file_name_to_search = `obs-${path.parse(file_name).name}`; - let allObservations = [], - uniqueObservations = []; - let allSpecimens = [], - uniqueSpecimens = []; - let allActivities = [], - uniqueActivities = []; - let allVisits = [], - uniqueVisits = []; - try { - let observations = ( - await this.axiosInstance.get("/v2/observations?limit=1000") - ).data.domainObjects; - const relatedData = observations.filter((observation) => - observation.activity.loggerFileName.includes(file_name_to_search), - ); - - relatedData.forEach((observation) => { - allObservations.push({ - id: observation.id, - customID: observation.customId, - }); - allSpecimens.push({ - id: observation.specimen.id, - customID: observation.specimen.name, - }); - allActivities.push({ - id: observation.activity.id, - customID: observation.activity.customId, - }); - allVisits.push({ - id: observation.fieldVisit.id, - startTime: observation.fieldVisit.startTime, - }); - }); + async deleteRelatedData(fileName: string) { + const guidsToDelete: any = await this.prisma.aqi_imported_data.findMany({ + where: { + file_name: fileName, + }, + }); - uniqueObservations = this.getUnique("obs", allObservations); - uniqueSpecimens = this.getUnique("specimen", allSpecimens); - uniqueActivities = this.getUnique("activity", allActivities); - uniqueVisits = this.getUnique("visit", allVisits); + // Delete all the observations from the list of imported guids + if (guidsToDelete[0].imported_guids.observations.length > 0) { + try { + let deletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v2/observations?ids=${guidsToDelete[0].imported_guids.observations}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQI_ACCESS_TOKEN, + }, + }, + ); + console.log("AQI OBS DELETION: " + deletion.data); + } catch (err) { + console.error(`API call to delete AQI observation failed: `, err); + } + } - // Delete all the observations for the activities imported from AQI - if (uniqueObservations.length > 0) { + // Delete all the specimens for the activities imported from AQI and the PSQL db + if (guidsToDelete[0].imported_guids.specimens.length > 0) { + for (const specimen of guidsToDelete[0].imported_guids.specimens) { try { - let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v2/observations?specimenIds=${uniqueSpecimens}`, + let aqiDeletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v1/specimens/${specimen}`, { headers: { Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, @@ -442,107 +429,91 @@ export class AqiApiService { }, }, ); - console.log("AQI OBS DELETION: " + deletion); + console.log("AQI SPECIMEN DELETION: " + aqiDeletion.data); } catch (err) { - console.error(`API call to delete AQI observation failed: `, err); + console.error(`API call to delete AQI specimen failed: `, err); } } - - // Delete all the specimens for the activities imported from AQI and the PSQL db - if (uniqueSpecimens.length > 0) { - for (const specimen of uniqueSpecimens) { - try { - let aqiDeletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v1/specimens/${specimen}`, - { - headers: { - Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, - "x-api-key": process.env.AQI_ACCESS_TOKEN, - }, - }, - ); - console.log("AQI SPECIMEN DELETION: " + aqiDeletion); - } catch (err) { - console.error(`API call to delete AQI specimen failed: `, err); - } - } - try { - const dbDeletion = await this.prisma.aqi_specimens.deleteMany({ - where: { - aqi_specimens_id: { - in: uniqueSpecimens, - }, + try { + const dbDeletion = await this.prisma.aqi_specimens.deleteMany({ + where: { + aqi_specimens_id: { + in: guidsToDelete[0].imported_guids.specimens, }, - }); - console.log("DB SPECIMEN DELETION: " + dbDeletion); - } catch (err) { - console.error(`API call to delete DB specimen failed: `, err); - } + }, + }); + console.log("DB SPECIMEN DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB specimen failed: `, err); } + } - // Delete all the activities for the visits imported - if (uniqueActivities.length > 0) { - try { - let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v1/activities?ids=${uniqueActivities}`, - { - headers: { - Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, - "x-api-key": process.env.AQ, - }, + // Delete all the activities for the visits imported + if (guidsToDelete[0].imported_guids.activities.length > 0) { + try { + let deletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v1/activities?ids=${guidsToDelete[0].imported_guids.activities}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQ, }, - ); - console.log("AQI ACTIVITY DELETION: " + deletion); - } catch (err) { - console.error(`API call to delete DB activity failed: `, err); - } + }, + ); + console.log("AQI ACTIVITY DELETION: " + deletion.data); + } catch (err) { + console.error(`API call to delete DB activity failed: `, err); + } - try { - const dbDeletion = await this.prisma.aqi_field_activities.deleteMany({ - where: { - aqi_field_activities_id: { - in: uniqueActivities, - }, + try { + const dbDeletion = await this.prisma.aqi_field_activities.deleteMany({ + where: { + aqi_field_activities_id: { + in: guidsToDelete[0].imported_guids.activities, }, - }); - console.log("DB ACTIVITY DELETION: " + dbDeletion); - } catch (err) { - console.error(`API call to delete DB activities failed: `, err); - } + }, + }); + console.log("DB ACTIVITY DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB activities failed: `, err); } + } - // Delete all the visits for the visits imported - if (uniqueVisits.length > 0) { - try { - let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${uniqueVisits}`, - { - headers: { - Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, - "x-api-key": process.env.AQI_ACCESS_TOKEN, - }, + // Delete all the visits for the visits imported + if (guidsToDelete[0].imported_guids.visits.length > 0) { + try { + let deletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${guidsToDelete[0].imported_guids.visits}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQI_ACCESS_TOKEN, }, - ); - console.log("AQI VISIT DELETION: " + deletion); - } catch (err) { - console.error(`API call to delete AQI visit failed: `, err); - } + }, + ); + console.log("AQI VISIT DELETION: " + deletion.data); + } catch (err) { + console.error(`API call to delete AQI visit failed: `, err); + } - try { - const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({ - where: { - aqi_field_visits_id: { - in: uniqueVisits, - }, + try { + const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({ + where: { + aqi_field_visits_id: { + in: guidsToDelete[0].imported_guids.visits, }, - }); - console.log("DB VISIT DELETION: " + dbDeletion); - } catch (err) { - console.error(`API call to delete DB visits failed: `, err); - } + }, + }); + console.log("DB VISIT DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB visits failed: `, err); } - } catch (err) { - console.error(`API call to fetch Observations failed: `, err); } + + await this.prisma.aqi_imported_data.deleteMany({ + where: { + file_name: fileName, + }, + }); } } diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index f73fd5ca..78fcc676 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -115,6 +115,7 @@ export default function Dashboard() { handleOpen( params.row.original_file_name, params.row.submission_id, + params.row.file_name ) } > @@ -130,6 +131,7 @@ export default function Dashboard() { handleOpen( params.row.original_file_name, params.row.submission_id, + params.row.file_name ) } > @@ -275,7 +277,6 @@ export default function Dashboard() { }; const handleCloseAndSubmit = async () => { - console.log("here"); handleClose(); await handleSearch(undefined); }; @@ -514,7 +515,7 @@ export default function Dashboard() { Are you sure you want to delete{" "} - {currentItem ? currentItem.file_name : ""} ? + {currentItem ? currentItem.original_file_name : ""} ? @@ -554,8 +555,8 @@ function useHandleOpen() { const [open, setOpen] = useState(false); const [currentItem, setCurrentItem] = useState({}); - const handleOpen = (file_name: string, submission_id: string) => { - setCurrentItem({ file_name, submission_id }); + const handleOpen = (original_file_name: string, submission_id: string, file_name: string) => { + setCurrentItem({ original_file_name, submission_id, file_name }); setOpen(true); }; const handleClose = () => { From 53b392abfce2307e06f23a7f5f5faa3434b51942 Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:21:46 -0700 Subject: [PATCH 14/18] Updates to patch procedure --- .../file_parse_and_validation.service.ts | 155 ++++++++++++++---- 1 file changed, 120 insertions(+), 35 deletions(-) diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index 4725da30..0e88c5c3 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -891,7 +891,7 @@ export class FileParseValidateService { record.LocationID, record.FieldVisitStartTime, ]); - if (visitExists != null) { + if (visitExists !== null && visitExists !== undefined) { existingGUIDS["visit"] = visitExists; let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Visit": "Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in AQI Field Visits"}}`; errorLogs.push(JSON.parse(errorLog)); @@ -902,7 +902,7 @@ export class FileParseValidateService { "aqi_field_activities", [record.ActivityName, record.FieldVisitStartTime, record.LocationID], ); - if (activityExists != null) { + if (activityExists !== null && activityExists !== undefined) { existingGUIDS["activity"] = activityExists; let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Activity": "Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in AQI Activities"}}`; errorLogs.push(JSON.parse(errorLog)); @@ -915,7 +915,7 @@ export class FileParseValidateService { record.ActivityName, record.LocationID, ]); - if (specimenExists != null) { + if (specimenExists !== null && specimenExists !== undefined) { existingGUIDS["specimen"] = specimenExists; let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Specimen": "Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in AQI Specimen"}}`; errorLogs.push(JSON.parse(errorLog)); @@ -1003,9 +1003,9 @@ export class FileParseValidateService { await this.prisma.$transaction(async (prisma) => { await prisma.aqi_imported_data.create({ - data: imported_guids_data + data: imported_guids_data, }); - }) + }); } async parseFile( @@ -1115,15 +1115,29 @@ export class FileParseValidateService { } else if ( localValidationResults[0].some((item) => item.type === "WARN") ) { + let visitInfo = [], + expandedVisitInfo = []; + let activityInfo = [], + expandedActivityInfo = []; + let specimenInfo = []; + const { existingVisitGUIDS, existingActivityGUIDS, existingSpecimenGUIDS, } = localValidationResults[1].reduce( (acc, { existingGUIDS }) => { - acc.existingVisitGUIDS.push(existingGUIDS.visit); - acc.existingActivityGUIDS.push(existingGUIDS.activity); - acc.existingSpecimenGUIDS.push(existingGUIDS.specimen); + if (existingGUIDS.visit != null) { + acc.existingVisitGUIDS.push(existingGUIDS.visit); + } + + if (existingGUIDS.activity != null) { + acc.existingActivityGUIDS.push(existingGUIDS.activity); + } + + if (existingGUIDS.specimen != null) { + acc.existingSpecimenGUIDS.push(existingGUIDS.specimen); + } return acc; }, { @@ -1133,42 +1147,113 @@ export class FileParseValidateService { }, ); - const allVisitsWithGUIDS = allFieldVisits.map((visit, index) => { - return { - id: existingVisitGUIDS[index], - ...visit, - }; - }); + if (existingVisitGUIDS.length > 0) { + // Do a PUT to update the existing visit record + const allVisitsWithGUIDS = allFieldVisits.map((visit, index) => { + return { + id: existingVisitGUIDS[index], + ...visit, + }; + }); + + const uniqueVisitsWithIDsAndCounts = + this.getUniqueWithCounts(allVisitsWithGUIDS); + await this.fieldVisitJson(uniqueVisitsWithIDsAndCounts, "put"); + } else { + // Do a POST to insert a new visit record. Keep track of the newly inserted GUIDs for potential activity insertions + const uniqueVisitsWithCounts = + this.getUniqueWithCounts(allFieldVisits); + visitInfo = await this.fieldVisitJson(uniqueVisitsWithCounts, "post"); + expandedVisitInfo = this.expandList(visitInfo); + } - const uniqueVisitsWithIDsAndCounts = - this.getUniqueWithCounts(allVisitsWithGUIDS); - await this.fieldVisitJson(uniqueVisitsWithIDsAndCounts, "put"); + if (existingActivityGUIDS.length > 0) { + // Do a PUT to update the existing activity record + const allActivitiesWithGUIDS = allFieldActivities.map( + (activity, index) => { + return { + id: existingActivityGUIDS[index], + ...activity, + }; + }, + ); + + const uniqueActivitiesWithIDsAndCounts = this.getUniqueWithCounts( + allActivitiesWithGUIDS, + ); + await this.fieldActivityJson(uniqueActivitiesWithIDsAndCounts, "put"); + } else { + // Do a POST to insert a new activity record. Keep track of the newly inserted GUIDs for potential specimen insertions + allFieldActivities = allFieldActivities.map((obj2, index) => { + const obj1 = expandedVisitInfo[index]; + return { ...obj2, ...obj1 }; + }); + + const uniqueActivitiesWithCounts = + this.getUniqueWithCounts(allFieldActivities); + activityInfo = await this.fieldActivityJson( + uniqueActivitiesWithCounts, + "post", + ); + expandedActivityInfo = this.expandList(activityInfo); + } - const allActivitiesWithGUIDS = allFieldActivities.map( - (activity, index) => { + if (existingSpecimenGUIDS.length > 0) { + // Do a PUT to update the existing specimen record + const allSpecimensWithGUIDS = allSpecimens.map((specimen, index) => { return { - id: existingActivityGUIDS[index], - ...activity, + id: existingSpecimenGUIDS[index], + ...specimen, }; - }, + }); + + const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( + allSpecimensWithGUIDS, + ); + await this.specimensJson(uniqueSpecimensWithIDsAndCounts, "put"); + } else { + // Do a POST to insert a new specimen record. Keep track of the newly inserted GUIDs for potential observation insertions + allSpecimens = allSpecimens.map((obj2, index) => { + const obj1 = expandedActivityInfo[index]; + return { ...obj2, ...obj1 }; + }); + const uniqueSpecimensWithCounts = + this.getUniqueWithCounts(allSpecimens); + specimenInfo = await this.specimensJson( + uniqueSpecimensWithCounts, + "post", + ); + } + + await this.aqiService.importObservations(ObsFilePath, "import"); + + await this.fileSubmissionsService.updateFileStatus( + file_submission_id, + "SUBMITTED", ); - const uniqueActivitiesWithIDsAndCounts = this.getUniqueWithCounts( - allActivitiesWithGUIDS, + // Save the created GUIDs to aqi_inserted_elements + await this.saveAQIInsertedElements( + fileName, + originalFileName, + visitInfo, + activityInfo, + specimenInfo, ); - await this.fieldActivityJson(uniqueActivitiesWithIDsAndCounts, "put"); - const allSpecimensWithGUIDS = allSpecimens.map((specimen, index) => { - return { - id: existingSpecimenGUIDS[index], - ...specimen, - }; - }); + const file_error_log_data = { + file_submission_id: file_submission_id, + file_name: fileName, + original_file_name: originalFileName, + file_operation_code: file_operation_code, + ministry_contact: uniqueMinistryContacts.join(", "), + error_log: localValidationResults[0], + create_utc_timestamp: new Date(), + }; - const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( - allSpecimensWithGUIDS, - ); - await this.specimensJson(uniqueSpecimensWithIDsAndCounts, "put"); + await this.prisma.file_error_logs.create({ + data: file_error_log_data, + }); } else if ( !(await localValidationResults[0]).includes("ERROR") && !(await localValidationResults[0]).includes("WARN") From 64c728490fa590d94a982651d4b4276e925a00a8 Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 15 Oct 2024 10:50:11 -0700 Subject: [PATCH 15/18] Updating deletion logic, update existing record logic and warning logs logic --- backend/src/aqi_api/aqi_api.service.ts | 73 +++++++++---------- .../file_error_logs.service.ts | 3 + .../file_parse_and_validation.service.ts | 57 ++++++++++++--- 3 files changed, 85 insertions(+), 48 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 139d2bbc..7ba87c88 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -397,7 +397,7 @@ export class AqiApiService { file_name: fileName, }, }); - + // Delete all the observations from the list of imported guids if (guidsToDelete[0].imported_guids.observations.length > 0) { try { @@ -430,22 +430,21 @@ export class AqiApiService { }, ); console.log("AQI SPECIMEN DELETION: " + aqiDeletion.data); + + try { + const dbDeletion = await this.prisma.aqi_specimens.delete({ + where: { + aqi_specimens_id: specimen, + }, + }); + console.log("DB SPECIMEN DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB specimen failed: `, err); + } } catch (err) { console.error(`API call to delete AQI specimen failed: `, err); } } - try { - const dbDeletion = await this.prisma.aqi_specimens.deleteMany({ - where: { - aqi_specimens_id: { - in: guidsToDelete[0].imported_guids.specimens, - }, - }, - }); - console.log("DB SPECIMEN DELETION: " + dbDeletion); - } catch (err) { - console.error(`API call to delete DB specimen failed: `, err); - } } // Delete all the activities for the visits imported @@ -461,21 +460,21 @@ export class AqiApiService { }, ); console.log("AQI ACTIVITY DELETION: " + deletion.data); - } catch (err) { - console.error(`API call to delete DB activity failed: `, err); - } - try { - const dbDeletion = await this.prisma.aqi_field_activities.deleteMany({ - where: { - aqi_field_activities_id: { - in: guidsToDelete[0].imported_guids.activities, + try { + const dbDeletion = await this.prisma.aqi_field_activities.deleteMany({ + where: { + aqi_field_activities_id: { + in: guidsToDelete[0].imported_guids.activities, + }, }, - }, - }); - console.log("DB ACTIVITY DELETION: " + dbDeletion); + }); + console.log("DB ACTIVITY DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB activities failed: `, err); + } } catch (err) { - console.error(`API call to delete DB activities failed: `, err); + console.error(`API call to delete DB activity failed: `, err); } } @@ -492,21 +491,21 @@ export class AqiApiService { }, ); console.log("AQI VISIT DELETION: " + deletion.data); - } catch (err) { - console.error(`API call to delete AQI visit failed: `, err); - } - try { - const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({ - where: { - aqi_field_visits_id: { - in: guidsToDelete[0].imported_guids.visits, + try { + const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({ + where: { + aqi_field_visits_id: { + in: guidsToDelete[0].imported_guids.visits, + }, }, - }, - }); - console.log("DB VISIT DELETION: " + dbDeletion); + }); + console.log("DB VISIT DELETION: " + dbDeletion); + } catch (err) { + console.error(`API call to delete DB visits failed: `, err); + } } catch (err) { - console.error(`API call to delete DB visits failed: `, err); + console.error(`API call to delete AQI visit failed: `, err); } } diff --git a/backend/src/file_error_logs/file_error_logs.service.ts b/backend/src/file_error_logs/file_error_logs.service.ts index 841f14a3..70749f04 100644 --- a/backend/src/file_error_logs/file_error_logs.service.ts +++ b/backend/src/file_error_logs/file_error_logs.service.ts @@ -20,6 +20,9 @@ export class FileErrorLogsService { where: { file_submission_id: file_submission_id, }, + orderBy: { + create_utc_timestamp: "desc", + } }); const formattedMessage = formulateErrorFile(fileLogs); diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index 0e88c5c3..b82f8fe8 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -360,13 +360,12 @@ export class FileParseValidateService { Object.assign(postData, { notes: row.rec.FieldVisitComments }); Object.assign(postData, { planningStatus: row.rec.PlanningStatus }); - if (apiType === "post") { - let currentVisitAndLoc: any = {}; - - Object.assign(currentVisitAndLoc, { - samplingLocation: postData.samplingLocation, - }); + let currentVisitAndLoc: any = {}; + Object.assign(currentVisitAndLoc, { + samplingLocation: postData.samplingLocation, + }); + if (apiType === "post") { Object.assign(currentVisitAndLoc, { fieldVisit: await this.aqiService.fieldVisits(postData), }); @@ -378,6 +377,14 @@ export class FileParseValidateService { } else if (apiType === "put") { const GUIDtoUpdate = row.rec.id; await this.aqiService.putFieldVisits(GUIDtoUpdate, postData); + Object.assign(currentVisitAndLoc, { + fieldVisit: GUIDtoUpdate, + }); + visitAndLocId.push({ + rec: currentVisitAndLoc, + count: row.count, + positions: row.positions, + }); } } @@ -461,8 +468,9 @@ export class FileParseValidateService { }); Object.assign(postData, { customId: row.rec.ActivityName }); + let currentActivity: any = {}; + if (apiType === "post") { - let currentActivity = {}; Object.assign(currentActivity, { activity: { id: await this.aqiService.fieldActivities(postData), @@ -478,6 +486,18 @@ export class FileParseValidateService { } else { const GUIDtoUpdate = row.rec.id; await this.aqiService.putFieldActivities(GUIDtoUpdate, postData); + Object.assign(currentActivity, { + activity: { + id: GUIDtoUpdate, + customId: row.rec.ActivityName, + startTime: row.rec.ObservedDateTime, + }, + }); + activityId.push({ + rec: currentActivity, + count: row.count, + positions: row.positions, + }); } } return activityId; @@ -547,8 +567,9 @@ export class FileParseValidateService { Object.assign(postData, { activity: row.rec.activity }); Object.assign(postData, extendedAttribs); + let currentSpecimen: any = {}; + if (apiType === "post") { - let currentSpecimen = {}; Object.assign(currentSpecimen, { specimen: { id: await this.aqiService.fieldSpecimens(postData), @@ -564,6 +585,18 @@ export class FileParseValidateService { } else if (apiType === "put") { const GUIDtoUpdate = row.rec.id; await this.aqiService.putSpecimens(GUIDtoUpdate, postData); + Object.assign(currentSpecimen, { + specimen: { + id: GUIDtoUpdate, + customId: row.rec.SpecimenName, + startTime: row.rec.ObservedDateTime, + } + }); + specimenIds.push({ + rec: currentSpecimen, + count: row.count, + positions: row.positions, + }); } } return specimenIds; @@ -1158,7 +1191,8 @@ export class FileParseValidateService { const uniqueVisitsWithIDsAndCounts = this.getUniqueWithCounts(allVisitsWithGUIDS); - await this.fieldVisitJson(uniqueVisitsWithIDsAndCounts, "put"); + visitInfo = await this.fieldVisitJson(uniqueVisitsWithIDsAndCounts, "put"); + expandedVisitInfo = this.expandList(visitInfo); } else { // Do a POST to insert a new visit record. Keep track of the newly inserted GUIDs for potential activity insertions const uniqueVisitsWithCounts = @@ -1181,7 +1215,8 @@ export class FileParseValidateService { const uniqueActivitiesWithIDsAndCounts = this.getUniqueWithCounts( allActivitiesWithGUIDS, ); - await this.fieldActivityJson(uniqueActivitiesWithIDsAndCounts, "put"); + activityInfo = await this.fieldActivityJson(uniqueActivitiesWithIDsAndCounts, "put"); + expandedActivityInfo = this.expandList(activityInfo); } else { // Do a POST to insert a new activity record. Keep track of the newly inserted GUIDs for potential specimen insertions allFieldActivities = allFieldActivities.map((obj2, index) => { @@ -1210,7 +1245,7 @@ export class FileParseValidateService { const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( allSpecimensWithGUIDS, ); - await this.specimensJson(uniqueSpecimensWithIDsAndCounts, "put"); + specimenInfo = await this.specimensJson(uniqueSpecimensWithIDsAndCounts, "put"); } else { // Do a POST to insert a new specimen record. Keep track of the newly inserted GUIDs for potential observation insertions allSpecimens = allSpecimens.map((obj2, index) => { From 40e15a6a74538e70d919655a2ff1146ee48c1b23 Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 15 Oct 2024 11:40:01 -0700 Subject: [PATCH 16/18] updating file import metrics in db after submission --- .../file_parse_and_validation.service.ts | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index b82f8fe8..bf190100 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -590,7 +590,7 @@ export class FileParseValidateService { id: GUIDtoUpdate, customId: row.rec.SpecimenName, startTime: row.rec.ObservedDateTime, - } + }, }); specimenIds.push({ rec: currentSpecimen, @@ -1003,6 +1003,7 @@ export class FileParseValidateService { } async saveAQIInsertedElements( + file_submission_id: string, fileName: string, originalFileName: string, visitInfo: any[], @@ -1039,6 +1040,19 @@ export class FileParseValidateService { data: imported_guids_data, }); }); + + //Update the number of samples and results imported from the file + await this.prisma.$transaction(async (prisma) => { + const updateStatus = await this.prisma.file_submission.update({ + where: { + submission_id: file_submission_id, + }, + data: { + sample_count: activityGUIDS.length, + results_count: observationGUIDS.length, + }, + }); + }); } async parseFile( @@ -1191,7 +1205,10 @@ export class FileParseValidateService { const uniqueVisitsWithIDsAndCounts = this.getUniqueWithCounts(allVisitsWithGUIDS); - visitInfo = await this.fieldVisitJson(uniqueVisitsWithIDsAndCounts, "put"); + visitInfo = await this.fieldVisitJson( + uniqueVisitsWithIDsAndCounts, + "put", + ); expandedVisitInfo = this.expandList(visitInfo); } else { // Do a POST to insert a new visit record. Keep track of the newly inserted GUIDs for potential activity insertions @@ -1215,7 +1232,10 @@ export class FileParseValidateService { const uniqueActivitiesWithIDsAndCounts = this.getUniqueWithCounts( allActivitiesWithGUIDS, ); - activityInfo = await this.fieldActivityJson(uniqueActivitiesWithIDsAndCounts, "put"); + activityInfo = await this.fieldActivityJson( + uniqueActivitiesWithIDsAndCounts, + "put", + ); expandedActivityInfo = this.expandList(activityInfo); } else { // Do a POST to insert a new activity record. Keep track of the newly inserted GUIDs for potential specimen insertions @@ -1245,7 +1265,10 @@ export class FileParseValidateService { const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( allSpecimensWithGUIDS, ); - specimenInfo = await this.specimensJson(uniqueSpecimensWithIDsAndCounts, "put"); + specimenInfo = await this.specimensJson( + uniqueSpecimensWithIDsAndCounts, + "put", + ); } else { // Do a POST to insert a new specimen record. Keep track of the newly inserted GUIDs for potential observation insertions allSpecimens = allSpecimens.map((obj2, index) => { @@ -1269,6 +1292,7 @@ export class FileParseValidateService { // Save the created GUIDs to aqi_inserted_elements await this.saveAQIInsertedElements( + file_submission_id, fileName, originalFileName, visitInfo, @@ -1360,6 +1384,7 @@ export class FileParseValidateService { // Save the created GUIDs to aqi_inserted_elements await this.saveAQIInsertedElements( + file_submission_id, fileName, originalFileName, visitInfo, From d6f8f002c94d27ce311f6bb682d0f15d84746e58 Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 15 Oct 2024 15:54:52 -0700 Subject: [PATCH 17/18] adding comments and updating some case conditions to handle errors, warnings and successful imports --- .../file_parse_and_validation.service.ts | 445 +++++++++--------- 1 file changed, 235 insertions(+), 210 deletions(-) diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index bf190100..39774cd4 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -1146,6 +1146,7 @@ export class FileParseValidateService { if (localValidationResults[0].some((item) => item.type === "ERROR")) { /* + * If there are any errors then * Set the file status to 'REJECTED' * Save the error logs to the database table * Send the an email to the submitter and the ministry contact that is inside the file @@ -1159,224 +1160,156 @@ export class FileParseValidateService { localValidationResults[0], ); return; - } else if ( - localValidationResults[0].some((item) => item.type === "WARN") - ) { - let visitInfo = [], - expandedVisitInfo = []; - let activityInfo = [], - expandedActivityInfo = []; - let specimenInfo = []; - - const { - existingVisitGUIDS, - existingActivityGUIDS, - existingSpecimenGUIDS, - } = localValidationResults[1].reduce( - (acc, { existingGUIDS }) => { - if (existingGUIDS.visit != null) { - acc.existingVisitGUIDS.push(existingGUIDS.visit); - } - - if (existingGUIDS.activity != null) { - acc.existingActivityGUIDS.push(existingGUIDS.activity); - } - - if (existingGUIDS.specimen != null) { - acc.existingSpecimenGUIDS.push(existingGUIDS.specimen); - } - return acc; - }, - { - existingVisitGUIDS: [] as string[], - existingActivityGUIDS: [] as string[], - existingSpecimenGUIDS: [] as string[], - }, - ); - - if (existingVisitGUIDS.length > 0) { - // Do a PUT to update the existing visit record - const allVisitsWithGUIDS = allFieldVisits.map((visit, index) => { - return { - id: existingVisitGUIDS[index], - ...visit, - }; - }); - - const uniqueVisitsWithIDsAndCounts = - this.getUniqueWithCounts(allVisitsWithGUIDS); - visitInfo = await this.fieldVisitJson( - uniqueVisitsWithIDsAndCounts, - "put", + } else { + /* + * If there are no errors then + * Check to see if there are any WARNINGS + * If WARNINGS + * Proceed with the PATCH logic + */ + if (localValidationResults[0].some((item) => item.type === "WARN")) { + let visitInfo = [], + expandedVisitInfo = []; + let activityInfo = [], + expandedActivityInfo = []; + let specimenInfo = []; + + + // Get three seprated lists for the existing GUIDS for visits, acticities and specimens + const { + existingVisitGUIDS, + existingActivityGUIDS, + existingSpecimenGUIDS, + } = localValidationResults[1].reduce( + (acc, { existingGUIDS }) => { + if (existingGUIDS.visit != null) { + acc.existingVisitGUIDS.push(existingGUIDS.visit); + } + + if (existingGUIDS.activity != null) { + acc.existingActivityGUIDS.push(existingGUIDS.activity); + } + + if (existingGUIDS.specimen != null) { + acc.existingSpecimenGUIDS.push(existingGUIDS.specimen); + } + return acc; + }, + { + existingVisitGUIDS: [] as string[], + existingActivityGUIDS: [] as string[], + existingSpecimenGUIDS: [] as string[], + }, ); - expandedVisitInfo = this.expandList(visitInfo); - } else { - // Do a POST to insert a new visit record. Keep track of the newly inserted GUIDs for potential activity insertions - const uniqueVisitsWithCounts = - this.getUniqueWithCounts(allFieldVisits); - visitInfo = await this.fieldVisitJson(uniqueVisitsWithCounts, "post"); - expandedVisitInfo = this.expandList(visitInfo); - } - if (existingActivityGUIDS.length > 0) { - // Do a PUT to update the existing activity record - const allActivitiesWithGUIDS = allFieldActivities.map( - (activity, index) => { + // If the visit to import already exists, add the corresponding GUID to each record object + if (existingVisitGUIDS.length > 0) { + // Do a PUT to update the existing visit record + const allVisitsWithGUIDS = allFieldVisits.map((visit, index) => { return { - id: existingActivityGUIDS[index], - ...activity, + id: existingVisitGUIDS[index], + ...visit, }; - }, - ); - - const uniqueActivitiesWithIDsAndCounts = this.getUniqueWithCounts( - allActivitiesWithGUIDS, - ); - activityInfo = await this.fieldActivityJson( - uniqueActivitiesWithIDsAndCounts, - "put", - ); - expandedActivityInfo = this.expandList(activityInfo); - } else { - // Do a POST to insert a new activity record. Keep track of the newly inserted GUIDs for potential specimen insertions - allFieldActivities = allFieldActivities.map((obj2, index) => { - const obj1 = expandedVisitInfo[index]; - return { ...obj2, ...obj1 }; - }); - - const uniqueActivitiesWithCounts = - this.getUniqueWithCounts(allFieldActivities); - activityInfo = await this.fieldActivityJson( - uniqueActivitiesWithCounts, - "post", - ); - expandedActivityInfo = this.expandList(activityInfo); - } - - if (existingSpecimenGUIDS.length > 0) { - // Do a PUT to update the existing specimen record - const allSpecimensWithGUIDS = allSpecimens.map((specimen, index) => { - return { - id: existingSpecimenGUIDS[index], - ...specimen, - }; - }); - - const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( - allSpecimensWithGUIDS, - ); - specimenInfo = await this.specimensJson( - uniqueSpecimensWithIDsAndCounts, - "put", - ); - } else { - // Do a POST to insert a new specimen record. Keep track of the newly inserted GUIDs for potential observation insertions - allSpecimens = allSpecimens.map((obj2, index) => { - const obj1 = expandedActivityInfo[index]; - return { ...obj2, ...obj1 }; - }); - const uniqueSpecimensWithCounts = - this.getUniqueWithCounts(allSpecimens); - specimenInfo = await this.specimensJson( - uniqueSpecimensWithCounts, - "post", - ); - } - - await this.aqiService.importObservations(ObsFilePath, "import"); - - await this.fileSubmissionsService.updateFileStatus( - file_submission_id, - "SUBMITTED", - ); - - // Save the created GUIDs to aqi_inserted_elements - await this.saveAQIInsertedElements( - file_submission_id, - fileName, - originalFileName, - visitInfo, - activityInfo, - specimenInfo, - ); - - const file_error_log_data = { - file_submission_id: file_submission_id, - file_name: fileName, - original_file_name: originalFileName, - file_operation_code: file_operation_code, - ministry_contact: uniqueMinistryContacts.join(", "), - error_log: localValidationResults[0], - create_utc_timestamp: new Date(), - }; - - await this.prisma.file_error_logs.create({ - data: file_error_log_data, - }); - } else if ( - !(await localValidationResults[0]).includes("ERROR") && - !(await localValidationResults[0]).includes("WARN") - ) { - await this.fileSubmissionsService.updateFileStatus( - file_submission_id, - "VALIDATED", - ); + }); + + // Find the unique records with the visit GUIDS and send a PUT request with that data + const uniqueVisitsWithIDsAndCounts = + this.getUniqueWithCounts(allVisitsWithGUIDS); + visitInfo = await this.fieldVisitJson( + uniqueVisitsWithIDsAndCounts, + "put", + ); + // Expand the returned list for potential relational computation for activities + expandedVisitInfo = this.expandList(visitInfo); + } else { + // If visits don't already exist --> Do a POST to insert a new visit record. Keep track of the newly inserted GUIDs for potential activity insertions + const uniqueVisitsWithCounts = + this.getUniqueWithCounts(allFieldVisits); + visitInfo = await this.fieldVisitJson( + uniqueVisitsWithCounts, + "post", + ); + // Expand the returned list for potential relational computation for activities + expandedVisitInfo = this.expandList(visitInfo); + } - if (file_operation_code === "VALIDATE") { - return; - } else { - /* - * If the local validation passed then split the file into 4 and process with the AQI API calls - * Get unique records to prevent redundant API calls - * Post the unique records to the API - * Expand the returned list of object - this will be used for finding unique activities - */ - const uniqueVisitsWithCounts = - this.getUniqueWithCounts(allFieldVisits); - let visitInfo = await this.fieldVisitJson( - uniqueVisitsWithCounts, - "post", - ); - let expandedVisitInfo = this.expandList(visitInfo); - - /* - * Merge the expanded visitInfo with allFieldActivities - * Collapse allFieldActivities with a dupe count - * Post the unique records to the API - * Expand the returned list of object - this will be used for finding unique specimens - */ - - allFieldActivities = allFieldActivities.map((obj2, index) => { - const obj1 = expandedVisitInfo[index]; - return { ...obj2, ...obj1 }; - }); + // If the activity to import already exists, add the corresponding GUID to each record object + if (existingActivityGUIDS.length > 0) { + // Do a PUT to update the existing activity record + const allActivitiesWithGUIDS = allFieldActivities.map( + (activity, index) => { + return { + id: existingActivityGUIDS[index], + ...activity, + }; + }, + ); + + // Find the unique records with the activity GUIDS and send a PUT request with that data + const uniqueActivitiesWithIDsAndCounts = this.getUniqueWithCounts( + allActivitiesWithGUIDS, + ); + activityInfo = await this.fieldActivityJson( + uniqueActivitiesWithIDsAndCounts, + "put", + ); + // Expand the returned list for potential relational computation for specimen + expandedActivityInfo = this.expandList(activityInfo); + } else { + // If the activities don't already exist --> Do a POST to insert a new activity record. Keep track of the newly inserted GUIDs for potential specimen insertions + allFieldActivities = allFieldActivities.map((obj2, index) => { + const obj1 = expandedVisitInfo[index]; + return { ...obj2, ...obj1 }; + }); + + const uniqueActivitiesWithCounts = + this.getUniqueWithCounts(allFieldActivities); + activityInfo = await this.fieldActivityJson( + uniqueActivitiesWithCounts, + "post", + ); + // Expand the returned list for potential relational computation for specimen + expandedActivityInfo = this.expandList(activityInfo); + } - const uniqueActivitiesWithCounts = - this.getUniqueWithCounts(allFieldActivities); - let activityInfo = await this.fieldActivityJson( - uniqueActivitiesWithCounts, - "post", - ); - let expandedActivityInfo = this.expandList(activityInfo); - - /* - * Merge the expanded activityInfo with allSpecimens - * Collapse allSpecimens with a dupe count - * Post the unique records to the API - */ - allSpecimens = allSpecimens.map((obj2, index) => { - const obj1 = expandedActivityInfo[index]; - return { ...obj2, ...obj1 }; - }); - const uniqueSpecimensWithCounts = - this.getUniqueWithCounts(allSpecimens); - let specimenInfo = await this.specimensJson( - uniqueSpecimensWithCounts, - "post", - ); + // If the specimen to import already exists, add the corresponding GUID to each record object + if (existingSpecimenGUIDS.length > 0) { + // Do a PUT to update the existing specimen record + const allSpecimensWithGUIDS = allSpecimens.map( + (specimen, index) => { + return { + id: existingSpecimenGUIDS[index], + ...specimen, + }; + }, + ); + + // Find the unique records with the specimen GUIDS and send a PUT request with that data + const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( + allSpecimensWithGUIDS, + ); + specimenInfo = await this.specimensJson( + uniqueSpecimensWithIDsAndCounts, + "put", + ); + } else { + //If the specimens don't already exist --> Do a POST to insert a new specimen record. Keep track of the newly inserted GUIDs for potential observation insertions + allSpecimens = allSpecimens.map((obj2, index) => { + const obj1 = expandedActivityInfo[index]; + return { ...obj2, ...obj1 }; + }); + const uniqueSpecimensWithCounts = + this.getUniqueWithCounts(allSpecimens); + specimenInfo = await this.specimensJson( + uniqueSpecimensWithCounts, + "post", + ); + } + // Import the observations await this.aqiService.importObservations(ObsFilePath, "import"); + // Update file submission status await this.fileSubmissionsService.updateFileStatus( file_submission_id, "SUBMITTED", @@ -1392,6 +1325,7 @@ export class FileParseValidateService { specimenInfo, ); + // Create a record for the file log const file_error_log_data = { file_submission_id: file_submission_id, file_name: fileName, @@ -1405,7 +1339,98 @@ export class FileParseValidateService { await this.prisma.file_error_logs.create({ data: file_error_log_data, }); - return; + } else { + // If there are no errors or warnings + await this.fileSubmissionsService.updateFileStatus( + file_submission_id, + "VALIDATED", + ); + + if (file_operation_code === "VALIDATE") { + return; + } else { + /* + * If the local validation passed then split the file into 4 and process with the AQI API calls + * Get unique records to prevent redundant API calls + * Post the unique records to the API + * Expand the returned list of object - this will be used for finding unique activities + */ + const uniqueVisitsWithCounts = + this.getUniqueWithCounts(allFieldVisits); + let visitInfo = await this.fieldVisitJson( + uniqueVisitsWithCounts, + "post", + ); + let expandedVisitInfo = this.expandList(visitInfo); + + /* + * Merge the expanded visitInfo with allFieldActivities + * Collapse allFieldActivities with a dupe count + * Post the unique records to the API + * Expand the returned list of object - this will be used for finding unique specimens + */ + + allFieldActivities = allFieldActivities.map((obj2, index) => { + const obj1 = expandedVisitInfo[index]; + return { ...obj2, ...obj1 }; + }); + + const uniqueActivitiesWithCounts = + this.getUniqueWithCounts(allFieldActivities); + let activityInfo = await this.fieldActivityJson( + uniqueActivitiesWithCounts, + "post", + ); + let expandedActivityInfo = this.expandList(activityInfo); + + /* + * Merge the expanded activityInfo with allSpecimens + * Collapse allSpecimens with a dupe count + * Post the unique records to the API + */ + allSpecimens = allSpecimens.map((obj2, index) => { + const obj1 = expandedActivityInfo[index]; + return { ...obj2, ...obj1 }; + }); + const uniqueSpecimensWithCounts = + this.getUniqueWithCounts(allSpecimens); + let specimenInfo = await this.specimensJson( + uniqueSpecimensWithCounts, + "post", + ); + + await this.aqiService.importObservations(ObsFilePath, "import"); + + await this.fileSubmissionsService.updateFileStatus( + file_submission_id, + "SUBMITTED", + ); + + // Save the created GUIDs to aqi_inserted_elements + await this.saveAQIInsertedElements( + file_submission_id, + fileName, + originalFileName, + visitInfo, + activityInfo, + specimenInfo, + ); + + const file_error_log_data = { + file_submission_id: file_submission_id, + file_name: fileName, + original_file_name: originalFileName, + file_operation_code: file_operation_code, + ministry_contact: uniqueMinistryContacts.join(", "), + error_log: localValidationResults[0], + create_utc_timestamp: new Date(), + }; + + await this.prisma.file_error_logs.create({ + data: file_error_log_data, + }); + return; + } } } } From cbc3d923b2b5fb7c7251a3d1c044be41422b0866 Mon Sep 17 00:00:00 2001 From: vmanawat <109625428+vmanawat@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:02:00 -0700 Subject: [PATCH 18/18] updating console.error to this.logger.error --- backend/src/aqi_api/aqi_api.service.ts | 40 +++--- backend/src/cron-job/cron-job.service.ts | 4 +- .../file_submissions.service.ts | 8 +- .../src/objectStore/objectStore.service.ts | 2 +- ...E-2b6dc1ec-b903-4e5f-b219-0d07dfdfd6fb.csv | 132 ++++++++++++++++++ 5 files changed, 160 insertions(+), 26 deletions(-) create mode 100644 backend/src/tempObsFiles/obs-SUCCESS_TEST_FILE-2b6dc1ec-b903-4e5f-b219-0d07dfdfd6fb.csv diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 7ba87c88..69b31f8b 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -30,7 +30,7 @@ export class AqiApiService { ); return response.data.id; } catch (err) { - console.error( + this.logger.error( "API CALL TO POST Field Visits failed: ", err.response.data.message, ); @@ -48,7 +48,7 @@ export class AqiApiService { ); return response.data.id; } catch (err) { - console.error( + this.logger.error( "API CALL TO PUT Field Visits failed: ", err.response.data.message, ); @@ -63,7 +63,7 @@ export class AqiApiService { ); return response.data.id; } catch (err) { - console.error( + this.logger.error( "API CALL TO POST Activities failed: ", err.response.data.message, ); @@ -81,7 +81,7 @@ export class AqiApiService { ); return response.data.id; } catch (err) { - console.error( + this.logger.error( "API CALL TO PUT Field Activities failed: ", err.response.data.message, ); @@ -96,7 +96,7 @@ export class AqiApiService { ); return response.data.id; } catch (err) { - console.error( + this.logger.error( "API CALL TO POST Specimens failed: ", err.response.data.message, ); @@ -114,7 +114,7 @@ export class AqiApiService { ); return response.data.id; } catch (err) { - console.error( + this.logger.error( "API CALL TO PUT Specimens failed: ", err.response.data.message, ); @@ -136,7 +136,7 @@ export class AqiApiService { .map((observation) => observation.id); return relatedData; } catch (err) { - console.error("API CALL TO GET Observations from File failed: ", err); + this.logger.error("API CALL TO GET Observations from File failed: ", err); } } @@ -189,7 +189,7 @@ export class AqiApiService { return errorMessages; } } catch (err) { - console.error("API call to Observation Import failed: ", err); + this.logger.error("API call to Observation Import failed: ", err); } } @@ -232,7 +232,7 @@ export class AqiApiService { console.warn("409 Conflict: Continuing without failing"); return axiosError.response.data; } else { - console.error( + this.logger.error( "API CALL TO Observations Status failed: ", err.response, ); @@ -270,7 +270,7 @@ export class AqiApiService { return false; } } catch (err) { - console.error(`API CALL TO ${dbTable} failed: `, err); + this.logger.error(`API CALL TO ${dbTable} failed: `, err); } } @@ -290,7 +290,7 @@ export class AqiApiService { return null; } } catch (err) { - console.error(`API CALL TO ${dbTable} failed: `, err); + this.logger.error(`API CALL TO ${dbTable} failed: `, err); } } else if (dbTable == "aqi_field_activities") { try { @@ -307,7 +307,7 @@ export class AqiApiService { return null; } } catch (err) { - console.error(`API CALL TO ${dbTable} failed: `, err); + this.logger.error(`API CALL TO ${dbTable} failed: `, err); } } else if (dbTable == "aqi_specimens") { try { @@ -325,7 +325,7 @@ export class AqiApiService { return null; } } catch (err) { - console.error(`API CALL TO ${dbTable} failed: `, err); + this.logger.error(`API CALL TO ${dbTable} failed: `, err); } } } @@ -412,7 +412,7 @@ export class AqiApiService { ); console.log("AQI OBS DELETION: " + deletion.data); } catch (err) { - console.error(`API call to delete AQI observation failed: `, err); + this.logger.error(`API call to delete AQI observation failed: `, err); } } @@ -439,10 +439,10 @@ export class AqiApiService { }); console.log("DB SPECIMEN DELETION: " + dbDeletion); } catch (err) { - console.error(`API call to delete DB specimen failed: `, err); + this.logger.error(`API call to delete DB specimen failed: `, err); } } catch (err) { - console.error(`API call to delete AQI specimen failed: `, err); + this.logger.error(`API call to delete AQI specimen failed: `, err); } } } @@ -471,10 +471,10 @@ export class AqiApiService { }); console.log("DB ACTIVITY DELETION: " + dbDeletion); } catch (err) { - console.error(`API call to delete DB activities failed: `, err); + this.logger.error(`API call to delete DB activities failed: `, err); } } catch (err) { - console.error(`API call to delete DB activity failed: `, err); + this.logger.error(`API call to delete DB activity failed: `, err); } } @@ -502,10 +502,10 @@ export class AqiApiService { }); console.log("DB VISIT DELETION: " + dbDeletion); } catch (err) { - console.error(`API call to delete DB visits failed: `, err); + this.logger.error(`API call to delete DB visits failed: `, err); } } catch (err) { - console.error(`API call to delete AQI visit failed: `, err); + this.logger.error(`API call to delete AQI visit failed: `, err); } } diff --git a/backend/src/cron-job/cron-job.service.ts b/backend/src/cron-job/cron-job.service.ts index bf0052b2..f0d9fc1a 100644 --- a/backend/src/cron-job/cron-job.service.ts +++ b/backend/src/cron-job/cron-job.service.ts @@ -256,7 +256,7 @@ export class CronJobService { this.logger.log(`-`); return; } catch (err) { - console.error(`Error updating #### ${dbTable} #### table`, error); + this.logger.error(`Error updating #### ${dbTable} #### table`, error); } } @@ -305,7 +305,7 @@ export class CronJobService { this.dataPullDownComplete = true; } catch (error) { this.dataPullDownComplete = false; - console.error(`Error updating database for ${api.endpoint}`, error); + this.logger.error(`Error updating database for ${api.endpoint}`, error); } } diff --git a/backend/src/file_submissions/file_submissions.service.ts b/backend/src/file_submissions/file_submissions.service.ts index 2f21e8c7..b3277806 100644 --- a/backend/src/file_submissions/file_submissions.service.ts +++ b/backend/src/file_submissions/file_submissions.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from "@nestjs/common"; +import { Injectable, Logger } from "@nestjs/common"; import { CreateFileSubmissionDto } from "./dto/create-file_submission.dto"; import { UpdateFileSubmissionDto } from "./dto/update-file_submission.dto"; import { PrismaService } from "nestjs-prisma"; @@ -15,6 +15,8 @@ export class FileSubmissionsService { private prisma: PrismaService, private readonly objectStore: ObjectStoreService, private readonly aqiService: AqiApiService, + private readonly logger = new Logger(FileSubmissionsService.name), + ) {} async create(body: any, file: Express.Multer.File) { @@ -219,7 +221,7 @@ export class FileSubmissionsService { }); return true; } catch (err) { - console.error(`Error deleting file: ${err.message}`); + this.logger.error(`Error deleting file: ${err.message}`); return false; } } @@ -229,7 +231,7 @@ export class FileSubmissionsService { const fileBinary = await this.objectStore.getFileData(fileName); return fileBinary; } catch (err) { - console.error(`Error fetching file from S3: ${err.message}`); + this.logger.error(`Error fetching file from S3: ${err.message}`); throw err; } } diff --git a/backend/src/objectStore/objectStore.service.ts b/backend/src/objectStore/objectStore.service.ts index 94a636a0..e7705c46 100644 --- a/backend/src/objectStore/objectStore.service.ts +++ b/backend/src/objectStore/objectStore.service.ts @@ -44,7 +44,7 @@ export class ObjectStoreService { return response.data; } catch (error) { - console.error('Error fetching the file:', error.message); + this.logger.error('Error fetching the file:', error.message); throw error; } } diff --git a/backend/src/tempObsFiles/obs-SUCCESS_TEST_FILE-2b6dc1ec-b903-4e5f-b219-0d07dfdfd6fb.csv b/backend/src/tempObsFiles/obs-SUCCESS_TEST_FILE-2b6dc1ec-b903-4e5f-b219-0d07dfdfd6fb.csv new file mode 100644 index 00000000..4da23010 --- /dev/null +++ b/backend/src/tempObsFiles/obs-SUCCESS_TEST_FILE-2b6dc1ec-b903-4e5f-b219-0d07dfdfd6fb.csv @@ -0,0 +1,132 @@ +Observation ID,Location ID,Observed Property ID,Observed DateTime,Analyzed DateTime,Depth,Depth Unit,Data Classification,Result Value,Result Unit,Source Of Rounded Value,Rounded Value,Rounding Specification,Result Status,Result Grade,Medium,Activity ID,Activity Name,Collection Method,Field: Device ID,Field: Device Type,Field: Comment,Lab: Specimen Name,Lab: Analysis Method,Lab: Detection Condition,Lab: Limit Type,Lab: MDL,Lab: MRL,Lab: Quality Flag,Lab: Received DateTime,Lab: Prepared DateTime,Lab: Sample Fraction,Lab: From Laboratory,Lab: Sample ID,Lab: Dilution Factor,Lab: Comment,QC: Type,QC: Source Sample ID,EA_FileID +,0270800,621 - Anacystis aeruginosa,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3120 - Asterionella formosa,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,62,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,619 - Anacystis limneticus,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3521 - Achnanthes minutissima,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4721 - Amphora ovalis,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,8276 - Arthrodesmus,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6308 - Botryococcus braunii,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,10411 - Ceratium hirundinella,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1670 - Chrysosphaerella longispina,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,336,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,59417 - Conochilus,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,220,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,7848 - Cosmarium,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2439 - Cyclotella,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4795 - Cymbella,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1522 - Dinobryon bavaricum,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,20,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83838 - Diaphanosoma brachyurum,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,44,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1534 - Dinobryon divergens,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,1022,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83887 - Daphnia longiremis,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,1848,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83891 - Daphnia rosea,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,1320,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1517 - Dinobryon sertularia,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,88789 - Diacyclops thomasi,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,968,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83873 - Daphnia,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,176,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5020 - Epithemia turgida,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3337 - Eunotia,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2933 - Fragilaria crotonensis,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,20,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4564 - Frustulia,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6356 - Gloeocystis ampla,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,10031 - Gymnodinium,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,6,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83957 - Holopedium gibberum,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,528,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,58486 - Kellicottia longispina,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,44,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,894 - Lyngbya limnetica,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,591483 - Aulacoseira italica,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2290 - Melosira,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,7055 - Mougeotia,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,6,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3649 - Navicula,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,954 - Oscillatoria tenuis,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,8959 - Oedogonium,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2016 - Ophiocytium,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,917 - Oscillatoria,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,5.6,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,5.6,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,10331 - Peridinium inconspicuum,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4428 - Pinnularia,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1175 - Pseudanabaena,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5041 - Rhopalodia gibba,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6127 - Scenedesmus bijuga,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,7549 - Staurastrum paradoxum,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,8472 - Spondylosium planum,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,11,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6110 - Scenedesmus quadricauda,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3023 - Synedra ulna,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1656 - Synura uvella,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1348 - Scytonema,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6996 - Spirogyra,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,7440 - Staurastrum,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3242 - Tabellaria fenestrata,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3247 - Tabellaria flocculosa,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,59297 - Testudinella,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,616,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5661 - Tetraedron,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6426 - Ulothrix variabilis,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,88530 - Cyclopoida,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,572,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,861 - Nostocales,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2930 - Pennales,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,58239 - Rotifera,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,10,metre,LAB,309,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317735,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6994 - Zygnematales,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,34,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6417 - Ulothrix,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,8313 - Xanthidium,2016-08-25T00:00:00-08:00,2016-08-25T00:00:00-08:00,0.5,metre,LAB,2.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3324679,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,2.8,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,UNK - Anacystis elachista,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3120 - Asterionella formosa,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,619 - Anacystis limneticus,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,53.2,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3521 - Achnanthes minutissima,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1100 - Anabaena,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,8276 - Arthrodesmus,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6308 - Botryococcus braunii,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,331224 - Chroomonas acuta,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,20,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4803 - Cymbella affinis,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2460 - Cyclotella bodanica,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4884 - Cymbella minuta,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,10635 - Cryptomonas,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,28,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6235 - Crucigenia quadrata,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,11,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6226 - Crucigenia tetrapedia,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2439 - Cyclotella,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4795 - Cymbella,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1534 - Dinobryon divergens,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83891 - Daphnia rosea,2016-08-27T10:55:00-08:00,2016-08-27T10:55:00-08:00,10,metre,LAB,1526,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317734,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1517 - Dinobryon sertularia,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,93.8,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,88789 - Diacyclops thomasi,2016-08-27T10:55:00-08:00,2016-08-27T10:55:00-08:00,10,metre,LAB,35,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317734,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83873 - Daphnia,2016-08-27T10:55:00-08:00,2016-08-27T10:55:00-08:00,10,metre,LAB,186,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317734,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3214 - Diatoma,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1515 - Dinobryon,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,3,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4325 - Diploneis,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,9413 - Elakatothrix gelatinosa,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,6,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5020 - Epithemia turgida,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5005 - Epithemia,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3337 - Eunotia,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2933 - Fragilaria crotonensis,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4565 - Frustulia rhomboides,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2932 - Fragilaria,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6356 - Gloeocystis ampla,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,50.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4971 - Gomphonema olivaceum,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4911 - Gomphonema,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,83957 - Holopedium gibberum,2016-08-27T10:55:00-08:00,2016-08-27T10:55:00-08:00,10,metre,LAB,105,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317734,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,58360 - Keratella cochlearis,2016-08-27T10:55:00-08:00,2016-08-27T10:55:00-08:00,10,metre,LAB,47,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317734,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,58486 - Kellicottia longispina,2016-08-27T10:55:00-08:00,2016-08-27T10:55:00-08:00,10,metre,LAB,408,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317734,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1764 - Kephyrion,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,UNK - Limnothrix,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,870 - Lyngbya,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,591483 - Aulacoseira italica,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,28,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5099 - Nitzschia acicularis,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3952 - Navicula radiosa,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3649 - Navicula,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5838 - Oocystis parva,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,954 - Oscillatoria tenuis,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1455 - Ochromonas,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,8959 - Oedogonium,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5827 - Oocystis,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,917 - Oscillatoria,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6032 - Pediastrum boryanum,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,10331 - Peridinium inconspicuum,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,24,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4428 - Pinnularia,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,5041 - Rhopalodia gibba,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6127 - Scenedesmus bijuga,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6145 - Scenedesmus denticulatus,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,11,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6256 - Selenastrum minutum,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,3,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,8472 - Spondylosium planum,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6249 - Selenastrum,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,1053 - Spirulina,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,7440 - Staurastrum,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,4127 - Stauroneis,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3242 - Tabellaria fenestrata,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,3247 - Tabellaria flocculosa,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,9187 - Tetraspora,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,88530 - Cyclopoida,2016-08-27T10:55:00-08:00,2016-08-27T10:55:00-08:00,10,metre,LAB,221,to/s,,,,Preliminary,Unknown,Animal - Zooplankton,,3317734,Grab,,,rigohreioghetiogh,Zooplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,2930 - Pennales,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,,,,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx +,0270800,6417 - Ulothrix,2016-08-27T10:50:00-08:00,2016-08-27T10:50:00-08:00,0.5,metre,LAB,1.4,cells/mL,,,,Preliminary,Unknown,Plant - Phytoplankton,,3317818,Grab,,,rigohreioghetiogh,Phytoplankton,,NOT_DETECTED,LOWER,1.4,,,,,,FES,,,,,,SUCCESS_TEST_FILE.xlsx