Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EMSEDT186: Data Validation #44

Merged
merged 11 commits into from
Jan 9, 2025
Merged

EMSEDT186: Data Validation #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ac6b0ae
uncommenting data pull down flag
vmanawat Dec 10, 2024
2062687
fixed spinning wheel, PUT procedure and DELETE procedure
vmanawat Dec 12, 2024
e3865bc
updating the delete to make it more readable and isolated
vmanawat Dec 17, 2024
8417538
await for deleting and file processing + new validation
vmanawat Dec 19, 2024
c71875a
Added new validation for Tissue Type
vmanawat Dec 19, 2024
6285e7e
fixing deletion procedure to make sure it happens sequentially
vmanawat Dec 26, 2024
d36ea79
Updated importer for vertical profiles
vmanawat Dec 30, 2024
5453841
Added a flag to ensure that the files are processed sequentially
vmanawat Dec 31, 2024
2108f64
fixing some validation rules
vmanawat Jan 9, 2025
39d3a82
New code table and validation rule for sampling agency
vmanawat Jan 9, 2025
bced6c4
removing temp obs file
vmanawat Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions backend/prisma/schema.prisma
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,3 +296,21 @@ model aqi_units_xref {
update_user_id String @db.VarChar(200)
update_utc_timestamp DateTime @db.Timestamp(6)
}

model aqi_tissue_types {
aqi_tissue_types_id String @id @db.Uuid
custom_id String @db.VarChar(200)
create_user_id String @db.VarChar(200)
create_utc_timestamp DateTime @db.Timestamp(6)
update_user_id String @db.VarChar(200)
update_utc_timestamp DateTime @db.Timestamp(6)
}

model aqi_sampling_agency {
aqi_sampling_agency_id String @id @db.Uuid
custom_id String @db.VarChar(200)
create_user_id String @db.VarChar(200)
create_utc_timestamp DateTime @db.Timestamp(6)
update_user_id String @db.VarChar(200)
update_utc_timestamp DateTime @db.Timestamp(6)
}
169 changes: 128 additions & 41 deletions backend/src/aqi_api/aqi_api.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ export class AqiApiService {
);
return response.data.id;
} catch (err) {
console.log(body);
this.logger.error(
"API CALL TO POST Field Visits failed: ",
err.response.data.message,
Expand Down Expand Up @@ -510,12 +509,13 @@ export class AqiApiService {
}

mergeErrorMessages(localErrors: any[], remoteErrors: any[]) {
const map = new Map<number, any>();
const map = new Map<string, any>();

const mergeItem = (item: any) => {
const exists = map.get(item.rowNum);
const key = `${item.rowNum}-${item.type}`;
const exists = map.get(key);
map.set(
item.rowNum,
key,
exists
? { ...exists, message: { ...exists.message, ...item.message } }
: item,
Expand Down Expand Up @@ -570,36 +570,38 @@ export class AqiApiService {
}
}

async deleteRelatedData(fileName: string) {
const guidsToDelete: any = await this.prisma.aqi_imported_data.findMany({
where: {
file_name: fileName,
},
});

console.log(guidsToDelete)
async ObservationDelete(obsData: any[]) {
if (obsData.length > 0) {
const batchSize = 50;
const observationBatches = [];
for (let i = 0; i < obsData.length; i += batchSize) {
observationBatches.push(obsData.slice(i, i + batchSize));
}

// Delete all the observations from the list of imported guids
if (guidsToDelete[0].imported_guids.observations.length > 0) {
try {
let deletion = await axios.delete(
`${process.env.AQI_BASE_URL}/v2/observations?ids=${guidsToDelete[0].imported_guids.observations}`,
{
headers: {
Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`,
"x-api-key": process.env.AQI_ACCESS_TOKEN,
observationBatches.forEach(async (batch) => {
try {
let deletion = await axios.delete(
`${process.env.AQI_BASE_URL}/v2/observations?ids=${batch}`,
{
headers: {
Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`,
"x-api-key": process.env.AQI_ACCESS_TOKEN,
},
},
},
);
this.logger.log("AQI OBS DELETION: " + deletion.data);
} catch (err) {
this.logger.error(`API call to delete AQI observation failed: `, err);
}
);
this.logger.log("AQI OBS DELETION: " + deletion.status);
} catch (err) {
this.logger.error(`API call to delete AQI observation failed: `, err);
}
});
}

// Delete all the specimens for the activities imported from AQI and the PSQL db
if (guidsToDelete[0].imported_guids.specimens.length > 0) {
for (const specimen of guidsToDelete[0].imported_guids.specimens) {
return new Promise((resolve) => setTimeout(resolve, 1000));
}

async SpecimenDelete(specimenData: any[]) {
if (specimenData.length > 0) {
for (const specimen of specimenData) {
try {
let aqiDeletion = await axios.delete(
`${process.env.AQI_BASE_URL}/v1/specimens/${specimen}`,
Expand All @@ -620,17 +622,25 @@ export class AqiApiService {
});
this.logger.log("DB SPECIMEN DELETION: " + dbDeletion);
} catch (err) {
this.logger.error(`API call to delete DB specimen failed: `, err);
if (err.code === "P2025") {
this.logger.log(
`Record with ID ${specimen} not found in DB. Record was deleted in AQI but skipping deletion from DB.`,
);
} else {
this.logger.error(`API call to delete DB specimen failed: `, err);
}
}
} catch (err) {
this.logger.error(`API call to delete AQI specimen failed: `, err);
}
}
}
return new Promise((resolve) => setTimeout(resolve, 1000));
}

// Delete all the activities for the visits imported
if (guidsToDelete[0].imported_guids.activities.length > 0) {
for (const activity of guidsToDelete[0].imported_guids.activities) {
async ActivityDelete(activityData: any[]) {
if (activityData.length > 0) {
for (const activity of activityData) {
try {
let aqiDeletion = await axios.delete(
`${process.env.AQI_BASE_URL}/v1/activities/${activity}`,
Expand All @@ -651,19 +661,27 @@ export class AqiApiService {
});
this.logger.log("DB ACTIVITY DELETION: " + dbDeletion);
} catch (err) {
this.logger.error(`API call to delete DB activity failed: `, err);
if (err.code === "P2025") {
this.logger.log(
`Record with ID ${activity} not found in DB. Record was deleted in AQI but skipping deletion from DB.`,
);
} else {
this.logger.error(`API call to delete DB activity failed: `, err);
}
}
} catch (err) {
this.logger.error(`API call to delete AQI activity failed: `, err);
}
}
}
return new Promise((resolve) => setTimeout(resolve, 1000));
}

// Delete all the visits for the visits imported
if (guidsToDelete[0].imported_guids.visits.length > 0) {
async VisitDelete(visitData: any[]) {
if (visitData.length > 0) {
try {
let deletion = await axios.delete(
`${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${guidsToDelete[0].imported_guids.visits}`,
`${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${visitData}`,
{
headers: {
Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`,
Expand All @@ -677,23 +695,92 @@ export class AqiApiService {
const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({
where: {
aqi_field_visits_id: {
in: guidsToDelete[0].imported_guids.visits,
in: visitData,
},
},
});
this.logger.log("DB VISIT DELETION: " + dbDeletion);
} catch (err) {
this.logger.error(`API call to delete DB visits failed: `, err);
if (err.code === "P2025") {
this.logger.log(
`Records with IDs ${visitData} not found in DB. Records were deleted in AQI but skipping deletion from DB.`,
);
} else {
this.logger.error(`API call to delete DB visits failed: `, err);
}
}
} catch (err) {
this.logger.error(`API call to delete AQI visit failed: `, err);
}
}
return new Promise((resolve) => setTimeout(resolve, 1000));
}

await this.prisma.aqi_imported_data.deleteMany({
async deleteRelatedData(fileName: string) {
const guidsToDelete: any = await this.prisma.aqi_imported_data.findMany({
where: {
file_name: fileName,
},
});

let successfulObs = false
let successfulSpecimen = false
let successfulActivity = false
let successfulVisit = false

// Delete all the observations in AQI that are in the list of imported guids
this.logger.log(
`Starting observation delete for file ${fileName}..............`,
);

await this.ObservationDelete(
guidsToDelete[0].imported_guids.observations,
).then(() => {
successfulObs = true
this.logger.log(`Finished observation delete for file ${fileName}`)
})

if (successfulObs){
// Delete all the specimens that were imported for the file from AQI and the PSQL db
this.logger.log(
`Starting specimen delete for file ${fileName}..............`,
);
await this.SpecimenDelete(guidsToDelete[0].imported_guids.specimens).then(
() => {
successfulSpecimen = true;
this.logger.log(`Finished specimen delete for file ${fileName}.`);
},
);
}

if (successfulSpecimen){
// Delete all the activities for the visits imported
this.logger.log(
`Starting activity delete for file ${fileName}..............`,
);
await this.ActivityDelete(guidsToDelete[0].imported_guids.activities).then(() => {
successfulActivity = true;
this.logger.log(`Finished activity delete for file ${fileName}.`);
});
}

if (successfulActivity){
// Delete all the visits for the visits imported
this.logger.log(`Starting visit delete for file ${fileName}..............`);
await this.VisitDelete(guidsToDelete[0].imported_guids.visits).then(() => {
successfulVisit = true
this.logger.log(`Finished visit delete for file ${fileName}.`);
});
}

if (successfulObs && successfulSpecimen && successfulActivity && successfulVisit){
await this.prisma.aqi_imported_data.deleteMany({
where: {
file_name: fileName,
},
});
}else{
this.logger.error(`Error deleting related data for file ${fileName}.`);
}
}
}
Loading
Loading