Merge pull request #49 from bcgov/EMDEDT_186_Data_Validation

EMSEDT-186: data validation
bcgov · Jan 23, 2025 · a38d9cf · a38d9cf
2 parents b9c32b9 + 7891fa2
commit a38d9cf
Show file tree

Hide file tree

Showing 8 changed files with 731 additions and 62 deletions.
diff --git a/backend/package-lock.json b/backend/package-lock.json
diff --git a/backend/package.json b/backend/package.json
@@ -40,11 +40,13 @@
     "axios": "^1.7.2",
     "basic-ftp": "^5.0.5",
     "class-validator": "^0.14.1",
+    "csv-parse": "^5.6.0",
     "csv-parser": "^3.0.0",
     "csv-writer": "^1.6.0",
     "dotenv": "^16.0.1",
     "exceljs": "^4.4.0",
     "express-prom-bundle": "^7.0.0",
+    "fast-csv": "^5.0.2",
     "file-type": "^19.4.0",
     "helmet": "^7.0.0",
     "jwks-rsa": "^3.1.0",

diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts
@@ -687,38 +687,38 @@ export class AqiApiService {
 
   async VisitDelete(visitData: any[]) {
     if (visitData.length > 0) {
-      try {
-        let deletion = await axios.delete(
-          `${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${visitData}`,
-          {
-            headers: {
-              Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`,
-              "x-api-key": process.env.AQI_ACCESS_TOKEN,
-            },
-          },
-        );
-        this.logger.log("AQI VISIT DELETION: " + deletion.data);
-
+      for (const visit of visitData) {
         try {
-          const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({
-            where: {
-              aqi_field_visits_id: {
-                in: visitData,
+          let deletion = await axios.delete(
+            `${process.env.AQI_BASE_URL}/v1/fieldvisits/${visit}`,
+            {
+              headers: {
+                Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`,
+                "x-api-key": process.env.AQI_ACCESS_TOKEN,
               },
             },
-          });
-          this.logger.log("DB VISIT DELETION: " + dbDeletion);
-        } catch (err) {
-          if (err.code === "P2025") {
-            this.logger.log(
-              `Records with IDs ${visitData} not found in DB. Records were deleted in AQI but skipping deletion from DB.`,
-            );
-          } else {
-            this.logger.error(`API call to delete DB visits failed: `, err);
+          );
+          this.logger.log("AQI VISIT DELETION: " + deletion.data);
+
+          try {
+            const dbDeletion = await this.prisma.aqi_field_visits.delete({
+              where: {
+                aqi_field_visits_id: visit
+              },
+            });
+            this.logger.log("DB VISIT DELETION: " + dbDeletion);
+          } catch (err) {
+            if (err.code === "P2025") {
+              this.logger.log(
+                `Records with IDs ${visitData} not found in DB. Records were deleted in AQI but skipping deletion from DB.`,
+              );
+            } else {
+              this.logger.error(`API call to delete DB visits failed: `, err);
+            }
           }
+        } catch (err) {
+          this.logger.error(`API call to delete AQI visit failed: `, err);
         }
-      } catch (err) {
-        this.logger.error(`API call to delete AQI visit failed: `, err);
       }
     }
     return new Promise((resolve) => setTimeout(resolve, 1000));

diff --git a/backend/src/cron-job/cron-job.service.ts b/backend/src/cron-job/cron-job.service.ts
@@ -353,6 +353,11 @@ export class CronJobService {
 
   @Cron(CronExpression.EVERY_5_MINUTES)
   private async fetchAQSSData() {
+    if (this.isProcessing){
+      this.logger.log("Skipping cron procedure of data pull down: File processing underway.");
+      return;
+    }
+
     this.logger.log(`Starting Code Table Cron Job`);
     axios.defaults.method = "GET";
     axios.defaults.headers.common["Authorization"] =
@@ -576,7 +581,7 @@ export class CronJobService {
       for each file returned, change the status to INPROGRESS and go to the parser
     // */
     if (!this.dataPullDownComplete) {
-      this.logger.warn("Data pull down from AQSS did not complete");
+      this.logger.warn("Data pull down from AQI did not complete");
       return;
     }
 
@@ -594,7 +599,7 @@ export class CronJobService {
 
   async processFiles(files) {
     if (this.isProcessing){
-      this.logger.log("Skipping cron execution: Already processing files.");
+      this.logger.log("Skipping cron procedure of file processing: Already processing files.");
       return;
     }