From 553d7896adb141c6cf5a66207e59149e62a41168 Mon Sep 17 00:00:00 2001
From: Ru Chern Chong <ruchern.chong@sgcarstrends.com>
Date: Mon, 16 Dec 2024 18:17:48 +0800
Subject: [PATCH] Fix processing of CSV

---
 src/lib/updater.ts           |  2 +-
 src/utils/createUniqueKey.ts |  4 +--
 src/utils/processCSV.ts      | 53 +++++++++++++++++-------------------
 3 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/src/lib/updater.ts b/src/lib/updater.ts
index 8fe82b4..7efe0ef 100644
--- a/src/lib/updater.ts
+++ b/src/lib/updater.ts
@@ -78,7 +78,7 @@ export const updater = async <T extends PgTable>({
 	);
 
 	// Check against the existing records for new non-duplicated entries
-	const newRecords = processedData.filter((record: Record<string, any>) => {
+	const newRecords = processedData.filter((record) => {
 		const identifier = createUniqueKey(record, keyFields);
 		return !existingKeys.has(identifier);
 	});
diff --git a/src/utils/createUniqueKey.ts b/src/utils/createUniqueKey.ts
index 10bc216..cabdedf 100644
--- a/src/utils/createUniqueKey.ts
+++ b/src/utils/createUniqueKey.ts
@@ -1,4 +1,4 @@
 export const createUniqueKey = (
-	item: Record<string, any>,
+	record: Record<string, any>,
 	keyFields: string[],
-) => keyFields.map((field) => item[field]).join("-");
+) => keyFields.map((field) => record[field]).join("-");
diff --git a/src/utils/processCSV.ts b/src/utils/processCSV.ts
index 30fb011..4299bbc 100644
--- a/src/utils/processCSV.ts
+++ b/src/utils/processCSV.ts
@@ -1,35 +1,32 @@
 import fs from "node:fs";
 import Papa from "papaparse";
 
-export const processCSV = async (
-	filePath: string,
-): Promise<Record<string, any>[]> => {
+export const processCSV = async (filePath: string) => {
 	const fileContent = fs.readFileSync(filePath, "utf-8");
 
-	return new Promise<Record<string, any>[]>((resolve, reject) => {
-		Papa.parse(fileContent, {
-			header: true,
-			dynamicTyping: true,
-			skipEmptyLines: true,
-			transform: (value) => {
-				if (/\d+,\d+/.test(value)) {
-					if (value === "") {
-						return 0;
-					}
-					const cleanValue = value.replace(/,/g, "");
-					return cleanValue.includes(".")
-						? Number.parseFloat(cleanValue)
-						: Number.parseInt(cleanValue, 10);
-				}
-				return value;
-			},
-			complete: ({ data }: { data: Record<string, any>[] }) => {
-				resolve(data);
-			},
-			error: (error: Error) => {
-				console.error(error);
-				reject(error);
-			},
-		});
+	const { data } = Papa.parse(fileContent, {
+		header: true,
+		dynamicTyping: true,
+		skipEmptyLines: true,
+		transform: (value, field) => {
+			// Handle empty number field
+			if (field === "number" && value === "") {
+				return 0;
+			}
+
+			// Clean up make field
+			if (field === "make") {
+				return value.trim().replace(/\./g, "");
+			}
+
+			// Handle numeric values with commas
+			if (/\d+,\d+/.test(value)) {
+				return Number.parseInt(value.replace(/,/g, ""), 10);
+			}
+
+			return value;
+		},
 	});
+
+	return data;
 };