feat:RMP summary data now on postgres

package.json

@@ -58,6 +58,7 @@
     "expo-updates": "0.25.25",
     "expo-web-browser": "~13.0.3",
     "express": "^4.21.0",
+    "fs-extra": "^11.2.0",
     "jsonwebtoken": "^9.0.2",
     "lucide-react-native": "^0.359.0",
     "nativewind": "^4.0.22",

prisma/schema.prisma

@@ -58,11 +58,14 @@ model UserRegistrationDetails {
 
 // this model will be used to store the summary format of the data
 model RateMyProfessorDataSummary {
-  id            String @id @unique @default(uuid()) //this will randomly generate an unique id
-  professorName String
-  numRatings    String
-  rating        Float
-  department    String
+  id             String @id @unique @default(uuid()) //this will randomly generate an unique id
+  professorName  String
+  numRatings     Int
+  avgRatings     Float
+  avgDifficulty  Float
+  // this is a percentage value stored in the form of a string
+  wouldTakeAgain String
+  department     String
 }
 
 // this model will be used to store the data about the professor and the comments relevant to the particular professor

src/lib/api/server.ts

@@ -5,7 +5,8 @@ import onboardingRouter from './onboarding/onboarding.routes';
 //import { createMap, gatherSummaryByDepartment } from '../utils/RMPScraper';
 //import { gatherRMPSummary } from '../utils/RMPScraper';
 import { department_list, department_professor_object } from '../utils/data/constants';
-import { completeProfessorSummary, createMap } from '../utils/RMPScraper';
+import { completeProfessorSummary, createMap, sendToDatabase } from '../utils/RMPScraper';
+import fs from 'fs';
 
 const app = express();
 // ** needed to add express.json()
@@ -23,11 +24,24 @@ app.get('/test', (req, res) => {
   res.send('Hello from A!').status(200);
 });
 
-app.listen('4001', async () => {
+// this is temporary, just to get a better understanding of what the stored data looks like
+const saveToJson = async () => {
   // TODO : call on the final function here
   //await gatherSummaryByDepartment(inputMap, 'american studies'); --> this function gets called in the scraper itself
   const inputMap = await createMap(department_list, department_professor_object);
   const result: string[][] = await completeProfessorSummary(inputMap);
-  console.log(`Result is ${JSON.stringify(result)}`); // tested and worked
-  console.log(`Listening to port 4001`);
+  const test = sendToDatabase(result);
+  console.log(`Current JSON Data : ${test}`);
+  // test using the newly created function
+  const result_json = JSON.stringify(result);
+  fs.writeFile('results.json', result_json, (err) => {
+    if (err) {
+      console.error('Error writting to file : ', err);
+    } else {
+      console.log('successfull!');
+    }
+  });
+};
+app.listen('4001', async () => {
+  await saveToJson();
 });

src/lib/utils/RMPScraper.ts

@@ -4,49 +4,37 @@
 
 // TODO : Integreate nur's machine learning scraped data later as something to be more scalable.
 import * as rmp from 'ratemyprofessor-api';
-import { department_professor_object_type } from './data/constants';
+import { department_professor_object_type } from './data/constants'; // interface for professor object
 import { db } from './db';
-// Given the scope of this project, the ideal thing would be to simply use the text file nur has and store the values via a hashmap
 
 // we then parse the hashmap and then store the data as a cache
-
 // This is intended to provide a summary for the data from rate my professor.
 // we want to be able to dynamically retrieve the data for the professor
-
 // in terms of flow, we will retrieve the name of the professor from the previous mode, and then plug in that information into this function, and execute it for each of the flatlist values we need
 
+// randomize data for list of professors
 export const arrayShuffler = (inputArray: string[]) => {
   for (let iterator = inputArray.length - 1; iterator > 0; iterator--) {
-    // determine a random index to serve as the shuffler
     const secondIterator = Math.floor(Math.random() * (iterator + 1));
-    // perform a simple in-place swap
     [inputArray[iterator], inputArray[secondIterator]] = [
       inputArray[secondIterator],
       inputArray[iterator],
     ];
-    // TODO : Remove later
-    console.log(`Shuffled Array : ${inputArray}`);
     return inputArray;
   }
 };
 
 export const createMap = (inputArray: string[], inputObject: department_professor_object_type) => {
-  // TODO : Modify based on what's listed on RMP
   // list of professors per division
-  // NOTE : Names of professors are case sensetive
   // we will use this within the teacher's list itself
-
   // we will have this as in inner function for now
   // INTENT : randomize the ordering of the department through which we search and render
   // unfortunately, for now, the hashmap will need to be re-constructed every time the re-rendering occurs
   // NOTE : the shuffling logic should take place when the map is created, so that the keys within the map matches
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  const shuffledArray: string[] | undefined = arrayShuffler(inputArray);
 
-  // NOTE : the reason this is hardcoded right now is because we have no method of actually collecting data on them
+  const shuffledArray: string[] | undefined = arrayShuffler(inputArray);
   // once the setup for collecting data on particular professors is setup, we won't need this, however, the hashamp will still be helpful
   // these datas should be globally accessible as well
-  // TODO : make these data globally accessible and migrate them somehwere else
   // this is how we dynamically set interface types in typeScript
   //@see https://stackoverflow.com/questions/55831886/typescript-an-index-signature-parameter-must-be-a-string-or-number-when-try
 
@@ -58,7 +46,6 @@ export const createMap = (inputArray: string[], inputObject: department_professo
       department_professor_map.set(
         // ensure that all the keys are set to lower case
         // this will reduce the issue regarding case sensetive issues arising
-        // TODO : see how to remove trailing commas and whitespaces if possible as well --> we can achieve this using another functon as well
         shuffledArray[i].toLowerCase(),
         inputObject[inputArray[i]]
       );
@@ -68,8 +55,8 @@ export const createMap = (inputArray: string[], inputObject: department_professo
   return department_professor_map;
 };
 
-// TODO : Later on, if this project is expanded for other collges, we can simply have a parameter that will be updated based on the form input
 export const getSpecificProfessorData = async (professorName: string) => {
+  // school name will remain unchnaged
   const school = await rmp.searchSchool('City College of New York');
 
   if (school !== undefined) {
@@ -80,13 +67,16 @@ export const getSpecificProfessorData = async (professorName: string) => {
   }
 };
 
+// not very useful of a function
 export const getSchoolId = async (nameOfSchool: string) => {
   const school = await rmp.searchSchool(nameOfSchool);
   if (school) {
     return school[0].node.id.toString();
   }
   return 'undefined';
 };
+
+// not very useful of a function
 export const searchByProfessor = async (professorName: string, nameOfSchool: string) => {
   const searchResults = await rmp.searchProfessorsAtSchoolId(
     professorName,
@@ -109,7 +99,9 @@ export const gatherSummaryByDepartment = async (
   inputMap: Map<string, string[]>,
   department: string
 ) => {
+  // variable where data is being stored in array format
   const departmentResult: any = [];
+  // search to check if department exist
   const doesKeyExist: boolean = inputMap.has(department);
   if (doesKeyExist) {
     const teacherListByDepartment = inputMap.get(department);
@@ -164,11 +156,23 @@ export const sendToDatabase = async (data: string[][]) => {
   // @see https://www.prisma.io/docs/orm/prisma-client/setup-and-configuration/introduction#3-importing-prisma-client
 
   // NOTE : we are working with list of objects in this case
-  for (let iterator = 0; iterator < data.length; iterator++) {
-    /*
-    const newProfessorSummary = await db.RateMyProfessorDataSummary.create({
-
-    }) */
+  // first array contains the array containing professors regarding a particular department
+
+  // list of keys are --> avgRating,avgDifficulty,wouldTakeAgainPercent,numRatings,formattedName,department,link
+  for (const row of data) {
+    // item in this case is the json object
+    for (const item of row) {
+      const professors_data = await db.rateMyProfessorDataSummary.create({
+        data: {
+          professorName: item.formattedName,
+          avgRatings: parseFloat(item.avgRating),
+          avgDifficulty: parseFloat(item.avgDifficulty),
+          numRatings: parseInt(item.numRatings),
+          wouldTakeAgain: item.wouldTakeAgainPercent + '%',
+          department: item.department,
+        },
+      });
+    }
   }
 };
 /**