Merge pull request #184 from GDSC-PKNU-Official/fix/#182

Fix/#182: 공지사항 크롤링 비동기처리 추가 및 무한 슬랙알림 문제 해결

src/db/data/noticeHandler.ts

@@ -17,6 +17,7 @@ export interface PushNoti {
 
 interface NotiLink {
   link: string;
+  rep_yn: boolean;
 }
 
 export const saveDepartmentToDB = async (college: College[]): Promise<void> => {
@@ -74,81 +75,119 @@ const convertAllNoticeToNormalNotice = async (
   }
 };
 
-const convertSpecificNoticeToPinnedNotice = async (
+const convertSpecificNoticePinned = async (
   tableName: string,
   noticeLink: string,
+  isPinned: boolean,
+  connection?: PoolConnection,
 ): Promise<void> => {
-  const query = `UPDATE ${tableName} SET rep_yn = true WHERE link = '${noticeLink}';`;
+  const query = `UPDATE ${tableName} SET rep_yn = ${isPinned} WHERE link = '${noticeLink}';`;
 
   try {
-    await db.execute(query);
+    if (connection) await connection.execute(query);
+    else await db.execute(query);
   } catch (error) {
     console.log(error.message + '고정 공지로 변경 실패');
     // notificationToSlack(error.message + '\n 고정 공지로 변경 실패');
   }
 };
 
-export const saveMajorNoticeToDB = async (
-  connection?: PoolConnection,
-): Promise<PushNoti> => {
-  await convertAllNoticeToNormalNotice('major_notices', connection);
+export const saveMajorNoticeToDB = async (): Promise<PushNoti> => {
+  // await convertAllNoticeToNormalNotice('major_notices', connection);
   const query = 'SELECT * FROM departments;';
-  const colleges = await selectQuery<College[]>(query, connection);
-
-  const getNotiLinkQuery = `SELECT link FROM major_notices;`;
-  const noticeLinksInDB = (
-    await selectQuery<NotiLink[]>(getNotiLinkQuery, connection)
-  ).map((noticeLink) => noticeLink.link);
+  const colleges = await selectQuery<College[]>(query);
 
-  const savePromises: Promise<void>[] = [];
   const newNoticeMajor: PushNoti = {};
+  const failedMajor: number[] = [];
 
-  for (const college of colleges) {
-    console.log(college.id);
-    const noticeLink = await noticeCrawling(college);
-    const noticeLists = await noticeListCrawling(noticeLink);
-
-    const normalNotices = noticeLists.normalNotice;
-    const pinnedNotices = noticeLists.pinnedNotice;
-
-    if (normalNotices.length === 0) {
-      notificationToSlack(`${noticeLink} 크롤링 실패`);
-      continue;
-    }
-
-    for (const notice of normalNotices) {
-      const result = await noticeContentCrawling(notice);
-      if (result.link === '') {
-        // notificationToSlack(`${notice} 콘텐츠 크롤링 실패`);
-        continue;
+  const savePromises = colleges.map(async (college) => {
+    const connection = await db.getConnection();
+    await connection.beginTransaction();
+    try {
+      const noticeLink = await noticeCrawling(college);
+      const noticeLists = await noticeListCrawling(noticeLink);
+      const pinnedNotices = noticeLists.pinnedNotice;
+      const normalNotices = noticeLists.normalNotice;
+      if (normalNotices.length === 0) {
+        notificationToSlack(`${noticeLink} 크롤링 실패`);
+        connection.release();
+        return;
       }
 
-      if (noticeLinksInDB.includes(result.link)) continue;
-      if (!newNoticeMajor[college.id]) newNoticeMajor[college.id] = [];
-      newNoticeMajor[college.id].push(result.title);
-      savePromises.push(saveMajorNotice(result, college.id, false, connection));
-    }
-
-    if (pinnedNotices) {
-      for (const notice of pinnedNotices) {
+      const getNotiLinkQuery = `SELECT link, rep_yn FROM major_notices WHERE department_id = '${college.id}'`;
+      const noticeDataInDB = await selectQuery<NotiLink[]>(
+        getNotiLinkQuery,
+        connection,
+      );
+      const noticeLinksInDB = noticeDataInDB.map((noti) => noti.link);
+      const pinnedNoticeLinksInDB = noticeDataInDB
+        .filter((noti) => noti.rep_yn)
+        .map((noti) => noti.link);
+
+      for (const notice of normalNotices) {
         const result = await noticeContentCrawling(notice);
         if (result.link === '') {
-          notificationToSlack(`${notice} 콘텐츠 크롤링 실패`);
+          // notificationToSlack(`${notice} 콘텐츠 크롤링 실패`);
           continue;
         }
 
-        if (!noticeLinksInDB.includes(result.link)) {
-          savePromises.push(
-            saveMajorNotice(result, college.id, true, connection),
+        if (noticeLinksInDB.includes(result.link)) return;
+        if (!newNoticeMajor[college.id]) newNoticeMajor[college.id] = [];
+        newNoticeMajor[college.id].push(result.title);
+        saveMajorNotice(result, college.id, false, connection);
+        noticeLinksInDB.push(result.link);
+      }
+
+      if (pinnedNotices) {
+        await pinnedNoticeLinksInDB
+          .filter((noti) => !pinnedNotices.includes(noti))
+          .map(
+            async (noti) =>
+              await convertSpecificNoticePinned(
+                'major_notices',
+                noti,
+                false,
+                connection,
+              ),
           );
-          continue;
+        for (const notice of pinnedNotices) {
+          const result = await noticeContentCrawling(notice);
+          if (result.link === '') {
+            notificationToSlack(`${notice} 콘텐츠 크롤링 실패`);
+            continue;
+          }
+
+          if (!noticeLinksInDB.includes(result.link)) {
+            saveMajorNotice(result, college.id, true, connection);
+            continue;
+          }
+
+          if (!pinnedNoticeLinksInDB.includes(result.link))
+            convertSpecificNoticePinned(
+              'major_notices',
+              result.link,
+              true,
+              connection,
+            );
         }
-        convertSpecificNoticeToPinnedNotice('major_notices', result.link);
       }
+
+      connection.commit();
+    } catch (error) {
+      notificationToSlack(college.id + '크롤링 실패' + error.message);
+      failedMajor.push(college.id);
+      connection.rollback();
+    } finally {
+      connection.release();
     }
-  }
+  });
 
   await Promise.all(savePromises);
+  if (failedMajor.length !== 0) {
+    const failedMajorList = failedMajor.join();
+    notificationToSlack('크롤링 실패한 학과: ' + failedMajorList);
+  }
+
   return newNoticeMajor;
 };
 
@@ -188,7 +227,7 @@ export const saveSchoolNoticeToDB = async (): Promise<void> => {
 
   for (const noticeLink of pinnedNotices) {
     if (schoolNoticeLinksInDB.includes(noticeLink)) {
-      await convertSpecificNoticeToPinnedNotice('notices', noticeLink);
+      await convertSpecificNoticePinned('notices', noticeLink, true);
       continue;
     }
 

src/hooks/cronNoticeCrawling.ts

@@ -21,34 +21,34 @@ const pushToUsers = async (pushNotiToUserLists: PushNoti) => {
   if (pushedUserCount.length !== 0) notificationToSlack(pushedUserCount);
 };
 
-const cronNoticeCrawling = async () => {
-  const connection = await db.getConnection();
-  await connection.beginTransaction();
+const cronNoticeCrawling = async (reTryCount = 0) => {
   try {
-    const pushNotiToUserLists = await saveMajorNoticeToDB(connection);
+    const pushNotiToUserLists = await saveMajorNoticeToDB();
     const today = new Date();
     const year = today.getFullYear();
     const month = today.getMonth() + 1; // 월은 0부터 시작하므로 1을 더해줍니다.
     const day = today.getDate();
     notificationToSlack(`${year}-${month}-${day} 크롤링 완료`);
-    await connection.commit();
     pushToUsers(pushNotiToUserLists);
   } catch (error) {
-    await connection.rollback();
-    notificationToSlack(error.message);
-    cronNoticeCrawling();
-  } finally {
-    connection.release();
+    if (reTryCount >= 2) {
+      notificationToSlack(error.message);
+      return;
+    }
+    cronNoticeCrawling(reTryCount + 1);
   }
 };
 
-const cronExtracurricularCrawling = async () => {
+const cronExtracurricularCrawling = async (reTryCount = 0) => {
   try {
     await saveSchoolNoticeToDB();
     await saveLanguageNoticeToDB();
     await saveWhalebeToDB();
   } catch (error) {
-    notificationToSlack(error.message);
+    if (reTryCount >= 2) {
+      notificationToSlack(error.message);
+      return;
+    }
     cronExtracurricularCrawling();
   }
 };