Merge pull request #149 from GDSC-PKNU-Official/feat/#148

Feat/#148: 웨일비 데이터 크롤링 시 모집기간과 운영기간을 크롤링 해 DB에 저장하도록 추가
GDSC-PKNU-Official · Dec 1, 2023 · df957e0 · df957e0
2 parents 9f87751 + c6cf984
commit df957e0
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 18 deletions.
diff --git a/src/@types/college.ts b/src/@types/college.ts
@@ -23,7 +23,8 @@ export type NoticeBoolean = 1 | 0;
 
 export interface WhalebeData {
   title: string;
-  date: string;
-  imgUrl: string;
+  operating_period: string;
+  recruitment_period: string;
+  imgurl: string;
   link: string;
 }
diff --git a/src/apis/notice/service.ts b/src/apis/notice/service.ts
@@ -71,7 +71,10 @@ export const getWhalebe = async (): Promise<WhalebeData[]> => {
     ).padStart(2, '0')}.${String(today.getDate()).padStart(2, '0')}`;
 
     const filteredData = whalebeData
-      .filter((data) => data.date >= todayString)
+      .filter(
+        (data) =>
+          data.recruitment_period.split('~')[1].trim() >= todayString || data,
+      )
       .slice(0, 7);
     return filteredData;
   } catch (error) {

diff --git a/src/crawling/whalebeCrawling.ts b/src/crawling/whalebeCrawling.ts
@@ -1,12 +1,6 @@
 import axios from 'axios';
 import * as cheerio from 'cheerio';
-
-export interface WhalebeData {
-  title: string;
-  date: string;
-  imgUrl: string;
-  link: string;
-}
+import { WhalebeData } from 'src/@types/college';
 
 const sliceURL = (link: string): string => {
   if (link.startsWith('location.href=')) {
@@ -29,20 +23,33 @@ export const whalebeCrawling = async (): Promise<WhalebeData[]> => {
   if (programs.length < 1) return;
 
   programs.each((_, element) => {
-    const imgUrl = hostname + $(element).find('img').attr('src');
+    const imgurl = hostname + $(element).find('img').attr('src');
     const link = hostname + sliceURL($(element).find('div').attr('onclick'));
     const title = $(element).find('.card-title').text().trim();
-    const date = $(element)
+    const recruitment_period = $(element)
       .find('.app_date')
       .find('.col-12')
       .first()
+      .find('span')
+      .eq(1)
       .text()
-      .split('~')[1]
-      .trim();
-    const tmpData = {
+      .trim()
+      .replace('&nbsp;', '');
+    const operating_period = $(element)
+      .find('.app_date')
+      .find('.col-12')
+      .eq(1)
+      .find('span')
+      .eq(1)
+      .text()
+      .trim()
+      .replace('&nbsp;', '');
+
+    const tmpData: WhalebeData = {
       title,
-      date,
-      imgUrl,
+      recruitment_period,
+      operating_period,
+      imgurl,
       link,
     };
     whalebeData.push(tmpData);

diff --git a/src/db/data/noticeHandler.ts b/src/db/data/noticeHandler.ts
@@ -204,7 +204,13 @@ export const saveWhalebeToDB = async (): Promise<void> => {
 
   // TODO: 웨일비 크롤링하는 데이터 추가해야함
   const promises = whalebeDatas.map((data) => {
-    const values = [data.title, data.link, 'tmp', 'tmp2', data.imgUrl];
+    const values = [
+      data.title,
+      data.link,
+      data.operating_period,
+      data.recruitment_period,
+      data.imgurl,
+    ];
 
     return db.execute(query, values);
   });