Merge pull request #102 from GDSC-PKNU-21-22/feat/#101

Feat/#101: 웨일비 데이터 크롤링 추가

src/apis/notice/controller.ts

@@ -1,4 +1,4 @@
-import { getNotices, getSchoolNotices } from '@apis/notice/service';
+import { getNotices, getSchoolNotices, getWhalebe } from '@apis/notice/service';
 import express, { Request, Response } from 'express';
 
 const router = express.Router();
@@ -17,4 +17,13 @@ router.get('/', async (req: Request, res: Response) => {
   }
 });
 
+router.get('/whalebe', async (req: Request, res: Response) => {
+  try {
+    const whalebeData = await getWhalebe();
+    res.json(whalebeData);
+  } catch (err) {
+    res.json();
+  }
+});
+
 export default router;

src/apis/notice/service.ts

@@ -1,5 +1,7 @@
+import { WhalebeData } from '@crawling/whalebeCrawling';
 import db from '@db/index';
 import { Notice } from 'src/@types/college';
+import notificationToSlack from 'src/hooks/notificateToSlack';
 
 interface SeparateNoti {
   고정: Notice[];
@@ -42,3 +44,13 @@ export const getSchoolNotices = async (): Promise<SeparateNoti> => {
   };
   return notices;
 };
+
+export const getWhalebe = async (): Promise<WhalebeData[]> => {
+  const query = 'SELECT * FROM 웨일비;';
+  return new Promise<WhalebeData[]>((resolve) => {
+    db.query(query, (err, res) => {
+      if (err) notificationToSlack('웨일비 조회 실패');
+      resolve(res as WhalebeData[]);
+    });
+  });
+};

src/crawling/whalebeCrawling.ts

@@ -0,0 +1,39 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+
+export interface WhalebeData {
+  title: string;
+  date: string;
+  imgUrl: string;
+}
+
+export const whalebeCrawling = async (): Promise<WhalebeData[]> => {
+  const hostname = 'https://whalebe.pknu.ac.kr';
+  const whalebeLink = hostname + '/main';
+  const whalebeData: WhalebeData[] = [];
+
+  const response = await axios.get(whalebeLink);
+  const $ = cheerio.load(response.data);
+
+  const programs = $('ul.px-0').find('li');
+  if (programs.length < 1) return;
+
+  programs.each((_, element) => {
+    const imgUrl = hostname + $(element).find('img').attr('src');
+    const title = $(element).find('.card-title').text();
+    const date = $(element)
+      .find('.app_date')
+      .find('.col-12')
+      .first()
+      .text()
+      .split('~')[1]
+      .trim();
+    const tmpData = {
+      title,
+      date,
+      imgUrl,
+    };
+    whalebeData.push(tmpData);
+  });
+  return whalebeData;
+};

src/db/data/handler.ts

@@ -3,6 +3,7 @@ import {
   noticeCrawling,
   noticeListCrawling,
 } from '@crawling/noticeCrawling';
+import { whalebeCrawling } from '@crawling/whalebeCrawling';
 import { RowDataPacket } from 'mysql2';
 import { College, Notice } from 'src/@types/college';
 import db from 'src/db';
@@ -205,3 +206,20 @@ export const saveSchoolNoticeToDB = async (): Promise<void> => {
 
   await Promise.all(savePromises);
 };
+
+export const saveWhalebeToDB = async (): Promise<void> => {
+  const query = 'INSERT INTO 웨일비 (title, date, imgUrl) VALUES (?, ?, ?)';
+  const whalebeDatas = await whalebeCrawling();
+
+  const promises = whalebeDatas.map((data) => {
+    const values = [data.title, data.date, data.imgUrl];
+
+    return new Promise<void>((resolve) => {
+      db.query(query, values, () => {
+        resolve();
+      });
+    });
+  });
+
+  Promise.all(promises);
+};

src/db/table/createTables.ts

@@ -102,8 +102,26 @@ const createSchoolNoticeTable = () => {
   }
 };
 
+const createWhalebeDataTable = () => {
+  const createTableQuery = `CREATE TABLE 웨일비 (
+    id INT PRIMARY KEY AUTO_INCREMENT,
+    title VARCHAR(255) NOT NULL UNIQUE,
+    date VARCHAR(255) NOT NULL,
+    imgUrl VARCHAR(255) NOT NULL
+        );`;
+
+  db.query(createTableQuery, (error) => {
+    if (error) {
+      console.log('웨일비 DB 생성 실패', error);
+      return;
+    }
+    console.log('웨일비 테이블 생성 성공!');
+  });
+};
+
 const createAllTables = (college: College[]) => {
   createDepartmentTable();
+  createWhalebeDataTable();
   createGraduationTable();
   createSchoolNoticeTable();
   createNoticeTable(college);

src/hooks/cronNoticeCrawling.ts

@@ -1,5 +1,9 @@
 import { pushNotification } from '@apis/subscribe/service';
-import { saveNoticeToDB, saveSchoolNoticeToDB } from '@db/data/handler';
+import {
+  saveNoticeToDB,
+  saveSchoolNoticeToDB,
+  saveWhalebeToDB,
+} from '@db/data/handler';
 import cron from 'node-cron';
 import notificationToSlack from 'src/hooks/notificateToSlack';
 
@@ -16,6 +20,7 @@ cron.schedule('0 3 * * *', async () => {
   const majors = await saveNoticeToDB();
   await saveNoticeToDB();
   await saveSchoolNoticeToDB();
+  await saveWhalebeToDB();
   const today = new Date();
   const year = today.getFullYear();
   const month = today.getMonth() + 1; // 월은 0부터 시작하므로 1을 더해줍니다.

src/hooks/startCrawlingData.ts

@@ -4,6 +4,7 @@ import {
   saveDepartmentToDB,
   saveNoticeToDB,
   saveSchoolNoticeToDB,
+  saveWhalebeToDB,
 } from '@db/data/handler';
 import db from '@db/index';
 import createNoticeTable from '@db/table/createTables';
@@ -22,6 +23,7 @@ export const initialCrawling = () => {
         await saveDepartmentToDB(collegeList);
         await saveGraduationRequirementToDB();
         await saveSchoolNoticeToDB();
+        await saveWhalebeToDB();
         await saveNoticeToDB();
       }
     });

src/index.ts

@@ -4,6 +4,8 @@ import noticeRouter from '@apis/notice/controller';
 import subscriptionRouter from '@apis/subscribe/controller';
 import suggestionRouter from '@apis/suggestion/controller';
 import env from '@config';
+import { saveWhalebeToDB } from '@db/data/handler';
+import db from '@db/index';
 import { corsOptions } from '@middlewares/cors';
 import errorHandler from '@middlewares/error-handler';
 import cors from 'cors';
@@ -39,3 +41,27 @@ app.listen(env.SERVER_PORT, () => {
 });
 
 webpush();
+
+const forDeployedServer = () => {
+  // 이 함수는 현재 배포되어있는 서버를 위해 사용되는 로직이며 최초 서버에 배포되는 1회만 실행되도록 하기위한 함수에요
+  // 그렇기에 아래에 작성된 코드들은 배포서버에 배포되면 다음 배포전 수정해주세요!!
+
+  // 웨일비 관련 테이블 생성 후 데이터 삽입
+  const createTableQuery = `CREATE TABLE 웨일비 (
+    id INT PRIMARY KEY AUTO_INCREMENT,
+    title VARCHAR(255) NOT NULL UNIQUE,
+    date VARCHAR(255) NOT NULL,
+    imgUrl VARCHAR(255) NOT NULL
+        );`;
+
+  db.query(createTableQuery, (error) => {
+    if (error) {
+      console.log('웨일비 DB 생성 실패', error);
+      return;
+    }
+    console.log('웨일비 테이블 생성 성공!');
+    saveWhalebeToDB();
+  });
+};
+
+forDeployedServer();