From 06b494aa2a183f71b86ee907b0b986e4899cdff8 Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Sun, 17 Sep 2023 22:26:33 +0900 Subject: [PATCH 1/7] =?UTF-8?q?Feat:=20=EC=9B=A8=EC=9D=BC=EB=B9=84=20?= =?UTF-8?q?=ED=8E=98=EC=9D=B4=EC=A7=80=20=ED=81=AC=EB=A1=A4=EB=A7=81?= =?UTF-8?q?=ED=95=98=EB=8A=94=20=ED=95=A8=EC=88=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/crawling/whalebeCrawling.ts | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/crawling/whalebeCrawling.ts diff --git a/src/crawling/whalebeCrawling.ts b/src/crawling/whalebeCrawling.ts new file mode 100644 index 00000000..54cf21a7 --- /dev/null +++ b/src/crawling/whalebeCrawling.ts @@ -0,0 +1,39 @@ +import axios from 'axios'; +import * as cheerio from 'cheerio'; + +interface WhalebeData { + title: string; + date: string; + imgUrl: string; +} + +export const whalebeCrawling = async (): Promise => { + const hostname = 'https://whalebe.pknu.ac.kr'; + const whalebeLink = hostname + '/main'; + const whalebeData: WhalebeData[] = []; + + const response = await axios.get(whalebeLink); + const $ = cheerio.load(response.data); + + const programs = $('ul.px-0').find('li'); + if (programs.length < 1) return; + + programs.each((_, element) => { + const imgUrl = hostname + $(element).find('img').attr('src'); + const title = $(element).find('.card-title').text(); + const date = $(element) + .find('.app_date') + .find('.col-12') + .first() + .text() + .split('~')[1] + .trim(); + const tmpData = { + title, + date, + imgUrl, + }; + whalebeData.push(tmpData); + }); + return whalebeData; +}; From cfdb3a47fec7c60d955b824cdf4ba1fcc2aec0e8 Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Sun, 17 Sep 2023 23:16:38 +0900 Subject: [PATCH 2/7] =?UTF-8?q?Feat:=20=EC=9B=A8=EC=9D=BC=EB=B9=84=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EB=A5=BC=20=EC=A0=80=EC=9E=A5?= =?UTF-8?q?=ED=95=A0=20DB=20=EC=83=9D=EC=84=B1=ED=95=98=EB=8A=94=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/table/createTables.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/db/table/createTables.ts b/src/db/table/createTables.ts index e100e3b0..845517a9 100644 --- a/src/db/table/createTables.ts +++ b/src/db/table/createTables.ts @@ -102,8 +102,26 @@ const createSchoolNoticeTable = () => { } }; +const createWhalebeDataTable = () => { + const createTableQuery = `CREATE TABLE 웨일비 ( + id INT PRIMARY KEY AUTO_INCREMENT, + title VARCHAR(255) NOT NULL UNIQUE, + date VARCHAR(255) NOT NULL, + imgUrl VARCHAR(255) NOT NULL + );`; + + db.query(createTableQuery, (error) => { + if (error) { + console.log('웨일비 DB 생성 실패', error); + return; + } + console.log('웨일비 테이블 생성 성공!'); + }); +}; + const createAllTables = (college: College[]) => { createDepartmentTable(); + createWhalebeDataTable(); createGraduationTable(); createSchoolNoticeTable(); createNoticeTable(college); From e874dacbd2067156db3e4fc98cf7c7dabeea277e Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Mon, 18 Sep 2023 00:03:32 +0900 Subject: [PATCH 3/7] =?UTF-8?q?Feat:=20=EC=9B=A8=EC=9D=BC=EB=B9=84=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EB=A5=BC=20DB=20=EC=97=90=20?= =?UTF-8?q?=EC=A0=80=EC=9E=A5=ED=95=98=EB=8A=94=20=ED=95=A8=EC=88=98=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/data/handler.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/db/data/handler.ts b/src/db/data/handler.ts index b59e6db6..78bba19c 100644 --- a/src/db/data/handler.ts +++ b/src/db/data/handler.ts @@ -3,6 +3,7 @@ import { noticeCrawling, noticeListCrawling, } from '@crawling/noticeCrawling'; +import { whalebeCrawling } from '@crawling/whalebeCrawling'; import { RowDataPacket } from 'mysql2'; import { College, Notice } from 'src/@types/college'; import db from 'src/db'; @@ -205,3 +206,20 @@ export const saveSchoolNoticeToDB = async (): Promise => { await Promise.all(savePromises); }; + +export const saveWhalebeToDB = async (): Promise => { + const query = 'INSERT INTO 웨일비 (title, date, imgUrl) VALUES (?, ?, ?)'; + const whalebeDatas = await whalebeCrawling(); + + const promises = whalebeDatas.map((data) => { + const values = [data.title, data.date, data.imgUrl]; + + return new Promise((resolve) => { + db.query(query, values, () => { + resolve(); + }); + }); + }); + + Promise.all(promises); +}; From db5ef90f51a910682fc116ab17c5d95068aa1fbd Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Mon, 18 Sep 2023 00:04:18 +0900 Subject: [PATCH 4/7] =?UTF-8?q?Feat:=20=EC=9B=A8=EC=9D=BC=EB=B9=84=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EB=A5=BC=20=EB=B0=98=ED=99=98?= =?UTF-8?q?=ED=95=98=EB=8A=94=20=EC=97=94=EB=93=9C=ED=8F=AC=EC=9D=B8?= =?UTF-8?q?=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/notice/controller.ts | 11 ++++++++++- src/apis/notice/service.ts | 12 ++++++++++++ src/crawling/whalebeCrawling.ts | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/apis/notice/controller.ts b/src/apis/notice/controller.ts index 851755c6..dc130cad 100644 --- a/src/apis/notice/controller.ts +++ b/src/apis/notice/controller.ts @@ -1,4 +1,4 @@ -import { getNotices, getSchoolNotices } from '@apis/notice/service'; +import { getNotices, getSchoolNotices, getWhalebe } from '@apis/notice/service'; import express, { Request, Response } from 'express'; const router = express.Router(); @@ -17,4 +17,13 @@ router.get('/', async (req: Request, res: Response) => { } }); +router.get('/whalebe', async (req: Request, res: Response) => { + try { + const whalebeData = await getWhalebe(); + res.json(whalebeData); + } catch (err) { + res.json(); + } +}); + export default router; diff --git a/src/apis/notice/service.ts b/src/apis/notice/service.ts index e20a2668..ec521390 100644 --- a/src/apis/notice/service.ts +++ b/src/apis/notice/service.ts @@ -1,5 +1,7 @@ +import { WhalebeData } from '@crawling/whalebeCrawling'; import db from '@db/index'; import { Notice } from 'src/@types/college'; +import notificationToSlack from 'src/hooks/notificateToSlack'; interface SeparateNoti { 고정: Notice[]; @@ -42,3 +44,13 @@ export const getSchoolNotices = async (): Promise => { }; return notices; }; + +export const getWhalebe = async (): Promise => { + const query = 'SELECT * FROM 웨일비;'; + return new Promise((resolve) => { + db.query(query, (err, res) => { + if (err) notificationToSlack('웨일비 조회 실패'); + resolve(res as WhalebeData[]); + }); + }); +}; diff --git a/src/crawling/whalebeCrawling.ts b/src/crawling/whalebeCrawling.ts index 54cf21a7..aaa5d8f9 100644 --- a/src/crawling/whalebeCrawling.ts +++ b/src/crawling/whalebeCrawling.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import * as cheerio from 'cheerio'; -interface WhalebeData { +export interface WhalebeData { title: string; date: string; imgUrl: string; From c9ef75bc216a291e7dc53103eb04fe147e3aae4c Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Mon, 18 Sep 2023 00:04:56 +0900 Subject: [PATCH 5/7] =?UTF-8?q?Feat:=20=EB=A7=A4=EC=9D=BC=20=EC=9B=A8?= =?UTF-8?q?=EC=9D=BC=EB=B9=84=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=ED=81=AC?= =?UTF-8?q?=EB=A1=A4=EB=A7=81=ED=95=98=EB=8F=84=EB=A1=9D=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hooks/cronNoticeCrawling.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hooks/cronNoticeCrawling.ts b/src/hooks/cronNoticeCrawling.ts index 599d23ab..56f52fd9 100644 --- a/src/hooks/cronNoticeCrawling.ts +++ b/src/hooks/cronNoticeCrawling.ts @@ -1,5 +1,9 @@ import { pushNotification } from '@apis/subscribe/service'; -import { saveNoticeToDB, saveSchoolNoticeToDB } from '@db/data/handler'; +import { + saveNoticeToDB, + saveSchoolNoticeToDB, + saveWhalebeToDB, +} from '@db/data/handler'; import cron from 'node-cron'; import notificationToSlack from 'src/hooks/notificateToSlack'; @@ -16,6 +20,7 @@ cron.schedule('0 3 * * *', async () => { const majors = await saveNoticeToDB(); await saveNoticeToDB(); await saveSchoolNoticeToDB(); + await saveWhalebeToDB(); const today = new Date(); const year = today.getFullYear(); const month = today.getMonth() + 1; // 월은 0부터 시작하므로 1을 더해줍니다. From 63d252f68403ee827e88df522fb9376bd6a6b087 Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Mon, 18 Sep 2023 00:05:42 +0900 Subject: [PATCH 6/7] =?UTF-8?q?Feat:=20=EC=84=9C=EB=B2=84=20=EC=B5=9C?= =?UTF-8?q?=EC=B4=88=20=EC=8B=A4=ED=96=89=20=EC=8B=9C=20=EC=9B=A8=EC=9D=BC?= =?UTF-8?q?=EB=B9=84=20=EB=8D=B0=EC=9D=B4=ED=84=B0=EB=A5=BC=20=EC=A0=80?= =?UTF-8?q?=EC=9E=A5=ED=95=98=EB=8A=94=20=EB=A1=9C=EC=A7=81=20=EC=8B=9C?= =?UTF-8?q?=EC=9E=91=ED=95=98=EB=8F=84=EB=A1=9D=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/hooks/startCrawlingData.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hooks/startCrawlingData.ts b/src/hooks/startCrawlingData.ts index 7a730dfc..bdd8b500 100644 --- a/src/hooks/startCrawlingData.ts +++ b/src/hooks/startCrawlingData.ts @@ -4,6 +4,7 @@ import { saveDepartmentToDB, saveNoticeToDB, saveSchoolNoticeToDB, + saveWhalebeToDB, } from '@db/data/handler'; import db from '@db/index'; import createNoticeTable from '@db/table/createTables'; @@ -22,6 +23,7 @@ export const initialCrawling = () => { await saveDepartmentToDB(collegeList); await saveGraduationRequirementToDB(); await saveSchoolNoticeToDB(); + await saveWhalebeToDB(); await saveNoticeToDB(); } }); From a5f466e08e35a02cce5e028fcdce73291ddd69d3 Mon Sep 17 00:00:00 2001 From: Lee sang Yeop Date: Mon, 18 Sep 2023 00:12:00 +0900 Subject: [PATCH 7/7] =?UTF-8?q?Feat:=20=EB=B0=B0=ED=8F=AC=EB=90=98?= =?UTF-8?q?=EC=96=B4=20=EC=9E=88=EB=8A=94=20=EC=84=9C=EB=B2=84=EB=A5=BC=20?= =?UTF-8?q?=EC=9C=84=ED=95=B4=20=EC=9B=A8=EC=9D=BC=EB=B9=84=20DB=20?= =?UTF-8?q?=ED=85=8C=EC=9D=B4=EB=B8=94=20=EC=83=9D=EC=84=B1=20=ED=9B=84=20?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=84=B0=EC=82=BD=EC=9E=85=ED=95=98=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/index.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/index.ts b/src/index.ts index a69eda63..2f3c4403 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,8 @@ import noticeRouter from '@apis/notice/controller'; import subscriptionRouter from '@apis/subscribe/controller'; import suggestionRouter from '@apis/suggestion/controller'; import env from '@config'; +import { saveWhalebeToDB } from '@db/data/handler'; +import db from '@db/index'; import { corsOptions } from '@middlewares/cors'; import errorHandler from '@middlewares/error-handler'; import cors from 'cors'; @@ -39,3 +41,27 @@ app.listen(env.SERVER_PORT, () => { }); webpush(); + +const forDeployedServer = () => { + // 이 함수는 현재 배포되어있는 서버를 위해 사용되는 로직이며 최초 서버에 배포되는 1회만 실행되도록 하기위한 함수에요 + // 그렇기에 아래에 작성된 코드들은 배포서버에 배포되면 다음 배포전 수정해주세요!! + + // 웨일비 관련 테이블 생성 후 데이터 삽입 + const createTableQuery = `CREATE TABLE 웨일비 ( + id INT PRIMARY KEY AUTO_INCREMENT, + title VARCHAR(255) NOT NULL UNIQUE, + date VARCHAR(255) NOT NULL, + imgUrl VARCHAR(255) NOT NULL + );`; + + db.query(createTableQuery, (error) => { + if (error) { + console.log('웨일비 DB 생성 실패', error); + return; + } + console.log('웨일비 테이블 생성 성공!'); + saveWhalebeToDB(); + }); +}; + +forDeployedServer();