diff --git a/src/crawling/noticeCrawling.ts b/src/crawling/noticeCrawling.ts index 49341ca8..b2c8a76f 100644 --- a/src/crawling/noticeCrawling.ts +++ b/src/crawling/noticeCrawling.ts @@ -74,6 +74,14 @@ export const noticeListCrawling = async ( const pinnedNotice: string[] = []; const normalNotice: string[] = []; + if (link === 'http://geoinfo.pknu.ac.kr/05piazza/08.php') { + const noticePage2Link = + 'http://geoinfo.pknu.ac.kr/05piazza/08.php?p=2&key=&keyword=&bbscode=cate0501&reCategory='; + const noticePage2Lists = await noticeListCrawling(noticePage2Link, link); + pinnedNotice.push(...noticePage2Lists.pinnedNotice); + normalNotice.push(...noticePage2Lists.normalNotice); + } + tableData.each((index, element) => { const anchorElement = $(element).find('a'); let tmpLink = anchorElement.attr('href'); @@ -86,7 +94,7 @@ export const noticeListCrawling = async ( .text() .match(/\d{4}[-.]\d{2}[-.]\d{2}/); - if (link === 'http://geoinfo.pknu.ac.kr/05piazza/08.php') { + if (link.startsWith('http://geoinfo.pknu.ac.kr/05piazza/08.php')) { // 공간정보시스템공학과 if ($(element).find('td').first().text().trim() === '공지') pinnedNotice.push(tmpLink); diff --git a/src/crawling/whalebeCrawling.ts b/src/crawling/whalebeCrawling.ts index aaa5d8f9..6e7e08a5 100644 --- a/src/crawling/whalebeCrawling.ts +++ b/src/crawling/whalebeCrawling.ts @@ -5,8 +5,18 @@ export interface WhalebeData { title: string; date: string; imgUrl: string; + link: string; } +const sliceURL = (link: string): string => { + if (link.startsWith('location.href=')) { + link = link.slice(15); + link = link.slice(0, -2); + } + + return link; +}; + export const whalebeCrawling = async (): Promise => { const hostname = 'https://whalebe.pknu.ac.kr'; const whalebeLink = hostname + '/main'; @@ -20,7 +30,8 @@ export const whalebeCrawling = async (): Promise => { programs.each((_, element) => { const imgUrl = hostname + $(element).find('img').attr('src'); - const title = $(element).find('.card-title').text(); + const link = hostname + sliceURL($(element).find('div').attr('onclick')); + const title = $(element).find('.card-title').text().trim(); const date = $(element) .find('.app_date') .find('.col-12') @@ -32,6 +43,7 @@ export const whalebeCrawling = async (): Promise => { title, date, imgUrl, + link, }; whalebeData.push(tmpData); }); diff --git a/src/db/data/noticeHandler.ts b/src/db/data/noticeHandler.ts index 7764eb52..2cdbd9d0 100644 --- a/src/db/data/noticeHandler.ts +++ b/src/db/data/noticeHandler.ts @@ -50,6 +50,15 @@ const saveNotice = (notice: Notice, major: string): Promise => { }); }; +const deleteNotice = (major: string, noticeLinks: string[], mode: string) => { + const deleteQuery = `DELETE FROM ${major}${mode} WHERE link = ?`; + for (const link of noticeLinks) { + db.query(deleteQuery, [link], (err) => { + if (err) notificationToSlack(`${major}${mode} 공지사항 삭제 실패`); + }); + } +}; + export const saveNoticeToDB = async (): Promise => { const selectQuery = 'SELECT * FROM departments;'; const results = await new Promise((resolve) => { @@ -97,6 +106,7 @@ export const saveNoticeToDB = async (): Promise => { return; } const rows = res as RowDataPacket[]; + const deleteNotiLinks: string[] = []; let pinnedNotiLink: string[] = []; if (Array.isArray(rows) && rows.length > 0) @@ -112,6 +122,13 @@ export const saveNoticeToDB = async (): Promise => { savePromises.push(saveNotice(result, major + '고정')); } } + + for (const noticeLink of pinnedNotiLink) { + if (!noticeLists.pinnedNotice.includes(noticeLink)) { + deleteNotiLinks.push(noticeLink); + } + } + deleteNotice(major, deleteNotiLinks, '고정'); }); } @@ -219,11 +236,12 @@ export const saveSchoolNoticeToDB = async (): Promise => { }; export const saveWhalebeToDB = async (): Promise => { - const query = 'INSERT INTO 웨일비 (title, date, imgUrl) VALUES (?, ?, ?)'; + const query = + 'INSERT INTO 웨일비 (title, date, imgUrl, link) VALUES (?, ?, ?, ?)'; const whalebeDatas = await whalebeCrawling(); const promises = whalebeDatas.map((data) => { - const values = [data.title, data.date, data.imgUrl]; + const values = [data.title, data.date, data.imgUrl, data.link]; return new Promise((resolve) => { db.query(query, values, () => { diff --git a/src/db/table/createTables.ts b/src/db/table/createTables.ts index 7a010496..758cde14 100644 --- a/src/db/table/createTables.ts +++ b/src/db/table/createTables.ts @@ -107,7 +107,8 @@ const createWhalebeDataTable = () => { id INT PRIMARY KEY AUTO_INCREMENT, title VARCHAR(255) NOT NULL UNIQUE, date VARCHAR(255) NOT NULL, - imgUrl VARCHAR(255) NOT NULL + imgUrl VARCHAR(255) NOT NULL, + link VARCHAR(255) NOT NULL );`; db.query(createTableQuery, (error) => { diff --git a/src/index.ts b/src/index.ts index 81bdce5f..9ed4272f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import subscriptionRouter from '@apis/subscribe/controller'; import suggestionRouter from '@apis/suggestion/controller'; import env from '@config'; import { saveLanguageNoticeToDB } from '@db/data/languageHandler'; +import { saveWhalebeToDB } from '@db/data/noticeHandler'; import db from '@db/index'; import { corsOptions } from '@middlewares/cors'; import errorHandler from '@middlewares/error-handler'; @@ -12,6 +13,7 @@ import cors from 'cors'; import express, { Request, Response } from 'express'; import morgan from 'morgan'; import webpush from 'src/config/webpush'; +import notificationToSlack from 'src/hooks/notificateToSlack'; import { initialCrawling } from 'src/hooks/startCrawlingData'; import './hooks/cronNoticeCrawling'; @@ -42,7 +44,7 @@ app.listen(env.SERVER_PORT, () => { webpush(); -const handleDeployToServer = () => { +const handleDeployToServer = async () => { // 이 함수는 현재 배포되어있는 서버를 위해 사용되는 로직이며 최초 서버에 배포되는 1회만 실행되도록 하기위한 함수에요 // 그렇기에 아래에 작성된 코드들은 배포서버에 배포되면 다음 배포전 수정해주세요!! @@ -67,6 +69,20 @@ const handleDeployToServer = () => { }; createLanguageDataTable(); + + const addColumnQuery = `ALTER TABLE 웨일비 ADD link VARCHAR(255)`; + db.query(addColumnQuery, (err) => { + if (err) { + notificationToSlack('배포시 최초 핸들러 함수 에러'); + return; + } + const query = `delete from 웨일비;`; + db.query(query, (err) => { + if (!err) { + saveWhalebeToDB(); + } + }); + }); }; -handleDeployToServer(); +// handleDeployToServer();