From af7fc18c6a4612e84f81a9ce27d0690284b34fb5 Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sun, 7 Jan 2024 15:50:00 +0700 Subject: [PATCH] WIP --- package.json | 13 +++- src/0 - types/replay.d.ts | 22 +++++- src/0 - utils/cronitorHelper.ts | 4 +- .../filterPlayersByTotalPlayedGames.ts | 10 ++- src/0 - utils/generateBasicFolders.ts | 6 +- src/0 - utils/isDev.ts | 3 + src/0 - utils/logger.ts | 7 +- src/0 - utils/namesHelper/prepareNamesList.ts | 12 ++- src/0 - utils/namesHelper/utils/types.ts | 1 + src/0 - utils/paths.ts | 4 +- src/1 - replays/parseReplays.ts | 2 +- src/2 - parseReplayInfo/getEntities.ts | 5 +- src/3 - statistics/global/add.ts | 4 +- src/4 - output/index.ts | 12 +-- src/index.ts | 8 +- src/jobs/generateMaceListHTML/index.ts | 2 +- src/jobs/generateMissionMakersList/index.ts | 2 +- src/jobs/prepareReplaysList/index.ts | 4 +- src/schedule.ts | 78 +++++++++++-------- tsconfig.json | 2 +- 20 files changed, 135 insertions(+), 66 deletions(-) create mode 100644 src/0 - utils/isDev.ts diff --git a/package.json b/package.json index df78107..ae5339d 100644 --- a/package.json +++ b/package.json @@ -7,12 +7,21 @@ "scripts": { "build-dist": "rimraf dist && rimraf output && tsc --project tsconfig.build.json", "schedule": "yarn build-dist && node dist/schedule.js", - "schedule-prod": "yarn build-dist && pm2 start dist/schedule.js", + "schedule-prod": "yarn build-dist && pm2 start dist/schedule.js --cron-restart=\"59 0 * * *\"", "lint-files": "eslint ./src/**/*.ts", "lint-tests": "eslint ./src/**/*.test.ts", "lint": "yarn lint-files && yarn lint-tests", "test": "jest --silent src/!tests", - "test:watch": "jest src/!tests --watch" + "test:watch": "jest src/!tests --watch", + + "generate-replays-list": "yarn build-dist && node dist/jobs/prepareReplaysList/start.js", + "generate-replays-list-dev": "yarn build-dist && NODE_ENV=development node dist/jobs/prepareReplaysList/start.js", + + "parse": "yarn build-dist && node ./dist/start.js", + "parse-dev": "yarn build-dist && NODE_ENV=development node ./dist/start.js", + + "parse-new-year": "yarn build-dist && node ./dist/!yearStatistics/index.js", + "parse-new-year-dev": "yarn build-dist && NODE_ENV=development node ./dist/!yearStatistics/index.js" }, "author": "Afgan0r", "devDependencies": { diff --git a/src/0 - types/replay.d.ts b/src/0 - types/replay.d.ts index 32c0990..f976e77 100644 --- a/src/0 - types/replay.d.ts +++ b/src/0 - types/replay.d.ts @@ -33,7 +33,7 @@ type PlayerPosition = [ isPlayer: 0 | 1, ]; type VehiclePosition = [ - pos: Position3D, + pos: Position3D | undefined, direction: number, isAlive: 0 | 1, playersInside: number[], // order like this [driver, gunner, commander, turrets, cargo] @@ -46,6 +46,8 @@ type CommonEntity = { startFrameNum: number; }; +type MinifiedCommonEntity = Omit; + type PlayerEntity = CommonEntity & { type: 'unit'; description: string; @@ -55,12 +57,25 @@ type PlayerEntity = CommonEntity & { positions: PlayerPosition[]; }; +type MinifiedPlayerEntity = MinifiedCommonEntity & { + type: 'unit'; + description: string; + isPlayer: 0 | 1; + side: EntitySide; + group: string; +}; + type VehicleEntity = CommonEntity & { type: 'vehicle'; class: RawVehicleClass; positions: VehiclePosition[]; }; +type MinifiedVehicleEntity = MinifiedCommonEntity & { + type:'vehicle'; + class: RawVehicleClass; +}; + type ReplayInfo = { playersCount: number[]; endFrame: number; @@ -74,6 +89,10 @@ type ReplayInfo = { worldName: string; }; +type MinifiedReplayInfo = Pick & { + entities: Array; +}; + type PlayerInfo = { id: EntityId; name: PlayerName; @@ -97,6 +116,7 @@ type VehicleInfo = { id: EntityId; name: EntityName; class: RawVehicleClass; + positions: VehiclePosition[]; }; type VehicleList = Record; diff --git a/src/0 - utils/cronitorHelper.ts b/src/0 - utils/cronitorHelper.ts index 467895a..ece8235 100644 --- a/src/0 - utils/cronitorHelper.ts +++ b/src/0 - utils/cronitorHelper.ts @@ -11,7 +11,7 @@ const getCronitorURL = (jobKey: JobKey) => ( `https://cronitor.link/p/${apiKey}/${jobsName[jobKey]}` ); -export const pingMonitor = async ( +export const pingMonitor = ( jobKey: JobKey, state: 'run' | 'complete' | 'fail' | 'ok', message?: string, @@ -28,5 +28,5 @@ export const pingMonitor = async ( { arrayFormat: 'repeat', encode: false }, ); - request(`${url}?${params}`, {}, 5); + return request(`${url}?${params}`, {}, 5); }; diff --git a/src/0 - utils/filterPlayersByTotalPlayedGames.ts b/src/0 - utils/filterPlayersByTotalPlayedGames.ts index 9026cd9..0312fbe 100644 --- a/src/0 - utils/filterPlayersByTotalPlayedGames.ts +++ b/src/0 - utils/filterPlayersByTotalPlayedGames.ts @@ -2,16 +2,20 @@ type Params = { statistics: GlobalPlayerStatistics[]; gamesCount: number; type?: 'remove' | 'not show'; + isNewYearStats?: boolean; }; const filterPlayersByTotalPlayedGames = ({ statistics, gamesCount, type, + isNewYearStats = false, }: Params) => { - const minGamesCount = gamesCount < 125 - ? (15 * gamesCount) / 100 // 15% - : 20; + let minGamesCount = 20; + + if (isNewYearStats || (!isNewYearStats && gamesCount < 125)) { + minGamesCount = (15 * gamesCount) / 100; // 15% + } const condition = (count) => count >= minGamesCount; diff --git a/src/0 - utils/generateBasicFolders.ts b/src/0 - utils/generateBasicFolders.ts index 4a912cd..4b7fbb7 100644 --- a/src/0 - utils/generateBasicFolders.ts +++ b/src/0 - utils/generateBasicFolders.ts @@ -2,10 +2,6 @@ import fs from 'fs-extra'; import { basicPaths } from './paths'; -const generateBasicFolders = () => ( - basicPaths.forEach((dir) => { - fs.ensureDirSync(dir); - }) -); +const generateBasicFolders = () => basicPaths.forEach((dir) => { fs.ensureDirSync(dir); }); export default generateBasicFolders; diff --git a/src/0 - utils/isDev.ts b/src/0 - utils/isDev.ts new file mode 100644 index 0000000..0b25637 --- /dev/null +++ b/src/0 - utils/isDev.ts @@ -0,0 +1,3 @@ +const isDev = process.env.NODE_ENV === 'development'; + +export default isDev; diff --git a/src/0 - utils/logger.ts b/src/0 - utils/logger.ts index d490d06..377edcf 100644 --- a/src/0 - utils/logger.ts +++ b/src/0 - utils/logger.ts @@ -7,7 +7,7 @@ import { dayjsUTC } from './dayjs'; import { dateFormat } from './namesHelper/utils/consts'; import { logsPath } from './paths'; -const pinoPrettyDefaultOptions = { colorize: true, colorizeObjects: true }; +const pinoPrettyDefaultOptions = { sync: true, colorize: true, colorizeObjects: true }; const getTransport = () => { fs.ensureDirSync(logsPath); @@ -34,4 +34,9 @@ const getTransport = () => { const logger = pino(getTransport()); +process.on('uncaughtException', (err) => { + logger.fatal('Uncaught exception detected', err); + process.exit(1); +}); + export default logger; diff --git a/src/0 - utils/namesHelper/prepareNamesList.ts b/src/0 - utils/namesHelper/prepareNamesList.ts index 22d9d96..65bfd0c 100644 --- a/src/0 - utils/namesHelper/prepareNamesList.ts +++ b/src/0 - utils/namesHelper/prepareNamesList.ts @@ -8,8 +8,8 @@ import { v4 as uuid } from 'uuid'; import { getNamesList, setNamesList } from '.'; import { dayjsUTC, dayjsUnix } from '../dayjs'; -import { configPath } from '../paths'; import logger from '../logger'; +import { configPath } from '../paths'; import pipe from '../pipe'; import { findNameInfo } from './findNameInfo'; import moscowDateToUTC from './moscowDateToUTC'; @@ -71,8 +71,10 @@ export const prepareNamesList = (): void => { const processedRecords = pipe(filter, order)(records); processedRecords.forEach((record) => { - let oldName = record['Старый позывной'].toLowerCase(); - let newName = record['Новый позывной'].toLowerCase(); + const originalOldName = record['Старый позывной'].trim(); + const originalNewName = record['Новый позывной'].trim(); + let oldName = originalOldName.toLowerCase(); + let newName = originalNewName.toLowerCase(); const date = moscowDateToUTC(record['Дата смены ника']); const formattedDate = date.format(dateFormat); @@ -90,24 +92,28 @@ export const prepareNamesList = (): void => { if (!oldNameInfo) { newNamesList[oldName] = { id, + name: originalOldName, fromDate: dayjsUnix(1).format(dateFormat), endDate: formattedDate, }; newNamesList[newName] = { id, + name: originalNewName, fromDate: formattedDate, endDate: dayjsUTC('2100-01-01').endOf('day').format(dateFormat), }; } else { newNamesList[oldName] = { id, + name: originalOldName, fromDate: oldNameInfo.info.fromDate, endDate: formattedDate, }; newNamesList[newName] = { id, + name: originalNewName, fromDate: formattedDate, endDate: dayjsUTC('2100-01-01').endOf('day').format(dateFormat), }; diff --git a/src/0 - utils/namesHelper/utils/types.ts b/src/0 - utils/namesHelper/utils/types.ts index b64d1d5..39f52cd 100644 --- a/src/0 - utils/namesHelper/utils/types.ts +++ b/src/0 - utils/namesHelper/utils/types.ts @@ -1,5 +1,6 @@ export type NameInfo = { id: PlayerId; + name: string; fromDate: string; endDate: string; }; diff --git a/src/0 - utils/paths.ts b/src/0 - utils/paths.ts index fa2ccd4..5d19f04 100644 --- a/src/0 - utils/paths.ts +++ b/src/0 - utils/paths.ts @@ -1,7 +1,9 @@ import os from 'os'; import path from 'path'; -const statsPath = path.join(os.homedir(), 'sg_stats'); +import isDev from './isDev'; + +const statsPath = path.join(os.homedir(), isDev ? 'dev_sg_stats' : 'sg_stats'); export const rawReplaysPath = path.join(statsPath, 'raw_replays'); diff --git a/src/1 - replays/parseReplays.ts b/src/1 - replays/parseReplays.ts index 73b2def..a2bb61e 100644 --- a/src/1 - replays/parseReplays.ts +++ b/src/1 - replays/parseReplays.ts @@ -56,7 +56,7 @@ const parseReplays = async ( replays: Replay[], gameType: GameType, ): Promise => { - const limit = pLimit(gameType === 'mace' ? 30 : 10); + const limit = pLimit(gameType === 'mace' ? 50 : 25); const parsedReplays = await Promise.all( replays.map( (replay) => limit( diff --git a/src/2 - parseReplayInfo/getEntities.ts b/src/2 - parseReplayInfo/getEntities.ts index d3364ff..50b888c 100644 --- a/src/2 - parseReplayInfo/getEntities.ts +++ b/src/2 - parseReplayInfo/getEntities.ts @@ -37,12 +37,15 @@ const getEntities = ({ entities, events }: ReplayInfo): VehiclesWithPlayersList } if (entity.type === 'vehicle') { - const { id, name, class: vehicleClass } = entity; + const { + id, name, class: vehicleClass, positions, + } = entity; vehicles[id] = { id, name, class: vehicleClass, + positions, }; } }); diff --git a/src/3 - statistics/global/add.ts b/src/3 - statistics/global/add.ts index 9263717..5e8da42 100644 --- a/src/3 - statistics/global/add.ts +++ b/src/3 - statistics/global/add.ts @@ -1,17 +1,17 @@ -import fs from 'fs-extra'; import path from 'path'; import { Dayjs } from 'dayjs'; +import fs from 'fs-extra'; import { round } from 'lodash'; import calculateKDRatio from '../../0 - utils/calculateKDRatio'; import calculateScore from '../../0 - utils/calculateScore'; import { dayjsUTC } from '../../0 - utils/dayjs'; -import { configPath } from '../../0 - utils/paths'; import getPlayerName from '../../0 - utils/getPlayerName'; import { isInInterval } from '../../0 - utils/isInInterval'; import mergeOtherPlayers from '../../0 - utils/mergeOtherPlayers'; import { getPlayerId } from '../../0 - utils/namesHelper/getId'; +import { configPath } from '../../0 - utils/paths'; import { unionWeaponsStatistic } from '../../0 - utils/weaponsStatistic'; import { defaultStatistics } from '../consts'; import addPlayerGameResultToWeekStatistics from './addToResultsByWeek'; diff --git a/src/4 - output/index.ts b/src/4 - output/index.ts index b1b613f..9b8736b 100644 --- a/src/4 - output/index.ts +++ b/src/4 - output/index.ts @@ -29,17 +29,17 @@ const generateOutput = async (statistics: StatisticsForOutput): Promise => generalRotationsStats.push(omit(rotation, 'stats')); generateRotationJSONOutput(rotation, index, folderName); }); - } - fs.writeFileSync( - path.join(folderPath, rotationsGeneralInfoFileName), - JSON.stringify(generalRotationsStats, null, '\t'), - ); + fs.writeFileSync( + path.join(folderPath, rotationsGeneralInfoFileName), + JSON.stringify(generalRotationsStats, null, '\t'), + ); + } }); await archiveFiles(folderNames); - fs.rmdirSync(resultsPath, { recursive: true }); + fs.removeSync(resultsPath); fs.moveSync(tempResultsPath, resultsPath); }; diff --git a/src/index.ts b/src/index.ts index 8609fc7..d16112c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ +import fs from 'fs-extra'; + import { gameTypes } from './0 - consts'; import { dayjsUTC } from './0 - utils/dayjs'; import filterPlayersByTotalPlayedGames from './0 - utils/filterPlayersByTotalPlayedGames'; @@ -5,6 +7,7 @@ import formatGameType from './0 - utils/formatGameType'; import generateBasicFolders from './0 - utils/generateBasicFolders'; import logger from './0 - utils/logger'; import { prepareNamesList } from './0 - utils/namesHelper/prepareNamesList'; +import { tempResultsPath } from './0 - utils/paths'; import getReplays from './1 - replays/getReplays'; import parseReplays from './1 - replays/parseReplays'; import calculateGlobalStatistics from './3 - statistics/global'; @@ -55,8 +58,11 @@ const countStatistics = ( const startParsingReplays = async () => { generateBasicFolders(); + fs.mkdirSync(tempResultsPath); prepareNamesList(); + logger.info('Started parsing replays.'); + const [sgParsedReplays, maceParsedReplays, smParsedReplays] = await Promise.all( gameTypes.map((gameType) => getParsedReplays(gameType)), ); @@ -74,7 +80,7 @@ const startParsingReplays = async () => { logger.info('All statistics collected, start generating output files.'); - generateOutput({ + await generateOutput({ sg: { ...sgStats }, mace: { ...maceStats }, sm: { ...smStats }, diff --git a/src/jobs/generateMaceListHTML/index.ts b/src/jobs/generateMaceListHTML/index.ts index 7526c91..8ebebcd 100644 --- a/src/jobs/generateMaceListHTML/index.ts +++ b/src/jobs/generateMaceListHTML/index.ts @@ -5,9 +5,9 @@ import { JSDOM } from 'jsdom'; import { dayjsUTC } from '../../0 - utils/dayjs'; import defaultDateFormat from '../../0 - utils/defaultDateFormat'; -import { listsPath, replaysListPath } from '../../0 - utils/paths'; import generateBasicHTML from '../../0 - utils/generateBasicHTML'; import logger from '../../0 - utils/logger'; +import { listsPath, replaysListPath } from '../../0 - utils/paths'; import body from './utils/body'; type MaceReplayItem = { diff --git a/src/jobs/generateMissionMakersList/index.ts b/src/jobs/generateMissionMakersList/index.ts index e982abf..a9b7f44 100644 --- a/src/jobs/generateMissionMakersList/index.ts +++ b/src/jobs/generateMissionMakersList/index.ts @@ -5,9 +5,9 @@ import { JSDOM } from 'jsdom'; import { dayjsUTC } from '../../0 - utils/dayjs'; import defaultDateFormat from '../../0 - utils/defaultDateFormat'; -import { listsPath } from '../../0 - utils/paths'; import generateBasicHTML from '../../0 - utils/generateBasicHTML'; import logger from '../../0 - utils/logger'; +import { listsPath } from '../../0 - utils/paths'; import body from './utils/body'; import fetchTeamPage from './utils/requestTeamPage'; diff --git a/src/jobs/prepareReplaysList/index.ts b/src/jobs/prepareReplaysList/index.ts index 06e1ade..c05502f 100644 --- a/src/jobs/prepareReplaysList/index.ts +++ b/src/jobs/prepareReplaysList/index.ts @@ -3,8 +3,9 @@ import fs from 'fs-extra'; import { union } from 'lodash'; -import { replaysListPath } from '../../0 - utils/paths'; +import generateBasicFolders from '../../0 - utils/generateBasicFolders'; import logger from '../../0 - utils/logger'; +import { replaysListPath } from '../../0 - utils/paths'; import { defaultEmptyOutput, excludeReplaysPath, includeReplaysPath } from './consts'; import parseReplaysOnPage from './parseReplaysOnPage'; import checks from './utils/checks'; @@ -44,6 +45,7 @@ const readExcludeReplays = (): ConfigExcludeReplays => { }; const startFetchingReplays = async () => { + generateBasicFolders(); const replaysList = readReplaysListFile(); const includeReplays = readIncludeReplays(); const excludeReplays = readExcludeReplays(); diff --git a/src/schedule.ts b/src/schedule.ts index 8de3cb4..706cd5c 100644 --- a/src/schedule.ts +++ b/src/schedule.ts @@ -4,9 +4,9 @@ import fs from 'fs-extra'; import startParsingReplays from '.'; // import { pingMonitor } from './0 - utils/cronitorHelper'; -import { tempResultsPath } from './0 - utils/paths'; import generateBasicFolders from './0 - utils/generateBasicFolders'; import logger from './0 - utils/logger'; +import { tempResultsPath } from './0 - utils/paths'; import generateMaceList from './jobs/generateMaceListHTML'; import generateMissionMakersList from './jobs/generateMissionMakersList'; import startFetchingReplays from './jobs/prepareReplaysList'; @@ -15,27 +15,33 @@ generateBasicFolders(); Cron( '*/5 * * * *', - { - protect: true, - catch: (err: Error) => { - logger.error(`Error during fetching mission makers list. Trace: ${err.stack}`); - // pingMonitor('missionMakersFetcher', 'fail', err.message); - }, - }, + { protect: true }, async () => { // pingMonitor('missionMakersFetcher', 'run'); - await generateMissionMakersList(); + + try { + await generateMissionMakersList(); + } catch (err) { + logger.error(`Error during fetching mission makers list. Trace: ${err.stack}`); + // pingMonitor('missionMakersFetcher', 'fail', err.message); + + // return; + } + // pingMonitor('missionMakersFetcher', 'complete'); }, ); -const generateMaceListJob = () => { +const generateMaceListJob = async () => { // pingMonitor('maceListGenerator', 'run'); try { generateMaceList(); - } catch (e) { - // pingMonitor('maceListGenerator', 'fail', e.message); + } catch (err) { + logger.error(`Error during mace list generation. Trace: ${err.stack}`); + // pingMonitor('maceListGenerator', 'fail', err.message); + + // return; } // pingMonitor('maceListGenerator', 'complete'); @@ -43,16 +49,19 @@ const generateMaceListJob = () => { const replaysFetcherJob = Cron( '*/5 * * * *', - { - protect: true, - catch: (err: Error) => { - logger.error(`Error during fetching replays list. Trace: ${err.stack}`); - // pingMonitor('replaysFetcher', 'fail', err.message); - }, - }, + { protect: true }, async () => { // pingMonitor('replaysFetcher', 'run'); - await startFetchingReplays(); + + try { + await startFetchingReplays(); + } catch (err) { + logger.error(`Error during fetching replays list. Trace: ${err.stack}`); + // pingMonitor('replaysFetcher', 'fail', err.message); + + return; + } + // pingMonitor('replaysFetcher', 'complete'); generateMaceListJob(); @@ -72,17 +81,8 @@ const waitReplaysFetchingToFinish = async (): Promise => ( Cron( '4 */1 * * *', - { - protect: true, - catch: (err: Error) => { - logger.error(`Error during parsing replays list. Trace: ${err.stack}`); - fs.rmdirSync(tempResultsPath, { recursive: true }); - // pingMonitor('replaysParser', 'fail', err.message); - }, - }, + { protect: true }, async () => { - // pingMonitor('replaysParser', 'run'); - if (replaysFetcherJob.isBusy()) { const beforeMsg = 'Replays list is preparing, waiting...'; const afterMsg = 'Replays list is finished.'; @@ -96,9 +96,21 @@ Cron( // pingMonitor('replaysParser', 'ok', afterMsg); } - fs.mkdirSync(tempResultsPath); - await startParsingReplays(); - fs.rmdirSync(tempResultsPath, { recursive: true }); + // pingMonitor('replaysParser', 'run'); + + try { + fs.removeSync(tempResultsPath); + + await startParsingReplays(); + } catch (err) { + logger.error(`Error during parsing replays list. Trace: ${err.stack}`); + + fs.removeSync(tempResultsPath); + // pingMonitor('replaysParser', 'fail', err.message); + + // return; + } + // pingMonitor('replaysParser', 'complete'); }, ); diff --git a/tsconfig.json b/tsconfig.json index c0b200d..19d7985 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,4 +20,4 @@ "ts-node": { "esm": true, } -} \ No newline at end of file +}