WIP

package.json

@@ -7,12 +7,21 @@
   "scripts": {
     "build-dist": "rimraf dist && rimraf output && tsc --project tsconfig.build.json",
     "schedule": "yarn build-dist && node dist/schedule.js",
-    "schedule-prod": "yarn build-dist && pm2 start dist/schedule.js",
+    "schedule-prod": "yarn build-dist && pm2 start dist/schedule.js --cron-restart=\"59 0 * * *\"",
     "lint-files": "eslint ./src/**/*.ts",
     "lint-tests": "eslint ./src/**/*.test.ts",
     "lint": "yarn lint-files && yarn lint-tests",
     "test": "jest --silent src/!tests",
-    "test:watch": "jest src/!tests --watch"
+    "test:watch": "jest src/!tests --watch",
+
+    "generate-replays-list": "yarn build-dist && node dist/jobs/prepareReplaysList/start.js",
+    "generate-replays-list-dev": "yarn build-dist && NODE_ENV=development node dist/jobs/prepareReplaysList/start.js",
+
+    "parse": "yarn build-dist && node ./dist/start.js",
+    "parse-dev": "yarn build-dist && NODE_ENV=development node ./dist/start.js",
+
+    "parse-new-year": "yarn build-dist && node ./dist/!yearStatistics/index.js",
+    "parse-new-year-dev": "yarn build-dist && NODE_ENV=development node ./dist/!yearStatistics/index.js"
   },
   "author": "Afgan0r",
   "devDependencies": {

src/0 - types/replay.d.ts

@@ -33,7 +33,7 @@ type PlayerPosition = [
   isPlayer: 0 | 1,
 ];
 type VehiclePosition = [
-  pos: Position3D,
+  pos: Position3D | undefined,
   direction: number,
   isAlive: 0 | 1,
   playersInside: number[], // order like this [driver, gunner, commander, turrets, cargo]
@@ -46,6 +46,8 @@ type CommonEntity = {
   startFrameNum: number;
 };
 
+type MinifiedCommonEntity = Omit<CommonEntity, 'framesFired' | 'startFrameNum'>;
+
 type PlayerEntity = CommonEntity & {
   type: 'unit';
   description: string;
@@ -55,12 +57,25 @@ type PlayerEntity = CommonEntity & {
   positions: PlayerPosition[];
 };
 
+type MinifiedPlayerEntity = MinifiedCommonEntity & {
+  type: 'unit';
+  description: string;
+  isPlayer: 0 | 1;
+  side: EntitySide;
+  group: string;
+};
+
 type VehicleEntity = CommonEntity & {
   type: 'vehicle';
   class: RawVehicleClass;
   positions: VehiclePosition[];
 };
 
+type MinifiedVehicleEntity = MinifiedCommonEntity & {
+  type:'vehicle';
+  class: RawVehicleClass;
+};
+
 type ReplayInfo = {
   playersCount: number[];
   endFrame: number;
@@ -74,6 +89,10 @@ type ReplayInfo = {
   worldName: string;
 };
 
+type MinifiedReplayInfo = Pick<ReplayInfo, 'playersCount' | 'endFrame' | 'events' | 'missionAuthor' | 'missionName' | 'worldName'> & {
+  entities: Array<MinifiedPlayerEntity | MinifiedVehicleEntity>;
+};
+
 type PlayerInfo = {
   id: EntityId;
   name: PlayerName;
@@ -97,6 +116,7 @@ type VehicleInfo = {
   id: EntityId;
   name: EntityName;
   class: RawVehicleClass;
+  positions: VehiclePosition[];
 };
 
 type VehicleList = Record<EntityId, VehicleInfo>;

src/0 - utils/cronitorHelper.ts

@@ -11,7 +11,7 @@ const getCronitorURL = (jobKey: JobKey) => (
   `https://cronitor.link/p/${apiKey}/${jobsName[jobKey]}`
 );
 
-export const pingMonitor = async (
+export const pingMonitor = (
   jobKey: JobKey,
   state: 'run' | 'complete' | 'fail' | 'ok',
   message?: string,
@@ -28,5 +28,5 @@ export const pingMonitor = async (
     { arrayFormat: 'repeat', encode: false },
   );
 
-  request(`${url}?${params}`, {}, 5);
+  return request(`${url}?${params}`, {}, 5);
 };

src/0 - utils/filterPlayersByTotalPlayedGames.ts

@@ -2,16 +2,20 @@ type Params = {
   statistics: GlobalPlayerStatistics[];
   gamesCount: number;
   type?: 'remove' | 'not show';
+  isNewYearStats?: boolean;
 };
 
 const filterPlayersByTotalPlayedGames = ({
   statistics,
   gamesCount,
   type,
+  isNewYearStats = false,
 }: Params) => {
-  const minGamesCount = gamesCount < 125
-    ? (15 * gamesCount) / 100 // 15%
-    : 20;
+  let minGamesCount = 20;
+
+  if (isNewYearStats || (!isNewYearStats && gamesCount < 125)) {
+    minGamesCount = (15 * gamesCount) / 100; // 15%
+  }
 
   const condition = (count) => count >= minGamesCount;
 

src/0 - utils/generateBasicFolders.ts

@@ -2,10 +2,6 @@ import fs from 'fs-extra';
 
 import { basicPaths } from './paths';
 
-const generateBasicFolders = () => (
-  basicPaths.forEach((dir) => {
-    fs.ensureDirSync(dir);
-  })
-);
+const generateBasicFolders = () => basicPaths.forEach((dir) => { fs.ensureDirSync(dir); });
 
 export default generateBasicFolders;

src/0 - utils/isDev.ts

@@ -0,0 +1,3 @@
+const isDev = process.env.NODE_ENV === 'development';
+
+export default isDev;

src/0 - utils/logger.ts

@@ -7,7 +7,7 @@ import { dayjsUTC } from './dayjs';
 import { dateFormat } from './namesHelper/utils/consts';
 import { logsPath } from './paths';
 
-const pinoPrettyDefaultOptions = { colorize: true, colorizeObjects: true };
+const pinoPrettyDefaultOptions = { sync: true, colorize: true, colorizeObjects: true };
 
 const getTransport = () => {
   fs.ensureDirSync(logsPath);
@@ -34,4 +34,9 @@ const getTransport = () => {
 
 const logger = pino(getTransport());
 
+process.on('uncaughtException', (err) => {
+  logger.fatal('Uncaught exception detected', err);
+  process.exit(1);
+});
+
 export default logger;

src/0 - utils/namesHelper/prepareNamesList.ts

@@ -8,8 +8,8 @@ import { v4 as uuid } from 'uuid';
 import { getNamesList, setNamesList } from '.';
 
 import { dayjsUTC, dayjsUnix } from '../dayjs';
-import { configPath } from '../paths';
 import logger from '../logger';
+import { configPath } from '../paths';
 import pipe from '../pipe';
 import { findNameInfo } from './findNameInfo';
 import moscowDateToUTC from './moscowDateToUTC';
@@ -71,8 +71,10 @@ export const prepareNamesList = (): void => {
   const processedRecords = pipe(filter, order)(records);
 
   processedRecords.forEach((record) => {
-    let oldName = record['Старый позывной'].toLowerCase();
-    let newName = record['Новый позывной'].toLowerCase();
+    const originalOldName = record['Старый позывной'].trim();
+    const originalNewName = record['Новый позывной'].trim();
+    let oldName = originalOldName.toLowerCase();
+    let newName = originalNewName.toLowerCase();
 
     const date = moscowDateToUTC(record['Дата смены ника']);
     const formattedDate = date.format(dateFormat);
@@ -90,24 +92,28 @@ export const prepareNamesList = (): void => {
     if (!oldNameInfo) {
       newNamesList[oldName] = {
         id,
+        name: originalOldName,
         fromDate: dayjsUnix(1).format(dateFormat),
         endDate: formattedDate,
       };
 
       newNamesList[newName] = {
         id,
+        name: originalNewName,
         fromDate: formattedDate,
         endDate: dayjsUTC('2100-01-01').endOf('day').format(dateFormat),
       };
     } else {
       newNamesList[oldName] = {
         id,
+        name: originalOldName,
         fromDate: oldNameInfo.info.fromDate,
         endDate: formattedDate,
       };
 
       newNamesList[newName] = {
         id,
+        name: originalNewName,
         fromDate: formattedDate,
         endDate: dayjsUTC('2100-01-01').endOf('day').format(dateFormat),
       };

src/0 - utils/namesHelper/utils/types.ts

@@ -1,5 +1,6 @@
 export type NameInfo = {
   id: PlayerId;
+  name: string;
   fromDate: string;
   endDate: string;
 };

src/0 - utils/paths.ts

@@ -1,7 +1,9 @@
 import os from 'os';
 import path from 'path';
 
-const statsPath = path.join(os.homedir(), 'sg_stats');
+import isDev from './isDev';
+
+const statsPath = path.join(os.homedir(), isDev ? 'dev_sg_stats' : 'sg_stats');
 
 export const rawReplaysPath = path.join(statsPath, 'raw_replays');
 

src/1 - replays/parseReplays.ts

@@ -56,7 +56,7 @@ const parseReplays = async (
   replays: Replay[],
   gameType: GameType,
 ): Promise<PlayersGameResult[]> => {
-  const limit = pLimit(gameType === 'mace' ? 30 : 10);
+  const limit = pLimit(gameType === 'mace' ? 50 : 25);
   const parsedReplays = await Promise.all(
     replays.map(
       (replay) => limit(

src/2 - parseReplayInfo/getEntities.ts

@@ -37,12 +37,15 @@ const getEntities = ({ entities, events }: ReplayInfo): VehiclesWithPlayersList
     }
 
     if (entity.type === 'vehicle') {
-      const { id, name, class: vehicleClass } = entity;
+      const {
+        id, name, class: vehicleClass, positions,
+      } = entity;
 
       vehicles[id] = {
         id,
         name,
         class: vehicleClass,
+        positions,
       };
     }
   });

src/3 - statistics/global/add.ts

@@ -1,17 +1,17 @@
-import fs from 'fs-extra';
 import path from 'path';
 
 import { Dayjs } from 'dayjs';
+import fs from 'fs-extra';
 import { round } from 'lodash';
 
 import calculateKDRatio from '../../0 - utils/calculateKDRatio';
 import calculateScore from '../../0 - utils/calculateScore';
 import { dayjsUTC } from '../../0 - utils/dayjs';
-import { configPath } from '../../0 - utils/paths';
 import getPlayerName from '../../0 - utils/getPlayerName';
 import { isInInterval } from '../../0 - utils/isInInterval';
 import mergeOtherPlayers from '../../0 - utils/mergeOtherPlayers';
 import { getPlayerId } from '../../0 - utils/namesHelper/getId';
+import { configPath } from '../../0 - utils/paths';
 import { unionWeaponsStatistic } from '../../0 - utils/weaponsStatistic';
 import { defaultStatistics } from '../consts';
 import addPlayerGameResultToWeekStatistics from './addToResultsByWeek';

src/4 - output/index.ts

@@ -29,17 +29,17 @@ const generateOutput = async (statistics: StatisticsForOutput): Promise<void> =>
         generalRotationsStats.push(omit(rotation, 'stats'));
         generateRotationJSONOutput(rotation, index, folderName);
       });
-    }
 
-    fs.writeFileSync(
-      path.join(folderPath, rotationsGeneralInfoFileName),
-      JSON.stringify(generalRotationsStats, null, '\t'),
-    );
+      fs.writeFileSync(
+        path.join(folderPath, rotationsGeneralInfoFileName),
+        JSON.stringify(generalRotationsStats, null, '\t'),
+      );
+    }
   });
 
   await archiveFiles(folderNames);
 
-  fs.rmdirSync(resultsPath, { recursive: true });
+  fs.removeSync(resultsPath);
   fs.moveSync(tempResultsPath, resultsPath);
 };
 

src/index.ts

@@ -1,10 +1,13 @@
+import fs from 'fs-extra';
+
 import { gameTypes } from './0 - consts';
 import { dayjsUTC } from './0 - utils/dayjs';
 import filterPlayersByTotalPlayedGames from './0 - utils/filterPlayersByTotalPlayedGames';
 import formatGameType from './0 - utils/formatGameType';
 import generateBasicFolders from './0 - utils/generateBasicFolders';
 import logger from './0 - utils/logger';
 import { prepareNamesList } from './0 - utils/namesHelper/prepareNamesList';
+import { tempResultsPath } from './0 - utils/paths';
 import getReplays from './1 - replays/getReplays';
 import parseReplays from './1 - replays/parseReplays';
 import calculateGlobalStatistics from './3 - statistics/global';
@@ -55,8 +58,11 @@ const countStatistics = (
 
 const startParsingReplays = async () => {
   generateBasicFolders();
+  fs.mkdirSync(tempResultsPath);
   prepareNamesList();
 
+  logger.info('Started parsing replays.');
+
   const [sgParsedReplays, maceParsedReplays, smParsedReplays] = await Promise.all(
     gameTypes.map((gameType) => getParsedReplays(gameType)),
   );
@@ -74,7 +80,7 @@ const startParsingReplays = async () => {
 
   logger.info('All statistics collected, start generating output files.');
 
-  generateOutput({
+  await generateOutput({
     sg: { ...sgStats },
     mace: { ...maceStats },
     sm: { ...smStats },

src/jobs/generateMaceListHTML/index.ts

@@ -5,9 +5,9 @@ import { JSDOM } from 'jsdom';
 
 import { dayjsUTC } from '../../0 - utils/dayjs';
 import defaultDateFormat from '../../0 - utils/defaultDateFormat';
-import { listsPath, replaysListPath } from '../../0 - utils/paths';
 import generateBasicHTML from '../../0 - utils/generateBasicHTML';
 import logger from '../../0 - utils/logger';
+import { listsPath, replaysListPath } from '../../0 - utils/paths';
 import body from './utils/body';
 
 type MaceReplayItem = {

src/jobs/generateMissionMakersList/index.ts

@@ -5,9 +5,9 @@ import { JSDOM } from 'jsdom';
 
 import { dayjsUTC } from '../../0 - utils/dayjs';
 import defaultDateFormat from '../../0 - utils/defaultDateFormat';
-import { listsPath } from '../../0 - utils/paths';
 import generateBasicHTML from '../../0 - utils/generateBasicHTML';
 import logger from '../../0 - utils/logger';
+import { listsPath } from '../../0 - utils/paths';
 import body from './utils/body';
 import fetchTeamPage from './utils/requestTeamPage';
 

src/jobs/prepareReplaysList/index.ts

@@ -3,8 +3,9 @@
 import fs from 'fs-extra';
 import { union } from 'lodash';
 
-import { replaysListPath } from '../../0 - utils/paths';
+import generateBasicFolders from '../../0 - utils/generateBasicFolders';
 import logger from '../../0 - utils/logger';
+import { replaysListPath } from '../../0 - utils/paths';
 import { defaultEmptyOutput, excludeReplaysPath, includeReplaysPath } from './consts';
 import parseReplaysOnPage from './parseReplaysOnPage';
 import checks from './utils/checks';
@@ -44,6 +45,7 @@ const readExcludeReplays = (): ConfigExcludeReplays => {
 };
 
 const startFetchingReplays = async () => {
+  generateBasicFolders();
   const replaysList = readReplaysListFile();
   const includeReplays = readIncludeReplays();
   const excludeReplays = readExcludeReplays();