add timeout and abortsignal support for queries

src/tarkov-data-manager/data/quest_delays.json

@@ -0,0 +1,102 @@
+{
+    "5ae3270f86f77445ba41d4dd": {
+        "min": 75600,
+        "max": 75600
+    },
+    "665eeca92f7aedcc900b0437": {
+        "min": 43200,
+        "max": 46800
+    },
+    "639872fa9b4fb827b200d8e5": {
+        "min": 75600,
+        "max": 75600
+    },
+    "639872fe8871e1272b10ccf6": {
+        "min": 75600,
+        "max": 75600
+    },
+    "63a5cf262964a7488f5243ce": {
+        "min": 75600,
+        "max": 75600
+    },
+    "625d6ffaf7308432be1d44c5": {
+        "min": 3600,
+        "max": 32400
+    },
+    "625d70031ed3bb5bcc5bd9e5": {
+        "min": 86400,
+        "max": 86400
+    },
+    "625d7005a4eb80027c4f2e09": {
+        "min": 86400,
+        "max": 86400
+    },
+    "639135cd8ba6894d155e77cb": {
+        "min": 86400,
+        "max": 86400
+    },
+    "63966fbeea19ac7ed845db2e": {
+        "min": 36000,
+        "max": 36000
+    },
+    "63966fccac6f8f3c677b9d89": {
+        "min": 36000,
+        "max": 36000
+    },
+    "63966fd9ea19ac7ed845db30": {
+        "min": 36000,
+        "max": 36000
+    },
+    "63966fe7ea74a47c2d3fc0e6": {
+        "min": 36000,
+        "max": 36000
+    },
+    "63966ff54c3ef01b6f3ffad8": {
+        "min": 36000,
+        "max": 36000
+    },
+    "639670029113f06a7c3b2377": {
+        "min": 36000,
+        "max": 36000
+    },
+    "6396700fea19ac7ed845db32": {
+        "min": 36000,
+        "max": 36000
+    },
+    "6396701b9113f06a7c3b2379": {
+        "min": 36000,
+        "max": 36000
+    },
+    "63967028c4a91c5cb76abd81": {
+        "min": 36000,
+        "max": 36000
+    },
+    "64f1cc571a5f313cb144bf90": {
+        "min": 36000,
+        "max": 36000
+    },
+    "626bd75d5bef5d7d590bd415": {
+        "min": 75600,
+        "max": 79200
+    },
+    "639873003693c63d86328f25": {
+        "min": 75600,
+        "max": 75600
+    },
+    "5b47825886f77468074618d3": {
+        "min": 75600,
+        "max": 75600
+    },
+    "64f83bb69878a0569d6ecfbe": {
+        "min": 75600,
+        "max": 82800
+    },
+    "64f83bcdde58fc437700d8fa": {
+        "min": 75600,
+        "max": 82800
+    },
+    "64f83bd983cfca080a362c82": {
+        "min": 75600,
+        "max": 82800
+    }
+}

src/tarkov-data-manager/jobs/index.mjs

@@ -182,6 +182,18 @@ const jobManager = {
         await Promise.allSettled([promise]);
         console.log('Startup jobs complete');
     },
+    abortJob: async (jobName) => {
+        if (!jobs[jobName]) {
+            return  Promise.reject(new Error(`${jobName} is not a valid job`));
+        }
+        if (!jobs[jobName].running) {
+            return Promise.reject (new Error(`${jobName} is not running`));
+        }
+        return new Promise((resolve) => {
+            emitter.once(`jobComplete_${jobName}`, resolve);
+            jobs[jobName].abortController.abort();
+        });
+    },
     stop: () => {
         return schedule.gracefulShutdown();
     },

src/tarkov-data-manager/jobs/update-historical-prices.mjs

@@ -3,7 +3,7 @@ import  path from 'node:path';
 
 import DataJob from '../modules/data-job.mjs';
 
-const historicalPriceDays = 7;
+const historicalPriceDays = 30;
 
 class UpdateHistoricalPricesJob extends DataJob {
     constructor(options) {

src/tarkov-data-manager/jobs/update-item-cache.mjs

@@ -62,12 +62,14 @@ class UpdateItemCacheJob extends DataJob {
 
         const priceFields = [
             'lastLowPrice',
+            'lastLowAvgPrice',
             'avg24hPrice',
             'low24hPrice',
             'high24hPrice',
             'changeLast48h',
             'changeLast48hPercent',
             'lastOfferCount',
+            'updated',
         ];
         this.logger.log('Processing items...');
         for (const [key, value] of this.itemMap.entries()) {

src/tarkov-data-manager/jobs/update-quests.mjs

@@ -34,6 +34,7 @@ class UpdateQuestsJob extends DataJob {
             this.changedQuests,
             this.removedQuests,
             this.neededKeys,
+            this.questDelays,
             this.questConfig,
             this.s3Images,
         ] = await Promise.all([
@@ -61,6 +62,7 @@ class UpdateQuestsJob extends DataJob {
             fs.readFile(path.join(import.meta.dirname, '..', 'data', 'changed_quests.json')).then(json => JSON.parse(json)),
             fs.readFile(path.join(import.meta.dirname, '..', 'data', 'removed_quests.json')).then(json => JSON.parse(json)),
             fs.readFile(path.join(import.meta.dirname, '..', 'data', 'needed_keys.json')).then(json => JSON.parse(json)),
+            fs.readFile(path.join(import.meta.dirname, '..', 'data', 'quest_delays.json')).then(json => JSON.parse(json)),
             tarkovData.questConfig(),
             getLocalBucketContents(),
         ]);
@@ -131,7 +133,6 @@ class UpdateQuestsJob extends DataJob {
             try {
                 this.logger.warn(`Adding missing quest ${quest.name} ${quest.id}...`);
                 quest.name = this.addTranslation(`${questId} name`);
-                quest.wikiLink = `https://escapefromtarkov.fandom.com/wiki/${encodeURIComponent(this.locales.en[`${questId} name`].replaceAll(' ', '_'))}`;
                 for (const obj of quest.objectives) {
                     obj.description = this.addTranslation(obj.id);
                     if (obj.type.endsWith('QuestItem')) {
@@ -265,6 +266,11 @@ class UpdateQuestsJob extends DataJob {
         for (const quest of quests.Task) {
             quest.normalizedName = this.normalizeName(this.locales.en[quest.name])+(quest.factionName !== 'Any' ? `-${this.normalizeName(quest.factionName)}` : '');
 
+            if (this.questDelays[quest.id]) {
+                quest.availableDelaySecondsMin = this.questDelays[quest.id].min;
+                quest.availableDelaySecondsMax = this.questDelays[quest.id].max;
+            }
+
             const removeReqs = [];
             for (const req of quest.taskRequirements) {
                 const questIncluded = quests.Task.some(q => q.id === req.task);
@@ -278,11 +284,13 @@ class UpdateQuestsJob extends DataJob {
 
             quest.minPlayerLevel = getQuestMinLevel(quest.id);
 
-            const trader = this.traders.find(t => t.name === quest.name);
-            const map = this.maps.find(m => m.name === quest.name);
+            const trader = this.traders.find(t => t.normalizedName === quest.normalizedName);
+            const map = this.maps.find(m => m.normalizedName === quest.normalizedName);
+            let wikiLinkSuffix = '';
             if (trader || map) {
-                quest.wikiLink = `https://escapefromtarkov.fandom.com/wiki/${encodeURIComponent(quest.name.replaceAll(' ', '_'))}_(quest)`;
+                wikiLinkSuffix = '_(quest)';
             }
+            quest.wikiLink = `https://escapefromtarkov.fandom.com/wiki/${encodeURIComponent(this.getTranslation(quest.name).replaceAll(' ', '_'))}${wikiLinkSuffix}`;
 
             quest.kappaRequired = false;
             quest.lightkeeperRequired = false;
@@ -950,7 +958,7 @@ class UpdateQuestsJob extends DataJob {
             traderName: this.locales.en[`${quest.traderId} Nickname`],
             location_id: locationId,
             locationName: locationName,
-            wikiLink: `https://escapefromtarkov.fandom.com/wiki/${encodeURIComponent(this.locales.en[`${questId} name`].replaceAll(' ', '_'))}`,
+            wikiLink: ``,
             minPlayerLevel: 0,
             taskRequirements: [],
             traderLevelRequirements: [],

src/tarkov-data-manager/modules/data-job.mjs

@@ -371,14 +371,26 @@ class DataJob {
         return this.translationHelper.hasTranslation(key, langCode);
     }
 
-    query = (sql, params) => {
-        const queryPromise = query(sql, params);
+    query = (sql, values, options) => {
+        if (typeof values === 'object' && !Array.isArray(values)) {
+            options = values;
+            values = undefined;
+        }
+        const queryPromise = query(sql, values, {signal: this.abortController.signal, ...options});
         this.queries.push(queryPromise);
         return queryPromise;
     }
 
-    batchQuery = (sql, params, batchCallback) => {
-        const queryPromise = batchQuery(sql, params, batchCallback);
+    batchQuery = (sql, values, batchCallback, options) => {
+        if (values && !Array.isArray(values)) {
+            batchCallback = values;
+            values = [];
+        }
+        if (!options && typeof batchCallback !== 'function') {
+            options = batchCallback;
+            batchCallback = undefined;
+        }
+        const queryPromise = batchQuery(sql, values, batchCallback, {signal: this.abortController.signal, ...options});
         this.queries.push(queryPromise);
         return queryPromise;
     }

src/tarkov-data-manager/modules/db-connection.mjs

@@ -59,30 +59,58 @@ const waitForConnections = () => {
 const dbConnection = {
     connection: pool,
     pool,
-    query: (sql, params) => {
+    query: async (sql, values, options = {}) => {
+        if (typeof values === 'object' && !Array.isArray(values)) {
+            options = values;
+            values = undefined;
+        }
+        let abortListener;
         return new Promise((resolve, reject) => {
+            abortListener = () => {
+                reject(new Error('Query aborted'));
+            };
+            options?.signal?.addEventListener('abort', abortListener, { once: true });
             try {
-                pool.query(sql, params, (error, results) => {
+                pool.query({sql, values, timeout: options?.timeout}, (error, results) => {
                     if (error) {
-                        reject(error);
-                        return;
+                        return reject(error);
                     }
-
                     resolve(results);
                 });
             } catch (error) {
                 reject(error);
             }
+        }).finally(() => {
+            options.signal?.removeEventListener('abort', abortListener);
         });
     },
-    batchQuery: async (sql, params = [], batchCallback) => {
+    batchQuery: async (sql, values = [], batchCallback, options = {}) => {
         const batchSize = dbConnection.maxQueryRows;
         let offset = 0;
         const results = [];
+        const queryStart = new Date();
+        if (values && !Array.isArray(values)) {
+            batchCallback = values;
+            values = [];
+        }
+        if (!options && typeof batchCallback !== 'function') {
+            options = batchCallback;
+            batchCallback = undefined;
+        }
+        let timeout = options.timeout;
         while (true) {
-            const batchParams = [...params, offset, batchSize]
-            const batchResults = await dbConnection.query(`${sql} LIMIT ?, ?
-            `, batchParams);
+            const batchValues = [...values, offset, batchSize];
+            let batchTimeout;
+            if (timeout) {
+                batchTimeout = timeout - (new Date() - queryStart);
+                if (batchTimeout <= 0) {
+                    return Promise.reject(new Error('Query inactivity timeout'));
+                }
+            }
+            if (options.signal?.aborted) {
+                return Promise.reject(new Error('Query aborted'));
+            }
+            const batchResults = await dbConnection.query(`${sql} LIMIT ?, ?`, batchValues, {timeout: batchTimeout, signal: options.signal});
             batchResults.forEach(r => results.push(r));
             if (batchCallback) {
                 batchCallback(batchResults, offset);

src/tarkov-data-manager/modules/remote-data.mjs

@@ -130,11 +130,11 @@ const methods = {
                 logger.timeEnd('item-24h-price-query');
             });
 
-            logger.time('item-last-low-price-query');
-            const lastLowPricePromise = query(`
+            logger.time('item-last-price-query');
+            const lastPricePromise = query(`
                 SELECT
                     a.item_id,
-                    MIN(a.price) AS price,
+                    a.price,
                     timestamp,
                     a.game_mode
                 FROM
@@ -151,10 +151,8 @@ const methods = {
                 ) b
                 ON
                     a.item_id = b.item_id AND a.timestamp = b.max_timestamp AND a.game_mode = b.game_mode
-                GROUP BY
-                    a.item_id, a.timestamp, a.game_mode;
             `).finally(() => {
-                logger.timeEnd('item-last-low-price-query');
+                logger.timeEnd('item-last-price-query');
             });
 
             logger.time('price-yesterday-query');
@@ -178,12 +176,12 @@ const methods = {
             const [
                 items,
                 price24hResults,
-                lastLowPriceResults,
+                lastPriceResults,
                 avgPriceYesterday,
             ] = await Promise.all([
                 itemsPromise,
                 price24hPromise,
-                lastLowPricePromise,
+                lastPricePromise,
                 avgPriceYesterdayPromise,
             ]);
 
@@ -192,6 +190,20 @@ const methods = {
                 1: {},
             };
 
+            const itemLastPrices = lastPriceResults.reduce((all, current) => {
+                if (!all[current.game_mode]) {
+                    all[current.game_mode] = {};
+                }
+                if (!all[current.game_mode][current.item_id]) {
+                    all[current.game_mode][current.item_id] = {
+                        timestamp: current.timestamp,
+                        prices: [],
+                    };
+                }
+                all[current.game_mode][current.item_id].prices.push(current.price);
+                return all;
+            }, {});
+
             price24hResults.forEach((resultRow) => {
                 if (!item24hPrices[resultRow.game_mode][resultRow.item_id]) {
                     item24hPrices[resultRow.game_mode][resultRow.item_id] = [];
@@ -208,12 +220,16 @@ const methods = {
                 for (const gameMode of gameModes) {
                     const fieldPrefix = gameMode.name === 'regular' ? '' : `${gameMode.name}_`;
 
-                    const lastLowData = lastLowPriceResults.find(row => row.item_id === itemId && row.game_mode === gameMode.value);
-                    if (lastLowData) {
-                        item[`${fieldPrefix}lastLowPrice`] = lastLowData.price;
-                        if (gameMode.name === 'regular') {
-                            item.updated = lastLowData.timestamp;
-                        }
+                    //const lastLowData = lastLowPriceResults.find(row => row.item_id === itemId && row.game_mode === gameMode.value);
+                    const lastData = itemLastPrices[gameMode.value]?.[itemId];
+                    if (lastData) {
+                        item[`${fieldPrefix}lastLowPrice`] = lastData.prices[0];
+                        const lastLowAvgPrices = lastData.prices.slice(0, 3);
+                        item[`${fieldPrefix}lastLowAvgPrice`] = Math.round(lastLowAvgPrices.reduce((total, price) => {
+                            return total + price;
+                        }, 0) / lastLowAvgPrices.length);
+                        const updatedField = `${fieldPrefix}updated`;
+                        item[updatedField] = lastData.timestamp;
                     }
 
                     item24hPrices[gameMode.value][itemId]?.sort();