Merge branch 'develop'

bmino · Sep 3, 2020 · 402d29f · 402d29f
2 parents be6e8c9 + 892f437
commit 402d29f
Show file tree

Hide file tree

Showing 12 changed files with 179 additions and 52 deletions.
diff --git a/.gitignore b/.gitignore
@@ -100,5 +100,5 @@ $RECYCLE.BIN/
 # Example Configuration
 !*.example
 
-src/sandbox/
-**/Sandbox*.js
+**/*Sandbox*.js
+**/*sandbox*.js
diff --git a/config/config.json.example b/config/config.json.example
@@ -34,15 +34,19 @@
     },
 
     "DEPTH": {
-        "SIZE": 50,
-        "PRUNE": true,
+        "SIZE": 50
+    },
+
+    "WEBSOCKETS": {
+        "BUNDLE_SIZE": 1,
         "INITIALIZATION_INTERVAL": 75
     },
 
     "TIMING": {
         "RECEIVE_WINDOW": 5000,
         "USE_SERVER_TIME": false,
-        "CALCULATION_COOLDOWN": 250
+        "CALCULATION_COOLDOWN": 250,
+        "STATUS_UPDATE_INTERVAL": 120000
     }
 
 }
diff --git a/config/readme.md b/config/readme.md
@@ -143,11 +143,17 @@ Upon each version update you should copy the new syntax from `config.json.exampl
     * `100`
     * `500`
 
-#### `DEPTH.PRUNE` (Boolean)
-* Default: `true`
-* Description: Remove depth cache entries with a depth greater than `DEPTH.SIZE` before each calculation cycle
 
-#### `DEPTH.INITIALIZATION_INTERVAL` (Number)
+---
+
+
+### `WEBSOCKETS`
+
+#### `WEBSOCKETS.BUNDLE_SIZE` (Number)
+* Default: `1`
+* Description: Number of tickers combined/included in each depth websocket
+
+#### `WEBSOCKETS.INITIALIZATION_INTERVAL` (Number)
 * Default: `75`
 * Description: Delay (ms) between the initialization of each depth websocket
 
@@ -168,3 +174,7 @@ Upon each version update you should copy the new syntax from `config.json.exampl
 #### `TIMING.CALCULATION_COOLDOWN` (Number)
 * Default: `250`
 * Description: Delay (ms) between completing calculations and starting another cycle
+
+#### `TIMING.STATUS_UPDATE_INTERVAL` (Number)
+* Default: `120000`
+* Description: Interval (ms) between each status update
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "binance-triangle-arbitrage",
-  "version": "5.5.1",
+  "version": "5.6.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/bmino/binance-triangle-arbitrage.git"

diff --git a/src/main/ArbitrageExecution.js b/src/main/ArbitrageExecution.js
@@ -2,6 +2,7 @@ const CONFIG = require('../../config/config');
 const logger = require('./Loggers');
 const BinanceApi = require('./BinanceApi');
 const CalculationNode = require('./CalculationNode');
+const Util = require('./Util');
 
 const ArbitrageExecution = {
 
@@ -64,8 +65,14 @@ const ArbitrageExecution = {
                 logger.execution.debug(`Observed Conversion:  ${actual.c.spent.toFixed(8)} ${symbol.c} into ${actual.a.earned.toFixed(8)} ${symbol.a}`);
                 logger.execution.debug(`Price Error:          ${variation.ca.toFixed(8)}%`);
 
-                logger.execution.trace(`Depth cache used for calculation:`);
-                logger.execution.trace(calculated.depth);
+                const prunedDepthSnapshot = {
+                    ab: Util.pruneSnapshot(calculated.depth.ab, CalculationNode.getOrderBookDepthRequirement(calculated.trade.ab.method, calculated.ab, calculated.depth.ab) + 1),
+                    bc: Util.pruneSnapshot(calculated.depth.bc, CalculationNode.getOrderBookDepthRequirement(calculated.trade.bc.method, calculated.bc, calculated.depth.bc) + 1),
+                    ca: Util.pruneSnapshot(calculated.depth.ca, CalculationNode.getOrderBookDepthRequirement(calculated.trade.ca.method, calculated.ca, calculated.depth.ca) + 1)
+                };
+
+                logger.execution.trace(`Pruned depth cache used for calculation:`);
+                logger.execution.trace(prunedDepthSnapshot);
 
                 const percent = {
                     a: actual.a.delta / actual.a.spent * 100,

diff --git a/src/main/BinanceApi.js b/src/main/BinanceApi.js
@@ -38,7 +38,7 @@ const BinanceApi = {
     getDepthSnapshots(tickers) {
         const depthSnapshot = {};
         tickers.forEach((ticker) => {
-            depthSnapshot[ticker] = binance.depthCache(ticker);
+            depthSnapshot[ticker] = { ...binance.depthCache(ticker) };
         });
         return depthSnapshot;
     },
@@ -101,6 +101,21 @@ const BinanceApi = {
         return binance.websockets.depthCacheStaggered(tickers, BinanceApi.sortDepthCache, limit, stagger);
     },
 
+    depthCacheWebsockets(tickers, limit, groupSize, stagger) {
+        let chain = null;
+
+        for (let i=0; i < tickers.length; i += groupSize) {
+            const tickerGroup = tickers.slice(i, i + groupSize);
+            let promise = () => new Promise( resolve => {
+                binance.websockets.depthCache( tickerGroup, BinanceApi.sortDepthCache, limit );
+                setTimeout( resolve, stagger );
+            } );
+            chain = chain ? chain.then( promise ) : promise();
+        }
+
+        return chain;
+    },
+
     sortDepthCache(ticker, depth) {
         depth.bids = binance.sortBids(depth.bids);
         depth.asks = binance.sortAsks(depth.asks);

diff --git a/src/main/CalculationNode.js b/src/main/CalculationNode.js
@@ -212,6 +212,32 @@ const CalculationNode = {
         }
     },
 
+    getOrderBookDepthRequirement(method, quantity, depthSnapshot) {
+        let exchanged = 0;
+        let i;
+        const bidRates = Object.keys(depthSnapshot.bids || {});
+        const askRates = Object.keys(depthSnapshot.asks || {});
+
+        if (method === 'Sell') {
+            for (i=0; i<bidRates.length; i++) {
+                exchanged += depthSnapshot.bids[bidRates[i]];
+                if (exchanged >= quantity) {
+                    return i+1;
+                }
+            }
+        } else if (method === 'Buy') {
+            for (i=0; i<askRates.length; i++) {
+                exchanged += depthSnapshot.asks[askRates[i]];
+                if (exchanged >= quantity) {
+                    return i+1;
+                }
+            }
+        } else {
+            throw new Error(`Unknown method: ${method}`);
+        }
+        return i;
+    },
+
     calculateDustless(trade, amount) {
         if (Number.isInteger(amount)) return amount;
         const amountString = amount.toFixed(12);

diff --git a/src/main/Main.js b/src/main/Main.js
@@ -1,5 +1,6 @@
 const CONFIG = require('../../config/config');
 const logger = require('./Loggers');
+const Util = require('./Util');
 const os = require('os');
 const BinanceApi = require('./BinanceApi');
 const MarketCache = require('./MarketCache');
@@ -8,27 +9,42 @@ const ArbitrageExecution = require('./ArbitrageExecution');
 const CalculationNode = require('./CalculationNode');
 const SpeedTest = require('./SpeedTest');
 
+let recentCalculationTimes = [];
+
 // Helps identify application startup
+logger.binance.info(logger.LINE);
 logger.execution.info(logger.LINE);
 logger.performance.info(logger.LINE);
 
 if (CONFIG.TRADING.ENABLED) console.log(`WARNING! Order execution is enabled!\n`);
 
+process.on('uncaughtException', handleError);
+
 checkConfig()
-    .then(SpeedTest.multiPing)
+    .then(() => {
+        console.log(`Checking latency ...`);
+        return SpeedTest.multiPing(5);
+    })
     .then((pings) => {
-        const msg = `Successfully pinged Binance in ${(pings.reduce((a,b) => a+b, 0) / pings.length).toFixed(0)} ms`;
+        const msg = `Experiencing ${Util.average(pings).toFixed(0)} ms of latency`;
         console.log(msg);
         logger.performance.info(msg);
     })
-    .then(BinanceApi.exchangeInfo)
+    .then(() => {
+        console.log(`Fetching exchange info ...`);
+        return BinanceApi.exchangeInfo();
+    })
     .then(exchangeInfo => MarketCache.initialize(exchangeInfo, CONFIG.TRADING.WHITELIST, CONFIG.INVESTMENT.BASE))
     .then(checkBalances)
     .then(() => {
         // Listen for depth updates
         const tickers = MarketCache.tickers.watching;
-        console.log(`Opening ${tickers.length} depth websockets ...`);
-        return BinanceApi.depthCacheStaggered(tickers, CONFIG.DEPTH.SIZE, CONFIG.DEPTH.INITIALIZATION_INTERVAL);
+        console.log(`Opening ${Math.ceil(tickers.length / CONFIG.WEBSOCKETS.BUNDLE_SIZE)} depth websockets ...`);
+        if (CONFIG.WEBSOCKETS.BUNDLE_SIZE === 1) {
+            return BinanceApi.depthCacheStaggered(tickers, CONFIG.DEPTH.SIZE, CONFIG.WEBSOCKETS.INITIALIZATION_INTERVAL);
+        } else {
+            return BinanceApi.depthCacheWebsockets(tickers, CONFIG.DEPTH.SIZE, CONFIG.WEBSOCKETS.BUNDLE_SIZE, CONFIG.WEBSOCKETS.INITIALIZATION_INTERVAL);
+        }
     })
     .then(() => {
         console.log();
@@ -39,43 +55,54 @@ checkConfig()
         console.log(`Log Level:              ${CONFIG.LOG.LEVEL}`);
         console.log();
 
-        logger.performance.debug(`Operating System: ${os.type()}`);
-        logger.performance.debug(`Cores Speeds: [${os.cpus().map(cpu => cpu.speed)}] MHz`);
+        logger.performance.debug(`Operating System: ${os.type()} ${os.release()}`);
+        logger.performance.debug(`System Total Memory: ${(os.totalmem() / 1073741824).toFixed(1)} GB`)
+        logger.performance.debug(`CPU Core Speeds: [${os.cpus().map(cpu => cpu.speed)}] MHz`);
 
-        // Allow time to read output before starting calculation cycles
-        setTimeout(calculateArbitrage, 5000);
+        // Allow time for depth caches to populate
+        setTimeout(calculateArbitrage, 6000);
+        setInterval(displayStatusUpdate, CONFIG.TIMING.STATUS_UPDATE_INTERVAL);
     })
-    .catch(console.error);
+    .catch(handleError);
 
 function calculateArbitrage() {
-    if (CONFIG.DEPTH.PRUNE) MarketCache.pruneDepthsAboveThreshold(CONFIG.DEPTH.SIZE);
+    const depthSnapshots = BinanceApi.getDepthSnapshots(MarketCache.tickers.watching);
+    MarketCache.pruneDepthCacheAboveThreshold(depthSnapshots, CONFIG.DEPTH.SIZE);
 
     const { calculationTime, successCount, errorCount, results } = CalculationNode.cycle(
         MarketCache.relationships,
-        BinanceApi.getDepthSnapshots(MarketCache.tickers.watching),
+        depthSnapshots,
         (e) => logger.performance.warn(e),
         ArbitrageExecution.executeCalculatedPosition
     );
 
+    recentCalculationTimes.push(calculationTime);
     if (CONFIG.HUD.ENABLED) refreshHUD(results);
+
     displayCalculationResults(successCount, errorCount, calculationTime);
     setTimeout(calculateArbitrage, CONFIG.TIMING.CALCULATION_COOLDOWN);
 }
 
 function displayCalculationResults(successCount, errorCount, calculationTime) {
+    if (errorCount === 0) return;
     const totalCalculations = successCount + errorCount;
+    logger.performance.warn(`Completed ${successCount}/${totalCalculations} (${((successCount/totalCalculations) * 100).toFixed(1)}%) calculations in ${calculationTime} ms`);
+}
 
-    if (errorCount > 0) {
-        logger.performance.warn(`Completed ${successCount}/${totalCalculations} (${((successCount/totalCalculations) * 100).toFixed(1)}%) calculations in ${calculationTime} ms`);
+function displayStatusUpdate() {
+    const tickersWithoutDepthUpdate = MarketCache.getWatchedTickersWithoutDepthCacheUpdate();
+    if (tickersWithoutDepthUpdate.length > 0) {
+        logger.performance.debug(`Tickers without a depth cache update: [${tickersWithoutDepthUpdate}]`);
     }
+    logger.performance.debug(`Latest ${recentCalculationTimes.length} calculation cycles averaging ${Util.average(recentCalculationTimes).toFixed(2)} ms`);
+    logger.performance.debug(`CPU 1 minute load averaging ${os.loadavg()[0].toFixed(1)}%`);
+    recentCalculationTimes = [];
+}
 
-    if (CalculationNode.cycleCount % 500 === 0) {
-        const tickersWithoutDepthUpdate = MarketCache.getWatchedTickersWithoutDepthCacheUpdate();
-        if (tickersWithoutDepthUpdate.length > 0) {
-            logger.performance.debug(`Tickers without a depth cache update: [${tickersWithoutDepthUpdate}]`);
-        }
-        logger.performance.debug(`Recent calculations completed in ${calculationTime} ms`);
-    }
+function handleError(err) {
+    console.error(err);
+    logger.binance.error(err);
+    process.exit(1);
 }
 
 function checkConfig() {
@@ -146,6 +173,16 @@ function checkConfig() {
         logger.execution.error(msg);
         throw new Error(msg);
     }
+    if (isNaN(CONFIG.WEBSOCKETS.BUNDLE_SIZE) || CONFIG.WEBSOCKETS.BUNDLE_SIZE <= 0) {
+        const msg = `Websocket bundle size (${CONFIG.WEBSOCKETS.BUNDLE_SIZE}) must be a positive integer`;
+        logger.execution.error(msg);
+        throw new Error(msg);
+    }
+    if (isNaN(CONFIG.WEBSOCKETS.INITIALIZATION_INTERVAL) || CONFIG.WEBSOCKETS.INITIALIZATION_INTERVAL < 0) {
+        const msg = `Websocket initialization interval (${CONFIG.WEBSOCKETS.INITIALIZATION_INTERVAL}) must be a positive integer`;
+        logger.execution.error(msg);
+        throw new Error(msg);
+    }
     if (CONFIG.TIMING.RECEIVE_WINDOW > 60000) {
         const msg = `Receive window (${CONFIG.TIMING.RECEIVE_WINDOW}) must be less than 60000`;
         logger.execution.error(msg);
@@ -161,6 +198,11 @@ function checkConfig() {
         logger.execution.error(msg);
         throw new Error(msg);
     }
+    if (CONFIG.TIMING.STATUS_UPDATE_INTERVAL <= 0) {
+        const msg = `Status update interval (${CONFIG.TIMING.STATUS_UPDATE_INTERVAL}) must be a positive value`;
+        logger.execution.error(msg);
+        throw new Error(msg);
+    }
 
     return Promise.resolve();
 }

diff --git a/src/main/MarketCache.js b/src/main/MarketCache.js
@@ -1,4 +1,5 @@
 const CONFIG = require('../../config/config.json');
+const Util = require('./Util');
 const BinanceApi = require('./BinanceApi');
 
 const MarketCache = {
@@ -36,22 +37,17 @@ const MarketCache = {
         MarketCache.tickers.watching = Array.from(uniqueTickers);
     },
 
-    pruneDepthsAboveThreshold(threshold=100) {
-        const prune = (depthSnapshot, threshold) => {
-            return Object.keys(depthSnapshot)
-                .slice(0, threshold)
-                .reduce((prunedDepthSnapshot, key) => {
-                    prunedDepthSnapshot[key] = depthSnapshot[key];
-                    return prunedDepthSnapshot;
-                }, {});
-        };
-        MarketCache.tickers.watching.forEach(ticker => {
-            const depth = BinanceApi.depthCache(ticker);
-            depth.bids = prune(depth.bids, threshold);
-            depth.asks = prune(depth.asks, threshold);
+    pruneDepthCacheAboveThreshold(depthCache, threshold) {
+        Object.values(depthCache).forEach(depth => {
+            depth.bids = Util.prune(depth.bids, threshold);
+            depth.asks = Util.prune(depth.asks, threshold);
         });
     },
 
+    getWatchedTickersWithoutDepthCacheUpdate() {
+        return MarketCache.tickers.watching.filter(ticker => !BinanceApi.depthCache(ticker).eventTime);
+    },
+
     getTradesFromSymbol(symbol1) {
         const trades = [];
         MarketCache.symbols.forEach(symbol2 => {
@@ -63,10 +59,6 @@ const MarketCache = {
         return trades;
     },
 
-    getWatchedTickersWithoutDepthCacheUpdate() {
-        return MarketCache.tickers.watching.filter(ticker => !BinanceApi.depthCache(ticker).eventTime);
-    },
-
     createTrade(a, b, c) {
         a = a.toUpperCase();
         b = b.toUpperCase();

diff --git a/src/main/SpeedTest.js b/src/main/SpeedTest.js
@@ -1,8 +1,10 @@
+const logger = require('./Loggers');
 const BinanceApi = require('./BinanceApi');
 
 const SpeedTest = {
 
     ping() {
+        logger.performance.debug(`Pinging the Binance API ...`);
         const before = Date.now();
         return BinanceApi.time()
             .then(() => Date.now() - before);