diff --git a/CHANGELOG b/CHANGELOG index b2a22aecf..661729ab0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +1.6.X +* New setting: lock_during_index + 1.6.1 * fixed last_txs setting * added hashrate_units setting diff --git a/UPGRADE b/UPGRADE index af8eb0d45..813e936fe 100644 --- a/UPGRADE +++ b/UPGRADE @@ -1,5 +1,10 @@ Note: All updates require the explorer to be restarted +1.6.1 -> 1.6.X +* Fix mismatching balances/sent/received, negative balances etc +* Add new settings to settings.json (see settings.json.template) + * lock_during_index + 1.6.0 -> 1.6.1 * Add new cryptsy_id and hashrate_units settings (see settings.json.template) * remove tmp/market.pid (if it exists) diff --git a/app.js b/app.js index 81dd91e5b..8f21a9292 100644 --- a/app.js +++ b/app.js @@ -120,6 +120,7 @@ app.set('youtube', settings.youtube); app.set('genesis_block', settings.genesis_block); app.set('index', settings.index); app.set('heavy', settings.heavy); +app.set('lock_during_index', settings.lock_during_index); app.set('txcount', settings.txcount); app.set('nethash', settings.nethash); app.set('nethash_units', settings.nethash_units); diff --git a/lib/database.js b/lib/database.js index fd9bb2d65..9ac8fa42e 100644 --- a/lib/database.js +++ b/lib/database.js @@ -8,6 +8,7 @@ var mongoose = require('mongoose') , Heavy = require('../models/heavy') , lib = require('./explorer') , settings = require('./settings') + , fs = require('fs') , poloniex = require('./markets/poloniex') , bittrex = require('./markets/bittrex') , bleutrade = require('./markets/bleutrade') @@ -230,6 +231,56 @@ function get_market_data(market, cb) { } } +function create_lock(cb) { + if (settings.lock_during_index == true) { + database = 'db_index'; + var fname = './tmp/' + database + '.pid'; + fs.appendFile(fname, process.pid, function (err) { + if (err) { + console.log("Error: unable to create %s", fname); + process.exit(1); + } else { + return cb(); + } + }); + } else { + return cb(); + } +} + +function remove_lock(cb) { + if (settings.lock_during_index == true) { + database = 'db_index'; + var fname = './tmp/' + database + '.pid'; + fs.unlink(fname, function (err){ + if(err) { + console.log("unable to remove lock: %s", fname); + process.exit(1); + } else { + return cb(); + } + }); + } else { + return cb(); + } +} + +function is_locked(cb) { + if (settings.lock_during_index == true) { + database = 'db_index'; + var fname = './tmp/' + database + '.pid'; + fs.exists(fname, function (exists){ + if(exists) { + return cb(true); + } else { + return cb(false); + } + }); + } else { + return cb(false); + } +} + module.exports = { // initialize DB connect: function(database, cb) { @@ -338,29 +389,43 @@ module.exports = { }, create_tx: function(txid, cb) { - save_tx(txid, function(err){ - if (err) { - return cb(err); - } else { - //console.log('tx stored: %s', txid); + is_locked(function (exists) { + if (exists) { + console.log("db_index lock file exists..."); return cb(); + } else { + save_tx(txid, function(err){ + if (err) { + return cb(err); + } else { + //console.log('tx stored: %s', txid); + return cb(); + } + }); } }); }, create_txs: function(block, cb) { - lib.syncLoop(block.tx.length, function (loop) { - var i = loop.iteration(); - save_tx(block.tx[i], function(err){ - if (err) { - loop.next(); - } else { - //console.log('tx stored: %s', block.tx[i]); - loop.next(); - } - }); - }, function(){ - return cb(); + is_locked(function (exists) { + if (exists) { + console.log("db_index lock file exists..."); + return cb(); + } else { + lib.syncLoop(block.tx.length, function (loop) { + var i = loop.iteration(); + save_tx(block.tx[i], function(err){ + if (err) { + loop.next(); + } else { + //console.log('tx stored: %s', block.tx[i]); + loop.next(); + } + }); + }, function(){ + return cb(); + }); + } }); }, @@ -619,64 +684,75 @@ module.exports = { // updates tx, address & richlist db's; called by sync.js update_tx_db: function(coin, start, end, timeout, cb) { - var complete = false; - lib.syncLoop((end - start) + 1, function (loop) { - var x = loop.iteration(); - if (x % 5000 === 0) { - Tx.find({}).where('blockindex').lt(start + x).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ - Stats.update({coin: coin}, { - last: start + x - 1, - last_txs: '' //not used anymore left to clear out existing objects - }, function() {}); - }); - } - lib.get_blockhash(start + x, function(blockhash){ - if (blockhash) { - lib.get_block(blockhash, function(block) { - if (block) { - lib.syncLoop(block.tx.length, function (subloop) { - var i = subloop.iteration(); - Tx.findOne({txid: block.tx[i]}, function(err, tx) { - if(tx) { - tx = null; - subloop.next(); - } else { - save_tx(block.tx[i], function(err){ - if (err) { - console.log(err); - } else { - console.log('%s: %s', block.height, block.tx[i]); - } - setTimeout( function(){ - tx = null; - subloop.next(); - }, timeout); + is_locked(function (exists) { + if (exists) { + console.log("db_index lock file exists..."); + return cb(); + } else { + create_lock(function (){ + var complete = false; + lib.syncLoop((end - start) + 1, function (loop) { + var x = loop.iteration(); + if (x % 5000 === 0) { + Tx.find({}).where('blockindex').lt(start + x).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ + Stats.update({coin: coin}, { + last: start + x - 1, + last_txs: '' //not used anymore left to clear out existing objects + }, function() {}); + }); + } + lib.get_blockhash(start + x, function(blockhash){ + if (blockhash) { + lib.get_block(blockhash, function(block) { + if (block) { + lib.syncLoop(block.tx.length, function (subloop) { + var i = subloop.iteration(); + Tx.findOne({txid: block.tx[i]}, function(err, tx) { + if(tx) { + tx = null; + subloop.next(); + } else { + save_tx(block.tx[i], function(err){ + if (err) { + console.log(err); + } else { + console.log('%s: %s', block.height, block.tx[i]); + } + setTimeout( function(){ + tx = null; + subloop.next(); + }, timeout); + }); + } + }); + }, function(){ + blockhash = null; + block = null; + loop.next(); }); + } else { + console.log('block not found: %s', blockhash); + loop.next(); } }); - }, function(){ - blockhash = null; - block = null; + } else { loop.next(); + } + }); + }, function(){ + Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ + Stats.update({coin: coin}, { + last: end, + last_txs: '' //not used anymore left to clear out existing objects + }, function() { + remove_lock(function(){ + return cb(); + }); }); - } else { - console.log('block not found: %s', blockhash); - loop.next(); - } + }); }); - } else { - loop.next(); - } - }); - }, function(){ - Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ - Stats.update({coin: coin}, { - last: end, - last_txs: '' //not used anymore left to clear out existing objects - }, function() { - return cb(); }); - }); + } }); }, diff --git a/lib/settings.js b/lib/settings.js index 668e7c7c1..ff3887dfb 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -126,6 +126,7 @@ exports.genesis_tx = "65f705d2f385dc85763a317b3ec000063003d6b039546af5d8195a5ec2 exports.genesis_block = "b2926a56ca64e0cd2430347e383f63ad7092f406088b9b86d6d68c2a34baef51"; exports.heavy = false; +exports.lock_during_index = false; exports.txcount = 100; exports.show_sent_received = true; exports.supply = "COINBASE"; diff --git a/settings.json.template b/settings.json.template index 0f898d68b..d252a869a 100644 --- a/settings.json.template +++ b/settings.json.template @@ -128,6 +128,9 @@ //heavy (enable/disable additional heavy features) "heavy": false, + //disable saving blocks & TXs via API during indexing. + "lock_during_index": false, + //amount of txs to index per address (stores latest n txs) "txcount": 100,