diff --git a/CHANGELOG b/CHANGELOG index b2a22aecf..661729ab0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +1.6.X +* New setting: lock_during_index + 1.6.1 * fixed last_txs setting * added hashrate_units setting diff --git a/README.md b/README.md index 7d57e276e..e2ccbb66b 100644 --- a/README.md +++ b/README.md @@ -114,9 +114,10 @@ Ensure mongodb is not exposed to the outside world via your mongo config or a fi **script is already running.** -If you receive this message when launching the sync script either a) a sync is currently in progress, or b) a previous sync was killed before it completed. If you are certian a sync is not in progress remove the index.pid from the tmp folder in the explorer root directory. +If you receive this message when launching the sync script either a) a sync is currently in progress, or b) a previous sync was killed before it completed. If you are certian a sync is not in progress remove the index.pid and db_index.pid from the tmp folder in the explorer root directory. rm tmp/index.pid + rm tmp/db_index.pid **exceeding stack size** diff --git a/UPGRADE b/UPGRADE index af8eb0d45..813e936fe 100644 --- a/UPGRADE +++ b/UPGRADE @@ -1,5 +1,10 @@ Note: All updates require the explorer to be restarted +1.6.1 -> 1.6.X +* Fix mismatching balances/sent/received, negative balances etc +* Add new settings to settings.json (see settings.json.template) + * lock_during_index + 1.6.0 -> 1.6.1 * Add new cryptsy_id and hashrate_units settings (see settings.json.template) * remove tmp/market.pid (if it exists) diff --git a/app.js b/app.js index f9b19b943..3af596316 100644 --- a/app.js +++ b/app.js @@ -120,6 +120,7 @@ app.set('youtube', settings.youtube); app.set('genesis_block', settings.genesis_block); app.set('index', settings.index); app.set('heavy', settings.heavy); +app.set('lock_during_index', settings.lock_during_index); app.set('txcount', settings.txcount); app.set('nethash', settings.nethash); app.set('nethash_units', settings.nethash_units); diff --git a/lib/database.js b/lib/database.js index dfd6c4dd3..548c4a346 100644 --- a/lib/database.js +++ b/lib/database.js @@ -8,6 +8,7 @@ var mongoose = require('mongoose') , Heavy = require('../models/heavy') , lib = require('./explorer') , settings = require('./settings') + , fs = require('fs') , poloniex = require('./markets/poloniex') , bittrex = require('./markets/bittrex') , bleutrade = require('./markets/bleutrade') @@ -241,6 +242,53 @@ function get_market_data(market, cb) { } } +function create_lock(lockfile, cb) { + if (settings.lock_during_index == true) { + var fname = './tmp/' + lockfile + '.pid'; + fs.appendFile(fname, process.pid, function (err) { + if (err) { + console.log("Error: unable to create %s", fname); + process.exit(1); + } else { + return cb(); + } + }); + } else { + return cb(); + } +} + +function remove_lock(lockfile, cb) { + if (settings.lock_during_index == true) { + var fname = './tmp/' + lockfile + '.pid'; + fs.unlink(fname, function (err){ + if(err) { + console.log("unable to remove lock: %s", fname); + process.exit(1); + } else { + return cb(); + } + }); + } else { + return cb(); + } +} + +function is_locked(lockfile, cb) { + if (settings.lock_during_index == true) { + var fname = './tmp/' + lockfile + '.pid'; + fs.exists(fname, function (exists){ + if(exists) { + return cb(true); + } else { + return cb(false); + } + }); + } else { + return cb(false); + } +} + module.exports = { // initialize DB connect: function(database, cb) { @@ -256,6 +304,16 @@ module.exports = { }); }, + is_locked: function(cb) { + is_locked("db_index", function (exists) { + if (exists) { + return cb(true); + } else { + return cb(false); + } + }); + }, + check_stats: function(coin, cb) { Stats.findOne({coin: coin}, function(err, stats) { if(stats) { @@ -349,29 +407,43 @@ module.exports = { }, create_tx: function(txid, cb) { - save_tx(txid, function(err){ - if (err) { - return cb(err); - } else { - //console.log('tx stored: %s', txid); + is_locked("db_index", function (exists) { + if (exists) { + console.log("db_index lock file exists..."); return cb(); + } else { + save_tx(txid, function(err){ + if (err) { + return cb(err); + } else { + //console.log('tx stored: %s', txid); + return cb(); + } + }); } }); }, create_txs: function(block, cb) { - lib.syncLoop(block.tx.length, function (loop) { - var i = loop.iteration(); - save_tx(block.tx[i], function(err){ - if (err) { - loop.next(); - } else { - //console.log('tx stored: %s', block.tx[i]); - loop.next(); - } - }); - }, function(){ - return cb(); + is_locked("db_index", function (exists) { + if (exists) { + console.log("db_index lock file exists..."); + return cb(); + } else { + lib.syncLoop(block.tx.length, function (loop) { + var i = loop.iteration(); + save_tx(block.tx[i], function(err){ + if (err) { + loop.next(); + } else { + //console.log('tx stored: %s', block.tx[i]); + loop.next(); + } + }); + }, function(){ + return cb(); + }); + } }); }, @@ -630,64 +702,75 @@ module.exports = { // updates tx, address & richlist db's; called by sync.js update_tx_db: function(coin, start, end, timeout, cb) { - var complete = false; - lib.syncLoop((end - start) + 1, function (loop) { - var x = loop.iteration(); - if (x % 5000 === 0) { - Tx.find({}).where('blockindex').lt(start + x).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ - Stats.update({coin: coin}, { - last: start + x - 1, - last_txs: '' //not used anymore left to clear out existing objects - }, function() {}); - }); - } - lib.get_blockhash(start + x, function(blockhash){ - if (blockhash) { - lib.get_block(blockhash, function(block) { - if (block) { - lib.syncLoop(block.tx.length, function (subloop) { - var i = subloop.iteration(); - Tx.findOne({txid: block.tx[i]}, function(err, tx) { - if(tx) { - tx = null; - subloop.next(); - } else { - save_tx(block.tx[i], function(err){ - if (err) { - console.log(err); - } else { - console.log('%s: %s', block.height, block.tx[i]); - } - setTimeout( function(){ - tx = null; - subloop.next(); - }, timeout); + is_locked("db_index", function (exists) { + if (exists) { + console.log("db_index lock file exists..."); + return cb(); + } else { + create_lock("db_index", function (){ + var complete = false; + lib.syncLoop((end - start) + 1, function (loop) { + var x = loop.iteration(); + if (x % 5000 === 0) { + Tx.find({}).where('blockindex').lt(start + x).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ + Stats.update({coin: coin}, { + last: start + x - 1, + last_txs: '' //not used anymore left to clear out existing objects + }, function() {}); + }); + } + lib.get_blockhash(start + x, function(blockhash){ + if (blockhash) { + lib.get_block(blockhash, function(block) { + if (block) { + lib.syncLoop(block.tx.length, function (subloop) { + var i = subloop.iteration(); + Tx.findOne({txid: block.tx[i]}, function(err, tx) { + if(tx) { + tx = null; + subloop.next(); + } else { + save_tx(block.tx[i], function(err){ + if (err) { + console.log(err); + } else { + console.log('%s: %s', block.height, block.tx[i]); + } + setTimeout( function(){ + tx = null; + subloop.next(); + }, timeout); + }); + } + }); + }, function(){ + blockhash = null; + block = null; + loop.next(); }); + } else { + console.log('block not found: %s', blockhash); + loop.next(); } }); - }, function(){ - blockhash = null; - block = null; + } else { loop.next(); + } + }); + }, function(){ + Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ + Stats.update({coin: coin}, { + last: end, + last_txs: '' //not used anymore left to clear out existing objects + }, function() { + remove_lock("db_index", function(){ + return cb(); + }); }); - } else { - console.log('block not found: %s', blockhash); - loop.next(); - } + }); }); - } else { - loop.next(); - } - }); - }, function(){ - Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){ - Stats.update({coin: coin}, { - last: end, - last_txs: '' //not used anymore left to clear out existing objects - }, function() { - return cb(); }); - }); + } }); }, diff --git a/lib/settings.js b/lib/settings.js index 668e7c7c1..ff3887dfb 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -126,6 +126,7 @@ exports.genesis_tx = "65f705d2f385dc85763a317b3ec000063003d6b039546af5d8195a5ec2 exports.genesis_block = "b2926a56ca64e0cd2430347e383f63ad7092f406088b9b86d6d68c2a34baef51"; exports.heavy = false; +exports.lock_during_index = false; exports.txcount = 100; exports.show_sent_received = true; exports.supply = "COINBASE"; diff --git a/routes/index.js b/routes/index.js index 73a5dc418..dd6fc2e24 100644 --- a/routes/index.js +++ b/routes/index.js @@ -89,7 +89,13 @@ function route_get_tx(res, txid) { } function route_get_index(res, error) { - res.render('index', { active: 'home', error: error, warning: null}); + db.is_locked(function(locked) { + if (locked) { + res.render('index', { active: 'home', error: error, warning: locale.initial_index_alert}); + } else { + res.render('index', { active: 'home', error: error, warning: null}); + } + }); } function route_get_address(res, hash, count) { diff --git a/settings.json.template b/settings.json.template index 0f898d68b..d252a869a 100644 --- a/settings.json.template +++ b/settings.json.template @@ -128,6 +128,9 @@ //heavy (enable/disable additional heavy features) "heavy": false, + //disable saving blocks & TXs via API during indexing. + "lock_during_index": false, + //amount of txs to index per address (stores latest n txs) "txcount": 100,