Skip to content

Commit

Permalink
Database indexing: add file locks
Browse files Browse the repository at this point in the history
  • Loading branch information
@TheHolyRoger
TheHolyRoger committed Dec 18, 2019
1 parent 4598051 commit f8289fe
Show file tree
Hide file tree
Showing 7 changed files with 171 additions and 69 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
1.6.X
* New setting: lock_during_index

1.6.1
* fixed last_txs setting
* added hashrate_units setting
Expand Down
5 changes: 5 additions & 0 deletions UPGRADE
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
Note: All updates require the explorer to be restarted

1.6.1 -> 1.6.X
* Fix mismatching balances/sent/received, negative balances etc
* Add new settings to settings.json (see settings.json.template)
* lock_during_index

1.6.0 -> 1.6.1
* Add new cryptsy_id and hashrate_units settings (see settings.json.template)
* remove tmp/market.pid (if it exists)
Expand Down
1 change: 1 addition & 0 deletions app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ app.set('youtube', settings.youtube);
app.set('genesis_block', settings.genesis_block);
app.set('index', settings.index);
app.set('heavy', settings.heavy);
app.set('lock_during_index', settings.lock_during_index);
app.set('txcount', settings.txcount);
app.set('nethash', settings.nethash);
app.set('nethash_units', settings.nethash_units);
Expand Down
219 changes: 151 additions & 68 deletions lib/database.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ var mongoose = require('mongoose')
, Heavy = require('../models/heavy')
, lib = require('./explorer')
, settings = require('./settings')
, fs = require('fs')
, poloniex = require('./markets/poloniex')
, bittrex = require('./markets/bittrex')
, bleutrade = require('./markets/bleutrade')
Expand Down Expand Up @@ -230,6 +231,53 @@ function get_market_data(market, cb) {
}
}

function create_lock(lockfile, cb) {
if (settings.lock_during_index == true) {
var fname = './tmp/' + lockfile + '.pid';
fs.appendFile(fname, process.pid, function (err) {
if (err) {
console.log("Error: unable to create %s", fname);
process.exit(1);
} else {
return cb();
}
});
} else {
return cb();
}
}

function remove_lock(lockfile, cb) {
if (settings.lock_during_index == true) {
var fname = './tmp/' + lockfile + '.pid';
fs.unlink(fname, function (err){
if(err) {
console.log("unable to remove lock: %s", fname);
process.exit(1);
} else {
return cb();
}
});
} else {
return cb();
}
}

function is_locked(lockfile, cb) {
if (settings.lock_during_index == true) {
var fname = './tmp/' + lockfile + '.pid';
fs.exists(fname, function (exists){
if(exists) {
return cb(true);
} else {
return cb(false);
}
});
} else {
return cb(false);
}
}

module.exports = {
// initialize DB
connect: function(database, cb) {
Expand All @@ -245,6 +293,16 @@ module.exports = {
});
},

is_locked: function(cb) {
is_locked("db_index", function (exists) {
if (exists) {
return cb(true);
} else {
return cb(false);
}
});
},

check_stats: function(coin, cb) {
Stats.findOne({coin: coin}, function(err, stats) {
if(stats) {
Expand Down Expand Up @@ -338,29 +396,43 @@ module.exports = {
},

create_tx: function(txid, cb) {
save_tx(txid, function(err){
if (err) {
return cb(err);
} else {
//console.log('tx stored: %s', txid);
is_locked("db_index", function (exists) {
if (exists) {
console.log("db_index lock file exists...");
return cb();
} else {
save_tx(txid, function(err){
if (err) {
return cb(err);
} else {
//console.log('tx stored: %s', txid);
return cb();
}
});
}
});
},

create_txs: function(block, cb) {
lib.syncLoop(block.tx.length, function (loop) {
var i = loop.iteration();
save_tx(block.tx[i], function(err){
if (err) {
loop.next();
} else {
//console.log('tx stored: %s', block.tx[i]);
loop.next();
}
});
}, function(){
return cb();
is_locked("db_index", function (exists) {
if (exists) {
console.log("db_index lock file exists...");
return cb();
} else {
lib.syncLoop(block.tx.length, function (loop) {
var i = loop.iteration();
save_tx(block.tx[i], function(err){
if (err) {
loop.next();
} else {
//console.log('tx stored: %s', block.tx[i]);
loop.next();
}
});
}, function(){
return cb();
});
}
});
},

Expand Down Expand Up @@ -619,64 +691,75 @@ module.exports = {

// updates tx, address & richlist db's; called by sync.js
update_tx_db: function(coin, start, end, timeout, cb) {
var complete = false;
lib.syncLoop((end - start) + 1, function (loop) {
var x = loop.iteration();
if (x % 5000 === 0) {
Tx.find({}).where('blockindex').lt(start + x).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){
Stats.update({coin: coin}, {
last: start + x - 1,
last_txs: '' //not used anymore left to clear out existing objects
}, function() {});
});
}
lib.get_blockhash(start + x, function(blockhash){
if (blockhash) {
lib.get_block(blockhash, function(block) {
if (block) {
lib.syncLoop(block.tx.length, function (subloop) {
var i = subloop.iteration();
Tx.findOne({txid: block.tx[i]}, function(err, tx) {
if(tx) {
tx = null;
subloop.next();
} else {
save_tx(block.tx[i], function(err){
if (err) {
console.log(err);
} else {
console.log('%s: %s', block.height, block.tx[i]);
}
setTimeout( function(){
tx = null;
subloop.next();
}, timeout);
is_locked("db_index", function (exists) {
if (exists) {
console.log("db_index lock file exists...");
return cb();
} else {
create_lock("db_index", function (){
var complete = false;
lib.syncLoop((end - start) + 1, function (loop) {
var x = loop.iteration();
if (x % 5000 === 0) {
Tx.find({}).where('blockindex').lt(start + x).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){
Stats.update({coin: coin}, {
last: start + x - 1,
last_txs: '' //not used anymore left to clear out existing objects
}, function() {});
});
}
lib.get_blockhash(start + x, function(blockhash){
if (blockhash) {
lib.get_block(blockhash, function(block) {
if (block) {
lib.syncLoop(block.tx.length, function (subloop) {
var i = subloop.iteration();
Tx.findOne({txid: block.tx[i]}, function(err, tx) {
if(tx) {
tx = null;
subloop.next();
} else {
save_tx(block.tx[i], function(err){
if (err) {
console.log(err);
} else {
console.log('%s: %s', block.height, block.tx[i]);
}
setTimeout( function(){
tx = null;
subloop.next();
}, timeout);
});
}
});
}, function(){
blockhash = null;
block = null;
loop.next();
});
} else {
console.log('block not found: %s', blockhash);
loop.next();
}
});
}, function(){
blockhash = null;
block = null;
} else {
loop.next();
}
});
}, function(){
Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){
Stats.update({coin: coin}, {
last: end,
last_txs: '' //not used anymore left to clear out existing objects
}, function() {
remove_lock("db_index", function(){
return cb();
});
});
} else {
console.log('block not found: %s', blockhash);
loop.next();
}
});
});
} else {
loop.next();
}
});
}, function(){
Tx.find({}).sort({timestamp: 'desc'}).limit(settings.index.last_txs).exec(function(err, txs){
Stats.update({coin: coin}, {
last: end,
last_txs: '' //not used anymore left to clear out existing objects
}, function() {
return cb();
});
});
}
});
},

Expand Down
1 change: 1 addition & 0 deletions lib/settings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ exports.genesis_tx = "65f705d2f385dc85763a317b3ec000063003d6b039546af5d8195a5ec2
exports.genesis_block = "b2926a56ca64e0cd2430347e383f63ad7092f406088b9b86d6d68c2a34baef51";

exports.heavy = false;
exports.lock_during_index = false;
exports.txcount = 100;
exports.show_sent_received = true;
exports.supply = "COINBASE";
Expand Down
8 changes: 7 additions & 1 deletion routes/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,13 @@ function route_get_tx(res, txid) {
}

function route_get_index(res, error) {
res.render('index', { active: 'home', error: error, warning: null});
db.is_locked(function(locked) {
if (locked) {
res.render('index', { active: 'home', error: error, warning: locale.initial_index_alert});
} else {
res.render('index', { active: 'home', error: error, warning: null});
}
});
}

function route_get_address(res, hash, count) {
Expand Down
3 changes: 3 additions & 0 deletions settings.json.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,9 @@
//heavy (enable/disable additional heavy features)
"heavy": false,

//disable saving blocks & TXs via API during indexing.
"lock_during_index": false,

//amount of txs to index per address (stores latest n txs)
"txcount": 100,

Expand Down

0 comments on commit f8289fe

Please sign in to comment.