Merge pull request iquidus#52 from iquidus/master

pull in latest from iquidus

.gitignore

@@ -30,3 +30,7 @@ settings.json
 *.ico
 *.png
 
+
+.DS_Store
+
+package-lock.json

CHANGELOG

@@ -1,3 +1,15 @@
+1.6.2
+* Indexing is now MUCH faster.
+* Added AddressTX model/collection - REQUIRES REINDEX
+* Added balance history to Address TX History
+* Added support for AJAX loading of Address TX History, using txcount setting for loading
+* Stopped maximum TX count trimming TX's in AddressTX collection
+* Removed tx_array from Address collection -> AddressTX
+* Fix mismatching balances/sent/received, negative balances etc
+* Added new file lock during database indexing: tmp/db_index.pid
+* New setting to lock during indexing: lock_during_index
+* Add setting to call bitcoin-core directly or use RPC during indexing
+
 1.6.1
 * fixed last_txs setting
 * added hashrate_units setting

README.md

@@ -1,24 +1,25 @@
-Iquidus Explorer - 1.6.1
+Iquidus Explorer - 1.6.2
 ================
 
 An open source block explorer written in node.js.
 
 ### See it in action
 
 *  [Deutsche eMark](http://b.emark.tk/)
-*  [Sphere](http://sphere.iquidus.io)
 *  [Vertcoin](http://explorer.vertcoin.info/)
-*  [Vivo](http://vivo.explorerz.top:3003)
+*  [TheHolyRogerCoin (ROGER) Explorer](https://explorer.theholyroger.com/)
+*  [CPUChain (CPU) Explorer](https://explorer.cpuchain.org/)
+*  [Omega Blockchain Explorer](http://explorer.omegablockchain.net/)
+*  [Sugarchain Explorer](https://1explorer.sugarchain.org/)
 *  [Florincoin](https://florincoin.info/info)
 *  [Maxcoin Explorer 1](https://explorer.maxcoinproject.net/)
-*  [Maxcoin Explorer 2](https://explorer2.maxcoinproject.net/)
 
 
-*note: If you would like your instance mentioned here contact me*
+*Note: If you would like your instance mentioned here contact me*
 
 ### Requires
 
-*  node.js >= 0.10.28
+*  node.js >= 0.10.28 (8.17.0 is advised for updated dependencies)
 *  mongodb 2.6.x
 *  *coind
 
@@ -36,7 +37,7 @@ Create user with read/write access:
 
     > db.createUser( { user: "iquidus", pwd: "3xp!0reR", roles: [ "readWrite" ] } )
 
-*note: If you're using mongo shell 2.4.x, use the following to create your user:
+*Note: If you're using mongo shell 2.4.x, use the following to create your user:
 
     > db.addUser( { user: "username", pwd: "password", roles: [ "readWrite"] })
 
@@ -58,7 +59,7 @@ Create user with read/write access:
 
     npm start
 
-*note: mongod must be running to start the explorer*
+*Note: mongod must be running to start the explorer*
 
 As of version 1.4.0 the explorer defaults to cluster mode, forking an instance of its process to each cpu core. This results in increased performance and stability. Load balancing gets automatically taken care of and any instances that for some reason die, will be restarted automatically. For testing/development (or if you just wish to) a single instance can be launched with
 
@@ -102,22 +103,22 @@ sync.js (located in scripts/) is used for updating the local databases. This scr
 
 ### Wallet
 
-Iquidus Explorer is intended to be generic so it can be used with any wallet following the usual standards. The wallet must be running with atleast the following flags
+Iquidus Explorer is intended to be generic, so it can be used with any wallet following the usual standards. The wallet must be running with atleast the following flags
 
     -daemon -txindex
+
+### Security
 
-### Donate
-
-    BTC: 168hdKA3fkccPtkxnX8hBrsxNubvk4udJi
-    JBS: JZp9893FMmrm1681bDuJBU7c6w11kyEY7D
+Ensure mongodb is not exposed to the outside world via your mongo config or a firewall to prevent outside tampering of the indexed chain data. 
 
 ### Known Issues
 
 **script is already running.**
 
-If you receive this message when launching the sync script either a) a sync is currently in progress, or b) a previous sync was killed before it completed. If you are certian a sync is not in progress remove the index.pid from the tmp folder in the explorer root directory.
+If you receive this message when launching the sync script either a) a sync is currently in progress, or b) a previous sync was killed before it completed. If you are certian a sync is not in progress remove the index.pid and db_index.pid from the tmp folder in the explorer root directory.
 
     rm tmp/index.pid
+    rm tmp/db_index.pid
 
 **exceeding stack size**
 

UPGRADE

@@ -1,5 +1,14 @@
 Note: All updates require the explorer to be restarted
 
+1.6.1 -> 1.6.2
+* remove tmp/db_index.pid (if it exists)
+* Add new settings to settings.json (see settings.json.template)
+  * lock_during_index
+  * use_rpc
+  * txcount_per_page
+  * index.txs_per_page
+* Reindex explorerdb (node --stack-size=15000 scripts/sync.js index reindex) to build new address TX collection
+
 1.6.0 -> 1.6.1
 * Add new cryptsy_id and hashrate_units settings (see settings.json.template)
 * remove tmp/market.pid (if it exists)

app.js

@@ -44,7 +44,7 @@ app.set('view engine', 'jade');
 app.use(favicon(path.join(__dirname, settings.favicon)));
 app.use(logger('dev'));
 app.use(bodyParser.json());
-app.use(bodyParser.urlencoded());
+app.use(bodyParser.urlencoded({ extended: true }));
 app.use(cookieParser());
 app.use(express.static(path.join(__dirname, 'public')));
 
@@ -58,28 +58,77 @@ app.use('/ext/getmoneysupply', function(req,res){
 });
 
 app.use('/ext/getaddress/:hash', function(req,res){
-  db.get_address(req.param('hash'), function(address){
+  db.get_address(req.params.hash, function(address){
     if (address) {
       var a_ext = {
         address: address.a_id,
         sent: (address.sent / 100000000),
         received: (address.received / 100000000),
         balance: (address.balance / 100000000).toString().replace(/(^-+)/mg, ''),
-        last_txs: address.txs,
       };
       res.send(a_ext);
     } else {
-      res.send({ error: 'address not found.', hash: req.param('hash')})
+      res.send({ error: 'address not found.', hash: req.params.hash})
+    }
+  });
+});
+
+app.use('/ext/gettx/:txid', function(req, res) {
+  var txid = req.params.txid;
+  db.get_tx(txid, function(tx) {
+    if (tx) {
+      lib.get_blockcount(function(blockcount) {
+        res.send({ active: 'tx', tx: tx, confirmations: settings.confirmations, blockcount: blockcount});
+      });
+    }
+    else {
+      lib.get_rawtransaction(txid, function(rtx) {
+        if (rtx.txid) {
+          lib.prepare_vin(rtx, function(vin) {
+            lib.prepare_vout(rtx.vout, rtx.txid, vin, function(rvout, rvin) {
+              lib.calculate_total(rvout, function(total){
+                if (!rtx.confirmations > 0) {
+                  var utx = {
+                    txid: rtx.txid,
+                    vin: rvin,
+                    vout: rvout,
+                    total: total.toFixed(8),
+                    timestamp: rtx.time,
+                    blockhash: '-',
+                    blockindex: -1,
+                  };
+                  res.send({ active: 'tx', tx: utx, confirmations: settings.confirmations, blockcount:-1});
+                } else {
+                  var utx = {
+                    txid: rtx.txid,
+                    vin: rvin,
+                    vout: rvout,
+                    total: total.toFixed(8),
+                    timestamp: rtx.time,
+                    blockhash: rtx.blockhash,
+                    blockindex: rtx.blockheight,
+                  };
+                  lib.get_blockcount(function(blockcount) {
+                    res.send({ active: 'tx', tx: utx, confirmations: settings.confirmations, blockcount: blockcount});
+                  });
+                }
+              });
+            });
+          });
+        } else {
+          res.send({ error: 'tx not found.', hash: txid});
+        }
+      });
     }
   });
 });
 
 app.use('/ext/getbalance/:hash', function(req,res){
-  db.get_address(req.param('hash'), function(address){
+  db.get_address(req.params.hash, function(address){
     if (address) {
       res.send((address.balance / 100000000).toString().replace(/(^-+)/mg, ''));
     } else {
-      res.send({ error: 'address not found.', hash: req.param('hash')})
+      res.send({ error: 'address not found.', hash: req.params.hash})
     }
   });
 });
@@ -94,12 +143,75 @@ app.use('/ext/getdistribution', function(req,res){
   });
 });
 
-app.use('/ext/getlasttxs/:min', function(req,res){
-  db.get_last_txs(settings.index.last_txs, (req.params.min * 100000000), function(txs){
-    res.send({data: txs});
+app.use('/ext/getlasttxsajax/:min', function(req,res){
+  if(typeof req.query.length === 'undefined' || isNaN(req.query.length) || req.query.length > settings.index.last_txs){
+    req.query.length = settings.index.last_txs;
+  }
+  if(typeof req.query.start === 'undefined' || isNaN(req.query.start) || req.query.start < 0){
+    req.query.start = 0;
+  }
+  if(typeof req.params.min === 'undefined' || isNaN(req.params.min ) || req.params.min  < 0){
+    req.params.min  = 0;
+  } else {
+    req.params.min  = (req.params.min * 100000000);
+  }
+  db.get_last_txs_ajax(req.query.start, req.query.length, req.params.min,function(txs, count){
+    var data = [];
+    for(i=0; i<txs.length; i++){
+      var row = [];
+      row.push(txs[i].blockindex);
+      row.push(txs[i].blockhash);
+      row.push(txs[i].txid);
+      row.push(txs[i].vout.length);
+      row.push((txs[i].total));
+      row.push(new Date((txs[i].timestamp) * 1000).toUTCString());
+      data.push(row);
+    }
+    res.json({"data":data, "draw": req.query.draw, "recordsTotal": count, "recordsFiltered": count});
   });
 });
 
+app.use('/ext/getaddresstxsajax/:address', function(req,res){
+    req.query.length = parseInt(req.query.length);
+    if(isNaN(req.query.length) || req.query.length > settings.txcount){
+        req.query.length = settings.txcount;
+    }
+    if(isNaN(req.query.start) || req.query.start < 0){
+        req.query.start = 0;
+    }
+    db.get_address_txs_ajax(req.params.address, req.query.start, req.query.length,function(txs, count){
+        var data = [];
+        for(i=0; i<txs.length; i++){
+            if(typeof txs[i].txid !== "undefined") {
+                var out = 0
+                var vin = 0
+
+                txs[i].vout.forEach(function (r) {
+                    if (r.addresses == req.params.address) {
+                        out += r.amount;
+                    }
+                });
+
+                txs[i].vin.forEach(function (s) {
+                    if (s.addresses == req.params.address) {
+                        vin += s.amount
+                    }
+                });
+
+                var row = [];
+                row.push(new Date((txs[i].timestamp) * 1000).toUTCString());
+                row.push(txs[i].txid);
+                row.push(out);
+                row.push(vin);
+                row.push(txs[i].balance);
+                data.push(row);
+            }
+        }
+
+        res.json({"data":data, "draw": req.query.draw, "recordsTotal": count, "recordsFiltered": count});
+    });
+});
+
 app.use('/ext/connections', function(req,res){
   db.get_peers(function(peers){
     res.send({data: peers});
@@ -114,13 +226,16 @@ app.set('locale', locale);
 app.set('display', settings.display);
 app.set('markets', settings.markets);
 app.set('twitter', settings.twitter);
-app.set('facebook', settings.youtube);
+app.set('facebook', settings.facebook); 
 app.set('googleplus', settings.googleplus);
 app.set('youtube', settings.youtube);
 app.set('genesis_block', settings.genesis_block);
 app.set('index', settings.index);
+app.set('use_rpc', settings.use_rpc);
 app.set('heavy', settings.heavy);
+app.set('lock_during_index', settings.lock_during_index);
 app.set('txcount', settings.txcount);
+app.set('txcount_per_page', settings.txcount_per_page);
 app.set('nethash', settings.nethash);
 app.set('nethash_units', settings.nethash_units);
 app.set('show_sent_received', settings.show_sent_received);

bin/cluster

@@ -11,11 +11,16 @@ if (cluster.isMaster) {
       //ensure workers exit cleanly 
       process.on('SIGINT', function() {
         console.log('Cluster shutting down..');
-        for (var id in cluster.workers) {
-          cluster.workers[id].kill();
-        }
-        // exit the master process
-        process.exit(0);
+        setTimeout(function() {
+          for (var id in cluster.workers) {
+            console.log('Worker shutting down (' + id + ')');
+            cluster.workers[id].kill();
+          }
+          setTimeout(function() {
+            // exit the master process
+            process.exit(0);
+          }, 3000);
+        }, 1000);
       });
 
       // Count the machine's CPUs
@@ -27,8 +32,13 @@ if (cluster.isMaster) {
       }
 
       // Listen for dying workers
-      cluster.on('exit', function () {
-        cluster.fork();
+      cluster.on('exit', function (worker, code, signal) {
+        if (worker['process']['exitCode'] === 0) {
+          console.log('Worker shut down.');
+        } else if ((signal != 'SIGINT') && (worker['process']['exitCode'] !== 0) && (worker.exitedAfterDisconnect !== true)) {
+          console.log('Cluster restarting...');
+          cluster.fork();
+        }
       });
     }
   });