Skip to content

Commit

Permalink
Better progress reporting during check phase + track more stats durin…
Browse files Browse the repository at this point in the history 
…g check delays

Ref #99
  • Loading branch information
@animetosho
animetosho committed Dec 6, 2022
1 parent 788ab98 commit 20f3843
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 12 deletions.
47 changes: 36 additions & 11 deletions bin/nyuu.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1395,7 +1395,7 @@ var filesToUpload = argv._;
'Total articles: ' + totalPieces + ' (' + friendlySize(totalSize) + ')',
'Articles read: ' + uploader.articlesRead + ' (' + toPercent(uploader.articlesRead/totalPieces) + ')' + (uploader.articlesReRead ? ' (+' + uploader.articlesReRead + ' re-read)':''),
'Articles posted: ' + uploader.articlesPosted + ' (' + toPercent(uploader.articlesPosted/totalPieces) + ')' + (uploader.articlesRePosted ? ' (+' + uploader.articlesRePosted + ' re-posted)':''),
uploader.numCheckConns ? 'Articles checked: ' + uploader.articlesChecked + ' (' + toPercent(uploader.articlesChecked/totalPieces) + ')' : false,
uploader.numCheckConns ? 'Articles checked: ' + uploader.articlesChecked + ' (' + toPercent(uploader.articlesChecked/totalPieces) + ')' + (uploader.articlesRechecked ? ' (+'+uploader.articlesRechecked+' re-checked)':'') : false,
'Errors skipped: ' + errorCount + ' across ' + uploader.articleErrors + ' article(s)',
'Upload Rate (network|real): ' + friendlySize(uploader.currentNetworkUploadBytes()/uploader.currentNetworkUploadTime()*1000) + '/s | ' + friendlySize(uploader.bytesPosted/(now-startTime)*1000) + '/s',
].filter(function(e){return e;});
Expand Down Expand Up @@ -1436,6 +1436,7 @@ var filesToUpload = argv._;
case 'stderr':
case 'stdout':
if(getProcessIndicator) break; // no need to double output =P
var mainPostingDone = false;
var ProgressRecorder = require('../lib/progrec');
var byteSamples = new ProgressRecorder(180);
var progressSamples = new ProgressRecorder(180);
Expand Down Expand Up @@ -1466,25 +1467,46 @@ var filesToUpload = argv._;
var LINE_WIDTH = 35;
var barSize = Math.floor(chkPerc*LINE_WIDTH);
var line = repeatChar('=', barSize) + repeatChar('-', Math.floor(pstPerc * LINE_WIDTH) - barSize);
return '\x1b[0G\x1B[0K ' + lpad(totPerc, 6) + ' [' + rpad(line, LINE_WIDTH) + ']' + (uploader.bytesPosted ?
' ' + friendlySize(speed) + '/s, ETA ' + eta
: '');
var suffix = '';
if(mainPostingDone) {
if(uploader.articlesPosted < totalPieces)
suffix = ' reposting ' + (totalPieces - uploader.articlesPosted) + ' article(s)';
else if(uploader.checkPending || uploader.checkRePending)
suffix = ' awaiting check on ' + (uploader.checkPending + uploader.checkRePending) + ' article(s)';
} else if(uploader.bytesPosted)
suffix = ' ' + friendlySize(speed) + '/s, ETA ' + eta;
return '\x1b[0G\x1B[0K ' + lpad(totPerc, 6) + ' [' + rpad(line, LINE_WIDTH) + ']' + suffix;
} else {
// extended display
var posted = '' + uploader.articlesChecked;
if(uploader.articlesChecked != uploader.articlesPosted)
posted += '+' + (uploader.articlesPosted - uploader.articlesChecked);
var ret = 'Posted: ' + posted + '/' + totalPieces + ' (' + totPerc + ') @ ' + friendlySize(speed) + '/s (network: ' + friendlySize(uploader.currentNetworkUploadBytes()/uploader.currentNetworkUploadTime()*1000) + '/s) ETA ' + eta;
if(ret.length > 80)
// if too long, strip the network post speed
ret = ret.replace(/ \(network\: [0-9.]+ [A-Zi]+\/s\)/, ',');
var ret = '';
if(mainPostingDone) {
ret = 'Checked: ' + uploader.articlesChecked + ' ('+toPercent(chkPerc) + ')';
if(uploader.checkPending || uploader.checkRePending) {
if(uploader.checkPending)
ret += ', ' + uploader.checkPending + (uploader.checkRePending ? '+'+uploader.checkRePending:'') + ' pending';
else
ret += ', ' + uploader.checkRePending + ' pending re-check';
}
if(uploader.articlesPosted < totalPieces)
ret += ', ' + (totalPieces - uploader.articlesPosted) + ' to re-post';
} else {
var posted = '' + uploader.articlesChecked;
if(uploader.articlesChecked != uploader.articlesPosted)
posted += '+' + (uploader.articlesPosted - uploader.articlesChecked);
ret = 'Posted: ' + posted + '/' + totalPieces + ' (' + totPerc + ') @ ' + friendlySize(speed) + '/s (network: ' + friendlySize(uploader.currentNetworkUploadBytes()/uploader.currentNetworkUploadTime()*1000) + '/s) ETA ' + eta;
if(ret.length > 80)
// if too long, strip the network post speed
ret = ret.replace(/ \(network\: [0-9.]+ [A-Zi]+\/s\)/, ',');
}
return '\x1b[0G\x1B[0K' + ret;
}
};
var prgTarget = prg.type.substr(0, 6);
var seInterval = setInterval(function() {
byteSamples.add(uploader.bytesPosted);
progressSamples.add((uploader.articlesChecked + uploader.articlesPosted)/2);
if(uploader.articlesPosted == totalPieces && uploader.numCheckConns)
mainPostingDone = true; // we've moved into a 'primarily checking' phase
process[prgTarget].write(getProcessIndicator());
}, 1000);
process.on('finished', function() {
Expand Down Expand Up @@ -1521,6 +1543,9 @@ var filesToUpload = argv._;
'Check queue size: ' + uploader.checkQueue.queue.length + ' + ' + uploader.checkQueue.pendingAdds + ' delayed' + ' (' + toPercent(Math.min((uploader.checkQueue.queue.length+uploader.checkQueue.pendingAdds)/uploader.checkQueue.size, 1)) + ' full)' + (uploader.checkQueue.hasFinished ? ' - finished' : ''),
'Check cache size: ' + uploader.checkCache.cacheSize + ' (' + toPercent(Math.min(uploader.checkCache.cacheSize/uploader.checkCache.size, 1)) + ' full)',
'Re-read queue size: ' + uploader.reloadQueue.queue.length,
'',
'Article activity: ' + uploader.postActive + ' posting, ' + uploader.checkActive + ' checking',
'Articles awaiting check: ' + uploader.checkPending + ' + ' + uploader.checkRePending + ' awaiting re-check',
'', ''
]).join('\r\n'));

Expand Down
19 changes: 18 additions & 1 deletion lib/uploader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,13 @@ Uploader.prototype = {
articlesPosted: 0,
articlesRePosted: 0,
bytesPosted: 0,
articlesChecked: 0,
articlesChecked: 0, // number of successfully checked articles
articlesRechecked: 0, // times an article had to be rechecked
checkPending: 0,
checkRePending: 0,
// the following two may seem a bit redundant, as you can get the figures from connection states, but might help track uploader-level state (since the connection could be different, e.g. in error state)
checkActive: 0,
postActive: 0,
articleErrors: 0,
totalUploadData: 0,
rawSpeedTime: null,
Expand Down Expand Up @@ -232,7 +238,9 @@ Uploader.prototype = {
return;
}
if(self.cancelled) return;
self.postActive++;
c.post(post, function(err, messageId) {
self.postActive--;
if(err) {
if(self.cancelled && err.code == 'cancelled') return;
dumpPost(self, post); // post.data guaranteed to be set here
Expand Down Expand Up @@ -264,6 +272,7 @@ Uploader.prototype = {

if(!self.checkCacheAdd(post, !!post.reload, function(cacheId) {
post.cacheId = cacheId;
self.checkPending++;
// TODO: careful with holding up the post queue if reusing post connections for checking
if(!self.checkQueue.add(chkOpts.delay, post, doPost)) {
if(!checkQueueFillWarned) NNTP.log.info('Check queue is now full - upload speed will be throttled to compensate');
Expand Down Expand Up @@ -362,6 +371,11 @@ Uploader.prototype = {
// TODO: assumes that this is a checking connection and not a posting one
return;
}
if(post.chkFailures) {
self.checkRePending--;
self.articlesRechecked++;
} else
self.checkPending--;
self._checkPost(c, post, doCheck);
});
};
Expand All @@ -370,7 +384,9 @@ Uploader.prototype = {
_checkPost: function(conn, post, cb) {
var self = this;
if(this.cancelled) return;
self.checkActive++;
conn.stat(post.messageId, function(err, info) {
self.checkActive--;
if(err && self.cancelled && err.code == 'cancelled') return;
if(!err && !info) post.chkFailures++;
(function(cb) {
Expand Down Expand Up @@ -434,6 +450,7 @@ Uploader.prototype = {
} else {
// reschedule check
if(NNTP.log) NNTP.log.debug('Post check failed to find posted article ' + post.messageId + '; retrying after ' +(self.opts.check.recheckDelay/1000)+ ' second(s)... (attempt ' + post.chkFailures + '/' + self.opts.check.tries + ')');
self.checkRePending++;
// note that we do a force add (i.e. don't wait for callback), this is because we don't want to stall the check queue in the event that it grows too big (we do want to stall the post queue though). Also, this is really just putting an item back onto the queue, so it never really increases the overall size (by much)
self.checkQueue.add(self.opts.check.recheckDelay, post);
return cb();
Expand Down

0 comments on commit 20f3843

Please sign in to comment.