[DO NOT MERGE, THIS BRANCH IS FOR TESTING ONLY] Investigation/enhanced worker messaging and external stats

docker/.env-sample

@@ -7,6 +7,9 @@ MYSQLDB_DOCKER_PORT=3306
 JETMON_LOCAL_PORT=7800
 JETMON_DOCKER_PORT=7800
 
+JETMON_STATUS_LOCAL_PORT=7802
+JETMON_STATUS_DOCKER_PORT=7802
+
 WPCOM_JETMON_AUTH_TOKEN=change_me
 
 VERIFLIER_LOCAL_PORT=7801

docker/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '3.8'
 
-services: 
+services:
     mysqldb:
         image: mysql:5.7
         restart: unless-stopped
@@ -15,7 +15,7 @@ services:
           - db:/var/lib/mysql
     jetmon:
         hostname: docker.jetmon.dev.com
-        build: 
+        build:
           context: ../
           dockerfile: docker/Dockerfile_jetmon
         env_file:
@@ -35,10 +35,11 @@ services:
           - WPCOM_JETMON_AUTH_TOKEN=$WPCOM_JETMON_AUTH_TOKEN
         ports:
           - $JETMON_LOCAL_PORT:$JETMON_DOCKER_PORT
+          - $JETMON_STATUS_LOCAL_PORT:$JETMON_STATUS_DOCKER_PORT
         depends_on:
           - mysqldb
     veriflier:
-        build: 
+        build:
           context: ../
           dockerfile: docker/Dockerfile_veriflier
         volumes:
@@ -49,6 +50,14 @@ services:
           - VERIFLIER_AUTH_TOKEN=$VERIFLIER_AUTH_TOKEN
           - VERIFLIER_PORT=$VERIFLIER_DOCKER_PORT
           - JETMON_PORT=$JETMON_DOCKER_PORT
+    php-apache:
+        image: php:apache
+        hostname: test.local
+        volumes:
+          - ./php-apache/html:/var/www/html
+        command: >
+          bash -c "a2enmod rewrite
+          && apache2-foreground"
 
-volumes: 
+volumes:
   db:

docker/php-apache/html/.htaccess

@@ -0,0 +1,8 @@
+<IfModule mod_rewrite.c>
+RewriteEngine On
+RewriteBase /
+RewriteRule ^index\.php$ - [L]
+RewriteCond %{REQUEST_FILENAME} !-f
+RewriteCond %{REQUEST_FILENAME} !-d
+RewriteRule . /index.php [L]
+</IfModule>

docker/php-apache/html/index.php

@@ -0,0 +1,5 @@
+<?php
+
+usleep( rand( 50000, 500000 ) );
+
+?>OK

lib/httpcheck.js

@@ -29,10 +29,12 @@ var config   = require( './config' );
 config.load();
 
 var arrCheck      = [];
-var running	      = false;
+var running       = false;
 var askedForWork  = false;
 var suicideSignal = false;
 var pointer       = 0;
+var activeChecks  = 0;
+var createdTime   = Date.now();
 
 var workerTotals = {};
 workerTotals[SITE_DOWN] = 0;
@@ -73,6 +75,7 @@ var HttpChecker = {
 			if ( pointerCurrentMax > arrCheck.length )
 				pointerCurrentMax = arrCheck.length;
 			for ( ; pointer < pointerCurrentMax ; pointer++ ) {
+				activeChecks++;
 				_watcher.http_check( arrCheck[ pointer ].monitor_url, DEFAULT_HTTP_PORT, pointer, HttpChecker.processResultsCallback );
 			}
 		}
@@ -82,6 +85,8 @@ var HttpChecker = {
 	},
 
 	processResultsCallback: function( serverArrayIndex, rtt, http_code ) {
+		activeChecks--;
+
 		var server = arrCheck[ serverArrayIndex ];
 		server.processed = true;
 		server.lastCheck = new Date().valueOf();	// we use set the value to the milliseconds value
@@ -118,6 +123,7 @@ var HttpChecker = {
 		workerTotals[server.site_status]++;
 
 		if ( pointer < arrCheck.length ) {
+			activeChecks++;
 			_watcher.http_check( arrCheck[ pointer ].monitor_url, DEFAULT_HTTP_PORT, pointer, HttpChecker.processResultsCallback );
 			pointer++;
 		} else {
@@ -160,6 +166,10 @@ var HttpChecker = {
 		}
 	},
 
+	getAge: function() {
+		return Date.now() - createdTime;
+	}
+
 };
 
 process.on( 'message', function( msg ) {
@@ -211,3 +221,18 @@ setTimeout( function() {
 			}
 }, 2000 );
 
+setInterval( function() {
+	var message = {
+		msgtype:      'stats',
+		worker_pid:   process.pid,
+		stats:        {
+			queueLength:  arrCheck.length,
+			pointer:      pointer,
+			activeChecks: activeChecks,
+			memoryUsage:  process.memoryUsage().rss
+		}
+	};
+
+	process.send( message );
+}, 1000 );
+

lib/jetmon.js

@@ -16,6 +16,8 @@ const NUM_SSL_SERVERS     = 4;
 const JETMON_CHECK        = 1;
 const VERIFLIER_CHECK     = 2;
 
+const STATUS_PORT         = 7802;
+
 const SECONDS = 1000;
 const MINUTES = 60 * SECONDS;
 const HOURS   = 60 * MINUTES;
@@ -59,6 +61,7 @@ var db_mysql = require( './database' );
 var wpcom    = require( './wpcom'    );
 var comms    = require( './comms'    );
 var cluster  = require( 'cluster'    );
+var http     = require( 'http'       );
 
 var gCountSuccess   = 0;
 var gCountError     = 0;
@@ -69,6 +72,7 @@ var arrObjects      = [];
 var localRetries    = [];
 var freeWorkers     = [];
 var arrWorkers      = [];
+var workerStats     = {};
 var gettingSites    = false;
 var endOfRound      = false;
 
@@ -135,6 +139,9 @@ function deleteWorker( pid ) {
 		if ( ( undefined != arrWorkers[count] ) &&
 			( arrWorkers[count].pid == pid ) ) {
 			arrWorkers.splice( count, 1 );
+			if ( workerStats[pid] ) {
+				delete( workerStats[pid] );
+			}
 			break;
 		}
 	}
@@ -362,6 +369,11 @@ function workerMsgCallback( msg ) {
 					// we have exhausted our local check limit, ask the verifliers to confirm
 					host_check_request( msg.server );
 				}
+			case 'stats':
+				if ( msg.stats ) {
+					workerStats[msg.worker_pid] = msg.stats;
+				}
+				break;
 			default:
 		}
 	}
@@ -536,6 +548,25 @@ for ( var i = 0; i < NUM_SSL_SERVERS; i++ ) {
 	ssl_server.on( 'message', sslWorkerCallBack );
 }
 
+http.createServer( function ( request, response ) {
+	response.writeHead( 200, {'Content-Type': 'application/json'} );
+
+	var status = {
+		'sites per second': sitesCount,
+		'sites in queue':   arrObjects.length,
+		'working':          ( arrWorkers.length - freeWorkers.length ),
+		'waiting':          freeWorkers.length,
+		'error':            gCountError,
+		'offline':          gCountOffline,
+		'total':            gCountSuccess,
+		'workers':          workerStats
+	};
+
+	response.write( JSON.stringify( status ) );
+
+	response.end();
+} ).listen( STATUS_PORT );
+
 // set a repeating 'tick' to perform clean-up and retries allocation
 setInterval( processQueuedRetries, SECONDS * 5 );