diff --git a/build/capataz_node.min.js b/build/capataz_node.min.js index 3f6a817..6829baa 100644 --- a/build/capataz_node.min.js +++ b/build/capataz_node.min.js @@ -1,2 +1,2 @@ -//! capataz 0.1.4 +//! capataz 0.1.5 !function(a){"use strict";var b=require("creatartis-base"),c=require("express"),d=require("path"),e=require("fs");a.__name__="capataz_node",a.__dependencies__={base:b,express:c};var f=b.copy,g=b.declare,h=b.initialize,i=b.Text,j=b.iterable,k=b.Iterable,l=b.Future,m=b.Logger,n=b.Statistics,o=a.Capataz=g({constructor:function(a){h(this.config={},a).integer("workerCount",{defaultValue:2,coerce:!0}).integer("useWebworkers",{defaultValue:1,coerce:!0}).bool("adjustWorkerCount",{defaultValue:!0,coerce:!0}).number("maxRetries",{defaultValue:100,coerce:!0}).integer("minDelay",{defaultValue:100,coerce:!0}).number("maxDelay",{defaultValue:9e5,coerce:!0}).number("desiredEvaluationTime",{defaultValue:1e4,coerce:!0}).number("maxTaskSize",{defaultValue:50,coerce:!0}).number("evaluationTimeGainFactor",{defaultValue:.9,coerce:!0}).number("maxScheduled",{defaultValue:5e3,coerce:!0}).integer("port",{defaultValue:8080,coerce:!0}).string("staticRoute",{defaultValue:"/capataz"}).string("serverFiles",{defaultValue:d.dirname(module.filename)+"/static"}).string("customFiles",{defaultValue:""}).object("routes",{defaultValue:{}}).func("authentication",{ignore:!0}).bool("compression",{defaultValue:!0,coerce:!0}).string("logFile",{defaultValue:i.formatDate(new Date,'"capataz-"yyyymmdd-hhnnss".log"')}),h(this,a).object("statistics",{defaultValue:new n}).object("logger",{defaultValue:m.ROOT}).object("store",{defaultValue:new q(a)}),this.logger&&(this.logger.appendToConsole(),this.config.logFile&&this.logger.appendToFile(this.config.logFile)),this.__startTime__=Date.now()},schedule:function(a){a.fun||raise("Cannot schedule ",JSON.stringify(a),", it has no `fun`!");var b=this.store.store({info:a.info,imports:a.imports,args:a.args,fun:a.fun,tags:a.tags}).future;return b.fail(this.logger.error.bind(this.logger)),b},taskSize:function(){var a=this.statistics.stat({key:"estimated_time"}),b=Math.max(1,a.average());return a.count()<=3?1:Math.round(Math.max(1,Math.min(this.config.maxTaskSize,this.config.desiredEvaluationTime/b)))},nextTask:function(a){return isNaN(a)&&(a=this.taskSize()),{serverStartTime:this.__startTime__,jobs:this.store.task(a).map(function(a){return{id:a.id,info:(a.info||"")+"",imports:a.imports||[],args:a.args||[],fun:a.fun+"",assignedSince:Date.now()}})}},postIsInvalid:function(a){return isNaN(a.assignedSince)||a.assignedSinceDate.now()||a.time<=0},processResult:function(a){var b,c=this.store.assigned(a.id);c?this.postIsInvalid(a)?(this.logger.warn("Posted result for ",a.id," is not valid. Ignoring POST."),b="invalid"):"undefined"!=typeof a.error?(b="rejected",c.future.reject(a.error)):(b="resolved",c.future.resolve(a.result)):(this.logger.debug("Job ",a.id," not found. Ignoring POST."),b="ignored"),c&&this.accountPost(a,c,b)},accountPost:function(a,b,c){"resolved"===c&&this.statistics.gain({key:"estimated_time"},a.time,this.config.evaluationTimeGainFactor),this.statistics.add(f({key:"evaluation_time",status:c,platform:a.clientPlatform,client:a.postedFrom},b&&b.tags),a.time),this.statistics.add(f({key:"roundtrip_time",status:c,platform:a.clientPlatform,client:a.postedFrom},b&&b.tags),Date.now()-a.assignedSince)},get_config:function(a,b){b.set("Cache-Control","max-age=0,no-cache,no-store"),b.json({jobURI:this.config.taskRoute,workerCount:this.config.workerCount,adjustWorkerCount:this.config.adjustWorkerCount,maxRetries:this.config.maxRetries,minDelay:this.config.minDelay,maxDelay:this.config.maxDelay,useWebworkers:this.config.useWebworkers})},get_task:function(a,b){var c=this.nextTask();return c.jobs.length>0?(b.set("Cache-Control","max-age=0,no-cache,no-store"),b.json(c),this.statistics.add({key:"jobs_per_task"},c.jobs.length)):(this.logger.debug("There are no pending jobs yet."),b.send(404,"There are no pending jobs yet. Please try again later.")),!0},post_task:function(a,b){var c=this,d=a.connection.remoteAddress+"";a.body&&Array.isArray(a.body.jobs)?(b.send("Thank you."),a.body.jobs.forEach(function(a){a.postedFrom=d,c.processResult(a)})):b.send(400,"Invalid post.")},get_stats:function(a,b){b.set("Cache-Control","max-age=0,no-cache,no-store"),b.json(this.statistics)},get_store:function(a,b){b.set("Cache-Control","max-age=0,no-cache,no-store"),b.json(this.store.status())},configureApp:function(a){var b=this.config;return a=a||c(),a.use(c.json()),b.compression&&a.use(c.compress()),a.get("/",function(a,c){c.redirect(b.staticRoute+"/index.html")}),a.get(b.routes.task||"/capataz/task.json",this.get_task.bind(this)),a.post(b.routes.task||"/capataz/task.json",this.post_task.bind(this)),a.get(b.routes.config||"/capataz/config.json",this.get_config.bind(this)),a.get(b.routes.stats||"/capataz/stats.json",this.get_stats.bind(this)),a.get(b.routes.store||"/capataz/store.json",this.get_store.bind(this)),"function"==typeof b.authentication&&[b.staticRoute,b.taskRoute,b.configRoute,b.statsRoute].forEach(function(d){a.use(d,c.basicAuth(b.authentication.bind(this,d)))}),a.use(b.staticRoute,c["static"](b.serverFiles)),b.customFiles.split("\n").forEach(function(d){d=d.trim(),d&&a.use(b.staticRoute,c["static"](d))}),a},"static run":function(a){var b=new o(a),c=b.config.port;return b.logger.info("Setting up the Capataz server."),b.expressApp=b.configureApp(a.app),b.expressApp.listen(c),b.logger.info("Server started and listening at port ",c,"."),b},scheduleAll:function(a,b,c){var d=j(a).__iter__(),e=this;return b=isNaN(b)?this.config.maxScheduled:Math.min(0|+b,this.config.maxScheduled),l.doWhile(function(){var a,f=[];try{for(var g=0;b>g;++g)a=e.schedule(d()),f.push(c?c(a):a)}catch(h){k.prototype.catchStop(h)}return f.length<1?!1:l.all(f)})}}),p=a.Store=g({constructor:function(a){h(this,a).number("maxScheduled",{defaultValue:5e3,coerce:!0}).object("statistics",{ignore:!0}),this.__count__=0},store:function(a){return b.raiseIf(Object.keys(this.__pending__).length>=this.maxScheduled,"Cannot schedule more jobs: the maximum amount (",this.maxScheduled,") has been reached!"),a.id=this.__count__=this.__count__+1&2147483647,a.future=new l,a.scheduledSince=Date.now(),a.assignedSince=1/0,a.resolvedSince=1/0,a.rejectedSince=1/0,this.statistics&&this.statistics.add(f({key:"scheduled"},a.tags)),a},task:b.objects.unimplemented("Store","task"),assigned:b.objects.unimplemented("Store","assigned"),status:b.objects.unimplemented("Store","store"),onJobResolve:function(a,b){a.result=b,a.resolvedSince=Date.now()},onJobReject:function(a,b){a.error=b,a.rejectedSince=Date.now()},__take__:function(a,b,c){var d=[];for(var e in a)if(a.hasOwnProperty(e)){if(--b<0)break;d.push([e,a[e]]),c&&delete a[e]}return d}}),q=a.MemoryStore=g(p,{constructor:function(a){p.call(this,a),h(this,a).bool("keepResolved",{defaultValue:!1,coerce:!0}).bool("keepRejected",{defaultValue:!1,coerce:!0}),this.__pending__={},this.__assigned__={},this.keepResolved&&(this.__resolved__={}),this.keepRejected&&(this.__rejected__={})},store:function(a){return a=p.prototype.store.call(this,a),a.future.done(this.onJobResolve.bind(this,a)),a.future.fail(this.onJobReject.bind(this,a)),this.__pending__[a.id]=a,a},task:function(a){var b=this,c=this.__take__(this.__pending__,a,!0).map(function(a){var c=a[0],d=a[1];return b.__assigned__[c]=d,d.assignedSince=Date.now(),d});return c.length=0&&"undefined"!=typeof Worker?(this.webworker=new Worker("capataz_worker.js"),this.webworker.onmessage=function(a){b.resolve(a.data||!0)}):a>0?b.reject("Webworkers are required but are not supported in this browser!"):b.resolve(!1),b},onWorkerMessage:function(a,b){var c=JSON.parse(b.data);c.hasOwnProperty("error")?a.reject(c.error):c.hasOwnProperty("result")&&a.resolve(c.result)},getTask:function(){var a=this;return base.Future.retrying(function(){return LOGGER.info(a.__number__+" < Requesting jobs."),base.HttpRequest.getJSON(CONFIG.jobURI).then(function(b){return b.serverStartTime>CONFIG.startTime&&(LOGGER.info(a.__number__+" > Definitions are outdated. Reloading..."),window.location.reload()),LOGGER.info(a.__number__+" > Received "+b.jobs.length+" jobs. E.g.: "+b.jobs[0].info),b},function(b){throw LOGGER.warn(a.__number__+" ! Job request failed (status: ",b.status," ",b.statusText,' "',b.responseText,'")!'),new Error("Job request failed!")})},CONFIG.maxRetries,CONFIG.minDelay,2,CONFIG.maxDelay).fail(function(){LOGGER.error(a.__number__+" ! Job request failed too many times! Not retrying anymore.")})},doWork:function(a){var b=this;return base.Future.sequence(a.jobs,function(a){return a.clientPlatform=navigator.platform,a.startedAt=Date.now(),b.doJob(a).then(function(c){return a.result=c,a.time=Date.now()-a.startedAt,LOGGER.debug(b.__number__+" > ",a.info," -> ",c),a},function(c){return a.error=c,a.time=Date.now()-a.startedAt,LOGGER.warn(b.__number__+" > ",a.info," !! ",c),a})}).then(function(){return a})},doJob:function doJob(job){var drudger=this;if(this.webworker){var future=new base.Future;return this.webworker.onmessage=this.onWorkerMessage.bind(this,future),this.webworker.postMessage(JSON.stringify(job)),future}return base.Future.invoke(eval,this,job.code)},postResults:function(a){var b=this;return base.Future.retrying(function(){return LOGGER.info(b.__number__+" < Posting results."),base.HttpRequest.postJSON(CONFIG.jobURI,a).fail(function(a){LOGGER.warn(b.__number__+" ! Posting failed: ",a.status," ",a.statusText," ",a.responseText,".")})},CONFIG.maxRetries,CONFIG.minDelay,2,CONFIG.maxDelay).fail(function(){LOGGER.error(b.__number__+" ! Job result post failed too many times! Not retrying anymore.")})},drudge:function(){var a=this;return base.Future.doWhile(function(){return a.getTask().then(a.doWork.bind(a)).then(a.postResults.bind(a))},function(){return!0}).fail(function(b){LOGGER.error(a.__number__+" Uncaught error on drudger! "+b)})}}),APP.start=function(){var a={};return window.location.search.substring(1).split("&").forEach(function(b){b=b.split("=").map(decodeURIComponent),2==b.length&&(a[b[0]]=b[1])}),base.HttpRequest.getJSON(a.configURI||"config.json").then(function(b){CONFIG=APP.CONFIG=base.copy(a,b,{jobURI:"task.json",workerCount:2,adjustWorkerCount:!0,maxRetries:50,minDelay:100,maxDelay:12e4,logLength:30}),CONFIG.startTime=Date.now(),LOGGER.appendToHtml("log",CONFIG.logLength);var c=CONFIG.adjustWorkerCount&&navigator.hardwareConcurrency?navigator.hardwareConcurrency:CONFIG.workerCount;return LOGGER.info("Starting "+c+" workers."),APP.drudgers=base.Iterable.range(c).map(function(){return new APP.Drudger}).toArray(),base.Future.sequence(APP.drudgers,function(a){return a.initialize(CONFIG.useWebworkers).done(a.drudge.bind(a))})})},"complete"===document.readyState?APP.start():window.addEventListener("load",APP.start,!1)}); \ No newline at end of file diff --git a/build/static/capataz_worker.js b/build/static/capataz_worker.js index 55bf045..2469af0 100644 --- a/build/static/capataz_worker.js +++ b/build/static/capataz_worker.js @@ -1,2 +1,2 @@ -//! capataz 0.1.4 +//! capataz 0.1.5 !function(self){"use strict";self.onmessage=function(a){var b=JSON.parse(a.data);b.error="Worker is not ready yet.",self.postMessage(JSON.stringify(b))},importScripts("require.js"),require(["creatartis-base"],function(base){self.base=base,self.onmessage=function onmessage(msg){var data=JSON.parse(msg.data),code='(function () {"use strict";\n return base.Future.imports.apply(this, '+JSON.stringify(data.imports||[])+").then(function (deps) {\n return ("+data.fun+").apply(this, deps.concat("+JSON.stringify(data.args||[])+"));\n });\n})()";base.Future.invoke(eval,self,code).then(function(a){data.result=a,self.postMessage(JSON.stringify(data))},function(a){data.error='Execution failed with "'+a+'"!\nCode:\n'+code+"\nCallstack:\n "+base.callStack(a).join("\n "),self.postMessage(JSON.stringify(data))})},self.postMessage("Ready")})}(self); \ No newline at end of file diff --git a/package.json b/package.json index 4e87842..5c17133 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "capataz", "description": "A framework to set up distributed algorithm via HTTP and Javascript.", "keywords": [ "distributed computing", "distributed algorithm" ], - "version": "0.1.4", + "version": "0.1.5", "author": { "name": "Leonardo Val", "email": "leonardo.val@creatartis.com"