diff --git a/mantis-examples/mantis-examples-sine-function/build.gradle b/mantis-examples/mantis-examples-sine-function/build.gradle index 5f5bf55a1..49a77c4ff 100644 --- a/mantis-examples/mantis-examples-sine-function/build.gradle +++ b/mantis-examples/mantis-examples-sine-function/build.gradle @@ -27,6 +27,6 @@ task execute(type:JavaExec) { classpath = sourceSets.main.runtimeClasspath } dependencies { - implementation project(':mantis-server:mantis-server-worker') + implementation project(':mantis-runtime-executor') implementation libraries.spectatorApi } diff --git a/mantis-server/mantis-server-worker/build.gradle b/mantis-runtime-executor/build.gradle similarity index 85% rename from mantis-server/mantis-server-worker/build.gradle rename to mantis-runtime-executor/build.gradle index 16e53a859..da2bd2605 100644 --- a/mantis-server/mantis-server-worker/build.gradle +++ b/mantis-runtime-executor/build.gradle @@ -14,20 +14,16 @@ * limitations under the License. */ -apply plugin: 'eu.appsatori.fatjar' ext { mantisRxControlVersion = '1.3.+' - mesosVersion = '1.7.2' } dependencies { api project(":mantis-runtime") api project(":mantis-runtime-loader") api project(":mantis-server:mantis-server-worker-client") - api project(":mantis-server:mantis-server-agent") - implementation "org.apache.mesos:mesos:$mesosVersion" implementation libraries.slf4jApi implementation libraries.slf4jLog4j12 implementation libraries.vavr @@ -38,7 +34,6 @@ dependencies { implementation libraries.httpClient implementation "io.mantisrx:mantis-rxcontrol:$mantisRxControlVersion" implementation "com.yahoo.datasketches:sketches-core:0.9.1" - implementation libraries.spectatorApi testImplementation libraries.junit4 @@ -48,10 +43,3 @@ dependencies { testImplementation(testFixtures(project(":mantis-common"))) testImplementation(testFixtures(project(":mantis-control-plane:mantis-control-plane-core"))) } - -// exclude any signed jars -fatJar { - exclude "META-INF/*.SF" - exclude "META-INF/*.DSA" - exclude "META-INF/*.RSA" -} diff --git a/mantis-server/mantis-server-worker/dependencies.lock b/mantis-runtime-executor/dependencies.lock similarity index 90% rename from mantis-server/mantis-server-worker/dependencies.lock rename to mantis-runtime-executor/dependencies.lock index daafda6d4..18d9946e4 100644 --- a/mantis-server/mantis-server-worker/dependencies.lock +++ b/mantis-runtime-executor/dependencies.lock @@ -23,9 +23,6 @@ "org.apache.httpcomponents:httpclient": { "locked": "4.5.14" }, - "org.apache.mesos:mesos": { - "locked": "1.7.2" - }, "org.slf4j:slf4j-api": { "locked": "1.7.0" }, @@ -109,7 +106,7 @@ "firstLevelTransitive": [ "io.mantisrx:mantis-control-plane-core" ], - "locked": "2.12.5" + "locked": "2.12.6" }, "junit:junit": { "locked": "4.11" @@ -254,21 +251,12 @@ "project": true }, "io.mantisrx:mantis-runtime-loader": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-rxcontrol": { "locked": "1.3.20" }, - "io.mantisrx:mantis-server-agent": { - "project": true - }, "io.mantisrx:mantis-server-worker-client": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-shaded": { @@ -315,7 +303,7 @@ "firstLevelTransitive": [ "io.mantisrx:mantis-control-plane-core" ], - "locked": "2.12.5" + "locked": "2.12.6" }, "nz.ac.waikato.cms.moa:moa": { "locked": "2017.06" @@ -414,7 +402,6 @@ }, "com.netflix.spectator:spectator-api": { "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent", "io.mantisrx:mantis-server-worker-client" ], "locked": "1.3.10" @@ -427,8 +414,7 @@ }, "com.spotify:completable-futures": { "firstLevelTransitive": [ - "io.mantisrx:mantis-control-plane-client", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-control-plane-client" ], "locked": "0.3.1" }, @@ -438,17 +424,10 @@ "commons-io:commons-io": { "firstLevelTransitive": [ "io.mantisrx:mantis-common", - "io.mantisrx:mantis-control-plane-core", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-control-plane-core" ], "locked": "2.11.0" }, - "io.github.resilience4j:resilience4j-retry": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], - "locked": "1.5.0" - }, "io.mantisrx:mantis-common": { "firstLevelTransitive": [ "io.mantisrx:mantis-control-plane-core", @@ -495,21 +474,12 @@ "project": true }, "io.mantisrx:mantis-runtime-loader": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-rxcontrol": { "locked": "1.3.20" }, - "io.mantisrx:mantis-server-agent": { - "project": true - }, "io.mantisrx:mantis-server-worker-client": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-shaded": { @@ -558,16 +528,16 @@ }, "io.vavr:vavr": { "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent", + "io.mantisrx:mantis-runtime-loader", "io.mantisrx:mantis-shaded" ], - "locked": "0.10.2" + "locked": "0.9.2" }, "joda-time:joda-time": { "firstLevelTransitive": [ "io.mantisrx:mantis-control-plane-core" ], - "locked": "2.12.5" + "locked": "2.12.6" }, "net.jcip:jcip-annotations": { "firstLevelTransitive": [ @@ -575,12 +545,6 @@ ], "locked": "1.0" }, - "net.lingala.zip4j:zip4j": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], - "locked": "2.9.0" - }, "nz.ac.waikato.cms.moa:moa": { "locked": "2017.06" }, @@ -596,12 +560,6 @@ ], "locked": "1.14.2" }, - "org.apache.hadoop:hadoop-common": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], - "locked": "2.7.7" - }, "org.apache.httpcomponents:httpclient": { "locked": "4.5.14" }, @@ -645,18 +603,16 @@ "firstLevelTransitive": [ "io.mantisrx:mantis-common", "io.mantisrx:mantis-remote-observable", - "io.mantisrx:mantis-runtime", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-runtime" ], "locked": "1.7.36" }, "org.slf4j:slf4j-log4j12": { "firstLevelTransitive": [ "io.mantisrx:mantis-common", - "io.mantisrx:mantis-remote-observable", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-remote-observable" ], - "locked": "1.7.10" + "locked": "1.7.0" }, "org.xerial.snappy:snappy-java": { "firstLevelTransitive": [ @@ -742,21 +698,12 @@ "project": true }, "io.mantisrx:mantis-runtime-loader": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-rxcontrol": { "locked": "1.3.20" }, - "io.mantisrx:mantis-server-agent": { - "project": true - }, "io.mantisrx:mantis-server-worker-client": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-shaded": { @@ -806,7 +753,7 @@ "firstLevelTransitive": [ "io.mantisrx:mantis-control-plane-core" ], - "locked": "2.12.5" + "locked": "2.12.6" }, "junit:junit": { "locked": "4.11" @@ -912,7 +859,6 @@ }, "com.netflix.spectator:spectator-api": { "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent", "io.mantisrx:mantis-server-worker-client" ], "locked": "1.3.10" @@ -925,8 +871,7 @@ }, "com.spotify:completable-futures": { "firstLevelTransitive": [ - "io.mantisrx:mantis-control-plane-client", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-control-plane-client" ], "locked": "0.3.1" }, @@ -936,17 +881,10 @@ "commons-io:commons-io": { "firstLevelTransitive": [ "io.mantisrx:mantis-common", - "io.mantisrx:mantis-control-plane-core", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-control-plane-core" ], "locked": "2.11.0" }, - "io.github.resilience4j:resilience4j-retry": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], - "locked": "1.5.0" - }, "io.mantisrx:mantis-common": { "firstLevelTransitive": [ "io.mantisrx:mantis-common", @@ -995,21 +933,12 @@ "project": true }, "io.mantisrx:mantis-runtime-loader": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-rxcontrol": { "locked": "1.3.20" }, - "io.mantisrx:mantis-server-agent": { - "project": true - }, "io.mantisrx:mantis-server-worker-client": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], "project": true }, "io.mantisrx:mantis-shaded": { @@ -1058,10 +987,10 @@ }, "io.vavr:vavr": { "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent", + "io.mantisrx:mantis-runtime-loader", "io.mantisrx:mantis-shaded" ], - "locked": "0.10.2" + "locked": "0.9.2" }, "io.vavr:vavr-test": { "locked": "0.9.2" @@ -1070,7 +999,7 @@ "firstLevelTransitive": [ "io.mantisrx:mantis-control-plane-core" ], - "locked": "2.12.5" + "locked": "2.12.6" }, "junit:junit": { "locked": "4.11" @@ -1084,12 +1013,6 @@ ], "locked": "1.0" }, - "net.lingala.zip4j:zip4j": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], - "locked": "2.9.0" - }, "nz.ac.waikato.cms.moa:moa": { "locked": "2017.06" }, @@ -1111,12 +1034,6 @@ ], "locked": "1.14.2" }, - "org.apache.hadoop:hadoop-common": { - "firstLevelTransitive": [ - "io.mantisrx:mantis-server-agent" - ], - "locked": "2.7.7" - }, "org.apache.httpcomponents:httpclient": { "locked": "4.5.14" }, @@ -1166,18 +1083,16 @@ "firstLevelTransitive": [ "io.mantisrx:mantis-common", "io.mantisrx:mantis-remote-observable", - "io.mantisrx:mantis-runtime", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-runtime" ], "locked": "1.7.36" }, "org.slf4j:slf4j-log4j12": { "firstLevelTransitive": [ "io.mantisrx:mantis-common", - "io.mantisrx:mantis-remote-observable", - "io.mantisrx:mantis-server-agent" + "io.mantisrx:mantis-remote-observable" ], - "locked": "1.7.10" + "locked": "1.7.0" }, "org.xerial.snappy:snappy-java": { "firstLevelTransitive": [ diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/DataDroppedPayloadSetter.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/DataDroppedPayloadSetter.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/DataDroppedPayloadSetter.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/DataDroppedPayloadSetter.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/DownloadJob.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/DownloadJob.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/DownloadJob.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/DownloadJob.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/ExecuteStageRequestService.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/ExecuteStageRequestService.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/ExecuteStageRequestService.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/ExecuteStageRequestService.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/ExecutionDetails.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/ExecutionDetails.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/ExecutionDetails.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/ExecutionDetails.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/Heartbeat.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/Heartbeat.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/Heartbeat.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/Heartbeat.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/InstantiationUtil.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/InstantiationUtil.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/InstantiationUtil.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/InstantiationUtil.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/MantisWorker.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/MantisWorker.java similarity index 95% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/MantisWorker.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/MantisWorker.java index 96cf00bfb..25b161691 100644 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/MantisWorker.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/MantisWorker.java @@ -31,8 +31,6 @@ import io.mantisrx.server.master.client.MantisMasterGateway; import io.mantisrx.server.worker.config.ConfigurationFactory; import io.mantisrx.server.worker.config.StaticPropertiesConfigurationFactory; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; -import io.mantisrx.server.worker.mesos.VirualMachineWorkerServiceMesosImpl; import io.mantisrx.shaded.com.google.common.collect.ImmutableList; import java.io.File; import java.io.FileInputStream; @@ -115,8 +113,6 @@ public void run() { // metrics PublishSubject executeStageSubject = PublishSubject.create(); - PublishSubject vmTaskStatusSubject = PublishSubject.create(); - mantisServices.add(new VirualMachineWorkerServiceMesosImpl(executeStageSubject, vmTaskStatusSubject)); // TODO(sundaram): inline services are hard to read. Would be good to refactor this. mantisServices.add(new Service() { private RuntimeTaskImpl runtimeTaskImpl; @@ -154,8 +150,6 @@ public void start() { Clock.systemDefaultZone())); runtimeTaskImpl.setJob(jobToRun); - vmStatusSubscription = - runtimeTaskImpl.getVMStatus().subscribe(vmTaskStatusSubject); runtimeTaskImpl.startAsync(); } catch (Exception ex) { logger.error("Failed to start task, request: {}", wrappedRequest, ex); diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/ResourceUsagePayloadSetter.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/ResourceUsagePayloadSetter.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/ResourceUsagePayloadSetter.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/ResourceUsagePayloadSetter.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/RunningWorker.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RunningWorker.java similarity index 89% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/RunningWorker.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RunningWorker.java index 2cb409296..a6247aaca 100644 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/RunningWorker.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RunningWorker.java @@ -28,8 +28,6 @@ import io.mantisrx.server.core.JobSchedulingInfo; import io.mantisrx.server.core.Status; import io.mantisrx.server.core.Status.TYPE; -import io.mantisrx.server.core.domain.WorkerId; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; import java.util.Iterator; import java.util.concurrent.CountDownLatch; import org.slf4j.Logger; @@ -61,7 +59,6 @@ public class RunningWorker { private final String jobName; private final int totalStages; private final int metricsPort; - private final Observer vmTaskStatusObserver; private final Observable stageTotalWorkersObservable; private final Observable jobSchedulingInfoObservable; private final Iterator ports; @@ -85,7 +82,6 @@ public RunningWorker(Builder builder) { this.jobName = builder.jobName; this.totalStages = builder.totalStages; this.totalStagesNet = this.totalStages - (builder.hasJobMaster ? 1 : 0); - this.vmTaskStatusObserver = builder.vmTaskStatusObserver; this.jobStatus = builder.jobStatus; this.stageTotalWorkersObservable = builder.stageTotalWorkersObservable; this.jobSchedulingInfoObservable = builder.jobSchedulingInfoObservable; @@ -126,10 +122,6 @@ public void call(Throwable t) { public void signalStartedInitiated() { logger.info("JobId: " + jobId + ", stage: " + stageNum + " workerIndex: " + workerIndex + " workerNumber: " + workerNum + "," + " signaling started initiated"); - vmTaskStatusObserver.onNext(new VirtualMachineTaskStatus( - new WorkerId(jobId, workerIndex, workerNum).getId(), - VirtualMachineTaskStatus.TYPE.STARTED, jobName + ", " + - String.format(STATUS_MESSAGE_FORMAT, stageNum, workerIndex, workerNum, "started"))); // indicate start success requestSubject.onNext(true); requestSubject.onCompleted(); @@ -153,11 +145,6 @@ public void signalCompleted() { MantisJobState.Completed)); // send complete status jobStatus.onCompleted(); - // send completed status to vm service - vmTaskStatusObserver.onNext(new VirtualMachineTaskStatus( - new WorkerId(jobId, workerIndex, workerNum).getId(), - VirtualMachineTaskStatus.TYPE.COMPLETED, jobName + ", " + - String.format(STATUS_MESSAGE_FORMAT, stageNum, workerIndex, workerNum, "completed"))); } public void signalFailed(Throwable t) { @@ -263,10 +250,6 @@ public int getTotalStagesNet() { return totalStagesNet; } - public Observer getVmTaskStatusObserver() { - return vmTaskStatusObserver; - } - @Override public String toString() { return "RunningWorker [" @@ -275,8 +258,7 @@ public String toString() { + ", stageNum=" + stageNum + ", workerNum=" + workerNum + ", workerIndex=" + workerIndex + ", jobName=" + jobName + ", totalStages=" + totalStages + ", metricsPort=" - + metricsPort + ", vmTaskStatusObserver=" - + vmTaskStatusObserver + ", ports=" + ports + + metricsPort + ", ports=" + ports + ", requestSubject=" + requestSubject + ", context=" + context + ", workerInfo=" + workerInfo + "]"; } @@ -298,7 +280,6 @@ public static class Builder { private int workerIndex; private String jobName; private int totalStages; - private Observer vmTaskStatusObserver; private Observable stageTotalWorkersObservable; private Observable jobSchedulingInfoObservable; private PublishSubject requestSubject; @@ -374,11 +355,6 @@ public Builder totalStages(int totalStages) { return this; } - public Builder vmTaskStatusObservable(Observer vmTaskStatusObserver) { - this.vmTaskStatusObserver = vmTaskStatusObserver; - return this; - } - public Builder hasJobMaster(boolean b) { this.hasJobMaster = b; return this; diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java similarity index 96% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java index f97457fec..a86c3f9df 100644 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java @@ -19,10 +19,10 @@ import io.mantisrx.runtime.Job; import io.mantisrx.runtime.loader.RuntimeTask; import io.mantisrx.runtime.loader.SinkSubscriptionStateHandler; +import io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector; import io.mantisrx.runtime.loader.config.WorkerConfiguration; import io.mantisrx.runtime.loader.config.WorkerConfigurationUtils; import io.mantisrx.runtime.loader.config.WorkerConfigurationWritable; -import io.mantisrx.server.agent.metrics.cgroups.CgroupsMetricsCollector; import io.mantisrx.server.core.ExecuteStageRequest; import io.mantisrx.server.core.Service; import io.mantisrx.server.core.Status; @@ -33,7 +33,6 @@ import io.mantisrx.server.master.client.MantisMasterGateway; import io.mantisrx.server.master.client.TaskStatusUpdateHandler; import io.mantisrx.server.worker.client.WorkerMetricsClient; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; import io.mantisrx.shaded.com.google.common.util.concurrent.AbstractIdleService; import java.io.IOException; import java.time.Clock; @@ -69,8 +68,6 @@ public class RuntimeTaskImpl extends AbstractIdleService implements RuntimeTask private final PublishSubject> tasksStatusSubject; - private final PublishSubject vmTaskStatusSubject = PublishSubject.create(); - private Optional mantisJob = Optional.empty(); private ExecuteStageRequest executeStageRequest; @@ -181,7 +178,6 @@ private void doRun() throws Exception { executeStageSubject, tasksStatusSubject, new WorkerExecutionOperationsNetworkStage( - vmTaskStatusSubject, masterMonitor, config, workerMetricsClient, @@ -229,10 +225,6 @@ protected Observable getStatus() { .flatMap((Func1, Observable>) status -> status); } - public Observable getVMStatus() { - return vmTaskStatusSubject; - } - public String getWorkerId() { return executeStageRequest.getWorkerId().getId(); } diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/TrackedExecuteStageRequest.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/TrackedExecuteStageRequest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/TrackedExecuteStageRequest.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/TrackedExecuteStageRequest.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/VirtualMachineWorkerService.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/VirtualMachineWorkerService.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/VirtualMachineWorkerService.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/VirtualMachineWorkerService.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperations.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperations.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperations.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperations.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java similarity index 99% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java index a44545f07..6658ea2e7 100644 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java @@ -56,7 +56,6 @@ import io.mantisrx.server.worker.jobmaster.AutoScaleMetricsConfig; import io.mantisrx.server.worker.jobmaster.JobMasterService; import io.mantisrx.server.worker.jobmaster.JobMasterStageConfig; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; import io.mantisrx.shaded.com.google.common.base.Splitter; import io.mantisrx.shaded.com.google.common.base.Strings; import io.reactivex.mantis.remote.observable.RemoteRxServer; @@ -99,7 +98,6 @@ public class WorkerExecutionOperationsNetworkStage implements WorkerExecutionOpe private final WorkerMetricsClient workerMetricsClient; private final AtomicReference heartbeatRef = new AtomicReference<>(); private final SinkSubscriptionStateHandler.Factory sinkSubscriptionStateHandlerFactory; - private final Observer vmTaskStatusObserver; private final MantisMasterGateway mantisMasterApi; private int connectionsPerEndpoint = 2; private boolean lookupSpectatorRegistry = true; @@ -112,13 +110,11 @@ public class WorkerExecutionOperationsNetworkStage implements WorkerExecutionOpe private Observer jobStatusObserver; public WorkerExecutionOperationsNetworkStage( - Observer vmTaskStatusObserver, MantisMasterGateway mantisMasterApi, WorkerConfiguration config, WorkerMetricsClient workerMetricsClient, SinkSubscriptionStateHandler.Factory sinkSubscriptionStateHandlerFactory, ClassLoader classLoader) { - this.vmTaskStatusObserver = vmTaskStatusObserver; this.mantisMasterApi = mantisMasterApi; this.config = config; this.workerMetricsClient = workerMetricsClient; @@ -309,7 +305,6 @@ public void executeStage(final ExecutionDetails setup) throws IOException { .jobStatusObserver(setup.getStatus()) .requestSubject(setup.getExecuteStageRequest().getRequestSubject()) .workerInfo(workerInfo) - .vmTaskStatusObservable(vmTaskStatusObserver) .hasJobMaster(executionRequest.getHasJobMaster()) .jobId(executionRequest.getJobId()); diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/WorkerIndexHistory.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerIndexHistory.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/WorkerIndexHistory.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerIndexHistory.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/config/ConfigurationFactory.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/config/ConfigurationFactory.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/config/ConfigurationFactory.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/config/ConfigurationFactory.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/config/ConfigurationProvider.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/config/ConfigurationProvider.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/config/ConfigurationProvider.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/config/ConfigurationProvider.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/config/StaticPropertiesConfigurationFactory.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/config/StaticPropertiesConfigurationFactory.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/config/StaticPropertiesConfigurationFactory.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/config/StaticPropertiesConfigurationFactory.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfig.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfig.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfig.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfig.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/GaugeData.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/GaugeData.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/GaugeData.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/GaugeData.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/JobAutoScaler.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/JobAutoScaler.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/JobAutoScaler.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/JobAutoScaler.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterService.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterService.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterService.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterService.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterStageConfig.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterStageConfig.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterStageConfig.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/JobMasterStageConfig.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/MetricAggregator.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/MetricAggregator.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/MetricAggregator.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/MetricAggregator.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/MetricData.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/MetricData.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/MetricData.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/MetricData.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscription.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscription.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscription.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscription.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/Util.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/Util.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/Util.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/Util.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandler.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandler.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandler.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandler.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricSubscription.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricSubscription.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricSubscription.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetricSubscription.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetrics.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetrics.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetrics.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/WorkerMetrics.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchAutoScaler.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchAutoScaler.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchAutoScaler.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchAutoScaler.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchConfiguration.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchConfiguration.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchConfiguration.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchConfiguration.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchControllerOutput.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchControllerOutput.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchControllerOutput.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchControllerOutput.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchPIDConfig.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchPIDConfig.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchPIDConfig.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/ClutchPIDConfig.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/experimental/MantisClutchConfigurationSelector.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/experimental/MantisClutchConfigurationSelector.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/experimental/MantisClutchConfigurationSelector.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/experimental/MantisClutchConfigurationSelector.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/ClutchRpsPIDConfig.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/ClutchRpsPIDConfig.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/ClutchRpsPIDConfig.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/ClutchRpsPIDConfig.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelector.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelector.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelector.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelector.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputer.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputer.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputer.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputer.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputer.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputer.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputer.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputer.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscaler.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscaler.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscaler.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscaler.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscalerConfig.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscalerConfig.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscalerConfig.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoscalerConfig.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/Controller.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/Controller.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/Controller.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/Controller.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/ClutchMantisStageActuator.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/ClutchMantisStageActuator.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/ClutchMantisStageActuator.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/ClutchMantisStageActuator.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuator.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuator.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuator.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuator.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDController.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDController.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDController.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDController.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Derivative.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Derivative.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Derivative.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Derivative.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputer.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputer.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputer.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputer.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Integrator.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Integrator.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Integrator.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/Integrator.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/TransformerWrapper.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/TransformerWrapper.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/TransformerWrapper.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/jobmaster/control/utils/TransformerWrapper.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/scheduling/JobSchedulingTracker.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/scheduling/JobSchedulingTracker.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/scheduling/JobSchedulingTracker.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/scheduling/JobSchedulingTracker.java diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/scheduling/WorkerIndexChange.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/scheduling/WorkerIndexChange.java similarity index 100% rename from mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/scheduling/WorkerIndexChange.java rename to mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/scheduling/WorkerIndexChange.java diff --git a/mantis-server/mantis-server-worker/src/main/resources/META-INF/services/io.mantisrx.runtime.loader.RuntimeTask b/mantis-runtime-executor/src/main/resources/META-INF/services/io.mantisrx.runtime.loader.RuntimeTask similarity index 100% rename from mantis-server/mantis-server-worker/src/main/resources/META-INF/services/io.mantisrx.runtime.loader.RuntimeTask rename to mantis-runtime-executor/src/main/resources/META-INF/services/io.mantisrx.runtime.loader.RuntimeTask diff --git a/mantis-server/mantis-server-worker/src/main/resources/log4j.properties b/mantis-runtime-executor/src/main/resources/log4j.properties similarity index 100% rename from mantis-server/mantis-server-worker/src/main/resources/log4j.properties rename to mantis-runtime-executor/src/main/resources/log4j.properties diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/DataDroppedPayloadSetterTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/DataDroppedPayloadSetterTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/DataDroppedPayloadSetterTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/DataDroppedPayloadSetterTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/HeartbeatTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/HeartbeatTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/HeartbeatTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/HeartbeatTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStageTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStageTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStageTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStageTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/config/WorkerConfigurationTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/config/WorkerConfigurationTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/config/WorkerConfigurationTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/config/WorkerConfigurationTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfigTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfigTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfigTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/AutoScaleMetricsConfigTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/JobAutoScalerTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/JobAutoScalerTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/JobAutoScalerTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/JobAutoScalerTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscriptionTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscriptionTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscriptionTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/SourceJobWorkerMetricsSubscriptionTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandlerTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandlerTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandlerTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/WorkerMetricHandlerTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelectorTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelectorTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelectorTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsClutchConfigurationSelectorTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputerTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputerTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputerTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsMetricComputerTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputerTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputerTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputerTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/clutch/rps/RpsScaleComputerTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoScalerTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoScalerTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoScalerTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/AdaptiveAutoScalerTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuatorTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuatorTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuatorTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/actuators/MantisStageActuatorTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDControllerTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDControllerTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDControllerTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/controllers/PIDControllerTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputerTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputerTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputerTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/ErrorComputerTest.java diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/IntegratorTest.java b/mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/IntegratorTest.java similarity index 100% rename from mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/IntegratorTest.java rename to mantis-runtime-executor/src/test/java/io/mantisrx/server/worker/jobmaster/control/utils/IntegratorTest.java diff --git a/mantis-runtime-loader/build.gradle b/mantis-runtime-loader/build.gradle index 88c7ad32c..50e61c9f3 100644 --- a/mantis-runtime-loader/build.gradle +++ b/mantis-runtime-loader/build.gradle @@ -16,6 +16,7 @@ dependencies { api project(":mantis-control-plane:mantis-control-plane-client") + implementation libraries.vavr testImplementation libraries.junit4 testImplementation libraries.mockitoAll diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/Cgroup.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/Cgroup.java similarity index 92% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/Cgroup.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/Cgroup.java index 5afae0827..4a64a5704 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/Cgroup.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/Cgroup.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import java.io.IOException; import java.util.List; diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CgroupImpl.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupImpl.java similarity index 98% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CgroupImpl.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupImpl.java index 8ee783640..998210d6a 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CgroupImpl.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.shaded.org.apache.curator.shaded.com.google.common.base.Preconditions; import io.vavr.Tuple2; diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CgroupsMetricsCollector.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupsMetricsCollector.java similarity index 96% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CgroupsMetricsCollector.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupsMetricsCollector.java index 84313b4c5..8fd2442a0 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CgroupsMetricsCollector.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupsMetricsCollector.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.MetricsCollector; import io.mantisrx.runtime.loader.config.Usage; diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CpuAcctsSubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CpuAcctsSubsystemProcess.java similarity index 97% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CpuAcctsSubsystemProcess.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CpuAcctsSubsystemProcess.java index adf54cea0..63569c0a4 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/CpuAcctsSubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CpuAcctsSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage; import java.io.IOException; diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/MemorySubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/MemorySubsystemProcess.java similarity index 97% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/MemorySubsystemProcess.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/MemorySubsystemProcess.java index d30a5178e..6519b8e4f 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/MemorySubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/MemorySubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage.UsageBuilder; import java.io.IOException; diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/NetworkSubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/NetworkSubsystemProcess.java similarity index 97% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/NetworkSubsystemProcess.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/NetworkSubsystemProcess.java index afe4596f1..9a3a3c93e 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/NetworkSubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/NetworkSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage.UsageBuilder; import java.io.IOException; diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/ProcFileReader.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/ProcFileReader.java similarity index 98% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/ProcFileReader.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/ProcFileReader.java index f6763e65b..d112eb829 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/ProcFileReader.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/ProcFileReader.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.shaded.com.google.common.base.Charsets; import java.io.Closeable; diff --git a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/SubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/SubsystemProcess.java similarity index 90% rename from mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/SubsystemProcess.java rename to mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/SubsystemProcess.java index 5b7dcbff7..8a234f61c 100644 --- a/mantis-server/mantis-server-agent/src/main/java/io/mantisrx/server/agent/metrics/cgroups/SubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/SubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage; import java.io.IOException; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java index 1a9736c64..52f460006 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java @@ -141,7 +141,7 @@ default Time getHeartbeatInterval() { Integer getBindPort(); @Config("mantis.taskexecutor.metrics.collector") - @Default("io.mantisrx.server.worker.mesos.MesosMetricsCollector") + @Default("io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector") MetricsCollector getUsageSupplier(); // ------------------------------------------------------------------------ diff --git a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestCgroup.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCgroup.java similarity index 84% rename from mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestCgroup.java rename to mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCgroup.java index d947baef0..5e7fb14e5 100644 --- a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestCgroup.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCgroup.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,12 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.shaded.com.google.common.collect.ImmutableMap; import io.mantisrx.shaded.com.google.common.io.Resources; import java.io.IOException; +import org.junit.Assert; import org.junit.Test; public class TestCgroup { @@ -31,15 +29,15 @@ public class TestCgroup { @Test public void testReadingStatFiles() throws IOException { - assertEquals( + Assert.assertEquals( ImmutableMap.of("user", 49692738L, "system", 4700825L), cgroup.getStats("cpuacct", "cpuacct.stat")); - assertTrue(cgroup.isV1()); + Assert.assertTrue(cgroup.isV1()); } @Test public void testReadingMetrics() throws IOException { - assertEquals( + Assert.assertEquals( 400000L, cgroup.getMetric("cpuacct", "cpu.cfs_quota_us").longValue() ); @@ -47,7 +45,7 @@ public void testReadingMetrics() throws IOException { @Test public void testLongOverflow() throws IOException { - assertEquals( + Assert.assertEquals( Long.MAX_VALUE, cgroup.getMetric("testlongoverflow", "verylongvalue").longValue() ); diff --git a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestCpuAcctsSubsystemProcess.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCpuAcctsSubsystemProcess.java similarity index 64% rename from mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestCpuAcctsSubsystemProcess.java rename to mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCpuAcctsSubsystemProcess.java index 0c0ff4ab9..568eb2ff0 100644 --- a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestCpuAcctsSubsystemProcess.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCpuAcctsSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,56 +14,53 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage; import io.mantisrx.shaded.com.google.common.collect.ImmutableMap; import io.mantisrx.shaded.com.google.common.io.Resources; import java.io.IOException; +import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; public class TestCpuAcctsSubsystemProcess { @Test public void testWhenCgroupsReturnsCorrectData() throws Exception { - final Cgroup cgroup = mock(Cgroup.class); + final Cgroup cgroup = Mockito.mock(Cgroup.class); final CpuAcctsSubsystemProcess process = new CpuAcctsSubsystemProcess(cgroup); - when(cgroup.isV1()).thenReturn(true); - when(cgroup.getStats("cpuacct", "cpuacct.stat")) + Mockito.when(cgroup.isV1()).thenReturn(true); + Mockito.when(cgroup.getStats("cpuacct", "cpuacct.stat")) .thenReturn(ImmutableMap.of("user", 43873627L, "system", 4185541L)); - when(cgroup.getStats("cpuacct", "cpuacct.stat")) + Mockito.when(cgroup.getStats("cpuacct", "cpuacct.stat")) .thenReturn(ImmutableMap.of("user", 43873627L, "system", 4185541L)); - when(cgroup.getMetric("cpuacct", "cpu.cfs_quota_us")) + Mockito.when(cgroup.getMetric("cpuacct", "cpu.cfs_quota_us")) .thenReturn(400000L); - when(cgroup.getMetric("cpuacct", "cpu.cfs_period_us")) + Mockito.when(cgroup.getMetric("cpuacct", "cpu.cfs_period_us")) .thenReturn(100000L); final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(4L, (long) usage.getCpusLimit()); - assertEquals(438736L, (long) usage.getCpusUserTimeSecs()); - assertEquals(41855L, (long) usage.getCpusSystemTimeSecs()); + Assert.assertEquals(4L, (long) usage.getCpusLimit()); + Assert.assertEquals(438736L, (long) usage.getCpusUserTimeSecs()); + Assert.assertEquals(41855L, (long) usage.getCpusSystemTimeSecs()); } @Test public void testCgroupsV2() throws IOException { final Cgroup cgroupv2 = new CgroupImpl(Resources.getResource("example2").getPath()); - assertFalse(cgroupv2.isV1()); + Assert.assertFalse(cgroupv2.isV1()); final CpuAcctsSubsystemProcess process = new CpuAcctsSubsystemProcess(cgroupv2); final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(2L, (long) usage.getCpusLimit()); - assertEquals(4231L, (long) usage.getCpusUserTimeSecs()); - assertEquals(1277L, (long) usage.getCpusSystemTimeSecs()); + Assert.assertEquals(2L, (long) usage.getCpusLimit()); + Assert.assertEquals(4231L, (long) usage.getCpusUserTimeSecs()); + Assert.assertEquals(1277L, (long) usage.getCpusSystemTimeSecs()); } } diff --git a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestMemorySubsystemProcess.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestMemorySubsystemProcess.java similarity index 80% rename from mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestMemorySubsystemProcess.java rename to mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestMemorySubsystemProcess.java index 1d00f4298..5e182f038 100644 --- a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestMemorySubsystemProcess.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestMemorySubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,29 +14,27 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage; import io.mantisrx.shaded.com.google.common.collect.ImmutableMap; import io.mantisrx.shaded.com.google.common.io.Resources; import java.io.IOException; +import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; public class TestMemorySubsystemProcess { - private final Cgroup cgroup = mock(Cgroup.class); + private final Cgroup cgroup = Mockito.mock(Cgroup.class); private final MemorySubsystemProcess process = new MemorySubsystemProcess(cgroup); @Test public void testHappyPath() throws Exception { - when(cgroup.isV1()).thenReturn(true); - when(cgroup.getMetric("memory", "memory.limit_in_bytes")) + Mockito.when(cgroup.isV1()).thenReturn(true); + Mockito.when(cgroup.getMetric("memory", "memory.limit_in_bytes")) .thenReturn(17179869184L); - when(cgroup.getStats("memory", "memory.stat")) + Mockito.when(cgroup.getStats("memory", "memory.stat")) .thenReturn( ImmutableMap.builder() .put("cache", 233472L) @@ -80,9 +78,9 @@ public void testHappyPath() throws Exception { final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(17179869184L, (long) usage.getMemLimit()); - assertEquals(14828109824L, (long) usage.getMemRssBytes()); - assertEquals(14828109824L, (long) usage.getMemAnonBytes()); + Assert.assertEquals(17179869184L, (long) usage.getMemLimit()); + Assert.assertEquals(14828109824L, (long) usage.getMemRssBytes()); + Assert.assertEquals(14828109824L, (long) usage.getMemAnonBytes()); } @Test @@ -94,8 +92,8 @@ public void testCgroupv2() throws IOException { final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(2147483648L, (long) usage.getMemLimit()); - assertEquals(1693843456L, (long) usage.getMemRssBytes()); - assertEquals(945483776L, (long) usage.getMemAnonBytes()); + Assert.assertEquals(2147483648L, (long) usage.getMemLimit()); + Assert.assertEquals(1693843456L, (long) usage.getMemRssBytes()); + Assert.assertEquals(945483776L, (long) usage.getMemAnonBytes()); } } diff --git a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestNetworkSubsystemProcess.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestNetworkSubsystemProcess.java similarity index 80% rename from mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestNetworkSubsystemProcess.java rename to mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestNetworkSubsystemProcess.java index 941a18bb0..5acfaee9b 100644 --- a/mantis-server/mantis-server-agent/src/test/java/io/mantisrx/server/agent/metrics/cgroups/TestNetworkSubsystemProcess.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestNetworkSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,11 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; - -import static org.junit.Assert.assertEquals; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage; import io.mantisrx.shaded.com.google.common.io.Resources; +import org.junit.Assert; import org.junit.Test; public class TestNetworkSubsystemProcess { @@ -31,7 +30,7 @@ public void testValidPath() throws Exception { final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(2861321009430L, (long) usage.getNetworkReadBytes()); - assertEquals(2731791728959L, (long) usage.getNetworkWriteBytes()); + Assert.assertEquals(2861321009430L, (long) usage.getNetworkReadBytes()); + Assert.assertEquals(2731791728959L, (long) usage.getNetworkWriteBytes()); } } diff --git a/mantis-server/mantis-server-agent/src/test/resources/example1/cpuacct/cpu.cfs_quota_us b/mantis-runtime-loader/src/test/resources/example1/cpuacct/cpu.cfs_quota_us similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example1/cpuacct/cpu.cfs_quota_us rename to mantis-runtime-loader/src/test/resources/example1/cpuacct/cpu.cfs_quota_us diff --git a/mantis-server/mantis-server-agent/src/test/resources/example1/cpuacct/cpuacct.stat b/mantis-runtime-loader/src/test/resources/example1/cpuacct/cpuacct.stat similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example1/cpuacct/cpuacct.stat rename to mantis-runtime-loader/src/test/resources/example1/cpuacct/cpuacct.stat diff --git a/mantis-server/mantis-server-agent/src/test/resources/example1/network/dev b/mantis-runtime-loader/src/test/resources/example1/network/dev similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example1/network/dev rename to mantis-runtime-loader/src/test/resources/example1/network/dev diff --git a/mantis-server/mantis-server-agent/src/test/resources/example1/testlongoverflow/verylongvalue b/mantis-runtime-loader/src/test/resources/example1/testlongoverflow/verylongvalue similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example1/testlongoverflow/verylongvalue rename to mantis-runtime-loader/src/test/resources/example1/testlongoverflow/verylongvalue diff --git a/mantis-server/mantis-server-agent/src/test/resources/example2/cpu.max b/mantis-runtime-loader/src/test/resources/example2/cpu.max similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example2/cpu.max rename to mantis-runtime-loader/src/test/resources/example2/cpu.max diff --git a/mantis-server/mantis-server-agent/src/test/resources/example2/cpu.stat b/mantis-runtime-loader/src/test/resources/example2/cpu.stat similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example2/cpu.stat rename to mantis-runtime-loader/src/test/resources/example2/cpu.stat diff --git a/mantis-server/mantis-server-agent/src/test/resources/example2/memory.current b/mantis-runtime-loader/src/test/resources/example2/memory.current similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example2/memory.current rename to mantis-runtime-loader/src/test/resources/example2/memory.current diff --git a/mantis-server/mantis-server-agent/src/test/resources/example2/memory.max b/mantis-runtime-loader/src/test/resources/example2/memory.max similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example2/memory.max rename to mantis-runtime-loader/src/test/resources/example2/memory.max diff --git a/mantis-server/mantis-server-agent/src/test/resources/example2/memory.stat b/mantis-runtime-loader/src/test/resources/example2/memory.stat similarity index 100% rename from mantis-server/mantis-server-agent/src/test/resources/example2/memory.stat rename to mantis-runtime-loader/src/test/resources/example2/memory.stat diff --git a/mantis-server/mantis-server-agent/build.gradle b/mantis-server/mantis-server-agent/build.gradle index 798ec4371..f72cf29c5 100644 --- a/mantis-server/mantis-server-agent/build.gradle +++ b/mantis-server/mantis-server-agent/build.gradle @@ -35,7 +35,7 @@ dependencies { testImplementation libraries.junit4 testImplementation libraries.mockitoAll - testImplementation project(":mantis-server:mantis-server-worker") + testImplementation project(":mantis-runtime-executor") testImplementation(testFixtures(project(":mantis-control-plane:mantis-control-plane-core"))) } diff --git a/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties b/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties index f74e2399e..26c564b27 100644 --- a/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties +++ b/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties @@ -18,7 +18,7 @@ mantis.taskexecutor.heartbeats.interval=3500 mantis.localmode=false -mantis.taskexecutor.metrics.collector=io.mantisrx.server.agent.metrics.cgroups.CgroupsMetricsCollector +mantis.taskexecutor.metrics.collector=io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector mantis.taskexecutor.rpc.port-range=5050 mantis.taskexecutor.blob-store.storage-dir=file:///apps/mantis/mantis-server-agent/mantis-artifacts/storage/ mantis.taskexecutor.blob-store.local-cache=/apps/mantis/mantis-server-agent/mantis-artifacts diff --git a/mantis-server/mantis-server-agent/src/main/resources/agent.properties b/mantis-server/mantis-server-agent/src/main/resources/agent.properties index f55bd5939..ef458ada5 100644 --- a/mantis-server/mantis-server-agent/src/main/resources/agent.properties +++ b/mantis-server/mantis-server-agent/src/main/resources/agent.properties @@ -16,7 +16,7 @@ mantis.taskexecutor.id=agent1 mantis.localmode=true -mantis.taskexecutor.metrics.collector=io.mantisrx.server.agent.metrics.cgroups.CgroupsMetricsCollector +mantis.taskexecutor.metrics.collector=io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector mantis.taskexecutor.rpc.port-range=5050 mantis.taskexecutor.blob-store.storage-dir=file:///tmp/mantis/mantis-artifacts/storage/ mantis.taskexecutor.blob-store.local-cache=/tmp/mantis/mantis-artifacts/cache diff --git a/mantis-server/mantis-server-worker/Dockerfile b/mantis-server/mantis-server-worker/Dockerfile deleted file mode 100644 index ea61a4e1a..000000000 --- a/mantis-server/mantis-server-worker/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM docker-hub.netflix.net/mesosphere/mesos-slave:1.3.2 - -MAINTAINER Mantis Developers - -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - -ENV MESOS_SYSTEMD_ENABLE_SUPPORT false - -# Setup mantis-agent environment -RUN mkdir -p /mnt/local/mantisWorkerInstall/bin/ -RUN mkdir -p /mnt/local/mantisWorkerInstall/libs/ -RUN mkdir -p /mnt/local/mantisWorkerInstall/jobs/ - -# NOTE: Assumes you're building in the mantis-server-worker directory -COPY ./src/main/resources/startup_docker.sh /mnt/local/mantisWorkerInstall/bin/ -COPY ./build/libs/mantis-server-worker*.jar /mnt/local/mantisWorkerInstall/libs/ -COPY worker-docker.properties /mnt/local/mantisWorkerInstall/jobs/ - -# Set up Java 8 -RUN apt-get update -RUN apt-get install -y --no-install-recommends software-properties-common -RUN apt-get install -y python3-software-properties debconf-utils -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0xB1998361219BD9C9 -RUN apt-add-repository "deb http://repos.azulsystems.com/ubuntu stable main" -RUN apt-get install -y unzip -RUN apt-get install wget -RUN apt-get update -RUN apt-get install -y zulu-8 - - -# Make it look like we're running in VPC to test networking -ENV EC2_VPC_ID="FakeVpc" - -# Present a ENI ResourceSet to the mesos. We expect the executor to handle this in test mode. -CMD ["mesos-slave", "--master=zk://127.0.0.1:2181/mesos", "--work_dir=/apps/mesos-agent", "--log_dir=/var/log/mesos/", "--logging_level=INFO", "--hostname=localmantisagent", "--attributes=region:laptop;asg:mantisagent-laptop-v001;stack:laptop;zone:laptopd;itype:macbook.pro;cluster:mantisagent-laptop;id:l-deadbeef;res:ResourceSet-ENIs-7-29"] diff --git a/mantis-server/mantis-server-worker/README.md b/mantis-server/mantis-server-worker/README.md deleted file mode 100644 index 81e247c70..000000000 --- a/mantis-server/mantis-server-worker/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Running MantisWorker locally in IntelliJ IDE - -1. Add project lombok plugin to IntelliJ -2. use VirtualMachineWorkerServiceLocalImpl instead of VirualMachineWorkerServiceMesosImpl in MantisWorker class -3. add sine-function libs(mantis-sdk/examples/sine-function/build/distributions/sine-function-1.0/lib/) to mantis-server-worker_main Module classpath -4. Bring up master in docker cluster with docker-compose -5. use the MantisWorker runConfiguration from mantisoss repo to run MantisWorker diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/VirtualMachineWorkerServiceLocalImpl.java b/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/VirtualMachineWorkerServiceLocalImpl.java deleted file mode 100644 index da8e9ee25..000000000 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/VirtualMachineWorkerServiceLocalImpl.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.mantisrx.server.worker; - -import io.mantisrx.common.WorkerPorts; -import io.mantisrx.runtime.MachineDefinition; -import io.mantisrx.runtime.MachineDefinitions; -import io.mantisrx.runtime.MantisJobDurationType; -import io.mantisrx.runtime.descriptor.SchedulingInfo; -import io.mantisrx.runtime.descriptor.StageScalingPolicy; -import io.mantisrx.runtime.descriptor.StageSchedulingInfo; -import io.mantisrx.runtime.parameter.Parameter; -import io.mantisrx.server.core.BaseService; -import io.mantisrx.server.core.ExecuteStageRequest; -import io.mantisrx.server.core.WorkerTopologyInfo; -import io.mantisrx.server.core.WrappedExecuteStageRequest; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus.TYPE; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import org.apache.mesos.MesosExecutorDriver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import rx.Observable; -import rx.Observer; -import rx.functions.Action0; -import rx.functions.Action1; -import rx.schedulers.Schedulers; -import rx.subjects.PublishSubject; - - -/* Local impl to fake a task launch from mesos to allow running a MantisWorker in IDE for development */ -public class VirtualMachineWorkerServiceLocalImpl extends BaseService implements VirtualMachineWorkerService { - - private static final Logger logger = LoggerFactory.getLogger(VirtualMachineWorkerServiceLocalImpl.class); - private final WorkerTopologyInfo.Data workerInfo; - private MesosExecutorDriver mesosDriver; - private ExecutorService executor; - private Observer executeStageRequestObserver; - private Observable vmTaskStatusObservable; - - public VirtualMachineWorkerServiceLocalImpl(final WorkerTopologyInfo.Data workerInfo, - Observer executeStageRequestObserver, - Observable vmTaskStatusObservable) { - this.workerInfo = workerInfo; - this.executeStageRequestObserver = executeStageRequestObserver; - this.vmTaskStatusObservable = vmTaskStatusObservable; - executor = Executors.newSingleThreadExecutor(new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r, "vm_worker_mesos_executor_thread"); - t.setDaemon(true); - return t; - } - }); - } - - - private WrappedExecuteStageRequest createExecuteStageRequest() throws MalformedURLException { - - // TODO make ExecuteStageRequest params configurable - final long timeoutToReportStartSec = 5; - final URL jobJarUrl = new URL("file:/Users/nmahilani/Projects/Mantis/mantis-sdk/examples/sine-function/build/distributions/sine-function-1.0.zip"); - final List ports = Arrays.asList(31015, 31013, 31014); - final List params = Collections.singletonList(new Parameter("useRandom", "true")); - final int numInstances = 1; - - // new MachineDefinition(2, 300, 200, 1024, 2), true)); - final Map schedulingInfoMap = new HashMap<>(); - final StageSchedulingInfo stage0SchedInfo = StageSchedulingInfo.builder() - .numberOfInstances(numInstances) - .machineDefinition(MachineDefinitions.micro()) - .build(); - final StageSchedulingInfo stage1SchedInfo = StageSchedulingInfo.builder() - .numberOfInstances(numInstances) - .machineDefinition(new MachineDefinition(2, 300, 200, 1024, 2)) - .scalingPolicy(new StageScalingPolicy(1, 1, 5, 1, 1, 30, - Collections.singletonMap(StageScalingPolicy.ScalingReason.Memory, - new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.Memory, 15.0, 25.0, new StageScalingPolicy.RollingCount(1, 2))))) - .scalable(true) - .build(); - - // schedulingInfoMap.put(0, stage0SchedInfo); - schedulingInfoMap.put(1, stage1SchedInfo); - - final SchedulingInfo schedInfo = new SchedulingInfo(schedulingInfoMap); - - final ExecuteStageRequest executeStageRequest = new ExecuteStageRequest(workerInfo.getJobName(), workerInfo.getJobId(), workerInfo.getWorkerIndex(), workerInfo.getWorkerNumber(), - jobJarUrl, workerInfo.getStageNumber(), workerInfo.getNumStages(), ports, timeoutToReportStartSec, workerInfo.getMetricsPort(), params, schedInfo, MantisJobDurationType.Transient, - 0, 0L, 0L, new WorkerPorts(Arrays.asList(7151, 7152, 7153, 7154, 7155)), Optional.empty(), - "user"); - - return new WrappedExecuteStageRequest(PublishSubject.create(), executeStageRequest); - } - - private void setupRequestFailureHandler(long waitSeconds, Observable requestObservable, - final Action0 errorHandler) { - requestObservable - .buffer(waitSeconds, TimeUnit.SECONDS, 1) - .take(1) - .subscribe(new Observer>() { - @Override - public void onCompleted() { - } - - @Override - public void onError(Throwable e) { - logger.error("onError called for request failure handler"); - errorHandler.call(); - } - - @Override - public void onNext(List booleans) { - logger.info("onNext called for request failure handler with items: " + - ((booleans == null) ? "-1" : booleans.size())); - if ((booleans == null) || booleans.isEmpty()) - errorHandler.call(); - } - }); - } - - @Override - public void start() { - logger.info("Starting VirtualMachineWorkerServiceLocalImpl"); - Schedulers.newThread().createWorker().schedule(new Action0() { - @Override - public void call() { - try { - WrappedExecuteStageRequest request = null; - request = createExecuteStageRequest(); - setupRequestFailureHandler(request.getRequest().getTimeoutToReportStart(), request.getRequestSubject(), - new Action0() { - @Override - public void call() { - logger.error("launch error"); - } - }); - logger.info("onNext'ing WrappedExecuteStageRequest: {}", request.toString()); - executeStageRequestObserver.onNext(request); - } catch (MalformedURLException e) { - e.printStackTrace(); - } - } - }, 2, TimeUnit.SECONDS); - - - // subscribe to vm task updates on current thread - vmTaskStatusObservable.subscribe(new Action1() { - @Override - public void call(VirtualMachineTaskStatus vmTaskStatus) { - TYPE type = vmTaskStatus.getType(); - if (type == TYPE.COMPLETED) { - logger.info("Got COMPLETED state for " + vmTaskStatus.getTaskId()); - } else if (type == TYPE.STARTED) { - logger.info("Would send RUNNING state to mesos, worker started for " + vmTaskStatus.getTaskId()); - } - } - }); - } - - @Override - public void shutdown() { - logger.info("Unregistering Mantis Worker with Mesos executor callbacks"); - mesosDriver.stop(); - executor.shutdown(); - } - - @Override - public void enterActiveMode() {} - -} diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/MesosExecutorCallbackHandler.java b/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/MesosExecutorCallbackHandler.java deleted file mode 100644 index a153ca1b6..000000000 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/MesosExecutorCallbackHandler.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.mantisrx.server.worker.mesos; - -import io.mantisrx.common.JsonSerializer; -import io.mantisrx.server.core.ExecuteStageRequest; -import io.mantisrx.server.core.WrappedExecuteStageRequest; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.mesos.Executor; -import org.apache.mesos.ExecutorDriver; -import org.apache.mesos.Protos; -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.FrameworkInfo; -import org.apache.mesos.Protos.SlaveInfo; -import org.apache.mesos.Protos.TaskID; -import org.apache.mesos.Protos.TaskInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import rx.Observable; -import rx.Observer; -import rx.functions.Action0; -import rx.subjects.PublishSubject; - - -public class MesosExecutorCallbackHandler implements Executor { - - private static final Logger logger = LoggerFactory.getLogger(MesosExecutorCallbackHandler.class); - private Observer executeStageRequestObserver; - private final JsonSerializer serializer = new JsonSerializer(); - - public MesosExecutorCallbackHandler(Observer executeStageRequestObserver) { - this.executeStageRequestObserver = executeStageRequestObserver; - } - - @Override - public void disconnected(ExecutorDriver arg0) { - // TODO Auto-generated method stub - } - - @Override - public void error(ExecutorDriver arg0, String msg) { - // TODO Auto-generated method stub - logger.error(msg); - } - - @Override - public void frameworkMessage(ExecutorDriver arg0, byte[] arg1) { - // TODO Auto-generated method stub - - } - - @Override - public void killTask(ExecutorDriver arg0, TaskID task) { - logger.info("Executor going to kill task " + task.getValue()); - executeStageRequestObserver.onCompleted(); - waitAndExit(); - } - - private void waitAndExit() { - // Allow some time for clean up and the completion report to be sent out before exiting. - // Until we define a better way to exit than to assume that the time we wait here is - // sufficient before a hard exit, we will live with it. - Thread t = new Thread() { - @Override - public void run() { - try {sleep(2000);} catch (InterruptedException ie) {} - System.exit(0); - } - }; - t.setDaemon(true); - t.start(); - } - - private WrappedExecuteStageRequest createExecuteStageRequest(TaskInfo task) { - // TODO - try { - byte[] jsonBytes = task.getData().toByteArray(); - logger.info("Received request {}", new String(jsonBytes)); - return new WrappedExecuteStageRequest( - PublishSubject.create(), - serializer.fromJson(jsonBytes, ExecuteStageRequest.class)); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return null; - } - - private void sendLaunchError(ExecutorDriver driver, final TaskInfo task) { - driver.sendStatusUpdate(Protos.TaskStatus.newBuilder() - .setTaskId(task.getTaskId()) - .setState(Protos.TaskState.TASK_FAILED).build()); - waitAndExit(); - } - - private void setupRequestFailureHandler(long waitSeconds, Observable requestObservable, - final Action0 errorHandler) { - requestObservable - .buffer(waitSeconds, TimeUnit.SECONDS, 1) - .take(1) - .subscribe(new Observer>() { - @Override - public void onCompleted() { - } - - @Override - public void onError(Throwable e) { - logger.error("onError called for request failure handler"); - errorHandler.call(); - } - - @Override - public void onNext(List booleans) { - logger.info("onNext called for request failure handler with items: " + - ((booleans == null) ? "-1" : booleans.size())); - if ((booleans == null) || booleans.isEmpty()) - errorHandler.call(); - } - }); - } - - @Override - public void launchTask(final ExecutorDriver driver, final TaskInfo task) { - WrappedExecuteStageRequest request = createExecuteStageRequest(task); - logger.info("Worker for task [" + task.getTaskId().getValue() + "] with startTimeout=" + - request.getRequest().getTimeoutToReportStart()); - setupRequestFailureHandler(request.getRequest().getTimeoutToReportStart(), request.getRequestSubject(), - new Action0() { - @Override - public void call() { - sendLaunchError(driver, task); - } - }); - executeStageRequestObserver.onNext(request); - - } - - @Override - public void registered(ExecutorDriver arg0, ExecutorInfo arg1, - FrameworkInfo arg2, SlaveInfo arg3) { - // TODO Auto-generated method stub - - } - - @Override - public void reregistered(ExecutorDriver arg0, SlaveInfo arg1) { - // TODO Auto-generated method stub - - } - - @Override - public void shutdown(ExecutorDriver arg0) { - // TODO Auto-generated method stub - } - -} diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/MesosMetricsCollector.java b/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/MesosMetricsCollector.java deleted file mode 100644 index 26ed822d4..000000000 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/MesosMetricsCollector.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.mantisrx.server.worker.mesos; - -import io.mantisrx.runtime.loader.config.MetricsCollector; -import io.mantisrx.runtime.loader.config.Usage; -import io.mantisrx.runtime.loader.config.WorkerConfiguration; -import io.mantisrx.shaded.com.google.common.base.Strings; -import io.netty.buffer.ByteBuf; -import io.reactivx.mantis.operators.OperatorOnErrorResumeNextViaFunction; -import java.nio.charset.Charset; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import mantis.io.reactivex.netty.RxNetty; -import mantis.io.reactivex.netty.protocol.http.client.HttpClient; -import mantis.io.reactivex.netty.protocol.http.client.HttpClientRequest; -import mantis.io.reactivex.netty.protocol.http.client.HttpClientResponse; -import org.json.JSONArray; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import rx.Observable; -import rx.functions.Func1; -import rx.functions.Func2; - -/** - * Mesos implementation of MetricsCollector that collects metrics using the statistics endpoint on the mesos agent. - * mesos statics endpoint link - */ -public class MesosMetricsCollector implements MetricsCollector { - private static final String MESOS_TASK_EXECUTOR_ID_KEY = "MESOS_EXECUTOR_ID"; - private static final Logger logger = LoggerFactory.getLogger(MesosMetricsCollector.class); - private static final long GET_TIMEOUT_SECS = 5; - private static final int MAX_REDIRECTS = 10; - private final int slavePort; - private final String taskId; - private final Func1, Observable> retryLogic = attempts -> attempts - .zipWith(Observable.range(1, 3), (Func2) (t1, integer) -> integer) - .flatMap((Func1>) integer -> { - long delay = 2L; - logger.info(": retrying conx after sleeping for " + delay + " secs"); - return Observable.timer(delay, TimeUnit.SECONDS); - }); - - @SuppressWarnings("unused") - public static MesosMetricsCollector valueOf(Properties properties) { - int slavePort = Integer.parseInt(properties.getProperty("mantis.agent.mesos.slave.port", "5051")); - String taskId = System.getenv(MESOS_TASK_EXECUTOR_ID_KEY); - return new MesosMetricsCollector(slavePort, taskId); - } - - public static MesosMetricsCollector valueOf(WorkerConfiguration workerConfiguration) { - int slavePort = workerConfiguration.getMesosSlavePort(); - String taskId = System.getenv(MESOS_TASK_EXECUTOR_ID_KEY); - return new MesosMetricsCollector(slavePort, taskId); - } - - MesosMetricsCollector(int slavePort, String taskId) { - logger.info("Creating MesosMetricsCollector to port {} of taskId: {}", slavePort, taskId); - - if (Strings.isNullOrEmpty(taskId)) { - // only log error to avoid breaking tests. - logger.error("Invalid task id for MesosMetricsCollector"); - } - this.slavePort = slavePort; - this.taskId = taskId; - } - - private String getUsageJson() { - String usageEndpoint = "monitor/statistics.json"; - final String url = "http://localhost:" + slavePort + "/" + usageEndpoint; - return RxNetty - .createHttpRequest(HttpClientRequest.createGet(url), new HttpClient.HttpClientConfig.Builder() - .setFollowRedirect(true).followRedirect(MAX_REDIRECTS).build()) - .lift(new OperatorOnErrorResumeNextViaFunction<>(t -> Observable.error(t))) - .timeout(GET_TIMEOUT_SECS, TimeUnit.SECONDS) - .retryWhen(retryLogic) - .flatMap((Func1, Observable>) r -> r.getContent()) - .map(o -> o.toString(Charset.defaultCharset())) - .doOnError(throwable -> logger.warn("Can't get resource usage from mesos slave endpoint (" + url + ") - " + throwable.getMessage(), throwable)) - .toBlocking() - .firstOrDefault(""); - } - - @Override - public Usage get() { - return getCurentUsage(taskId, getUsageJson()); - } - - static Usage getCurentUsage(String taskId, String usageJson) { - if (usageJson == null || usageJson.isEmpty()) { - logger.warn("Empty usage on task {}", taskId); - return null; - } - - JSONArray array = new JSONArray(usageJson); - if (array.length() == 0) - return null; - JSONObject obj = null; - for (int i = 0; i < array.length(); i++) { - JSONObject executor = array.getJSONObject(i); - if (executor != null) { - String id = executor.optString("executor_id"); - if (id != null && id.equals(taskId)) { - obj = executor.getJSONObject("statistics"); - break; - } - } - } - if (obj == null) - return null; - double cpus_limit = obj.optDouble("cpus_limit"); - if (Double.isNaN(cpus_limit)) { - cpus_limit = 0.0; - } - double cpus_system_time_secs = obj.optDouble("cpus_system_time_secs"); - if (Double.isNaN(cpus_system_time_secs)) { - logger.warn("Didn't get cpus_system_time_secs from mesos stats"); - cpus_system_time_secs = 0.0; - } - double cpus_user_time_secs = obj.optDouble("cpus_user_time_secs"); - if (Double.isNaN(cpus_user_time_secs)) { - logger.warn("Didn't get cpus_user_time_secs from mesos stats"); - cpus_user_time_secs = 0.0; - } - // Also, cpus_throttled_time_secs may be useful to notice when job is throttled, will look into it later - double mem_rss_bytes = obj.optDouble("mem_rss_bytes"); - if (Double.isNaN(mem_rss_bytes)) { - logger.warn("Couldn't get mem_rss_bytes from mesos stats"); - mem_rss_bytes = 0.0; - } - double mem_anon_bytes = obj.optDouble("mem_anon_bytes"); - if (Double.isNaN(mem_anon_bytes)) { - mem_anon_bytes = mem_rss_bytes; - } - double mem_limit = obj.optDouble("mem_limit_bytes"); - if (Double.isNaN(mem_limit)) - mem_limit = 0.0; - double network_read_bytes = obj.optDouble("net_rx_bytes"); - if (Double.isNaN(network_read_bytes)) - network_read_bytes = 0.0; - double network_write_bytes = obj.optDouble("net_tx_bytes"); - if (Double.isNaN(network_write_bytes)) - network_write_bytes = 0.0; - return new Usage(cpus_limit, cpus_system_time_secs, cpus_user_time_secs, mem_limit, mem_rss_bytes, mem_anon_bytes, - network_read_bytes, network_write_bytes); - } -} diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/VirtualMachineTaskStatus.java b/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/VirtualMachineTaskStatus.java deleted file mode 100644 index 82c845c60..000000000 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/VirtualMachineTaskStatus.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.mantisrx.server.worker.mesos; - -public class VirtualMachineTaskStatus { - - private String taskId; - private TYPE type; - private String message; - public VirtualMachineTaskStatus(String taskId, TYPE type, String message) { - this.taskId = taskId; - this.type = type; - this.message = message; - } - - public String getTaskId() { - return taskId; - } - - public String getMessage() { - return message; - } - - public TYPE getType() { - return type; - } - - public enum TYPE { - STARTED, COMPLETED, ERROR; - } -} diff --git a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/VirualMachineWorkerServiceMesosImpl.java b/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/VirualMachineWorkerServiceMesosImpl.java deleted file mode 100644 index e1622729b..000000000 --- a/mantis-server/mantis-server-worker/src/main/java/io/mantisrx/server/worker/mesos/VirualMachineWorkerServiceMesosImpl.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2019 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.mantisrx.server.worker.mesos; - -import io.mantisrx.server.core.BaseService; -import io.mantisrx.server.core.WrappedExecuteStageRequest; -import io.mantisrx.server.worker.VirtualMachineWorkerService; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus.TYPE; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import org.apache.mesos.MesosExecutorDriver; -import org.apache.mesos.Protos; -import org.apache.mesos.Protos.TaskID; -import org.apache.mesos.Protos.TaskState; -import org.apache.mesos.Protos.TaskStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import rx.Observable; -import rx.Observer; -import rx.functions.Action1; - - -public class VirualMachineWorkerServiceMesosImpl extends BaseService implements VirtualMachineWorkerService { - - private static final Logger logger = LoggerFactory.getLogger(VirualMachineWorkerServiceMesosImpl.class); - private MesosExecutorDriver mesosDriver; - private ExecutorService executor; - private Observer executeStageRequestObserver; - private Observable vmTaskStatusObservable; - - public VirualMachineWorkerServiceMesosImpl(Observer executeStageRequestObserver, - Observable vmTaskStatusObservable) { - this.executeStageRequestObserver = executeStageRequestObserver; - this.vmTaskStatusObservable = vmTaskStatusObservable; - executor = Executors.newSingleThreadExecutor(new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r, "vm_worker_mesos_executor_thread"); - t.setDaemon(true); - return t; - } - }); - } - - @Override - public void start() { - logger.info("Registering Mantis Worker with Mesos executor callbacks"); - mesosDriver = new MesosExecutorDriver(new MesosExecutorCallbackHandler(executeStageRequestObserver)); - // launch driver on background thread - logger.info("launch driver on background thread"); - executor.execute(new Runnable() { - @Override - public void run() { - try { - mesosDriver.run(); - } catch (Exception e) { - logger.error("Failed to register Mantis Worker with Mesos executor callbacks", e); - } - } - }); - // subscribe to vm task updates on current thread - logger.info("subscribe to vm task updates on current thread"); - vmTaskStatusObservable.subscribe(new Action1() { - @Override - public void call(VirtualMachineTaskStatus vmTaskStatus) { - TYPE type = vmTaskStatus.getType(); - if (type == TYPE.COMPLETED) { - Protos.Status status = mesosDriver.sendStatusUpdate(TaskStatus.newBuilder() - .setTaskId(TaskID.newBuilder().setValue(vmTaskStatus.getTaskId()).build()) - .setState(TaskState.TASK_FINISHED).build()); - logger.info("Sent COMPLETED state to mesos, driver status=" + status); - } else if (type == TYPE.STARTED) { - Protos.Status status = mesosDriver.sendStatusUpdate(TaskStatus.newBuilder() - .setTaskId(TaskID.newBuilder().setValue(vmTaskStatus.getTaskId()).build()) - .setState(TaskState.TASK_RUNNING).build()); - logger.info("Sent RUNNING state to mesos, driver status=" + status); - } - } - }); - } - - @Override - public void shutdown() { - logger.info("Unregistering Mantis Worker with Mesos executor callbacks"); - mesosDriver.stop(); - executor.shutdown(); - } - - @Override - public void enterActiveMode() {} - -} diff --git a/mantis-server/mantis-server-worker/src/main/resources/startup.sh b/mantis-server/mantis-server-worker/src/main/resources/startup.sh deleted file mode 100755 index 46446db44..000000000 --- a/mantis-server/mantis-server-worker/src/main/resources/startup.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh - -echo "Executing script to download file at: ${JOB_URL}, storing in /tmp/mantis-jobs/${JOB_NAME}/lib" -mkdir -p /tmp/mantis-jobs/${JOB_NAME}/lib -mkdir -p /logs/mantisjobs/${JOB_NAME}/${JOB_ID}/${WORKER_NUMBER} - -EXTRA_OPTS="-Dcom.sun.management.jmxremote=true \ --Dcom.sun.management.jmxremote.ssl=false \ --Dcom.sun.management.jmxremote.authenticate=false \ --Dcom.sun.management.jmxremote.host=localhost \ --Dcom.sun.management.jmxremote.port=$MANTIS_WORKER_DEBUG_PORT" - -JAVA_OPTS=" $EXTRA_OPTS \ --Xmx${JVM_MEMORY_MB}m \ --XX:+PrintGCDetails \ --XX:+PrintGCTimeStamps \ --XX:+HeapDumpOnOutOfMemoryError \ --XX:HeapDumpPath=/logs/mantisjobs/${JOB_NAME}/${JOB_ID}/${WORKER_NUMBER} \ --Xloggc:/logs/mantisjobs/${JOB_NAME}/${JOB_ID}/${WORKER_NUMBER}/gc.log \ --XX:MaxDirectMemorySize=256m" - -JVM_CLASSPATH="${WORKER_LIB_DIR}/*" -JOB_JARS_DIR="/tmp/mantis-jobs/${JOB_NAME}/lib" -JOB_PROVIDER_CLASS="" - -java -cp $JVM_CLASSPATH io.mantisrx.server.worker.DownloadJob ${JOB_URL} ${JOB_NAME} /tmp/mantis-jobs - -cd $JOB_JARS_DIR - -zipexists=`ls -l *.zip 2>/dev/null | wc -l` - -if [ $zipexists == 1 ] -then - sudo yum install -y unzip - mkdir zipExtract - unzip *.zip -d zipExtract - JOB_PROVIDER_CLASS=`cat zipExtract/*/config/io.mantisrx.runtime.MantisJobProvider` - echo "job provider class $JOB_PROVIDER_CLASS" - ZIP_LIB_DIR=`echo $JOB_JARS_DIR/zipExtract/*/lib` - JVM_CLASSPATH="$ZIP_LIB_DIR/*:$JVM_CLASSPATH" -else - JVM_CLASSPATH="$JVM_CLASSPATH:$JOB_JARS_DIR/*" -fi - -echo "Executing Mantis Worker java $JAVA_OPTS -cp $JVM_CLASSPATH -DMASTER_DESCRIPTION="${MASTER_DESCRIPTION}" -DJOB_PROVIDER_CLASS="$JOB_PROVIDER_CLASS" io.mantisrx.server.worker.MantisWorker" -java $JOB_PARAM_MANTIS_WORKER_JVM_OPTS $JAVA_OPTS -cp $JVM_CLASSPATH -DMASTER_DESCRIPTION="${MASTER_DESCRIPTION}" -DJOB_PROVIDER_CLASS="$JOB_PROVIDER_CLASS" io.mantisrx.server.worker.MantisWorker - - - - - - - - - diff --git a/mantis-server/mantis-server-worker/src/main/resources/startup_docker.sh b/mantis-server/mantis-server-worker/src/main/resources/startup_docker.sh deleted file mode 100755 index dee0e96c0..000000000 --- a/mantis-server/mantis-server-worker/src/main/resources/startup_docker.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -set -x -#172.16.186.7:7101 mantis api host port in docker -echo "Executing script to download file at: ${JOB_URL}, storing in /tmp/mantis-jobs/${JOB_NAME}/lib" -ARTIFACT_NAME=$(echo ${JOB_URL:7}) -JOB_URL=`echo \${JOB_URL} | sed 's|http://|&172.16.186.7:7101/api/v1/artifacts/|g'` -echo "Executing script to download file at: ${JOB_URL}, storing in /tmp/mantis-jobs/${JOB_NAME}/lib" - -mkdir -p /tmp/mantis-jobs/${JOB_NAME}/lib -mkdir -p /logs/mantisjobs/${JOB_NAME}/${JOB_ID}/${WORKER_NUMBER} - -EXTRA_OPTS="-Dcom.sun.management.jmxremote=true \ --Dcom.sun.management.jmxremote.ssl=false \ --Dcom.sun.management.jmxremote.authenticate=false \ --Dcom.sun.management.jmxremote.host=localhost \ --Dcom.sun.management.jmxremote.port=$MANTIS_WORKER_DEBUG_PORT" - -JAVA_OPTS=" $EXTRA_OPTS \ --Xmx${JVM_MEMORY_MB}m \ --XX:+PrintGCDetails \ --XX:+PrintGCTimeStamps \ --XX:+HeapDumpOnOutOfMemoryError \ --XX:HeapDumpPath=/logs/mantisjobs/${JOB_NAME}/${JOB_ID}/${WORKER_NUMBER} \ --Xloggc:/logs/mantisjobs/${JOB_NAME}/${JOB_ID}/${WORKER_NUMBER}/gc.log \ --XX:MaxDirectMemorySize=256m" - -JVM_CLASSPATH="${WORKER_LIB_DIR}/*" -JOB_JARS_DIR="/tmp/mantis-jobs/${JOB_NAME}/lib" -JOB_PROVIDER_CLASS="" -wget -v $JOB_URL -P "/tmp/mantis-jobs/${JOB_NAME}/lib" -#java -cp $JVM_CLASSPATH io.mantisrx.server.worker.DownloadJob ${JOB_URL} ${JOB_NAME} /tmp/mantis-jobs/${JOB_NAME}/lib -#cp -r /mnt/local/mantisWorkerInstall/jobs/* "/tmp/mantis-jobs/${JOB_NAME}/lib" -# Link worker.properties -cp -s /mnt/local/mantisWorkerInstall/jobs/* "/tmp/mantis-jobs/${JOB_NAME}/lib" -# Link mantis-worker.jar -cp -s /mnt/local/mantisWorkerInstall/libs/* "/tmp/mantis-jobs/${JOB_NAME}/lib" -cd $JOB_JARS_DIR - -zipexists=`ls -l $ARTIFACT_NAME 2>/dev/null | wc -l` - -if [ $zipexists = 1 ] -then - sudo apt install -y unzip - mkdir zipExtract - unzip $ARTIFACT_NAME -d zipExtract - JOB_PROVIDER_CLASS=`cat zipExtract/*/config/io.mantisrx.runtime.MantisJobProvider` - echo "job provider class $JOB_PROVIDER_CLASS" - ZIP_LIB_DIR=`echo $JOB_JARS_DIR/zipExtract/*/lib` - JVM_CLASSPATH="$ZIP_LIB_DIR/*:$JVM_CLASSPATH" -else - JVM_CLASSPATH="$JVM_CLASSPATH:$JOB_JARS_DIR/*" -fi - -echo "Executing Mantis Worker java $JAVA_OPTS -cp $JVM_CLASSPATH -DMASTER_DESCRIPTION="${MASTER_DESCRIPTION}" -DJOB_PROVIDER_CLASS="$JOB_PROVIDER_CLASS" io.mantisrx.server.worker.MantisWorker" -java $JOB_PARAM_MANTIS_WORKER_JVM_OPTS $JAVA_OPTS -cp $JVM_CLASSPATH -DMASTER_DESCRIPTION="${MASTER_DESCRIPTION}" -DJOB_PROVIDER_CLASS="$JOB_PROVIDER_CLASS" io.mantisrx.server.worker.MantisWorker -p /mnt/local/mantisWorkerInstall/jobs/worker-docker.properties - - - - - - - - - diff --git a/mantis-server/mantis-server-worker/src/main/resources/worker-local.properties b/mantis-server/mantis-server-worker/src/main/resources/worker-local.properties deleted file mode 100644 index bbab0623a..000000000 --- a/mantis-server/mantis-server-worker/src/main/resources/worker-local.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright 2019 Netflix, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -mantis.master.api=http://127.0.0.1:8100/api/ -mantis.master.apiportv2=8075 -mantis.zookeeper.connectionTimeMs=1000 -mantis.zookeeper.connection.retrySleepMs=100 -mantis.zookeeper.connection.retryCount=3 -mantis.zookeeper.connectString=127.0.0.1:2181 -mantis.zookeeper.root=/mantis/master/nmahilani -mantis.zookeeper.leader.announcement.path=/leader -mantis.localMode=true diff --git a/mantis-server/mantis-server-worker/src/main/resources/worker.properties b/mantis-server/mantis-server-worker/src/main/resources/worker.properties deleted file mode 100644 index 877650361..000000000 --- a/mantis-server/mantis-server-worker/src/main/resources/worker.properties +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2019 Netflix, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -mantis.master.api=http://localhost:7070/api/ -mantis.master.apiportv2=7102 -mantis.zookeeper.connectionTimeMs=1000 -mantis.zookeeper.connection.retrySleepMs=100 -mantis.zookeeper.connection.retryCount=3 -mantis.zookeeper.connectString=zkhost:2181,zkhost2:2181 -mantis.zookeeper.root=/mantis/master -mantis.zookeeper.leader.announcement.path=/leader diff --git a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/mesos/TestMesosMetricsCollector.java b/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/mesos/TestMesosMetricsCollector.java deleted file mode 100644 index c44b58b6b..000000000 --- a/mantis-server/mantis-server-worker/src/test/java/io/mantisrx/server/worker/mesos/TestMesosMetricsCollector.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2022 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.mantisrx.server.worker.mesos; - -import io.mantisrx.runtime.loader.config.Usage; -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; - -@Slf4j -public class TestMesosMetricsCollector { - private static final String stats3 = "[\n" + - "{\n" + - "executor_id: \"SpeedBump-66-worker-0-0\",\n" + - "executor_name: \"Mantis Worker Executor\",\n" + - "framework_id: \"MantisFramework\",\n" + - "source: \"Outliers-mock-84\",\n" + - "statistics: \n" + - "{\n" + - "cpus_limit: 1,\n" + - "cpus_system_time_secs: 0.11,\n" + - "cpus_user_time_secs: 2.16,\n" + - "mem_limit_bytes: 2147483648,\n" + - "mem_rss_bytes: 97460224,\n" + - "timestamp: 1420842205.86559\n" + - "}\n" + - "}\n" + - "]"; - private static final String stats4 = "[\n" + - "{\n" + - "executor_id: \"SpeedBump-66-worker-0-0\",\n" + - "executor_name: \"Mantis Worker Executor\",\n" + - "framework_id: \"MantisFramework\",\n" + - "source: \"Outliers-mock-84\",\n" + - "statistics: \n" + - "{\n" + - "cpus_limit: 1,\n" + - "cpus_system_time_secs: 0.13,\n" + - "cpus_user_time_secs: 3.16,\n" + - "mem_limit_bytes: 2147483648,\n" + - "mem_rss_bytes: 97460224,\n" + - "timestamp: 1420842205.86559\n" + - "}\n" + - "}\n" + - "]"; - private static final String stats5 = "[\n" + - "{\n" + - "executor_id: \"APIHystrixMetricsSource-5-worker-0-10\",\n" + - "executor_name: \"Mantis Worker Executor\",\n" + - "framework_id: \"MantisFramework\",\n" + - "source: \"APIHystrixMetricsSource-5\",\n" + - "statistics: {\n" + - "cpus_limit: 8,\n" + - "cpus_system_time_secs: 5.4,\n" + - "cpus_user_time_secs: 67.74,\n" + - "mem_anon_bytes: 1265774592,\n" + - "mem_file_bytes: 48386048,\n" + - "mem_limit_bytes: 10510925824,\n" + - "mem_mapped_file_bytes: 1232896,\n" + - "mem_rss_bytes: 1314697216,\n" + - "net_rx_bytes: 994208159,\n" + - "net_rx_dropped: 0,\n" + - "net_rx_errors: 0,\n" + - "net_rx_packets: 723567,\n" + - "net_tx_bytes: 195020860,\n" + - "net_tx_dropped: 0,\n" + - "net_tx_errors: 0,\n" + - "net_tx_packets: 564689,\n" + - "timestamp: 1421792142.02197\n" + - "}\n" + - "}\n" + - "]"; - - @Test - public void test() { - final Usage usage1 = MesosMetricsCollector.getCurentUsage("SpeedBump-66-worker-0-0", stats3); - final Usage usage2 = MesosMetricsCollector.getCurentUsage("SpeedBump-66-worker-0-0", stats4); - log.info("cpuUsr=" + (usage2.getCpusUserTimeSecs() - usage1.getCpusUserTimeSecs()) + ", rss=" + (usage1.getMemRssBytes() / (1024 * 1024))); - final Usage usage3 = MesosMetricsCollector.getCurentUsage("APIHystrixMetricsSource-5-worker-0-10", stats5); - log.info("network read MB: " + (usage3.getNetworkReadBytes() / (1024.0 * 1024.0)) + ", write MB=" + - (usage3.getNetworkWriteBytes() / (1024.0 * 1024.0))); - } - -} diff --git a/mantis-server/mantis-server-worker/worker-docker.properties b/mantis-server/mantis-server-worker/worker-docker.properties deleted file mode 100644 index 6c6dd4c81..000000000 --- a/mantis-server/mantis-server-worker/worker-docker.properties +++ /dev/null @@ -1,9 +0,0 @@ -mantis.master.api=http://mantismaster:8100/api/ -mantis.master.apiportv2=8075 -mantis.zookeeper.connectionTimeMs=1000 -mantis.zookeeper.connection.retrySleepMs=100 -mantis.zookeeper.connection.retryCount=3 -mantis.zookeeper.connectString=zookeeper:2181 -mantis.zookeeper.root=/mantis/master/nmahilani -mantis.zookeeper.leader.announcement.path=/leader -mantis.localMode=true \ No newline at end of file diff --git a/mantis-source-jobs/mantis-source-job-publish/build.gradle b/mantis-source-jobs/mantis-source-job-publish/build.gradle index 7a1e862d4..d2c9d0daa 100644 --- a/mantis-source-jobs/mantis-source-job-publish/build.gradle +++ b/mantis-source-jobs/mantis-source-job-publish/build.gradle @@ -16,10 +16,9 @@ apply plugin: "mantis" dependencies { - implementation project(":mantis-runtime") implementation project(":mantis-publish:mantis-publish-netty") implementation project(":mantis-connectors:mantis-connector-publish") - implementation project(':mantis-server:mantis-server-worker') + implementation project(':mantis-runtime-executor') implementation libraries.slf4jApi diff --git a/settings.gradle b/settings.gradle index 7b2163cf1..cb9ff8133 100644 --- a/settings.gradle +++ b/settings.gradle @@ -55,12 +55,12 @@ include ':mantis-publish:mantis-publish-netty-guice' include 'mantis-remote-observable' include 'mantis-runtime' include 'mantis-runtime-loader' +include 'mantis-runtime-executor' include 'mantis-source-jobs:mantis-source-job-kafka' include 'mantis-source-jobs:mantis-source-job-publish' include 'mantis-server:mantis-server-agent' -include 'mantis-server:mantis-server-worker' include 'mantis-server:mantis-server-worker-client' include 'mantis-shaded'