From c3cab7f13549a7062450b46ea96c49c3b7d34406 Mon Sep 17 00:00:00 2001 From: Andy Zhang <87735571+Andyz26@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:30:55 -0800 Subject: [PATCH] fix refs --- .../build.gradle | 2 +- mantis-runtime-executor/build.gradle | 12 - mantis-runtime-executor/dependencies.lock | 1104 +++++++++++++++++ .../mantisrx/server/worker/MantisWorker.java | 6 - .../mantisrx/server/worker/RunningWorker.java | 26 +- .../server/worker/RuntimeTaskImpl.java | 10 +- ...WorkerExecutionOperationsNetworkStage.java | 5 - mantis-runtime-loader/build.gradle | 1 + .../runtime/loader/cgroups/Cgroup.java | 4 +- .../runtime/loader/cgroups/CgroupImpl.java | 4 +- .../cgroups/CgroupsMetricsCollector.java | 4 +- .../cgroups/CpuAcctsSubsystemProcess.java | 4 +- .../cgroups/MemorySubsystemProcess.java | 4 +- .../cgroups/NetworkSubsystemProcess.java | 4 +- .../loader/cgroups/ProcFileReader.java | 4 +- .../loader/cgroups/SubsystemProcess.java | 4 +- .../loader/config/WorkerConfiguration.java | 2 +- .../runtime/loader/cgroups/TestCgroup.java | 18 +- .../cgroups/TestCpuAcctsSubsystemProcess.java | 40 +- .../cgroups/TestMemorySubsystemProcess.java | 33 +- .../cgroups/TestNetworkSubsystemProcess.java | 12 +- .../mantis-server-agent/build.gradle | 2 +- .../main/resources/agent-docker.properties | 2 +- .../src/main/resources/agent.properties | 2 +- .../mantis-source-job-publish/build.gradle | 3 +- settings.gradle | 2 +- 26 files changed, 1173 insertions(+), 141 deletions(-) create mode 100644 mantis-runtime-executor/dependencies.lock diff --git a/mantis-examples/mantis-examples-sine-function/build.gradle b/mantis-examples/mantis-examples-sine-function/build.gradle index 5f5bf55a1..49a77c4ff 100644 --- a/mantis-examples/mantis-examples-sine-function/build.gradle +++ b/mantis-examples/mantis-examples-sine-function/build.gradle @@ -27,6 +27,6 @@ task execute(type:JavaExec) { classpath = sourceSets.main.runtimeClasspath } dependencies { - implementation project(':mantis-server:mantis-server-worker') + implementation project(':mantis-runtime-executor') implementation libraries.spectatorApi } diff --git a/mantis-runtime-executor/build.gradle b/mantis-runtime-executor/build.gradle index 16e53a859..da2bd2605 100644 --- a/mantis-runtime-executor/build.gradle +++ b/mantis-runtime-executor/build.gradle @@ -14,20 +14,16 @@ * limitations under the License. */ -apply plugin: 'eu.appsatori.fatjar' ext { mantisRxControlVersion = '1.3.+' - mesosVersion = '1.7.2' } dependencies { api project(":mantis-runtime") api project(":mantis-runtime-loader") api project(":mantis-server:mantis-server-worker-client") - api project(":mantis-server:mantis-server-agent") - implementation "org.apache.mesos:mesos:$mesosVersion" implementation libraries.slf4jApi implementation libraries.slf4jLog4j12 implementation libraries.vavr @@ -38,7 +34,6 @@ dependencies { implementation libraries.httpClient implementation "io.mantisrx:mantis-rxcontrol:$mantisRxControlVersion" implementation "com.yahoo.datasketches:sketches-core:0.9.1" - implementation libraries.spectatorApi testImplementation libraries.junit4 @@ -48,10 +43,3 @@ dependencies { testImplementation(testFixtures(project(":mantis-common"))) testImplementation(testFixtures(project(":mantis-control-plane:mantis-control-plane-core"))) } - -// exclude any signed jars -fatJar { - exclude "META-INF/*.SF" - exclude "META-INF/*.DSA" - exclude "META-INF/*.RSA" -} diff --git a/mantis-runtime-executor/dependencies.lock b/mantis-runtime-executor/dependencies.lock new file mode 100644 index 000000000..18d9946e4 --- /dev/null +++ b/mantis-runtime-executor/dependencies.lock @@ -0,0 +1,1104 @@ +{ + "annotationProcessor": { + "org.projectlombok:lombok": { + "locked": "1.18.20" + } + }, + "baseline-exact-dependencies-main": { + "com.netflix.spectator:spectator-api": { + "locked": "1.3.10" + }, + "com.yahoo.datasketches:sketches-core": { + "locked": "0.9.1" + }, + "io.mantisrx:mantis-rxcontrol": { + "locked": "1.3.20" + }, + "io.vavr:vavr": { + "locked": "0.9.2" + }, + "nz.ac.waikato.cms.moa:moa": { + "locked": "2017.06" + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.5.14" + }, + "org.slf4j:slf4j-api": { + "locked": "1.7.0" + }, + "org.slf4j:slf4j-log4j12": { + "locked": "1.7.0" + } + }, + "baseline-exact-dependencies-test": { + "com.github.spullara.cli-parser:cli-parser": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.1.1" + }, + "com.google.code.findbugs:jsr305": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "3.0.1" + }, + "com.netflix:mantis-rxnetty": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "0.4.19.1" + }, + "io.mantisrx:mantis-common": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-control-plane-core" + ], + "project": true + }, + "io.mantisrx:mantis-common-serde": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "project": true + }, + "io.mantisrx:mantis-shaded": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-common-serde" + ], + "project": true + }, + "io.netty:netty-buffer": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-codec-http": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-transport-native-epoll": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.reactivex:rxjava": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.3.8" + }, + "io.vavr:vavr-test": { + "locked": "0.9.2" + }, + "joda-time:joda-time": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.12.6" + }, + "junit:junit": { + "locked": "4.11" + }, + "junit:junit-dep": { + "locked": "4.11" + }, + "org.apache.flink:flink-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.flink:flink-rpc-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.mesos:mesos": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.7.2" + }, + "org.hamcrest:hamcrest-all": { + "locked": "1.3" + }, + "org.hdrhistogram:HdrHistogram": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.1.12" + }, + "org.jctools:jctools-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.2.1" + }, + "org.json:json": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "20180813" + }, + "org.mockito:mockito-all": { + "locked": "2.0.2-beta" + }, + "org.skife.config:config-magic": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "0.11" + }, + "org.slf4j:slf4j-api": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.7.15" + }, + "org.slf4j:slf4j-log4j12": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.7.0" + }, + "org.xerial.snappy:snappy-java": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.1.10.5" + } + }, + "compileClasspath": { + "com.github.spullara.cli-parser:cli-parser": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.1.1" + }, + "com.google.code.findbugs:jsr305": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "3.0.1" + }, + "com.netflix.spectator:spectator-api": { + "locked": "1.3.10" + }, + "com.netflix:mantis-rxnetty": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "0.4.19.1" + }, + "com.yahoo.datasketches:sketches-core": { + "locked": "0.9.1" + }, + "io.mantisrx:mantis-common": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core", + "io.mantisrx:mantis-network", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-common-serde": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-client": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime-loader", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-network": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-remote-observable": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-runtime": { + "project": true + }, + "io.mantisrx:mantis-runtime-loader": { + "project": true + }, + "io.mantisrx:mantis-rxcontrol": { + "locked": "1.3.20" + }, + "io.mantisrx:mantis-server-worker-client": { + "project": true + }, + "io.mantisrx:mantis-shaded": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-common-serde" + ], + "project": true + }, + "io.netty:netty-buffer": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-codec-http": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-handler": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-network" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-transport-native-epoll": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.reactivex:rxjava": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.3.8" + }, + "io.vavr:vavr": { + "locked": "0.9.2" + }, + "joda-time:joda-time": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.12.6" + }, + "nz.ac.waikato.cms.moa:moa": { + "locked": "2017.06" + }, + "org.apache.flink:flink-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.flink:flink-rpc-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.5.14" + }, + "org.apache.mesos:mesos": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.7.2" + }, + "org.hdrhistogram:HdrHistogram": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.1.12" + }, + "org.jctools:jctools-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.2.1" + }, + "org.json:json": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "20180813" + }, + "org.projectlombok:lombok": { + "locked": "1.18.20" + }, + "org.skife.config:config-magic": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "0.11" + }, + "org.slf4j:slf4j-api": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-runtime" + ], + "locked": "1.7.15" + }, + "org.slf4j:slf4j-log4j12": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.7.0" + }, + "org.xerial.snappy:snappy-java": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.1.10.5" + } + }, + "lombok": { + "org.projectlombok:lombok": { + "locked": "1.18.20" + } + }, + "runtimeClasspath": { + "com.github.spullara.cli-parser:cli-parser": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.1.1" + }, + "com.google.code.findbugs:jsr305": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "3.0.1" + }, + "com.netflix.rxjava:rxjava-math": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-remote-observable" + ], + "locked": "0.20.6" + }, + "com.netflix.spectator:spectator-api": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-server-worker-client" + ], + "locked": "1.3.10" + }, + "com.netflix:mantis-rxnetty": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "0.4.19.1" + }, + "com.spotify:completable-futures": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client" + ], + "locked": "0.3.1" + }, + "com.yahoo.datasketches:sketches-core": { + "locked": "0.9.1" + }, + "commons-io:commons-io": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.11.0" + }, + "io.mantisrx:mantis-common": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core", + "io.mantisrx:mantis-network", + "io.mantisrx:mantis-remote-observable", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-common-serde": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-client": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime-loader", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-network": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-remote-observable": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-runtime": { + "project": true + }, + "io.mantisrx:mantis-runtime-loader": { + "project": true + }, + "io.mantisrx:mantis-rxcontrol": { + "locked": "1.3.20" + }, + "io.mantisrx:mantis-server-worker-client": { + "project": true + }, + "io.mantisrx:mantis-shaded": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-common-serde" + ], + "project": true + }, + "io.mantisrx:mql-jvm": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-network" + ], + "locked": "3.4.0" + }, + "io.netty:netty-buffer": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.60.Final" + }, + "io.netty:netty-codec-http": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.60.Final" + }, + "io.netty:netty-handler": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-network", + "io.mantisrx:mantis-remote-observable" + ], + "locked": "4.1.60.Final" + }, + "io.netty:netty-transport-native-epoll": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.60.Final" + }, + "io.reactivex:rxjava": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.3.8" + }, + "io.vavr:vavr": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime-loader", + "io.mantisrx:mantis-shaded" + ], + "locked": "0.9.2" + }, + "joda-time:joda-time": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.12.6" + }, + "net.jcip:jcip-annotations": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.0" + }, + "nz.ac.waikato.cms.moa:moa": { + "locked": "2017.06" + }, + "org.apache.flink:flink-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.flink:flink-rpc-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.5.14" + }, + "org.apache.mesos:mesos": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.7.2" + }, + "org.asynchttpclient:async-http-client": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client" + ], + "locked": "2.12.3" + }, + "org.hdrhistogram:HdrHistogram": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.1.12" + }, + "org.jctools:jctools-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.2.1" + }, + "org.json:json": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "20180813" + }, + "org.skife.config:config-magic": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "0.11" + }, + "org.slf4j:slf4j-api": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-remote-observable", + "io.mantisrx:mantis-runtime" + ], + "locked": "1.7.36" + }, + "org.slf4j:slf4j-log4j12": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-remote-observable" + ], + "locked": "1.7.0" + }, + "org.xerial.snappy:snappy-java": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.1.10.5" + } + }, + "testAnnotationProcessor": { + "org.projectlombok:lombok": { + "locked": "1.18.20" + } + }, + "testCompileClasspath": { + "com.github.spullara.cli-parser:cli-parser": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.1.1" + }, + "com.google.code.findbugs:jsr305": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "3.0.1" + }, + "com.netflix.spectator:spectator-api": { + "locked": "1.3.10" + }, + "com.netflix:mantis-rxnetty": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "0.4.19.1" + }, + "com.yahoo.datasketches:sketches-core": { + "locked": "0.9.1" + }, + "io.mantisrx:mantis-common": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-control-plane-core", + "io.mantisrx:mantis-network", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-common-serde": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-client": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime-loader", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-control-plane-core", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-network": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-remote-observable": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-runtime": { + "project": true + }, + "io.mantisrx:mantis-runtime-loader": { + "project": true + }, + "io.mantisrx:mantis-rxcontrol": { + "locked": "1.3.20" + }, + "io.mantisrx:mantis-server-worker-client": { + "project": true + }, + "io.mantisrx:mantis-shaded": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-common-serde" + ], + "project": true + }, + "io.netty:netty-buffer": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-codec-http": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-handler": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-network" + ], + "locked": "4.1.17.Final" + }, + "io.netty:netty-transport-native-epoll": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.17.Final" + }, + "io.reactivex:rxjava": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.3.8" + }, + "io.vavr:vavr": { + "locked": "0.9.2" + }, + "io.vavr:vavr-test": { + "locked": "0.9.2" + }, + "joda-time:joda-time": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.12.6" + }, + "junit:junit": { + "locked": "4.11" + }, + "junit:junit-dep": { + "locked": "4.11" + }, + "nz.ac.waikato.cms.moa:moa": { + "locked": "2017.06" + }, + "org.apache.flink:flink-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.flink:flink-rpc-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.5.14" + }, + "org.apache.mesos:mesos": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.7.2" + }, + "org.hamcrest:hamcrest-all": { + "locked": "1.3" + }, + "org.hdrhistogram:HdrHistogram": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.1.12" + }, + "org.jctools:jctools-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.2.1" + }, + "org.json:json": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "20180813" + }, + "org.mockito:mockito-all": { + "locked": "2.0.2-beta" + }, + "org.projectlombok:lombok": { + "locked": "1.18.20" + }, + "org.skife.config:config-magic": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "0.11" + }, + "org.slf4j:slf4j-api": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-runtime" + ], + "locked": "1.7.15" + }, + "org.slf4j:slf4j-log4j12": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.7.0" + }, + "org.xerial.snappy:snappy-java": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.1.10.5" + } + }, + "testRuntimeClasspath": { + "com.github.spullara.cli-parser:cli-parser": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.1.1" + }, + "com.google.code.findbugs:jsr305": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "3.0.1" + }, + "com.netflix.rxjava:rxjava-math": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-remote-observable" + ], + "locked": "0.20.6" + }, + "com.netflix.spectator:spectator-api": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-server-worker-client" + ], + "locked": "1.3.10" + }, + "com.netflix:mantis-rxnetty": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "0.4.19.1" + }, + "com.spotify:completable-futures": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client" + ], + "locked": "0.3.1" + }, + "com.yahoo.datasketches:sketches-core": { + "locked": "0.9.1" + }, + "commons-io:commons-io": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.11.0" + }, + "io.mantisrx:mantis-common": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-control-plane-core", + "io.mantisrx:mantis-network", + "io.mantisrx:mantis-remote-observable", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-common-serde": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-client": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime-loader", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-control-plane-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-control-plane-core", + "io.mantisrx:mantis-server-worker-client" + ], + "project": true + }, + "io.mantisrx:mantis-network": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-remote-observable": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client", + "io.mantisrx:mantis-runtime" + ], + "project": true + }, + "io.mantisrx:mantis-runtime": { + "project": true + }, + "io.mantisrx:mantis-runtime-loader": { + "project": true + }, + "io.mantisrx:mantis-rxcontrol": { + "locked": "1.3.20" + }, + "io.mantisrx:mantis-server-worker-client": { + "project": true + }, + "io.mantisrx:mantis-shaded": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-common-serde" + ], + "project": true + }, + "io.mantisrx:mql-jvm": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-network" + ], + "locked": "3.4.0" + }, + "io.netty:netty-buffer": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.60.Final" + }, + "io.netty:netty-codec-http": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.60.Final" + }, + "io.netty:netty-handler": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-network", + "io.mantisrx:mantis-remote-observable" + ], + "locked": "4.1.60.Final" + }, + "io.netty:netty-transport-native-epoll": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "4.1.60.Final" + }, + "io.reactivex:rxjava": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.3.8" + }, + "io.vavr:vavr": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-runtime-loader", + "io.mantisrx:mantis-shaded" + ], + "locked": "0.9.2" + }, + "io.vavr:vavr-test": { + "locked": "0.9.2" + }, + "joda-time:joda-time": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.12.6" + }, + "junit:junit": { + "locked": "4.11" + }, + "junit:junit-dep": { + "locked": "4.11" + }, + "net.jcip:jcip-annotations": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.0" + }, + "nz.ac.waikato.cms.moa:moa": { + "locked": "2017.06" + }, + "org.apache.flink:flink-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.flink:flink-rpc-akka": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.flink:flink-rpc-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.14.2" + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.5.14" + }, + "org.apache.mesos:mesos": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "1.7.2" + }, + "org.asynchttpclient:async-http-client": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-client" + ], + "locked": "2.12.3" + }, + "org.hamcrest:hamcrest-all": { + "locked": "1.3" + }, + "org.hdrhistogram:HdrHistogram": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "2.1.12" + }, + "org.jctools:jctools-core": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.2.1" + }, + "org.json:json": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "20180813" + }, + "org.mockito:mockito-all": { + "locked": "2.0.2-beta" + }, + "org.skife.config:config-magic": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-control-plane-core" + ], + "locked": "0.11" + }, + "org.slf4j:slf4j-api": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-remote-observable", + "io.mantisrx:mantis-runtime" + ], + "locked": "1.7.36" + }, + "org.slf4j:slf4j-log4j12": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common", + "io.mantisrx:mantis-remote-observable" + ], + "locked": "1.7.0" + }, + "org.xerial.snappy:snappy-java": { + "firstLevelTransitive": [ + "io.mantisrx:mantis-common" + ], + "locked": "1.1.10.5" + } + } +} \ No newline at end of file diff --git a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/MantisWorker.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/MantisWorker.java index 96cf00bfb..25b161691 100644 --- a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/MantisWorker.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/MantisWorker.java @@ -31,8 +31,6 @@ import io.mantisrx.server.master.client.MantisMasterGateway; import io.mantisrx.server.worker.config.ConfigurationFactory; import io.mantisrx.server.worker.config.StaticPropertiesConfigurationFactory; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; -import io.mantisrx.server.worker.mesos.VirualMachineWorkerServiceMesosImpl; import io.mantisrx.shaded.com.google.common.collect.ImmutableList; import java.io.File; import java.io.FileInputStream; @@ -115,8 +113,6 @@ public void run() { // metrics PublishSubject executeStageSubject = PublishSubject.create(); - PublishSubject vmTaskStatusSubject = PublishSubject.create(); - mantisServices.add(new VirualMachineWorkerServiceMesosImpl(executeStageSubject, vmTaskStatusSubject)); // TODO(sundaram): inline services are hard to read. Would be good to refactor this. mantisServices.add(new Service() { private RuntimeTaskImpl runtimeTaskImpl; @@ -154,8 +150,6 @@ public void start() { Clock.systemDefaultZone())); runtimeTaskImpl.setJob(jobToRun); - vmStatusSubscription = - runtimeTaskImpl.getVMStatus().subscribe(vmTaskStatusSubject); runtimeTaskImpl.startAsync(); } catch (Exception ex) { logger.error("Failed to start task, request: {}", wrappedRequest, ex); diff --git a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RunningWorker.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RunningWorker.java index 2cb409296..a6247aaca 100644 --- a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RunningWorker.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RunningWorker.java @@ -28,8 +28,6 @@ import io.mantisrx.server.core.JobSchedulingInfo; import io.mantisrx.server.core.Status; import io.mantisrx.server.core.Status.TYPE; -import io.mantisrx.server.core.domain.WorkerId; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; import java.util.Iterator; import java.util.concurrent.CountDownLatch; import org.slf4j.Logger; @@ -61,7 +59,6 @@ public class RunningWorker { private final String jobName; private final int totalStages; private final int metricsPort; - private final Observer vmTaskStatusObserver; private final Observable stageTotalWorkersObservable; private final Observable jobSchedulingInfoObservable; private final Iterator ports; @@ -85,7 +82,6 @@ public RunningWorker(Builder builder) { this.jobName = builder.jobName; this.totalStages = builder.totalStages; this.totalStagesNet = this.totalStages - (builder.hasJobMaster ? 1 : 0); - this.vmTaskStatusObserver = builder.vmTaskStatusObserver; this.jobStatus = builder.jobStatus; this.stageTotalWorkersObservable = builder.stageTotalWorkersObservable; this.jobSchedulingInfoObservable = builder.jobSchedulingInfoObservable; @@ -126,10 +122,6 @@ public void call(Throwable t) { public void signalStartedInitiated() { logger.info("JobId: " + jobId + ", stage: " + stageNum + " workerIndex: " + workerIndex + " workerNumber: " + workerNum + "," + " signaling started initiated"); - vmTaskStatusObserver.onNext(new VirtualMachineTaskStatus( - new WorkerId(jobId, workerIndex, workerNum).getId(), - VirtualMachineTaskStatus.TYPE.STARTED, jobName + ", " + - String.format(STATUS_MESSAGE_FORMAT, stageNum, workerIndex, workerNum, "started"))); // indicate start success requestSubject.onNext(true); requestSubject.onCompleted(); @@ -153,11 +145,6 @@ public void signalCompleted() { MantisJobState.Completed)); // send complete status jobStatus.onCompleted(); - // send completed status to vm service - vmTaskStatusObserver.onNext(new VirtualMachineTaskStatus( - new WorkerId(jobId, workerIndex, workerNum).getId(), - VirtualMachineTaskStatus.TYPE.COMPLETED, jobName + ", " + - String.format(STATUS_MESSAGE_FORMAT, stageNum, workerIndex, workerNum, "completed"))); } public void signalFailed(Throwable t) { @@ -263,10 +250,6 @@ public int getTotalStagesNet() { return totalStagesNet; } - public Observer getVmTaskStatusObserver() { - return vmTaskStatusObserver; - } - @Override public String toString() { return "RunningWorker [" @@ -275,8 +258,7 @@ public String toString() { + ", stageNum=" + stageNum + ", workerNum=" + workerNum + ", workerIndex=" + workerIndex + ", jobName=" + jobName + ", totalStages=" + totalStages + ", metricsPort=" - + metricsPort + ", vmTaskStatusObserver=" - + vmTaskStatusObserver + ", ports=" + ports + + metricsPort + ", ports=" + ports + ", requestSubject=" + requestSubject + ", context=" + context + ", workerInfo=" + workerInfo + "]"; } @@ -298,7 +280,6 @@ public static class Builder { private int workerIndex; private String jobName; private int totalStages; - private Observer vmTaskStatusObserver; private Observable stageTotalWorkersObservable; private Observable jobSchedulingInfoObservable; private PublishSubject requestSubject; @@ -374,11 +355,6 @@ public Builder totalStages(int totalStages) { return this; } - public Builder vmTaskStatusObservable(Observer vmTaskStatusObserver) { - this.vmTaskStatusObserver = vmTaskStatusObserver; - return this; - } - public Builder hasJobMaster(boolean b) { this.hasJobMaster = b; return this; diff --git a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java index f97457fec..a86c3f9df 100644 --- a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/RuntimeTaskImpl.java @@ -19,10 +19,10 @@ import io.mantisrx.runtime.Job; import io.mantisrx.runtime.loader.RuntimeTask; import io.mantisrx.runtime.loader.SinkSubscriptionStateHandler; +import io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector; import io.mantisrx.runtime.loader.config.WorkerConfiguration; import io.mantisrx.runtime.loader.config.WorkerConfigurationUtils; import io.mantisrx.runtime.loader.config.WorkerConfigurationWritable; -import io.mantisrx.server.agent.metrics.cgroups.CgroupsMetricsCollector; import io.mantisrx.server.core.ExecuteStageRequest; import io.mantisrx.server.core.Service; import io.mantisrx.server.core.Status; @@ -33,7 +33,6 @@ import io.mantisrx.server.master.client.MantisMasterGateway; import io.mantisrx.server.master.client.TaskStatusUpdateHandler; import io.mantisrx.server.worker.client.WorkerMetricsClient; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; import io.mantisrx.shaded.com.google.common.util.concurrent.AbstractIdleService; import java.io.IOException; import java.time.Clock; @@ -69,8 +68,6 @@ public class RuntimeTaskImpl extends AbstractIdleService implements RuntimeTask private final PublishSubject> tasksStatusSubject; - private final PublishSubject vmTaskStatusSubject = PublishSubject.create(); - private Optional mantisJob = Optional.empty(); private ExecuteStageRequest executeStageRequest; @@ -181,7 +178,6 @@ private void doRun() throws Exception { executeStageSubject, tasksStatusSubject, new WorkerExecutionOperationsNetworkStage( - vmTaskStatusSubject, masterMonitor, config, workerMetricsClient, @@ -229,10 +225,6 @@ protected Observable getStatus() { .flatMap((Func1, Observable>) status -> status); } - public Observable getVMStatus() { - return vmTaskStatusSubject; - } - public String getWorkerId() { return executeStageRequest.getWorkerId().getId(); } diff --git a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java index a44545f07..6658ea2e7 100644 --- a/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java +++ b/mantis-runtime-executor/src/main/java/io/mantisrx/server/worker/WorkerExecutionOperationsNetworkStage.java @@ -56,7 +56,6 @@ import io.mantisrx.server.worker.jobmaster.AutoScaleMetricsConfig; import io.mantisrx.server.worker.jobmaster.JobMasterService; import io.mantisrx.server.worker.jobmaster.JobMasterStageConfig; -import io.mantisrx.server.worker.mesos.VirtualMachineTaskStatus; import io.mantisrx.shaded.com.google.common.base.Splitter; import io.mantisrx.shaded.com.google.common.base.Strings; import io.reactivex.mantis.remote.observable.RemoteRxServer; @@ -99,7 +98,6 @@ public class WorkerExecutionOperationsNetworkStage implements WorkerExecutionOpe private final WorkerMetricsClient workerMetricsClient; private final AtomicReference heartbeatRef = new AtomicReference<>(); private final SinkSubscriptionStateHandler.Factory sinkSubscriptionStateHandlerFactory; - private final Observer vmTaskStatusObserver; private final MantisMasterGateway mantisMasterApi; private int connectionsPerEndpoint = 2; private boolean lookupSpectatorRegistry = true; @@ -112,13 +110,11 @@ public class WorkerExecutionOperationsNetworkStage implements WorkerExecutionOpe private Observer jobStatusObserver; public WorkerExecutionOperationsNetworkStage( - Observer vmTaskStatusObserver, MantisMasterGateway mantisMasterApi, WorkerConfiguration config, WorkerMetricsClient workerMetricsClient, SinkSubscriptionStateHandler.Factory sinkSubscriptionStateHandlerFactory, ClassLoader classLoader) { - this.vmTaskStatusObserver = vmTaskStatusObserver; this.mantisMasterApi = mantisMasterApi; this.config = config; this.workerMetricsClient = workerMetricsClient; @@ -309,7 +305,6 @@ public void executeStage(final ExecutionDetails setup) throws IOException { .jobStatusObserver(setup.getStatus()) .requestSubject(setup.getExecuteStageRequest().getRequestSubject()) .workerInfo(workerInfo) - .vmTaskStatusObservable(vmTaskStatusObserver) .hasJobMaster(executionRequest.getHasJobMaster()) .jobId(executionRequest.getJobId()); diff --git a/mantis-runtime-loader/build.gradle b/mantis-runtime-loader/build.gradle index 88c7ad32c..50e61c9f3 100644 --- a/mantis-runtime-loader/build.gradle +++ b/mantis-runtime-loader/build.gradle @@ -16,6 +16,7 @@ dependencies { api project(":mantis-control-plane:mantis-control-plane-client") + implementation libraries.vavr testImplementation libraries.junit4 testImplementation libraries.mockitoAll diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/Cgroup.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/Cgroup.java index 5afae0827..4a64a5704 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/Cgroup.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/Cgroup.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import java.io.IOException; import java.util.List; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupImpl.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupImpl.java index 8ee783640..998210d6a 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupImpl.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.shaded.org.apache.curator.shaded.com.google.common.base.Preconditions; import io.vavr.Tuple2; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupsMetricsCollector.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupsMetricsCollector.java index 84313b4c5..8fd2442a0 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupsMetricsCollector.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CgroupsMetricsCollector.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.MetricsCollector; import io.mantisrx.runtime.loader.config.Usage; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CpuAcctsSubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CpuAcctsSubsystemProcess.java index adf54cea0..63569c0a4 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CpuAcctsSubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/CpuAcctsSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage; import java.io.IOException; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/MemorySubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/MemorySubsystemProcess.java index d30a5178e..6519b8e4f 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/MemorySubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/MemorySubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage.UsageBuilder; import java.io.IOException; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/NetworkSubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/NetworkSubsystemProcess.java index afe4596f1..9a3a3c93e 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/NetworkSubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/NetworkSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage.UsageBuilder; import java.io.IOException; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/ProcFileReader.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/ProcFileReader.java index f6763e65b..d112eb829 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/ProcFileReader.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/ProcFileReader.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.shaded.com.google.common.base.Charsets; import java.io.Closeable; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/SubsystemProcess.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/SubsystemProcess.java index 5b7dcbff7..8a234f61c 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/SubsystemProcess.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/cgroups/SubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; import io.mantisrx.runtime.loader.config.Usage; import java.io.IOException; diff --git a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java index 1a9736c64..52f460006 100644 --- a/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java +++ b/mantis-runtime-loader/src/main/java/io/mantisrx/runtime/loader/config/WorkerConfiguration.java @@ -141,7 +141,7 @@ default Time getHeartbeatInterval() { Integer getBindPort(); @Config("mantis.taskexecutor.metrics.collector") - @Default("io.mantisrx.server.worker.mesos.MesosMetricsCollector") + @Default("io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector") MetricsCollector getUsageSupplier(); // ------------------------------------------------------------------------ diff --git a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCgroup.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCgroup.java index 6d20d1fc5..5e7fb14e5 100644 --- a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCgroup.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCgroup.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,12 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import io.mantisrx.runtime.loader.cgroups.Cgroup; -import io.mantisrx.runtime.loader.cgroups.CgroupImpl; import io.mantisrx.shaded.com.google.common.collect.ImmutableMap; import io.mantisrx.shaded.com.google.common.io.Resources; import java.io.IOException; +import org.junit.Assert; import org.junit.Test; public class TestCgroup { @@ -33,15 +29,15 @@ public class TestCgroup { @Test public void testReadingStatFiles() throws IOException { - assertEquals( + Assert.assertEquals( ImmutableMap.of("user", 49692738L, "system", 4700825L), cgroup.getStats("cpuacct", "cpuacct.stat")); - assertTrue(cgroup.isV1()); + Assert.assertTrue(cgroup.isV1()); } @Test public void testReadingMetrics() throws IOException { - assertEquals( + Assert.assertEquals( 400000L, cgroup.getMetric("cpuacct", "cpu.cfs_quota_us").longValue() ); @@ -49,7 +45,7 @@ public void testReadingMetrics() throws IOException { @Test public void testLongOverflow() throws IOException { - assertEquals( + Assert.assertEquals( Long.MAX_VALUE, cgroup.getMetric("testlongoverflow", "verylongvalue").longValue() ); diff --git a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCpuAcctsSubsystemProcess.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCpuAcctsSubsystemProcess.java index a41dd9c21..568eb2ff0 100644 --- a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCpuAcctsSubsystemProcess.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestCpuAcctsSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,59 +14,53 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.mantisrx.runtime.loader.cgroups.Cgroup; -import io.mantisrx.runtime.loader.cgroups.CgroupImpl; -import io.mantisrx.runtime.loader.cgroups.CpuAcctsSubsystemProcess; import io.mantisrx.runtime.loader.config.Usage; import io.mantisrx.shaded.com.google.common.collect.ImmutableMap; import io.mantisrx.shaded.com.google.common.io.Resources; import java.io.IOException; +import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; public class TestCpuAcctsSubsystemProcess { @Test public void testWhenCgroupsReturnsCorrectData() throws Exception { - final Cgroup cgroup = mock(Cgroup.class); + final Cgroup cgroup = Mockito.mock(Cgroup.class); final CpuAcctsSubsystemProcess process = new CpuAcctsSubsystemProcess(cgroup); - when(cgroup.isV1()).thenReturn(true); - when(cgroup.getStats("cpuacct", "cpuacct.stat")) + Mockito.when(cgroup.isV1()).thenReturn(true); + Mockito.when(cgroup.getStats("cpuacct", "cpuacct.stat")) .thenReturn(ImmutableMap.of("user", 43873627L, "system", 4185541L)); - when(cgroup.getStats("cpuacct", "cpuacct.stat")) + Mockito.when(cgroup.getStats("cpuacct", "cpuacct.stat")) .thenReturn(ImmutableMap.of("user", 43873627L, "system", 4185541L)); - when(cgroup.getMetric("cpuacct", "cpu.cfs_quota_us")) + Mockito.when(cgroup.getMetric("cpuacct", "cpu.cfs_quota_us")) .thenReturn(400000L); - when(cgroup.getMetric("cpuacct", "cpu.cfs_period_us")) + Mockito.when(cgroup.getMetric("cpuacct", "cpu.cfs_period_us")) .thenReturn(100000L); final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(4L, (long) usage.getCpusLimit()); - assertEquals(438736L, (long) usage.getCpusUserTimeSecs()); - assertEquals(41855L, (long) usage.getCpusSystemTimeSecs()); + Assert.assertEquals(4L, (long) usage.getCpusLimit()); + Assert.assertEquals(438736L, (long) usage.getCpusUserTimeSecs()); + Assert.assertEquals(41855L, (long) usage.getCpusSystemTimeSecs()); } @Test public void testCgroupsV2() throws IOException { final Cgroup cgroupv2 = new CgroupImpl(Resources.getResource("example2").getPath()); - assertFalse(cgroupv2.isV1()); + Assert.assertFalse(cgroupv2.isV1()); final CpuAcctsSubsystemProcess process = new CpuAcctsSubsystemProcess(cgroupv2); final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(2L, (long) usage.getCpusLimit()); - assertEquals(4231L, (long) usage.getCpusUserTimeSecs()); - assertEquals(1277L, (long) usage.getCpusSystemTimeSecs()); + Assert.assertEquals(2L, (long) usage.getCpusLimit()); + Assert.assertEquals(4231L, (long) usage.getCpusUserTimeSecs()); + Assert.assertEquals(1277L, (long) usage.getCpusSystemTimeSecs()); } } diff --git a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestMemorySubsystemProcess.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestMemorySubsystemProcess.java index 60902369a..5e182f038 100644 --- a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestMemorySubsystemProcess.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestMemorySubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,32 +14,27 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.mantisrx.runtime.loader.cgroups.Cgroup; -import io.mantisrx.runtime.loader.cgroups.CgroupImpl; -import io.mantisrx.runtime.loader.cgroups.MemorySubsystemProcess; import io.mantisrx.runtime.loader.config.Usage; import io.mantisrx.shaded.com.google.common.collect.ImmutableMap; import io.mantisrx.shaded.com.google.common.io.Resources; import java.io.IOException; +import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; public class TestMemorySubsystemProcess { - private final Cgroup cgroup = mock(Cgroup.class); + private final Cgroup cgroup = Mockito.mock(Cgroup.class); private final MemorySubsystemProcess process = new MemorySubsystemProcess(cgroup); @Test public void testHappyPath() throws Exception { - when(cgroup.isV1()).thenReturn(true); - when(cgroup.getMetric("memory", "memory.limit_in_bytes")) + Mockito.when(cgroup.isV1()).thenReturn(true); + Mockito.when(cgroup.getMetric("memory", "memory.limit_in_bytes")) .thenReturn(17179869184L); - when(cgroup.getStats("memory", "memory.stat")) + Mockito.when(cgroup.getStats("memory", "memory.stat")) .thenReturn( ImmutableMap.builder() .put("cache", 233472L) @@ -83,9 +78,9 @@ public void testHappyPath() throws Exception { final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(17179869184L, (long) usage.getMemLimit()); - assertEquals(14828109824L, (long) usage.getMemRssBytes()); - assertEquals(14828109824L, (long) usage.getMemAnonBytes()); + Assert.assertEquals(17179869184L, (long) usage.getMemLimit()); + Assert.assertEquals(14828109824L, (long) usage.getMemRssBytes()); + Assert.assertEquals(14828109824L, (long) usage.getMemAnonBytes()); } @Test @@ -97,8 +92,8 @@ public void testCgroupv2() throws IOException { final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(2147483648L, (long) usage.getMemLimit()); - assertEquals(1693843456L, (long) usage.getMemRssBytes()); - assertEquals(945483776L, (long) usage.getMemAnonBytes()); + Assert.assertEquals(2147483648L, (long) usage.getMemLimit()); + Assert.assertEquals(1693843456L, (long) usage.getMemRssBytes()); + Assert.assertEquals(945483776L, (long) usage.getMemAnonBytes()); } } diff --git a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestNetworkSubsystemProcess.java b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestNetworkSubsystemProcess.java index 439f9effe..5acfaee9b 100644 --- a/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestNetworkSubsystemProcess.java +++ b/mantis-runtime-loader/src/test/java/io/mantisrx/runtime/loader/cgroups/TestNetworkSubsystemProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Netflix, Inc. + * Copyright 2024 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,11 @@ * limitations under the License. */ -package io.mantisrx.server.agent.metrics.cgroups; +package io.mantisrx.runtime.loader.cgroups; -import static org.junit.Assert.assertEquals; - -import io.mantisrx.runtime.loader.cgroups.NetworkSubsystemProcess; import io.mantisrx.runtime.loader.config.Usage; import io.mantisrx.shaded.com.google.common.io.Resources; +import org.junit.Assert; import org.junit.Test; public class TestNetworkSubsystemProcess { @@ -32,7 +30,7 @@ public void testValidPath() throws Exception { final Usage.UsageBuilder usageBuilder = Usage.builder(); process.getUsage(usageBuilder); final Usage usage = usageBuilder.build(); - assertEquals(2861321009430L, (long) usage.getNetworkReadBytes()); - assertEquals(2731791728959L, (long) usage.getNetworkWriteBytes()); + Assert.assertEquals(2861321009430L, (long) usage.getNetworkReadBytes()); + Assert.assertEquals(2731791728959L, (long) usage.getNetworkWriteBytes()); } } diff --git a/mantis-server/mantis-server-agent/build.gradle b/mantis-server/mantis-server-agent/build.gradle index 798ec4371..f72cf29c5 100644 --- a/mantis-server/mantis-server-agent/build.gradle +++ b/mantis-server/mantis-server-agent/build.gradle @@ -35,7 +35,7 @@ dependencies { testImplementation libraries.junit4 testImplementation libraries.mockitoAll - testImplementation project(":mantis-server:mantis-server-worker") + testImplementation project(":mantis-runtime-executor") testImplementation(testFixtures(project(":mantis-control-plane:mantis-control-plane-core"))) } diff --git a/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties b/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties index f74e2399e..26c564b27 100644 --- a/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties +++ b/mantis-server/mantis-server-agent/src/main/resources/agent-docker.properties @@ -18,7 +18,7 @@ mantis.taskexecutor.heartbeats.interval=3500 mantis.localmode=false -mantis.taskexecutor.metrics.collector=io.mantisrx.server.agent.metrics.cgroups.CgroupsMetricsCollector +mantis.taskexecutor.metrics.collector=io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector mantis.taskexecutor.rpc.port-range=5050 mantis.taskexecutor.blob-store.storage-dir=file:///apps/mantis/mantis-server-agent/mantis-artifacts/storage/ mantis.taskexecutor.blob-store.local-cache=/apps/mantis/mantis-server-agent/mantis-artifacts diff --git a/mantis-server/mantis-server-agent/src/main/resources/agent.properties b/mantis-server/mantis-server-agent/src/main/resources/agent.properties index f55bd5939..ef458ada5 100644 --- a/mantis-server/mantis-server-agent/src/main/resources/agent.properties +++ b/mantis-server/mantis-server-agent/src/main/resources/agent.properties @@ -16,7 +16,7 @@ mantis.taskexecutor.id=agent1 mantis.localmode=true -mantis.taskexecutor.metrics.collector=io.mantisrx.server.agent.metrics.cgroups.CgroupsMetricsCollector +mantis.taskexecutor.metrics.collector=io.mantisrx.runtime.loader.cgroups.CgroupsMetricsCollector mantis.taskexecutor.rpc.port-range=5050 mantis.taskexecutor.blob-store.storage-dir=file:///tmp/mantis/mantis-artifacts/storage/ mantis.taskexecutor.blob-store.local-cache=/tmp/mantis/mantis-artifacts/cache diff --git a/mantis-source-jobs/mantis-source-job-publish/build.gradle b/mantis-source-jobs/mantis-source-job-publish/build.gradle index 7a1e862d4..d2c9d0daa 100644 --- a/mantis-source-jobs/mantis-source-job-publish/build.gradle +++ b/mantis-source-jobs/mantis-source-job-publish/build.gradle @@ -16,10 +16,9 @@ apply plugin: "mantis" dependencies { - implementation project(":mantis-runtime") implementation project(":mantis-publish:mantis-publish-netty") implementation project(":mantis-connectors:mantis-connector-publish") - implementation project(':mantis-server:mantis-server-worker') + implementation project(':mantis-runtime-executor') implementation libraries.slf4jApi diff --git a/settings.gradle b/settings.gradle index 7b2163cf1..cb9ff8133 100644 --- a/settings.gradle +++ b/settings.gradle @@ -55,12 +55,12 @@ include ':mantis-publish:mantis-publish-netty-guice' include 'mantis-remote-observable' include 'mantis-runtime' include 'mantis-runtime-loader' +include 'mantis-runtime-executor' include 'mantis-source-jobs:mantis-source-job-kafka' include 'mantis-source-jobs:mantis-source-job-publish' include 'mantis-server:mantis-server-agent' -include 'mantis-server:mantis-server-worker' include 'mantis-server:mantis-server-worker-client' include 'mantis-shaded'