From 0dfd8cd1f27ad551e4dd84b75d7da3dcde1e8c74 Mon Sep 17 00:00:00 2001 From: tpetr Date: Tue, 7 Oct 2014 14:08:14 -0400 Subject: [PATCH 01/93] expose task id as MESOS_TASK_ID env var in runner script --- SingularityExecutor/src/main/resources/runner.sh.hbs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SingularityExecutor/src/main/resources/runner.sh.hbs b/SingularityExecutor/src/main/resources/runner.sh.hbs index 475d93061a..3ae3baa562 100644 --- a/SingularityExecutor/src/main/resources/runner.sh.hbs +++ b/SingularityExecutor/src/main/resources/runner.sh.hbs @@ -4,6 +4,8 @@ # Task ID: {{{ taskId }}} # +export MESOS_TASK_ID={{{bashEscaped taskId}}} # I can't believe Mesos doesn't expose this. + # load system-wide profile.d if [[ -d /etc/profile.d ]]; then for i in /etc/profile.d/*.sh; do From dc1a4b7eb4f5b5c752bd44957ffa72fd09904348 Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 15 Oct 2014 17:04:08 +0100 Subject: [PATCH 02/93] Set missing properties in builder and clone lists / maps --- .../java/com/hubspot/singularity/SingularityDeploy.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java index ed0c830d7e..010e07df1d 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java @@ -107,6 +107,12 @@ public SingularityDeployBuilder toBuilder() { .setHealthcheckTimeoutSeconds(healthcheckTimeoutSeconds) .setSkipHealthchecksOnDeploy(skipHealthchecksOnDeploy) + .setConsiderHealthyAfterRunningForSeconds(considerHealthyAfterRunningForSeconds) + .setDeployHealthTimeoutSeconds(deployHealthTimeoutSeconds) + .setServiceBasePath(serviceBasePath) + .setLoadBalancerGroups(copyOfList(loadBalancerGroups)) + .setLoadBalancerOptions(copyOfMap(loadBalancerOptions)) + .setMetadata(copyOfMap(metadata)) .setVersion(version) .setTimestamp(timestamp) From 41268637a9a453dad016008d13e2bb7bcff6bcd1 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Fri, 24 Oct 2014 15:52:50 -0400 Subject: [PATCH 03/93] fix to not kill tasks when pausing --- SingularityUI/app/models/Request.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/SingularityUI/app/models/Request.coffee b/SingularityUI/app/models/Request.coffee index a44511d358..7dfff1f84b 100644 --- a/SingularityUI/app/models/Request.coffee +++ b/SingularityUI/app/models/Request.coffee @@ -46,8 +46,6 @@ class Request extends Model type: 'POST' pause: (killTasks) => - killTasks = if not killTasks then undefined else killTasks - $.ajax url: "#{ @url() }/pause" type: 'POST' From 1f63d2daac0c63843c42d7a63c18d48b406466ba Mon Sep 17 00:00:00 2001 From: tpetr Date: Tue, 7 Oct 2014 18:02:49 -0400 Subject: [PATCH 04/93] optional thread limits for tasks --- .../java/com/hubspot/deploy/ExecutorData.java | 12 +++- .../hubspot/deploy/ExecutorDataBuilder.java | 16 ++++- .../SingularityExecutorConfiguration.java | 61 ++++++++++++++++--- ...ingularityExecutorConfigurationLoader.java | 4 ++ .../config/SingularityExecutorModule.java | 4 +- .../handlebars/BashEscapedHelper.java | 3 + .../executor/handlebars/IfPresentHelper.java | 32 ++++++++++ .../executor/models/RunnerContext.java | 20 +++++- ...SingularityExecutorTaskProcessBuilder.java | 2 +- .../src/main/resources/runner.sh.hbs | 20 +++--- 10 files changed, 147 insertions(+), 27 deletions(-) create mode 100644 SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/IfPresentHelper.java diff --git a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java index 0b29b18b90..b72bb9e79c 100644 --- a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java +++ b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java @@ -24,12 +24,14 @@ public class ExecutorData { private final Optional loggingTag; private final Map loggingExtraFields; private final Optional sigKillProcessesAfterMillis; + private final Optional maxTaskThreads; @JsonCreator public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArtifacts") List embeddedArtifacts, @JsonProperty("externalArtifacts") List externalArtifacts, @JsonProperty("s3Artifacts") List s3Artifacts, @JsonProperty("successfulExitCodes") List successfulExitCodes, @JsonProperty("user") Optional user, @JsonProperty("runningSentinel") Optional runningSentinel, @JsonProperty("extraCmdLineArgs") List extraCmdLineArgs, @JsonProperty("loggingTag") Optional loggingTag, - @JsonProperty("loggingExtraFields") Map loggingExtraFields, @JsonProperty("sigKillProcessesAfterMillis") Optional sigKillProcessesAfterMillis) { + @JsonProperty("loggingExtraFields") Map loggingExtraFields, @JsonProperty("sigKillProcessesAfterMillis") Optional sigKillProcessesAfterMillis, + @JsonProperty("maxTaskThreads") Optional maxTaskThreads) { this.cmd = cmd; this.embeddedArtifacts = JavaUtils.nonNullImmutable(embeddedArtifacts); this.externalArtifacts = JavaUtils.nonNullImmutable(externalArtifacts); @@ -41,10 +43,11 @@ public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArti this.loggingTag = loggingTag; this.loggingExtraFields = JavaUtils.nonNullImmutable(loggingExtraFields); this.sigKillProcessesAfterMillis = sigKillProcessesAfterMillis; + this.maxTaskThreads = maxTaskThreads; } public ExecutorDataBuilder toBuilder() { - return new ExecutorDataBuilder(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, runningSentinel, user, extraCmdLineArgs, loggingTag, loggingExtraFields, sigKillProcessesAfterMillis); + return new ExecutorDataBuilder(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, runningSentinel, user, extraCmdLineArgs, loggingTag, loggingExtraFields, sigKillProcessesAfterMillis, maxTaskThreads); } public String getCmd() { @@ -91,6 +94,10 @@ public List getS3Artifacts() { return s3Artifacts; } + public Optional getMaxTaskThreads() { + return maxTaskThreads; + } + @Override public String toString() { return Objects.toStringHelper(this) @@ -105,6 +112,7 @@ public String toString() { .add("loggingTag", loggingTag) .add("loggingExtraFields", loggingExtraFields) .add("sigKillProcessesAfterMillis", sigKillProcessesAfterMillis) + .add("maxTaskThreads", maxTaskThreads) .toString(); } } diff --git a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java index 4e3dfc1c80..43d4873865 100644 --- a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java +++ b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java @@ -19,9 +19,10 @@ public class ExecutorDataBuilder { private Optional loggingTag; private Map loggingExtraFields; private Optional sigKillProcessesAfterMillis; + private Optional maxTaskThreads; public ExecutorDataBuilder(String cmd, List embeddedArtifacts, List externalArtifacts, List s3Artifacts, List successfulExitCodes, Optional runningSentinel, - Optional user, List extraCmdLineArgs, Optional loggingTag, Map loggingExtraFields, Optional sigKillProcessesAfterMillis) { + Optional user, List extraCmdLineArgs, Optional loggingTag, Map loggingExtraFields, Optional sigKillProcessesAfterMillis, Optional maxTaskThreads) { this.cmd = cmd; this.embeddedArtifacts = embeddedArtifacts; this.externalArtifacts = externalArtifacts; @@ -32,6 +33,7 @@ public ExecutorDataBuilder(String cmd, List embeddedArtifacts, this.extraCmdLineArgs = extraCmdLineArgs; this.loggingTag = loggingTag; this.loggingExtraFields = loggingExtraFields; + this.maxTaskThreads = maxTaskThreads; } public ExecutorDataBuilder() { @@ -39,7 +41,7 @@ public ExecutorDataBuilder() { } public ExecutorData build() { - return new ExecutorData(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, user, runningSentinel, extraCmdLineArgs, loggingTag, loggingExtraFields, sigKillProcessesAfterMillis); + return new ExecutorData(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, user, runningSentinel, extraCmdLineArgs, loggingTag, loggingExtraFields, sigKillProcessesAfterMillis, maxTaskThreads); } public Optional getLoggingTag() { @@ -141,6 +143,15 @@ public ExecutorDataBuilder setS3Artifacts(List s3Artifacts) { return this; } + public Optional getMaxTaskThreads() { + return maxTaskThreads; + } + + public ExecutorDataBuilder setMaxTaskThreads(Optional maxTaskThreads) { + this.maxTaskThreads = maxTaskThreads; + return this; + } + @Override public String toString() { return Objects.toStringHelper(this) @@ -155,6 +166,7 @@ public String toString() { .add("loggingTag", loggingTag) .add("loggingExtraFields", loggingExtraFields) .add("sigKillProcessesAfterMillis", sigKillProcessesAfterMillis) + .add("maxTaskThreads", maxTaskThreads) .toString(); } } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java index 777d514736..688a2c462e 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java @@ -4,6 +4,8 @@ import java.nio.file.Paths; import java.util.Arrays; +import com.google.common.base.Optional; +import com.google.common.base.Strings; import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; @@ -58,6 +60,8 @@ public class SingularityExecutorConfiguration { private final boolean useLocalDownloadService; private final long localDownloadServiceTimeoutMillis; + private final Optional maxTaskThreads; + @Inject public SingularityExecutorConfiguration( @Named(SingularityExecutorConfigurationLoader.GLOBAL_TASK_DEFINITION_DIRECTORY) String globalTaskDefinitionDirectory, @@ -91,7 +95,8 @@ public SingularityExecutorConfiguration( @Named(SingularityExecutorConfigurationLoader.TAIL_LOG_LINES_TO_SAVE) String tailLogLinesToSave, @Named(SingularityExecutorConfigurationLoader.TAIL_LOG_FILENAME) String serviceFinishedTailLog, @Named(SingularityExecutorConfigurationLoader.USE_LOCAL_DOWNLOAD_SERVICE) String useLocalDownloadService, - @Named(SingularityExecutorConfigurationLoader.LOCAL_DOWNLOAD_SERVICE_TIMEOUT_MILLIS) String localDownloadServiceTimeoutMillis + @Named(SingularityExecutorConfigurationLoader.LOCAL_DOWNLOAD_SERVICE_TIMEOUT_MILLIS) String localDownloadServiceTimeoutMillis, + @Named(SingularityExecutorConfigurationLoader.MAX_TASK_THREADS) String maxTaskThreadsAsString ) { this.executorBashLog = executorBashLog; this.globalTaskDefinitionDirectory = globalTaskDefinitionDirectory; @@ -129,6 +134,12 @@ public SingularityExecutorConfiguration( } this.useLocalDownloadService = Boolean.parseBoolean(useLocalDownloadService); this.localDownloadServiceTimeoutMillis = Long.parseLong(localDownloadServiceTimeoutMillis); + + if (Strings.isNullOrEmpty(maxTaskThreadsAsString)) { + this.maxTaskThreads = Optional.absent(); + } else { + this.maxTaskThreads = Optional.of(Integer.parseInt(maxTaskThreadsAsString)); + } } public boolean isUseLocalDownloadService() { @@ -272,16 +283,46 @@ public Path getTaskDefinitionPath(String taskId) { return Paths.get(getGlobalTaskDefinitionDirectory()).resolve(getSafeTaskIdForDirectory(taskId) + getGlobalTaskDefinitionSuffix()); } + public Optional getMaxTaskThreads() { + return maxTaskThreads; + } + @Override public String toString() { - return "SingularityExecutorConfiguration [executorJavaLog=" + executorJavaLog + ", executorBashLog=" + executorBashLog + ", serviceLog=" + serviceLog + ", defaultRunAsUser=" + defaultRunAsUser - + ", taskAppDirectory=" + taskAppDirectory + ", shutdownTimeoutWaitMillis=" + shutdownTimeoutWaitMillis + ", idleExecutorShutdownWaitMillis=" + idleExecutorShutdownWaitMillis + ", stopDriverAfterMillis=" + stopDriverAfterMillis - + ", globalTaskDefinitionDirectory=" + globalTaskDefinitionDirectory + ", globalTaskDefinitionSuffix=" + globalTaskDefinitionSuffix + ", hardKillAfterMillis=" + hardKillAfterMillis + ", killThreads=" + killThreads - + ", maxTaskMessageLength=" + maxTaskMessageLength + ", logrotateCommand=" + logrotateCommand + ", logrotateStateFile=" + logrotateStateFile + ", logrotateConfDirectory=" + logrotateConfDirectory + ", logrotateToDirectory=" - + logrotateToDirectory + ", logrotateMaxageDays=" + logrotateMaxageDays + ", logrotateCount=" + logrotateCount + ", logrotateDateformat=" + logrotateDateformat + ", logrotateExtrasDateformat=" + logrotateExtrasDateformat - + ", logrotateExtrasFiles=" + Arrays.toString(logrotateExtrasFiles) + ", logMetadataDirectory=" + logMetadataDirectory + ", logMetadataSuffix=" + logMetadataSuffix + ", tailLogLinesToSave=" + tailLogLinesToSave - + ", serviceFinishedTailLog=" + serviceFinishedTailLog + ", s3MetadataSuffix=" + s3MetadataSuffix + ", s3MetadataDirectory=" + s3MetadataDirectory + ", s3KeyPattern=" + s3KeyPattern + ", s3Bucket=" + s3Bucket - + ", useLocalDownloadService=" + useLocalDownloadService + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + "]"; + return "SingularityExecutorConfiguration [" + + "executorJavaLog='" + executorJavaLog + '\'' + + ", executorBashLog='" + executorBashLog + '\'' + + ", serviceLog='" + serviceLog + '\'' + + ", defaultRunAsUser='" + defaultRunAsUser + '\'' + + ", taskAppDirectory='" + taskAppDirectory + '\'' + + ", shutdownTimeoutWaitMillis=" + shutdownTimeoutWaitMillis + + ", idleExecutorShutdownWaitMillis=" + idleExecutorShutdownWaitMillis + + ", stopDriverAfterMillis=" + stopDriverAfterMillis + + ", globalTaskDefinitionDirectory='" + globalTaskDefinitionDirectory + '\'' + + ", globalTaskDefinitionSuffix='" + globalTaskDefinitionSuffix + '\'' + + ", hardKillAfterMillis=" + hardKillAfterMillis + + ", killThreads=" + killThreads + + ", maxTaskMessageLength=" + maxTaskMessageLength + + ", logrotateCommand='" + logrotateCommand + '\'' + + ", logrotateStateFile='" + logrotateStateFile + '\'' + + ", logrotateConfDirectory=" + logrotateConfDirectory + + ", logrotateToDirectory='" + logrotateToDirectory + '\'' + + ", logrotateMaxageDays='" + logrotateMaxageDays + '\'' + + ", logrotateCount='" + logrotateCount + '\'' + + ", logrotateDateformat='" + logrotateDateformat + '\'' + + ", logrotateExtrasDateformat='" + logrotateExtrasDateformat + '\'' + + ", logrotateExtrasFiles=" + Arrays.toString(logrotateExtrasFiles) + + ", logMetadataDirectory=" + logMetadataDirectory + + ", logMetadataSuffix='" + logMetadataSuffix + '\'' + + ", tailLogLinesToSave=" + tailLogLinesToSave + + ", serviceFinishedTailLog='" + serviceFinishedTailLog + '\'' + + ", s3MetadataSuffix='" + s3MetadataSuffix + '\'' + + ", s3MetadataDirectory=" + s3MetadataDirectory + + ", s3KeyPattern='" + s3KeyPattern + '\'' + + ", s3Bucket='" + s3Bucket + '\'' + + ", useLocalDownloadService=" + useLocalDownloadService + + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + + ", maxTaskThreads=" + maxTaskThreads + + ']'; } - } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java index 207ed7bc99..1ac0de6fbb 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java @@ -51,6 +51,8 @@ public class SingularityExecutorConfigurationLoader extends SingularityConfigura public static final String LOCAL_DOWNLOAD_SERVICE_TIMEOUT_MILLIS = "executor.local.download.service.timeout.millis"; + public static final String MAX_TASK_THREADS = "executor.max.task.threads"; + public SingularityExecutorConfigurationLoader() { super("/etc/singularity.executor.properties", Optional.of("singularity-executor.log")); } @@ -85,6 +87,8 @@ protected void bindDefaults(Properties properties) { properties.put(USE_LOCAL_DOWNLOAD_SERVICE, Boolean.toString(false)); properties.put(LOCAL_DOWNLOAD_SERVICE_TIMEOUT_MILLIS, Long.toString(TimeUnit.MINUTES.toMillis(3))); + + properties.put(MAX_TASK_THREADS, ""); } } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java index 7cad3aa522..590c6003a4 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java @@ -9,6 +9,7 @@ import com.google.inject.Singleton; import com.google.inject.name.Named; import com.hubspot.singularity.executor.handlebars.BashEscapedHelper; +import com.hubspot.singularity.executor.handlebars.IfPresentHelper; import com.ning.http.client.AsyncHttpClient; import com.ning.http.client.AsyncHttpClientConfig; @@ -58,7 +59,8 @@ public Template providesLogrotateTemplate(Handlebars handlebars) throws IOExcept public Handlebars providesHandlebars() { final Handlebars handlebars = new Handlebars(); - handlebars.registerHelper("bashEscaped", new BashEscapedHelper()); + handlebars.registerHelper(BashEscapedHelper.NAME, BashEscapedHelper.INSTANCE); + handlebars.registerHelper(IfPresentHelper.NAME, IfPresentHelper.INSTANCE); return handlebars; } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/BashEscapedHelper.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/BashEscapedHelper.java index 601ffa3591..7a0f8ac8a3 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/BashEscapedHelper.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/BashEscapedHelper.java @@ -6,6 +6,9 @@ import com.github.jknack.handlebars.Options; public class BashEscapedHelper implements Helper { + public static final Helper INSTANCE = new BashEscapedHelper(); + + public static final String NAME = "bashEscaped"; @Override public CharSequence apply(Object context, Options options) throws IOException { diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/IfPresentHelper.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/IfPresentHelper.java new file mode 100644 index 0000000000..d0e20b7281 --- /dev/null +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/handlebars/IfPresentHelper.java @@ -0,0 +1,32 @@ +package com.hubspot.singularity.executor.handlebars; + +import java.io.IOException; + +import com.github.jknack.handlebars.Helper; +import com.github.jknack.handlebars.Options; +import com.google.common.base.Optional; + +public class IfPresentHelper implements Helper { + public static final Helper INSTANCE = new IfPresentHelper(); + + public static final String NAME = "ifPresent"; + + @Override + public CharSequence apply(Object context, Options options) throws IOException { + if ((context != null) && (context instanceof Optional)) { + final Optional maybeContext = (Optional)context; + + if (maybeContext.isPresent()) { + return options.fn(maybeContext.get()); + } else { + return options.inverse(); + } + } else { + if (context != null) { + return options.fn(context); + } else { + return options.inverse(); + } + } + } +} diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java index 9148e01330..43ca2a03b5 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java @@ -1,5 +1,7 @@ package com.hubspot.singularity.executor.models; +import com.google.common.base.Optional; + public class RunnerContext { private final String cmd; @@ -7,13 +9,15 @@ public class RunnerContext { private final String logfile; private final String taskId; private final String taskAppDirectory; + private final Optional maxTaskThreads; - public RunnerContext(String cmd, String taskAppDirectory, String user, String logfile, String taskId) { + public RunnerContext(String cmd, String taskAppDirectory, String user, String logfile, String taskId, Optional maxTaskThreads) { this.cmd = cmd; this.user = user; this.logfile = logfile; this.taskId = taskId; this.taskAppDirectory = taskAppDirectory; + this.maxTaskThreads = maxTaskThreads; } public String getTaskId() { @@ -36,9 +40,19 @@ public String getLogfile() { return logfile; } + public Optional getMaxTaskThreads() { + return maxTaskThreads; + } + @Override public String toString() { - return "RunnerContext [cmd=" + cmd + ", user=" + user + ", logfile=" + logfile + ", taskId=" + taskId + ", taskAppDirectory=" + taskAppDirectory + "]"; + return "RunnerContext [" + + "cmd='" + cmd + '\'' + + ", user='" + user + '\'' + + ", logfile='" + logfile + '\'' + + ", taskId='" + taskId + '\'' + + ", taskAppDirectory='" + taskAppDirectory + '\'' + + ", maxTaskThreads=" + maxTaskThreads + + ']'; } - } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java index 1c08cb2841..39ff92cdc6 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java @@ -85,7 +85,7 @@ private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData execu task.getLog().info("Writing a runner script to execute {}", cmd); - templateManager.writeRunnerScript(getPath("runner.sh"), new RunnerContext(cmd, configuration.getTaskAppDirectory(), executorData.getUser().or(configuration.getDefaultRunAsUser()), configuration.getServiceLog(), task.getTaskId())); + templateManager.writeRunnerScript(getPath("runner.sh"), new RunnerContext(cmd, configuration.getTaskAppDirectory(), executorData.getUser().or(configuration.getDefaultRunAsUser()), configuration.getServiceLog(), task.getTaskId(), executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()))); List command = Lists.newArrayList(); command.add("bash"); diff --git a/SingularityExecutor/src/main/resources/runner.sh.hbs b/SingularityExecutor/src/main/resources/runner.sh.hbs index 3ae3baa562..6d4d736a6a 100644 --- a/SingularityExecutor/src/main/resources/runner.sh.hbs +++ b/SingularityExecutor/src/main/resources/runner.sh.hbs @@ -8,9 +8,10 @@ export MESOS_TASK_ID={{{bashEscaped taskId}}} # I can't believe Mesos doesn't e # load system-wide profile.d if [[ -d /etc/profile.d ]]; then + echo "Sourcing files from /etc/profile.d" for i in /etc/profile.d/*.sh; do source $i >/dev/null 2>&1 - echo "Sourced $i" + echo " Sourced $i" done else echo "No /etc/profile.d to source" @@ -18,18 +19,20 @@ fi # load env vars if [[ -f deploy.env ]]; then + echo "Sourcing deploy-specific environment variables" source deploy.env - echo "Sourced deploy-specific environment variables" else echo "No deploy-specific environment variables" fi # TODO: consider removing this when all artifacts have tmp folder created at build time? if [[ ! -d ./tmp ]]; then + echo "Creating tmp directory in sandbox" mkdir ./tmp sudo chown -R {{{ user }}} ./tmp fi +echo "Ensuring {{{ taskAppDirectory }}} is owned by {{{ user }}}" sudo chown -R {{{ user }}} {{{ taskAppDirectory }}} cd {{{ taskAppDirectory }}}/ @@ -44,12 +47,13 @@ else echo "No deploy-specific profile.d" fi -if [[ "$USE_ULIMIT" == "1" ]]; then - echo "Setting max memory via ulimit -m $(($DEPLOY_MEM * 1024))" - ulimit -m $(($DEPLOY_MEM * 1024)) -fi - -echo "Going to execute: {{{ cmd }}}" +{{#ifPresent maxTaskThreads}} +echo "Setting max threads via ulimit -u {{{this}}}" +ulimit -u {{{this}}} +{{else}} +echo "No max thread limit to set" +{{/ifPresent}} # execute command +echo "Executing: sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1" exec sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1 From e8111311cd7371ac995aef134a1a2b8075f5f401 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 11 Nov 2014 13:47:52 -0500 Subject: [PATCH 05/93] fix s3 prefix generation --- .../SingularityS3FormatHelper.java | 37 +++++------------ .../singularity/SingularityS3Test.java | 41 +++++++++++++++++++ 2 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java index 372fbdff13..2e335d8dd5 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java @@ -3,6 +3,7 @@ import java.util.Calendar; import java.util.Collection; import java.util.Collections; +import java.util.GregorianCalendar; import java.util.List; import java.util.Set; @@ -117,7 +118,7 @@ public static Collection getS3KeyPrefixes(String s3KeyFormat, String req return getS3KeyPrefixes(s3KeyFormat, DISALLOWED_FOR_REQUEST, start, end); } - private static Collection getS3KeyPrefixes(String s3KeyFormat, List disallowedKeys, long start, long end) { + private static Collection getS3KeyPrefixes(String s3KeyFormat, List disallowedKeys, long start, long end) { String trimKeyFormat = trimKeyFormat(s3KeyFormat, disallowedKeys); int indexOfY = trimKeyFormat.indexOf("%Y"); @@ -142,10 +143,15 @@ private static Collection getS3KeyPrefixes(String s3KeyFormat, List keyPrefixes = Sets.newHashSet(); - Calendar calendar = Calendar.getInstance(); + Calendar calendar = GregorianCalendar.getInstance(); calendar.setTimeInMillis(start); - while (calendar.getTimeInMillis() <= end) { + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MILLISECOND, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.HOUR_OF_DAY, 0); + + while (calendar.getTimeInMillis() < end) { if (indexOfY > -1) { keyBuilder.replace(indexOfY, indexOfY + 4, getYear(calendar.get(Calendar.YEAR))); } @@ -163,8 +169,10 @@ private static Collection getS3KeyPrefixes(String s3KeyFormat, List -1) { calendar.add(Calendar.DAY_OF_YEAR, 1); } else if (indexOfM > -1) { + calendar.set(Calendar.DAY_OF_MONTH, 1); calendar.add(Calendar.MONTH, 1); } else { + calendar.set(Calendar.MONTH, 0); calendar.add(Calendar.YEAR, 1); } } @@ -178,27 +186,4 @@ public static Collection getS3KeyPrefixes(String s3KeyFormat, Singularit return getS3KeyPrefixes(keyFormat, DISALLOWED_FOR_TASK, start, end); } - public static void main(String[] args) { - SingularityTaskId taskId = new SingularityTaskId("rid", "did", System.currentTimeMillis(), 1, "host", "rackid"); - Optional tag = Optional. absent(); - Calendar c = Calendar.getInstance(); - - c.add(Calendar.DAY_OF_YEAR, -1); - - final long now = System.currentTimeMillis(); - - System.out.println(getS3KeyPrefixes("%requestId/%Y/%m/%taskId_%index-%s%fileext", taskId, tag, c.getTimeInMillis(), System.currentTimeMillis())); - System.out.println(getS3KeyPrefixes("%Y/%m/%d", taskId, tag, c.getTimeInMillis(), System.currentTimeMillis())); - - System.out.println(getS3KeyPrefixes("%requestId/%Y/%m/%taskId_%index-%s%fileext", "requestId", "deployId", tag, c.getTimeInMillis(), System.currentTimeMillis())); - - - System.out.println(getS3KeyPrefixes("%requestId/%Y/%m/%taskId_%index-%s%fileext", "rid", c.getTimeInMillis(), System.currentTimeMillis())); - - - - System.out.println(System.currentTimeMillis() - now); - - } - } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java b/SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java new file mode 100644 index 0000000000..e3392d975b --- /dev/null +++ b/SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java @@ -0,0 +1,41 @@ +package com.hubspot.singularity; + +import java.util.Collection; + +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.base.Optional; + +public class SingularityS3Test { + + @Test + public void testS3FormatHelper() throws Exception { + SingularityTaskId taskId = new SingularityTaskId("rid", "did", 1, 1, "host", "rack"); + + long start = 1414610537117l; // Wed, 29 Oct 2014 19:22:17 GMT + long end = 1415724215000l; // Tue, 11 Nov 2014 16:43:35 GMT + + Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%taskId", taskId, Optional. absent(), start, end); + + Assert.assertTrue(prefixes.size() == 2); + + end = 1447265861000l; // Tue, 11 Nov 2015 16:43:35 GMT + + prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%taskId", taskId, Optional. absent(), start, end); + + Assert.assertTrue(prefixes.size() == 2); + + start = 1415750399999l; + end = 1415771999000l; + + prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%d/%taskId", taskId, Optional. absent(), start, end); + + System.out.println(prefixes); + Assert.assertTrue(prefixes.size() == 2); + + + + } + +} From eb1884784071e5a93021d32837649c5c1e7d4ef7 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Wed, 12 Nov 2014 17:59:54 -0500 Subject: [PATCH 06/93] don't overwrite tail of log --- .../executor/task/SingularityExecutorTaskLogManager.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index e688694a09..a8bd7cb6da 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -69,6 +69,13 @@ private void copyLogTail() { return; } + final Path tailOfLogPath = taskDefinition.getTaskDirectoryPath().resolve(configuration.getServiceFinishedTailLog()); + + if (Files.exists(tailOfLogPath)) { + log.debug("{} already existed, skipping tail", tailOfLogPath); + return; + } + final List cmd = ImmutableList.of( "tail", "-n", @@ -76,7 +83,7 @@ private void copyLogTail() { taskDefinition.getServiceLogOut()); try { - super.runCommand(cmd, Redirect.to(taskDefinition.getTaskDirectoryPath().resolve(configuration.getServiceFinishedTailLog()).toFile())); + super.runCommand(cmd, Redirect.to(tailOfLogPath.toFile())); } catch (Throwable t) { log.error("Failed saving tail of log {} to {}", taskDefinition.getServiceLogOut(), configuration.getServiceFinishedTailLog(), t); } From fa5dbcf538fa0f9f8c451a7c2b6d10162a7e84be Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Mon, 17 Nov 2014 14:53:26 -0500 Subject: [PATCH 07/93] improve s3 uploaded failure log message --- .../runner/base/config/SingularityRunnerBaseLogging.java | 4 ++-- .../hubspot/singularity/s3uploader/SingularityS3Uploader.java | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java index 54b2e68e59..839ed765a0 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java @@ -43,7 +43,7 @@ public SingularityRunnerBaseLogging( } public void printProperties(Logger rootLogger) { - rootLogger.debug("Loaded {} properties", properties.size()); + rootLogger.info("Loaded {} properties", properties.size()); List strKeys = Lists.newArrayListWithCapacity(properties.size()); for (Object object : properties.keySet()) { @@ -52,7 +52,7 @@ public void printProperties(Logger rootLogger) { Collections.sort(strKeys); for (String key : strKeys) { - rootLogger.debug(" {} -> {}", key, properties.get(key)); + rootLogger.info(" {} -> {}", key, properties.get(key)); } } diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index 997b74e80a..22ce4dfcd8 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -10,6 +10,7 @@ import java.util.Set; import org.jets3t.service.S3Service; +import org.jets3t.service.S3ServiceException; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; import org.slf4j.Logger; @@ -107,6 +108,9 @@ private void uploadBatch(List toUpload) { metrics.upload(); success++; Files.delete(file); + } catch (S3ServiceException se) { + metrics.error(); + LOG.warn("{} Couldn't upload due to {} ({}) - {}", logIdentifier, se.getErrorCode(), se.getResponseCode(), se.getErrorMessage()); } catch (Exception e) { metrics.error(); LOG.warn("{} Couldn't upload or delete {}", logIdentifier, file, e); From 2bdfbdc8bd0dcb2c7e5cdc9a7f44938f5364341f Mon Sep 17 00:00:00 2001 From: tpetr Date: Thu, 20 Nov 2014 22:43:43 -0500 Subject: [PATCH 08/93] dont use ulimit -u for task thread limits -- it's a user-level setting --- SingularityExecutor/src/main/resources/runner.sh.hbs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/SingularityExecutor/src/main/resources/runner.sh.hbs b/SingularityExecutor/src/main/resources/runner.sh.hbs index 6d4d736a6a..ec3afaa8e8 100644 --- a/SingularityExecutor/src/main/resources/runner.sh.hbs +++ b/SingularityExecutor/src/main/resources/runner.sh.hbs @@ -47,13 +47,6 @@ else echo "No deploy-specific profile.d" fi -{{#ifPresent maxTaskThreads}} -echo "Setting max threads via ulimit -u {{{this}}}" -ulimit -u {{{this}}} -{{else}} -echo "No max thread limit to set" -{{/ifPresent}} - # execute command echo "Executing: sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1" exec sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1 From 92510de56e2eba9db735fae2d93a16c7111ccd4b Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Thu, 18 Dec 2014 11:00:12 +0000 Subject: [PATCH 09/93] remove sandbox fetch code (to be replaced later) --- .../singularity/smtp/SingularityMailer.java | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java index 8bbd0a5c20..706b2de5d0 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java @@ -31,7 +31,6 @@ import com.google.inject.Inject; import com.google.inject.name.Named; import com.hubspot.mesos.JavaUtils; -import com.hubspot.mesos.json.MesosFileChunkObject; import com.hubspot.singularity.ExtendedTaskState; import com.hubspot.singularity.SingularityCloseable; import com.hubspot.singularity.SingularityCloser; @@ -46,7 +45,6 @@ import com.hubspot.singularity.config.EmailConfigurationEnums.EmailType; import com.hubspot.singularity.config.SMTPConfiguration; import com.hubspot.singularity.config.SingularityConfiguration; -import com.hubspot.singularity.data.SandboxManager; import com.hubspot.singularity.data.TaskManager; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; import com.ning.http.client.AsyncHttpClient; @@ -70,9 +68,6 @@ public class SingularityMailer implements SingularityCloseable { private final JadeTemplate requestInCooldownTemplate; private final JadeTemplate requestModifiedTemplate; - private final AsyncHttpClient asyncHttpClient; - private final ObjectMapper objectMapper; - private final Optional uiHostnameAndPath; private final Joiner adminJoiner; @@ -91,8 +86,6 @@ public SingularityMailer(SingularityConfiguration configuration, Optional toList, String subject) { return String.format("[to: %s, subject: %s]", toList, subject); } - private Optional getTaskLogFile(SingularityTaskId taskId, String filename) { - final Optional task = taskManager.getTask(taskId); - - if (!task.isPresent()) { - LOG.error("No task found for {}", taskId.getId()); - return Optional.absent(); - } - - final Optional directory = taskManager.getDirectory(taskId); - - if (!directory.isPresent()) { - LOG.error("No directory found for task {} to fetch logs", taskId); - return Optional.absent(); - } - - final String slaveHostname = task.get().getOffer().getHostname(); - - final String fullPath = String.format("%s/%s", directory.get(), filename); - - final Long logLength = Long.valueOf(maybeSmtpConfiguration.get().getTaskLogLength()); - - final SandboxManager sandboxManager = new SandboxManager(asyncHttpClient, objectMapper); - - final Optional logChunkObject; - - try { - logChunkObject = sandboxManager.read(slaveHostname, fullPath, Optional.of(0L), Optional.of(logLength)); - } catch (RuntimeException e) { - LOG.error("Sandboxmanager failed to read {}/{} on slave {}", directory, filename, slaveHostname, e); - return Optional.absent(); - } - - if (logChunkObject.isPresent()) { - return Optional.of(logChunkObject.get().getData().split("[\r\n]+")); - } else { - LOG.error("Singularity mailer failed to get {} log for {} task ", filename, taskId.getId()); - return Optional.absent(); - } - } - private void populateRequestEmailProperties(Builder templateProperties, SingularityRequest request) { templateProperties.put("requestId", request.getId()); templateProperties.put("singularityRequestLink", getSingularityRequestLink(request)); @@ -189,8 +142,6 @@ private void populateRequestEmailProperties(Builder templateProp private void populateTaskEmailProperties(Builder templateProperties, SingularityTaskId taskId, Collection taskHistory, ExtendedTaskState taskState) { templateProperties.put("singularityTaskLink", getSingularityTaskLink(taskId)); - templateProperties.put("stdout", getTaskLogFile(taskId, "stdout").or(new String[0])); - templateProperties.put("stderr", getTaskLogFile(taskId, "stderr").or(new String[0])); templateProperties.put("taskId", taskId.getId()); templateProperties.put("deployId", taskId.getDeployId()); From c5403e06293e662fdb3816352beea9dfea1ea64f Mon Sep 17 00:00:00 2001 From: Henning Schmiedehausen Date: Thu, 11 Dec 2014 19:02:23 -0800 Subject: [PATCH 10/93] Factor out all the little thread pools. Factor thread pool management and control into guice injected, managed pools. --- .../singularity/SingularityMainModule.java | 76 +++++++++++++++++-- .../singularity/SingularityServiceModule.java | 2 +- .../config/SingularityConfiguration.java | 20 +++++ .../scheduler/SingularityHealthchecker.java | 24 ++---- .../SingularityLeaderOnlyPoller.java | 13 ++-- .../scheduler/SingularityNewTaskChecker.java | 23 ++---- .../SingularityTaskReconciliation.java | 21 +---- .../scheduler/SingularityTestModule.java | 11 +-- 8 files changed, 119 insertions(+), 71 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java index 6b7cf60d70..85cd79a46b 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java @@ -1,13 +1,23 @@ package com.hubspot.singularity; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static com.google.inject.name.Names.named; + +import de.neuland.jade4j.parser.Parser; +import de.neuland.jade4j.parser.node.Node; +import de.neuland.jade4j.template.JadeTemplate; import io.dropwizard.jetty.HttpConnectorFactory; +import io.dropwizard.lifecycle.Managed; import io.dropwizard.server.SimpleServerFactory; import java.io.IOException; import java.net.SocketException; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import javax.inject.Inject; import javax.inject.Provider; @@ -26,6 +36,7 @@ import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.net.HostAndPort; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; @@ -56,10 +67,6 @@ import com.hubspot.singularity.smtp.SingularitySmtpSender; import com.ning.http.client.AsyncHttpClient; -import de.neuland.jade4j.parser.Parser; -import de.neuland.jade4j.parser.node.Node; -import de.neuland.jade4j.template.JadeTemplate; - public class SingularityMainModule implements Module { @@ -76,6 +83,22 @@ public class SingularityMainModule implements Module { public static final String HTTP_HOST_AND_PORT = "http.host.and.port"; + public static final String CORE_THREADPOOL_NAME = "_core_threadpool"; + public static final Named CORE_THREADPOOL_NAMED = Names.named(CORE_THREADPOOL_NAME); + + public static final String HEALTHCHECK_THREADPOOL_NAME = "_healthcheck_threadpool"; + public static final Named HEALTHCHECK_THREADPOOL_NAMED = Names.named(HEALTHCHECK_THREADPOOL_NAME); + + public static final String NEW_TASK_THREADPOOL_NAME = "_new_task_threadpool"; + public static final Named NEW_TASK_THREADPOOL_NAMED = Names.named(NEW_TASK_THREADPOOL_NAME); + + + private final SingularityConfiguration configuration; + + public SingularityMainModule(final SingularityConfiguration configuration) { + this.configuration = configuration; + } + @Override public void configure(Binder binder) { binder.bind(HostAndPort.class).annotatedWith(named(HTTP_HOST_AND_PORT)).toProvider(SingularityHostAndPortProvider.class).in(Scopes.SINGLETON); @@ -110,12 +133,23 @@ public void configure(Binder binder) { binder.bind(ObjectMapper.class).toProvider(DropwizardObjectMapperProvider.class).in(Scopes.SINGLETON); binder.bind(AsyncHttpClient.class).to(SingularityHttpClient.class).in(Scopes.SINGLETON); - binder.bind(ServerProvider.class).in(Scopes.SINGLETON); binder.bind(SingularityDropwizardHealthcheck.class).in(Scopes.SINGLETON); binder.bindConstant().annotatedWith(Names.named(SERVER_ID_PROPERTY)).to(UUID.randomUUID().toString()); + binder.bind(ScheduledExecutorService.class).annotatedWith(CORE_THREADPOOL_NAMED).toProvider(new SingularityManagedScheduledExecutorServiceProvider(configuration.getCoreThreadpoolSize(), + configuration.getThreadpoolShutdownDelayInSeconds(), + "core")).in(Scopes.SINGLETON); + + binder.bind(ScheduledExecutorService.class).annotatedWith(HEALTHCHECK_THREADPOOL_NAMED).toProvider(new SingularityManagedScheduledExecutorServiceProvider(configuration.getHealthcheckStartThreads(), + configuration.getThreadpoolShutdownDelayInSeconds(), + "healthcheck")).in(Scopes.SINGLETON); + + binder.bind(ScheduledExecutorService.class).annotatedWith(NEW_TASK_THREADPOOL_NAMED).toProvider(new SingularityManagedScheduledExecutorServiceProvider(configuration.getCheckNewTasksScheduledThreads(), + configuration.getThreadpoolShutdownDelayInSeconds(), + "check-new-task")).in(Scopes.SINGLETON); + try { binder.bindConstant().annotatedWith(Names.named(HOST_ADDRESS_PROPERTY)).to(JavaUtils.getHostAddress()); } catch (SocketException e) { @@ -147,6 +181,38 @@ public HostAndPort get() { } + public static class SingularityManagedScheduledExecutorServiceProvider implements Provider, Managed { + + private final AtomicBoolean stopped = new AtomicBoolean(); + private ScheduledExecutorService service; + + private final long timeoutInSeconds; + + SingularityManagedScheduledExecutorServiceProvider(final int poolSize, final long timeoutInSeconds, final String name) { + this.service = Executors.newScheduledThreadPool(poolSize, new ThreadFactoryBuilder().setNameFormat(name + "-pool-%d").setDaemon(true).build()); + this.timeoutInSeconds = timeoutInSeconds; + } + + @Override + public synchronized ScheduledExecutorService get() { + checkState(!stopped.get(), "already stopped"); + return service; + } + + @Override + public void start() throws Exception { + // Ignored + } + + @Override + public void stop() throws Exception { + if (!stopped.getAndSet(true)) { + service.shutdown(); + service.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS); + } + } + } + @Provides @Singleton public ZooKeeperConfiguration zooKeeperConfiguration(final SingularityConfiguration config) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java index 141f9ab187..c3c13ead8c 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java @@ -17,7 +17,7 @@ public class SingularityServiceModule extends ConfigurationAwareModule lockHolder; + private ScheduledExecutorService executorService; private LeaderLatch leaderLatch; private SingularityExceptionNotifier exceptionNotifier; private SingularityAbort abort; @@ -49,14 +48,15 @@ private SingularityLeaderOnlyPoller(long pollDelay, TimeUnit pollTimeUnit, Optio this.pollTimeUnit = pollTimeUnit; this.lockHolder = lockHolder; - this.executorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(getClass().getSimpleName() + "-%d").build()); } @Inject - void injectPollerDependencies(LeaderLatch leaderLatch, + void injectPollerDependencies(@Named(SingularityMainModule.CORE_THREADPOOL_NAME) ScheduledExecutorService executorService, + LeaderLatch leaderLatch, SingularityExceptionNotifier exceptionNotifier, SingularityAbort abort, SingularityMesosSchedulerDelegator mesosScheduler) { + this.executorService = executorService; this.leaderLatch = checkNotNull(leaderLatch, "leaderLatch is null"); this.exceptionNotifier = checkNotNull(exceptionNotifier, "exceptionNotifier is null"); this.abort = checkNotNull(abort, "abort is null"); @@ -134,6 +134,5 @@ protected boolean abortsOnError() { @Override public void stop() { - MoreExecutors.shutdownAndAwaitTermination(executorService, 1, TimeUnit.SECONDS); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java index 9e90173761..566a1095c9 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java @@ -1,10 +1,7 @@ package com.hubspot.singularity.scheduler; -import io.dropwizard.lifecycle.Managed; - import java.util.Collections; import java.util.Map; -import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -18,9 +15,8 @@ import com.google.common.base.Optional; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; +import com.google.inject.name.Named; import com.hubspot.baragon.models.BaragonRequestState; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.LoadBalancerRequestType; @@ -29,6 +25,7 @@ import com.hubspot.singularity.SingularityAbort.AbortReason; import com.hubspot.singularity.SingularityLoadBalancerUpdate; import com.hubspot.singularity.SingularityLoadBalancerUpdate.LoadBalancerMethod; +import com.hubspot.singularity.SingularityMainModule; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskCleanup; import com.hubspot.singularity.SingularityTaskCleanup.TaskCleanupType; @@ -46,7 +43,7 @@ * b/c we will use a queue to kill them. */ @Singleton -public class SingularityNewTaskChecker implements Managed { +public class SingularityNewTaskChecker { private static final Logger LOG = LoggerFactory.getLogger(SingularityNewTaskChecker.class); @@ -63,7 +60,8 @@ public class SingularityNewTaskChecker implements Managed { private final SingularityExceptionNotifier exceptionNotifier; @Inject - public SingularityNewTaskChecker(SingularityConfiguration configuration, LoadBalancerClient lbClient, TaskManager taskManager, SingularityExceptionNotifier exceptionNotifier, SingularityAbort abort) { + public SingularityNewTaskChecker(@Named(SingularityMainModule.NEW_TASK_THREADPOOL_NAME) ScheduledExecutorService executorService, + SingularityConfiguration configuration, LoadBalancerClient lbClient, TaskManager taskManager, SingularityExceptionNotifier exceptionNotifier, SingularityAbort abort) { this.configuration = configuration; this.taskManager = taskManager; this.lbClient = lbClient; @@ -72,20 +70,11 @@ public SingularityNewTaskChecker(SingularityConfiguration configuration, LoadBal this.taskIdToCheck = Maps.newConcurrentMap(); this.killAfterUnhealthyMillis = TimeUnit.SECONDS.toMillis(configuration.getKillAfterTasksDoNotRunDefaultSeconds()); - this.executorService = Executors.newScheduledThreadPool(configuration.getCheckNewTasksScheduledThreads(), new ThreadFactoryBuilder().setNameFormat("SingularityNewTaskChecker-%d").build()); + this.executorService = executorService; this.exceptionNotifier = exceptionNotifier; } - @Override - public void start() { - } - - @Override - public void stop() { - MoreExecutors.shutdownAndAwaitTermination(executorService, 1, TimeUnit.SECONDS); - } - private boolean hasHealthcheck(SingularityTask task) { return task.getTaskRequest().getDeploy().getHealthcheckUri().isPresent(); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java index 7560f7cd69..42d3cd05ce 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java @@ -1,11 +1,8 @@ package com.hubspot.singularity.scheduler; -import io.dropwizard.lifecycle.Managed; - import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -23,8 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import com.google.inject.name.Named; import com.hubspot.mesos.JavaUtils; @@ -39,7 +34,7 @@ import com.hubspot.singularity.sentry.SingularityExceptionNotifier; @Singleton -public class SingularityTaskReconciliation implements Managed { +public class SingularityTaskReconciliation { private static final Logger LOG = LoggerFactory.getLogger(SingularityTaskReconciliation.class); @@ -53,7 +48,8 @@ public class SingularityTaskReconciliation implements Managed { private final SchedulerDriverSupplier schedulerDriverSupplier; @Inject - public SingularityTaskReconciliation(SingularityExceptionNotifier exceptionNotifier, + public SingularityTaskReconciliation(@Named(SingularityMainModule.CORE_THREADPOOL_NAME) ScheduledExecutorService executorService, + SingularityExceptionNotifier exceptionNotifier, TaskManager taskManager, SingularityConfiguration configuration, @Named(SingularityMainModule.SERVER_ID_PROPERTY) String serverId, @@ -68,16 +64,7 @@ public SingularityTaskReconciliation(SingularityExceptionNotifier exceptionNotif this.schedulerDriverSupplier = schedulerDriverSupplier; this.isRunningReconciliation = new AtomicBoolean(false); - this.executorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("SingularityTaskReconciliation-%d").build()); - } - - @Override - public void start() { - } - - @Override - public void stop() { - MoreExecutors.shutdownAndAwaitTermination(executorService, 1, TimeUnit.SECONDS); + this.executorService = executorService; } enum ReconciliationState { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java index a97df6798c..fab4f7836b 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java @@ -33,8 +33,6 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Module; -import com.google.inject.Provides; -import com.google.inject.Singleton; import com.google.inject.Stage; import com.google.inject.TypeLiteral; import com.google.inject.util.Modules; @@ -102,8 +100,10 @@ public void configure(Binder mainBinder) { mainBinder.install(new GuiceBundle.GuiceEnforcerModule()); mainBinder.bind(TestingServer.class).toInstance(ts); + final SingularityConfiguration configuration = getSingularityConfigurationForTestingServer(ts); + mainBinder.bind(SingularityConfiguration.class).toInstance(configuration); - mainBinder.install(Modules.override(new SingularityMainModule()) + mainBinder.install(Modules.override(new SingularityMainModule(configuration)) .with(new Module() { @Override @@ -169,9 +169,7 @@ public void configure(Binder binder) { mainBinder.bind(RequestResource.class); } - @Provides - @Singleton - public SingularityConfiguration getTestingConfiguration(final TestingServer ts) { + private static SingularityConfiguration getSingularityConfigurationForTestingServer(final TestingServer ts) { SingularityConfiguration config = new SingularityConfiguration(); config.setLoadBalancerUri("test"); @@ -190,5 +188,4 @@ public SingularityConfiguration getTestingConfiguration(final TestingServer ts) return config; } - } From a19d0625aa158ba04079cb521c53f4c19fa6e467 Mon Sep 17 00:00:00 2001 From: Henning Schmiedehausen Date: Tue, 6 Jan 2015 17:40:34 -0800 Subject: [PATCH 11/93] factor out SingularityManagedScheduledExecutorServiceProvider --- .../singularity/SingularityMainModule.java | 39 ---------------- ...nagedScheduledExecutorServiceProvider.java | 46 +++++++++++++++++++ 2 files changed, 46 insertions(+), 39 deletions(-) create mode 100644 SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java index 85cd79a46b..bb02ec9d84 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java @@ -1,23 +1,17 @@ package com.hubspot.singularity; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; import static com.google.inject.name.Names.named; - import de.neuland.jade4j.parser.Parser; import de.neuland.jade4j.parser.node.Node; import de.neuland.jade4j.template.JadeTemplate; import io.dropwizard.jetty.HttpConnectorFactory; -import io.dropwizard.lifecycle.Managed; import io.dropwizard.server.SimpleServerFactory; import java.io.IOException; import java.net.SocketException; import java.util.UUID; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import javax.inject.Inject; import javax.inject.Provider; @@ -36,7 +30,6 @@ import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.net.HostAndPort; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; @@ -181,38 +174,6 @@ public HostAndPort get() { } - public static class SingularityManagedScheduledExecutorServiceProvider implements Provider, Managed { - - private final AtomicBoolean stopped = new AtomicBoolean(); - private ScheduledExecutorService service; - - private final long timeoutInSeconds; - - SingularityManagedScheduledExecutorServiceProvider(final int poolSize, final long timeoutInSeconds, final String name) { - this.service = Executors.newScheduledThreadPool(poolSize, new ThreadFactoryBuilder().setNameFormat(name + "-pool-%d").setDaemon(true).build()); - this.timeoutInSeconds = timeoutInSeconds; - } - - @Override - public synchronized ScheduledExecutorService get() { - checkState(!stopped.get(), "already stopped"); - return service; - } - - @Override - public void start() throws Exception { - // Ignored - } - - @Override - public void stop() throws Exception { - if (!stopped.getAndSet(true)) { - service.shutdown(); - service.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS); - } - } - } - @Provides @Singleton public ZooKeeperConfiguration zooKeeperConfiguration(final SingularityConfiguration config) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java new file mode 100644 index 0000000000..8bd2ad96d5 --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java @@ -0,0 +1,46 @@ +package com.hubspot.singularity; + +import static com.google.common.base.Preconditions.checkState; + +import io.dropwizard.lifecycle.Managed; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.inject.Provider; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +public class SingularityManagedScheduledExecutorServiceProvider implements Provider, Managed { + + private final AtomicBoolean stopped = new AtomicBoolean(); + private ScheduledExecutorService service; + + private final long timeoutInSeconds; + + public SingularityManagedScheduledExecutorServiceProvider(final int poolSize, final long timeoutInSeconds, final String name) { + this.service = Executors.newScheduledThreadPool(poolSize, new ThreadFactoryBuilder().setNameFormat(name + "-pool-%d").setDaemon(true).build()); + this.timeoutInSeconds = timeoutInSeconds; + } + + @Override + public synchronized ScheduledExecutorService get() { + checkState(!stopped.get(), "already stopped"); + return service; + } + + @Override + public void start() throws Exception { + // Ignored + } + + @Override + public void stop() throws Exception { + if (!stopped.getAndSet(true)) { + service.shutdown(); + service.awaitTermination(timeoutInSeconds, TimeUnit.SECONDS); + } + } +} From 68c99dc11b7264a3eccac5a808aa55d38a3c6926 Mon Sep 17 00:00:00 2001 From: tpetr Date: Wed, 21 Jan 2015 11:33:33 -0500 Subject: [PATCH 12/93] de-dupe sentry errors by setting a proper culprit value --- .../hubspot/singularity/SingularityAbort.java | 2 +- .../SingularityLeaderController.java | 6 +- .../config/SentryConfiguration.java | 29 ++++++++ .../SingularityMesosSchedulerDelegator.java | 2 +- .../scheduler/SingularityCleaner.java | 7 +- .../SingularityHealthcheckAsyncHandler.java | 4 +- .../scheduler/SingularityHealthchecker.java | 4 +- .../SingularityLeaderOnlyPoller.java | 2 +- .../scheduler/SingularityNewTaskChecker.java | 4 +- .../SingularityScheduledJobPoller.java | 4 +- .../SingularityTaskReconciliation.java | 2 +- .../sentry/NotifyingExceptionMapper.java | 2 +- .../NotifyingUncaughtExceptionManager.java | 2 +- .../sentry/SingularityExceptionNotifier.java | 72 +++++++++++++++++-- .../singularity/smtp/SingularityMailer.java | 6 +- .../smtp/SingularitySmtpSender.java | 2 +- 16 files changed, 119 insertions(+), 31 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java index 7df061e82c..bacd960453 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java @@ -96,7 +96,7 @@ private void sendAbortNotification(AbortReason abortReason) { sendAbortMail(message); - exceptionNotifier.notify(message); + exceptionNotifier.notify(message, getClass()); } private void sendAbortMail(final String message) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java index 5ec0a539dd..dac1b7f7ec 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java @@ -83,7 +83,7 @@ public void isLeader() { statePoller.wake(); } catch (Throwable t) { LOG.error("While starting driver", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); abort.abort(AbortReason.UNRECOVERABLE_ERROR); } @@ -122,7 +122,7 @@ public void notLeader() { statePoller.wake(); } catch (Throwable t) { LOG.error("While stopping driver", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); } finally { abort.abort(AbortReason.LOST_LEADERSHIP); } @@ -196,7 +196,7 @@ public void run() { LOG.trace("Caught interrupted exception, running the loop"); } catch (Throwable t) { LOG.error("Caught exception while saving state", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); } finally { lock.unlock(); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java index 6a673c604b..a82cf6375a 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java @@ -2,6 +2,11 @@ import org.hibernate.validator.constraints.NotEmpty; +import java.util.Arrays; +import java.util.List; + +import javax.validation.constraints.NotNull; + import com.fasterxml.jackson.annotation.JsonProperty; public class SentryConfiguration { @@ -13,6 +18,14 @@ public class SentryConfiguration { @JsonProperty("prefix") private String prefix = ""; + @JsonProperty + @NotNull + private List ignoredTraceSignatures = Arrays.asList("ch.qos.logback", "org.log4j", "sun.reflect"); + + @JsonProperty + @NotNull + private List singularityTraceSignatures = Arrays.asList("com.hubspot"); + public String getDsn() { return dsn; } @@ -28,4 +41,20 @@ public String getPrefix() { public void setPrefix(String prefix) { this.prefix = prefix; } + + public List getIgnoredTraceSignatures() { + return ignoredTraceSignatures; + } + + public void setIgnoredTraceSignatures(List ignoredTraceSignatures) { + this.ignoredTraceSignatures = ignoredTraceSignatures; + } + + public List getSingularityTraceSignatures() { + return singularityTraceSignatures; + } + + public void setSingularityTraceSignatures(List singularityTraceSignatures) { + this.singularityTraceSignatures = singularityTraceSignatures; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java index c9e76e8e52..79a32b3c04 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java @@ -92,7 +92,7 @@ public void notifyStopping() { private void handleUncaughtSchedulerException(Throwable t) { LOG.error("Scheduler threw an uncaught exception - exiting", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); abort.abort(AbortReason.UNRECOVERABLE_ERROR); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java index 3c1b76e8c2..14536315e9 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java @@ -38,6 +38,7 @@ import com.hubspot.singularity.data.DeployManager; import com.hubspot.singularity.data.RequestManager; import com.hubspot.singularity.data.TaskManager; +import com.hubspot.singularity.hooks.LbUpdateException; import com.hubspot.singularity.hooks.LoadBalancerClient; import com.hubspot.singularity.scheduler.SingularityDeployHealthHelper.DeployHealth; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; @@ -461,9 +462,9 @@ private CheckLBState checkLbState(SingularityTaskId taskId) { return CheckLBState.DONE; case FAILED: case CANCELED: - final String errorMsg = String.format("LB removal request for %s (%s) got unexpected response %s", lbAddUpdate.get(), loadBalancerRequestId, lbRemoveUpdate.getLoadBalancerState()); - LOG.error(errorMsg); - exceptionNotifier.notify(errorMsg); + final String errorMessage = String.format("LB removal request for %s (%s) got unexpected response %s", lbAddUpdate.get(), loadBalancerRequestId, lbRemoveUpdate.getLoadBalancerState()); + LOG.error(errorMessage); + exceptionNotifier.notify(new LbUpdateException(errorMessage), getClass()); return CheckLBState.RETRY; case UNKNOWN: case CANCELING: diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java index 6067e846c3..30e0b5a188 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java @@ -80,7 +80,7 @@ public void saveResult(Optional statusCode, Optional responseBo } } catch (Throwable t) { LOG.error("Caught throwable while saving health check result {}, will re-enqueue", result, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); reEnqueueOrAbort(task); } @@ -91,7 +91,7 @@ private void reEnqueueOrAbort(SingularityTask task) { healthchecker.enqueueHealthcheck(task); } catch (Throwable t) { LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); abort.abort(AbortReason.UNRECOVERABLE_ERROR); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java index 6008a5056d..2f37283197 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java @@ -116,7 +116,7 @@ public void run() { asyncHealthcheck(task); } catch (Throwable t) { LOG.error("Uncaught throwable in async healthcheck", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); } } @@ -187,7 +187,7 @@ private void asyncHealthcheck(final SingularityTask task) { http.prepareRequest(builder.build()).execute(handler); } catch (Throwable t) { LOG.debug("Exception while preparing healthcheck ({}) for task ({})", uri, task.getTaskId(), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); saveFailure(handler, String.format("Healthcheck failed due to exception: %s", t.getMessage())); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java index ea28fdded3..49a2e586d1 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java @@ -109,7 +109,7 @@ private void runActionIfLeaderAndMesosIsRunning() { runActionOnPoll(); } catch (Throwable t) { LOG.error("Caught an exception while running {}", getClass().getSimpleName(), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); if (abortsOnError()) { abort.abort(AbortReason.UNRECOVERABLE_ERROR); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java index 9e90173761..1d8c8e2d33 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java @@ -169,7 +169,7 @@ public void run() { checkTask(task); } catch (Throwable t) { LOG.error("Uncaught throwable in task check for task {}, re-enqueing", task, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, SingularityNewTaskChecker.class); reEnqueueCheckOrAbort(task); } @@ -182,7 +182,7 @@ private void reEnqueueCheckOrAbort(SingularityTask task) { reEnqueueCheck(task); } catch (Throwable t) { LOG.error("Uncaught throwable re-enqueuing task check for task {}, aborting", task, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); abort.abort(AbortReason.UNRECOVERABLE_ERROR); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java index b474ff17ec..6093d62621 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java @@ -115,7 +115,7 @@ private Optional getExpectedRuntime(SingularityRequestWithState request, S cronExpression = new CronExpression(request.getRequest().getQuartzScheduleSafe()); } catch (ParseException e) { LOG.warn("Unable to parse cron for {} ({})", taskId, request.getRequest().getQuartzScheduleSafe(), e); - exceptionNotifier.notify(e); + exceptionNotifier.notify(e, getClass()); return Optional.absent(); } @@ -125,7 +125,7 @@ private Optional getExpectedRuntime(SingularityRequestWithState request, S if (nextRunAtDate == null) { String msg = String.format("No next run date found for %s (%s)", taskId, request.getRequest().getQuartzScheduleSafe()); LOG.warn(msg); - exceptionNotifier.notify(msg); + exceptionNotifier.notify(msg, getClass()); return Optional.absent(); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java index 7560f7cd69..119fdd45c9 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java @@ -126,7 +126,7 @@ public void run() { checkReconciliation(driver, reconciliationStart, remainingTaskIds, numTimes + 1); } catch (Throwable t) { LOG.error("While checking for reconciliation tasks", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, SingularityTaskReconciliation.class); abort.abort(AbortReason.UNRECOVERABLE_ERROR); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java index 33853538d1..1c00becc9a 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java @@ -20,7 +20,7 @@ public Response toResponse(final Exception e) { final Response response = super.toResponse(e); if (response.getStatus() >= 500) { - notifier.notify(e); + notifier.notify(e, getClass()); } return response; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java index 2b803d7fec..f8a9e59296 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java @@ -17,6 +17,6 @@ public NotifyingUncaughtExceptionManager(SingularityExceptionNotifier notifier) @Override public void uncaughtException(Thread t, Throwable e) { LOG.error("Uncaught exception!", e); - notifier.notify(e); + notifier.notify(e, getClass()); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java b/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java index 0b59d25385..81d05ab06d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java @@ -11,8 +11,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; import com.google.inject.Inject; import com.hubspot.singularity.config.SentryConfiguration; @@ -23,6 +30,31 @@ public class SingularityExceptionNotifier { private final Optional raven; private final Optional sentryConfiguration; + private final Predicate isSingularityTraceSignature = new Predicate() { + @Override + public boolean apply(StackTraceElement input) { + for (String sig : sentryConfiguration.get().getSingularityTraceSignatures()) { + if (!input.getClassName().contains(sig)) { + return true; + } + } + + return false; + } + }; + + private final Predicate isIgnoredStackTraceElement = new Predicate() { + @Override + public boolean apply(StackTraceElement input) { + for (String sig : sentryConfiguration.get().getIgnoredTraceSignatures()) { + if (input.getClassName().contains(sig)) { + return true; + } + } + return false; + } + }; + @Inject public SingularityExceptionNotifier(Optional sentryConfiguration) { this.sentryConfiguration = sentryConfiguration; @@ -41,42 +73,47 @@ private String getPrefix() { return sentryConfiguration.get().getPrefix() + " "; } - public void notify(Throwable t) { + public void notify(Throwable t, Class logger) { if (!raven.isPresent()) { return; } try { - notify(raven.get(), t); + notify(raven.get(), t, logger); } catch (Throwable e) { LOG.error("Caught exception while trying to report {} to Sentry", t.getMessage(), e); } } - public void notify(String message) { + public void notify(String message, Class logger) { if (!raven.isPresent()) { return; } try { - notify(raven.get(), message); + notify(raven.get(), message, logger); } catch (Throwable e) { LOG.error("Caught exception while trying to report {} to Sentry", message, e); } } - private void notify(Raven raven, String message) { + private void notify(Raven raven, String message, Class logger) { final EventBuilder eventBuilder = new EventBuilder() .setMessage(getPrefix() + message) + .setLogger(logger.getName()) .setLevel(Event.Level.ERROR); sendEvent(raven, eventBuilder); } - private void notify(Raven raven, Throwable t) { + private void notify(Raven raven, Throwable t, Class logger) { + final String culprit = determineCulprit(t); + final EventBuilder eventBuilder = new EventBuilder() - .setMessage(getPrefix() + t.getMessage()) + .setCulprit(getPrefix() + culprit) + .setMessage(Strings.nullToEmpty(t.getMessage())) .setLevel(Event.Level.ERROR) + .setLogger(logger.getName()) .addSentryInterface(new ExceptionInterface(t)); sendEvent(raven, eventBuilder); @@ -88,4 +125,25 @@ private void sendEvent(Raven raven, final EventBuilder eventBuilder) { raven.sendEvent(eventBuilder.build()); } + private String determineCulprit(final Throwable exception) { + Throwable cause = exception; + String culprit = null; + + while (cause != null) { + final List elements = new ArrayList<>(cause.getStackTrace().length); + + if(elements.size() > 0){ + final StackTraceElement root = Iterables.tryFind(elements, Predicates.not(isIgnoredStackTraceElement)).or(elements.get(0)); + final Optional lastSingularityElement = Iterables.tryFind(elements, Predicates.and(isSingularityTraceSignature, Predicates.not(isIgnoredStackTraceElement))); + + final String singularityCulprit = lastSingularityElement.isPresent() ? String.format("%s.%s():%s", lastSingularityElement.get().getClassName(), lastSingularityElement.get().getMethodName(), lastSingularityElement.get().getLineNumber()) : ""; + culprit = String.format("%s.%s():%s:%s|%s", root.getClassName(), root.getMethodName(), root.getLineNumber(), Throwables.getRootCause(exception).getClass().getName(), singularityCulprit); + } + + cause = cause.getCause(); + } + + return culprit; + } + } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java index 5299e734ae..3874f3ed04 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java @@ -258,7 +258,7 @@ public void run() { prepareTaskCompletedMail(taskId, request, taskState); } catch (Throwable t) { LOG.error("While preparing task completed mail for {}", taskId, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, SingularityMailer.class); } } }); @@ -339,7 +339,7 @@ public void run() { prepareRequestMail(request, type, user); } catch (Throwable t) { LOG.error("While preparing request mail for {} / {}", request, type, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, SingularityMailer.class); } } }); @@ -397,7 +397,7 @@ public void run() { prepareRequestInCooldownMail(request); } catch (Throwable t) { LOG.error("While preparing request in cooldown mail for {}", request, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, SingularityMailer.class); } } }); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java index 686cdf9e0f..4b67939517 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java @@ -136,7 +136,7 @@ private void sendMail(List toList, List ccList, String subject, Transport.send(message); } catch (Throwable t) { LOG.warn("Unable to send message {}", getEmailLogFormat(toList, subject), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, getClass()); } } From 85bcfa90b7a68fb6523b3bcc3bae33148d38b71c Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 26 Jan 2015 11:41:58 -0500 Subject: [PATCH 13/93] dont bother with determinCulprit(), couldnt get it working --- .../config/SentryConfiguration.java | 24 -------- .../sentry/SingularityExceptionNotifier.java | 60 +++---------------- 2 files changed, 8 insertions(+), 76 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java index a82cf6375a..603693af8d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SentryConfiguration.java @@ -18,14 +18,6 @@ public class SentryConfiguration { @JsonProperty("prefix") private String prefix = ""; - @JsonProperty - @NotNull - private List ignoredTraceSignatures = Arrays.asList("ch.qos.logback", "org.log4j", "sun.reflect"); - - @JsonProperty - @NotNull - private List singularityTraceSignatures = Arrays.asList("com.hubspot"); - public String getDsn() { return dsn; } @@ -41,20 +33,4 @@ public String getPrefix() { public void setPrefix(String prefix) { this.prefix = prefix; } - - public List getIgnoredTraceSignatures() { - return ignoredTraceSignatures; - } - - public void setIgnoredTraceSignatures(List ignoredTraceSignatures) { - this.ignoredTraceSignatures = ignoredTraceSignatures; - } - - public List getSingularityTraceSignatures() { - return singularityTraceSignatures; - } - - public void setSingularityTraceSignatures(List singularityTraceSignatures) { - this.singularityTraceSignatures = singularityTraceSignatures; - } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java b/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java index 52fce895ec..3ac54fb591 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java @@ -34,31 +34,6 @@ public class SingularityExceptionNotifier { private final Optional raven; private final Optional sentryConfiguration; - private final Predicate isSingularityTraceSignature = new Predicate() { - @Override - public boolean apply(StackTraceElement input) { - for (String sig : sentryConfiguration.get().getSingularityTraceSignatures()) { - if (!input.getClassName().contains(sig)) { - return true; - } - } - - return false; - } - }; - - private final Predicate isIgnoredStackTraceElement = new Predicate() { - @Override - public boolean apply(StackTraceElement input) { - for (String sig : sentryConfiguration.get().getIgnoredTraceSignatures()) { - if (input.getClassName().contains(sig)) { - return true; - } - } - return false; - } - }; - @Inject public SingularityExceptionNotifier(Optional sentryConfiguration) { this.sentryConfiguration = sentryConfiguration; @@ -78,34 +53,13 @@ private String getPrefix() { } private String getCallingClassName(StackTraceElement[] stackTrace) { - if (stackTrace != null && stackTrace.length > 1) { - return stackTrace[1].getClassName(); + if (stackTrace != null && stackTrace.length > 2) { + return stackTrace[2].getClassName(); } else { return "(unknown)"; } } - private String determineCulprit(final Throwable exception) { - Throwable cause = exception; - String culprit = null; - - while (cause != null) { - final List elements = new ArrayList<>(cause.getStackTrace().length); - - if(elements.size() > 0){ - final StackTraceElement root = Iterables.tryFind(elements, Predicates.not(isIgnoredStackTraceElement)).or(elements.get(0)); - final Optional lastSingularityElement = Iterables.tryFind(elements, Predicates.and(isSingularityTraceSignature, Predicates.not(isIgnoredStackTraceElement))); - - final String singularityCulprit = lastSingularityElement.isPresent() ? String.format("%s.%s():%s", lastSingularityElement.get().getClassName(), lastSingularityElement.get().getMethodName(), lastSingularityElement.get().getLineNumber()) : ""; - culprit = String.format("%s.%s():%s:%s|%s", root.getClassName(), root.getMethodName(), root.getLineNumber(), Throwables.getRootCause(exception).getClass().getName(), singularityCulprit); - } - - cause = cause.getCause(); - } - - return culprit; - } - private void sendEvent(Raven raven, final EventBuilder eventBuilder) { raven.runBuilderHelpers(eventBuilder); @@ -117,13 +71,13 @@ public void notify(Throwable t, Map extraData) { return; } - final String culprit = determineCulprit(t); + final StackTraceElement[] currentThreadStackTrace = Thread.currentThread().getStackTrace(); final EventBuilder eventBuilder = new EventBuilder() - .setCulprit(getPrefix() + culprit) + .setCulprit(getPrefix() + t.getMessage()) .setMessage(Strings.nullToEmpty(t.getMessage())) .setLevel(Event.Level.ERROR) - .setLogger(getCallingClassName(Thread.currentThread().getStackTrace())) + .setLogger(getCallingClassName(currentThreadStackTrace)) .addSentryInterface(new ExceptionInterface(t)); if (extraData != null && !extraData.isEmpty()) { @@ -140,10 +94,12 @@ public void notify(String subject, Map extraData) { return; } + final StackTraceElement[] currentThreadStackTrace = Thread.currentThread().getStackTrace(); + final EventBuilder eventBuilder = new EventBuilder() .setMessage(getPrefix() + subject) .setLevel(Event.Level.ERROR) - .setLogger(getCallingClassName(Thread.currentThread().getStackTrace())); + .setLogger(getCallingClassName(currentThreadStackTrace)); if (extraData != null && !extraData.isEmpty()) { for (Map.Entry entry : extraData.entrySet()) { From 5b81fdf7d9e5886add9fd9d54d2b19fed4dbd054 Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 26 Jan 2015 11:55:01 -0500 Subject: [PATCH 14/93] fix import --- .../com/hubspot/singularity/scheduler/SingularityCleaner.java | 1 - 1 file changed, 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java index f8d242958a..56fb5b2a71 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java @@ -39,7 +39,6 @@ import com.hubspot.singularity.data.DeployManager; import com.hubspot.singularity.data.RequestManager; import com.hubspot.singularity.data.TaskManager; -import com.hubspot.singularity.hooks.LbUpdateException; import com.hubspot.singularity.hooks.LoadBalancerClient; import com.hubspot.singularity.scheduler.SingularityDeployHealthHelper.DeployHealth; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; From 6bb0e918afbabf4f16ac7a8ace3d276b35a5c8c1 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Mon, 26 Jan 2015 13:30:13 -0500 Subject: [PATCH 15/93] add thread factory --- .../singularity/SingularityMainModule.java | 15 ++--- ...anagedScheduledExecutorServiceFactory.java | 63 +++++++++++++++++++ ...nagedScheduledExecutorServiceProvider.java | 1 - .../SingularityLeaderOnlyPoller.java | 8 +-- .../SingularityTaskReconciliation.java | 5 +- 5 files changed, 74 insertions(+), 18 deletions(-) create mode 100644 SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceFactory.java diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java index bb02ec9d84..a474463ed6 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java @@ -2,9 +2,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.inject.name.Names.named; -import de.neuland.jade4j.parser.Parser; -import de.neuland.jade4j.parser.node.Node; -import de.neuland.jade4j.template.JadeTemplate; import io.dropwizard.jetty.HttpConnectorFactory; import io.dropwizard.server.SimpleServerFactory; @@ -60,6 +57,10 @@ import com.hubspot.singularity.smtp.SingularitySmtpSender; import com.ning.http.client.AsyncHttpClient; +import de.neuland.jade4j.parser.Parser; +import de.neuland.jade4j.parser.node.Node; +import de.neuland.jade4j.template.JadeTemplate; + public class SingularityMainModule implements Module { @@ -76,16 +77,12 @@ public class SingularityMainModule implements Module { public static final String HTTP_HOST_AND_PORT = "http.host.and.port"; - public static final String CORE_THREADPOOL_NAME = "_core_threadpool"; - public static final Named CORE_THREADPOOL_NAMED = Names.named(CORE_THREADPOOL_NAME); - public static final String HEALTHCHECK_THREADPOOL_NAME = "_healthcheck_threadpool"; public static final Named HEALTHCHECK_THREADPOOL_NAMED = Names.named(HEALTHCHECK_THREADPOOL_NAME); public static final String NEW_TASK_THREADPOOL_NAME = "_new_task_threadpool"; public static final Named NEW_TASK_THREADPOOL_NAMED = Names.named(NEW_TASK_THREADPOOL_NAME); - private final SingularityConfiguration configuration; public SingularityMainModule(final SingularityConfiguration configuration) { @@ -131,9 +128,7 @@ public void configure(Binder binder) { binder.bind(SingularityDropwizardHealthcheck.class).in(Scopes.SINGLETON); binder.bindConstant().annotatedWith(Names.named(SERVER_ID_PROPERTY)).to(UUID.randomUUID().toString()); - binder.bind(ScheduledExecutorService.class).annotatedWith(CORE_THREADPOOL_NAMED).toProvider(new SingularityManagedScheduledExecutorServiceProvider(configuration.getCoreThreadpoolSize(), - configuration.getThreadpoolShutdownDelayInSeconds(), - "core")).in(Scopes.SINGLETON); + binder.bind(SingularityManagedScheduledExecutorServiceFactory.class).in(Scopes.SINGLETON); binder.bind(ScheduledExecutorService.class).annotatedWith(HEALTHCHECK_THREADPOOL_NAMED).toProvider(new SingularityManagedScheduledExecutorServiceProvider(configuration.getHealthcheckStartThreads(), configuration.getThreadpoolShutdownDelayInSeconds(), diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceFactory.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceFactory.java new file mode 100644 index 0000000000..317355d812 --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceFactory.java @@ -0,0 +1,63 @@ +package com.hubspot.singularity; + +import static com.google.common.base.Preconditions.checkState; +import io.dropwizard.lifecycle.Managed; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.hubspot.singularity.config.SingularityConfiguration; + +@Singleton +public class SingularityManagedScheduledExecutorServiceFactory implements Managed { + + private final AtomicBoolean stopped = new AtomicBoolean(); + private final List executorPools = new ArrayList<>(); + + private final long timeoutInMillis; + + @Inject + public SingularityManagedScheduledExecutorServiceFactory(final SingularityConfiguration configuration) { + this.timeoutInMillis = TimeUnit.SECONDS.toMillis(configuration.getThreadpoolShutdownDelayInSeconds()); + } + + public synchronized ScheduledExecutorService get(String name) { + checkState(!stopped.get(), "already stopped"); + ScheduledExecutorService service = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(name + "-%d").setDaemon(true).build()); + executorPools.add(service); + return service; + } + + @Override + public void start() throws Exception { + // Ignored + } + + @Override + public void stop() throws Exception { + if (!stopped.getAndSet(true)) { + for (ScheduledExecutorService service : executorPools) { + service.shutdown(); + } + + long timeoutLeftInMillis = timeoutInMillis; + + for (ScheduledExecutorService service : executorPools) { + final long start = System.currentTimeMillis(); + + if (!service.awaitTermination(timeoutLeftInMillis, TimeUnit.MILLISECONDS)) { + return; + } + + timeoutLeftInMillis -= (System.currentTimeMillis() - start); + } + } + } +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java index 8bd2ad96d5..f6466b8023 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceProvider.java @@ -1,7 +1,6 @@ package com.hubspot.singularity; import static com.google.common.base.Preconditions.checkState; - import io.dropwizard.lifecycle.Managed; import java.util.concurrent.Executors; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java index 1b2fb49828..b03f9b396b 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java @@ -13,11 +13,10 @@ import com.google.common.base.Optional; import com.google.inject.Inject; -import com.google.inject.name.Named; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.SingularityAbort; import com.hubspot.singularity.SingularityAbort.AbortReason; -import com.hubspot.singularity.SingularityMainModule; +import com.hubspot.singularity.SingularityManagedScheduledExecutorServiceFactory; import com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; @@ -47,16 +46,15 @@ private SingularityLeaderOnlyPoller(long pollDelay, TimeUnit pollTimeUnit, Optio this.pollDelay = pollDelay; this.pollTimeUnit = pollTimeUnit; this.lockHolder = lockHolder; - } @Inject - void injectPollerDependencies(@Named(SingularityMainModule.CORE_THREADPOOL_NAME) ScheduledExecutorService executorService, + void injectPollerDependencies(SingularityManagedScheduledExecutorServiceFactory executorServiceFactory, LeaderLatch leaderLatch, SingularityExceptionNotifier exceptionNotifier, SingularityAbort abort, SingularityMesosSchedulerDelegator mesosScheduler) { - this.executorService = executorService; + this.executorService = executorServiceFactory.get(getClass().getSimpleName()); this.leaderLatch = checkNotNull(leaderLatch, "leaderLatch is null"); this.exceptionNotifier = checkNotNull(exceptionNotifier, "exceptionNotifier is null"); this.abort = checkNotNull(abort, "abort is null"); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java index 42d3cd05ce..ee675dcf76 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java @@ -26,6 +26,7 @@ import com.hubspot.singularity.SingularityAbort; import com.hubspot.singularity.SingularityAbort.AbortReason; import com.hubspot.singularity.SingularityMainModule; +import com.hubspot.singularity.SingularityManagedScheduledExecutorServiceFactory; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskStatusHolder; import com.hubspot.singularity.config.SingularityConfiguration; @@ -48,7 +49,7 @@ public class SingularityTaskReconciliation { private final SchedulerDriverSupplier schedulerDriverSupplier; @Inject - public SingularityTaskReconciliation(@Named(SingularityMainModule.CORE_THREADPOOL_NAME) ScheduledExecutorService executorService, + public SingularityTaskReconciliation(SingularityManagedScheduledExecutorServiceFactory executorServiceFactory, SingularityExceptionNotifier exceptionNotifier, TaskManager taskManager, SingularityConfiguration configuration, @@ -64,7 +65,7 @@ public SingularityTaskReconciliation(@Named(SingularityMainModule.CORE_THREADPOO this.schedulerDriverSupplier = schedulerDriverSupplier; this.isRunningReconciliation = new AtomicBoolean(false); - this.executorService = executorService; + this.executorService = executorServiceFactory.get(getClass().getSimpleName()); } enum ReconciliationState { From d6e8120ca5e310105faac8be619e1614c67d39fb Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Wed, 28 Jan 2015 14:55:44 -0500 Subject: [PATCH 16/93] Add docs for LogrotateTemplateContext. --- .../models/LogrotateTemplateContext.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java index c90ec910aa..5d7b59b4c8 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java @@ -3,11 +3,19 @@ import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; import com.hubspot.singularity.executor.task.SingularityExecutorTaskDefinition; +/** + * Handlebars context for generating logrotate.conf files. + * Check `man logrotate` for more information. + */ public class LogrotateTemplateContext { private final SingularityExecutorTaskDefinition taskDefinition; private final SingularityExecutorConfiguration configuration; + /** + * @param configuration configuration to pull from. + * @param taskDefinition information about the task we're writing the logrotate configs for. + */ public LogrotateTemplateContext(SingularityExecutorConfiguration configuration, SingularityExecutorTaskDefinition taskDefinition) { this.configuration = configuration; this.taskDefinition = taskDefinition; @@ -17,18 +25,35 @@ public String getRotateDateformat() { return configuration.getLogrotateDateformat(); } + /** + * Log files are rotated count times before being removed. + * @return String count. + */ public String getRotateCount() { return configuration.getLogrotateCount(); } + /** + * Remove rotated logs older than $count days. + * The age is only checked if the logfile is to be rotated. + * @return String days. + */ public String getMaxageDays() { return configuration.getLogrotateMaxageDays(); } + /** + * Logs are moved into $directory for rotation. + * @return String directory. + */ public String getRotateDirectory() { return configuration.getLogrotateToDirectory(); } + /** + * Extra files for logrotate to rotate. + * @return String[] filenames to rotate. + */ public String[] getExtrasFiles() { final String[] original = configuration.getLogrotateExtrasFiles(); final String[] transformed = new String[original.length]; @@ -40,10 +65,20 @@ public String[] getExtrasFiles() { return transformed; } + /** + * dateformat for extra files. + * Only %Y %m %d and %s specifiers are allowed. + * @return String dateformat (e.g. "-%Y%m%d%s"). + */ public String getExtrasDateformat() { return configuration.getLogrotateExtrasDateformat(); } + /** + * Default log to logrotate, defaults to service.log. + * This if this log doesn't exist, logrotate will return an error message. + * @return String filename to rotate. + */ public String getLogfile() { return taskDefinition.getServiceLogOut(); } From 7e4e15d422a72cef0fac8d072d642bda79a29bb2 Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Wed, 28 Jan 2015 14:59:44 -0500 Subject: [PATCH 17/93] Javadoc doesn't need types. --- .../executor/models/LogrotateTemplateContext.java | 12 ++++++------ .../task/SingularityExecutorTaskLogManager.java | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java index 5d7b59b4c8..3717643449 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java @@ -27,7 +27,7 @@ public String getRotateDateformat() { /** * Log files are rotated count times before being removed. - * @return String count. + * @return count. */ public String getRotateCount() { return configuration.getLogrotateCount(); @@ -36,7 +36,7 @@ public String getRotateCount() { /** * Remove rotated logs older than $count days. * The age is only checked if the logfile is to be rotated. - * @return String days. + * @return days. */ public String getMaxageDays() { return configuration.getLogrotateMaxageDays(); @@ -44,7 +44,7 @@ public String getMaxageDays() { /** * Logs are moved into $directory for rotation. - * @return String directory. + * @return directory. */ public String getRotateDirectory() { return configuration.getLogrotateToDirectory(); @@ -52,7 +52,7 @@ public String getRotateDirectory() { /** * Extra files for logrotate to rotate. - * @return String[] filenames to rotate. + * @return filenames to rotate. */ public String[] getExtrasFiles() { final String[] original = configuration.getLogrotateExtrasFiles(); @@ -68,7 +68,7 @@ public String[] getExtrasFiles() { /** * dateformat for extra files. * Only %Y %m %d and %s specifiers are allowed. - * @return String dateformat (e.g. "-%Y%m%d%s"). + * @return dateformat (e.g. "-%Y%m%d%s"). */ public String getExtrasDateformat() { return configuration.getLogrotateExtrasDateformat(); @@ -77,7 +77,7 @@ public String getExtrasDateformat() { /** * Default log to logrotate, defaults to service.log. * This if this log doesn't exist, logrotate will return an error message. - * @return String filename to rotate. + * @return filename to rotate. */ public String getLogfile() { return taskDefinition.getServiceLogOut(); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index aed305a579..f3552cb1f2 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -100,6 +100,10 @@ private boolean removeLogrotateFile() { return true; } + /** + * Trigger manual logrotate run. + * @return boolean + */ public boolean manualLogrotate() { if (!Files.exists(getLogrotateConfPath())) { log.info("{} did not exist, skipping manual logrotation", getLogrotateConfPath()); From 29640a425f1652aabe05ed3d809b809a30bd9efa Mon Sep 17 00:00:00 2001 From: Henning Schmiedehausen Date: Fri, 16 Jan 2015 16:22:28 -0800 Subject: [PATCH 18/93] Add switches for auto-deploy-id generation. Useful for testing, auto-generate the deploy id. --- .../api/SingularityDeployRequest.java | 6 +++++ .../config/SingularityConfiguration.java | 24 +++++++++++++++++ .../data/SingularityValidator.java | 27 ++++++++++++++++--- .../singularity/resources/DeployResource.java | 4 +-- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java index 942f2e9ab4..48fb079505 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java @@ -1,6 +1,7 @@ package com.hubspot.singularity.api; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Optional; import com.hubspot.singularity.SingularityDeploy; @@ -32,6 +33,11 @@ public Optional getUnpauseOnSuccessfulDeploy() { return unpauseOnSuccessfulDeploy; } + @JsonIgnore + public boolean isUnpauseOnSuccessfulDeploy() { + return unpauseOnSuccessfulDeploy.or(Boolean.FALSE); + } + @ApiModelProperty(required=true, value="The Singularity deploy object") public SingularityDeploy getDeploy() { return deploy; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index 7455baf5ae..62a36fb9eb 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -7,6 +7,8 @@ import java.util.concurrent.TimeUnit; import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -142,6 +144,12 @@ public class SingularityConfiguration extends Configuration { /** If true, the event system waits for all listeners having processed an event. */ private boolean waitForListeners = true; + private boolean createDeployIds = false; + + @Min(4) + @Max(32) + private int deployIdLength = 8; + @JsonProperty("zookeeper") @Valid private ZooKeeperConfiguration zooKeeperConfiguration; @@ -609,4 +617,20 @@ public void setListenerThreadpoolSize(int listenerThreadpoolSize) { public void setWaitForListeners(boolean waitForListeners) { this.waitForListeners = waitForListeners; } + + public boolean isCreateDeployIds() { + return createDeployIds; + } + + public void setCreateDeployIds(boolean createDeployIds) { + this.createDeployIds = createDeployIds; + } + + public int getDeployIdLength() { + return deployIdLength; + } + + public void setDeployIdLength(int deployIdLength) { + this.deployIdLength = deployIdLength; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java b/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java index 044d4e2f9d..625ef907b6 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java @@ -4,6 +4,7 @@ import static com.hubspot.singularity.WebExceptions.checkBadRequest; import java.util.List; +import java.util.UUID; import javax.inject.Singleton; @@ -13,6 +14,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.collect.Lists; +import com.google.common.hash.Hashing; import com.google.inject.Inject; import com.hubspot.mesos.Resources; import com.hubspot.mesos.SingularityDockerInfo; @@ -20,6 +22,7 @@ import com.hubspot.mesos.SingularityPortMappingType; import com.hubspot.singularity.ScheduleType; import com.hubspot.singularity.SingularityDeploy; +import com.hubspot.singularity.SingularityDeployBuilder; import com.hubspot.singularity.SingularityRequest; import com.hubspot.singularity.config.SingularityConfiguration; import com.hubspot.singularity.data.history.DeployHistoryHelper; @@ -39,6 +42,8 @@ public class SingularityValidator { private final int defaultMemoryMb; private final int maxMemoryMbPerInstance; private final boolean allowRequestsWithoutOwners; + private final boolean createDeployIds; + private final int deployIdLength; private final DeployHistoryHelper deployHistoryHelper; private final Resources defaultResources; @@ -47,6 +52,8 @@ public SingularityValidator(SingularityConfiguration configuration, DeployHistor this.maxDeployIdSize = configuration.getMaxDeployIdSize(); this.maxRequestIdSize = configuration.getMaxRequestIdSize(); this.allowRequestsWithoutOwners = configuration.isAllowRequestsWithoutOwners(); + this.createDeployIds = configuration.isCreateDeployIds(); + this.deployIdLength = configuration.getDeployIdLength(); this.deployHistoryHelper = deployHistoryHelper; this.defaultCpus = configuration.getMesosConfiguration().getDefaultCpus(); @@ -154,14 +161,20 @@ public SingularityRequest checkSingularityRequest(SingularityRequest request, Op return request.toBuilder().setQuartzSchedule(Optional.fromNullable(quartzSchedule)).build(); } - public void checkDeploy(SingularityRequest request, SingularityDeploy deploy) { - + public SingularityDeploy checkDeploy(SingularityRequest request, SingularityDeploy deploy) { checkNotNull(request, "request is null"); checkNotNull(deploy, "deploy is null"); String deployId = deploy.getId(); - checkBadRequest(deployId != null, "Id must not be null"); + if (deployId == null) { + checkBadRequest(createDeployIds, "Id must not be null"); + SingularityDeployBuilder builder = deploy.toBuilder(); + builder.setId(createUniqueDeployId()); + deploy = builder.build(); + deployId = deploy.getId(); + } + checkBadRequest(!deployId.contains("/") && !deployId.contains("-"), "Id must not be null and can not contain / or - characters"); checkBadRequest(deployId.length() < maxDeployIdSize, "Deploy id must be less than %s characters, it is %s (%s)", maxDeployIdSize, deployId.length(), deployId); checkBadRequest(deploy.getRequestId() != null && deploy.getRequestId().equals(request.getId()), "Deploy id must match request id"); @@ -184,6 +197,14 @@ public void checkDeploy(SingularityRequest request, SingularityDeploy deploy) { } checkBadRequest(deployHistoryHelper.isDeployIdAvailable(request.getId(), deployId), "Can not deploy a deploy that has already been deployed"); + + return deploy; + } + + private String createUniqueDeployId() { + UUID id = UUID.randomUUID(); + String result = Hashing.sha256().newHasher().putLong(id.getLeastSignificantBits()).putLong(id.getMostSignificantBits()).hash().toString(); + return result.substring(0, deployIdLength); } private void checkDocker(SingularityDeploy deploy) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java index 277ca555a3..8e05ed25a2 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java @@ -89,11 +89,11 @@ public SingularityRequestParent deploy(@ApiParam(required=true) SingularityDeplo SingularityRequestWithState requestWithState = fetchRequestWithState(requestId); SingularityRequest request = requestWithState.getRequest(); - if (!deployRequest.getUnpauseOnSuccessfulDeploy().isPresent() || !deployRequest.getUnpauseOnSuccessfulDeploy().get().booleanValue()) { + if (!deployRequest.isUnpauseOnSuccessfulDeploy()) { checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to deploy (it must be manually unpaused first)", requestWithState.getRequest().getId()); } - validator.checkDeploy(request, deploy); + deploy = validator.checkDeploy(request, deploy); final long now = System.currentTimeMillis(); From b25a40513e4c548ce5a763384ded01c574623e04 Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Fri, 30 Jan 2015 10:20:22 -0500 Subject: [PATCH 19/93] Add configuration for extra files backed up to S3. --- .../SingularityExecutorConfiguration.java | 19 ++++++++++++++++-- ...ingularityExecutorConfigurationLoader.java | 1 + .../SingularityExecutorTaskLogManager.java | 20 +++++++++++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java index 1100dd7513..26e330e31b 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java @@ -48,6 +48,11 @@ public class SingularityExecutorConfiguration { private final String logrotateExtrasDateformat; private final String[] logrotateExtrasFiles; + /** + * Extra files to backup to S3 besides the service log. + */ + private final String[] s3FilesToBackup; + private final Path logMetadataDirectory; private final String logMetadataSuffix; @@ -84,6 +89,7 @@ public SingularityExecutorConfiguration( @Named(SingularityExecutorConfigurationLoader.MAX_TASK_MESSAGE_LENGTH) String maxTaskMessageLength, @Named(SingularityRunnerBaseConfigurationLoader.LOG_METADATA_DIRECTORY) String logMetadataDirectory, @Named(SingularityRunnerBaseConfigurationLoader.LOG_METADATA_SUFFIX) String logMetadataSuffix, + @Named(SingularityExecutorConfigurationLoader.S3_FILES_TO_BACKUP) String s3FilesToBackup, @Named(SingularityExecutorConfigurationLoader.S3_UPLOADER_BUCKET) String s3Bucket, @Named(SingularityExecutorConfigurationLoader.S3_UPLOADER_PATTERN) String s3KeyPattern, @Named(SingularityRunnerBaseConfigurationLoader.S3_METADATA_DIRECTORY) String s3MetadataDirectory, @@ -126,6 +132,11 @@ public SingularityExecutorConfiguration( this.logrotateCount = logrotateCount; this.logrotateMaxageDays = logrotateMaxageDays; this.logrotateDateformat = logrotateDateformat; + if ((s3FilesToBackup != null) && (s3FilesToBackup.trim().length() > 0)) { + this.s3FilesToBackup = s3FilesToBackup.split(","); + } else { + this.s3FilesToBackup = new String[0]; + } this.s3Bucket = s3Bucket; this.s3KeyPattern = s3KeyPattern; this.s3MetadataSuffix = s3MetadataSuffix; @@ -273,6 +284,10 @@ public Path getS3MetadataDirectory() { return s3MetadataDirectory; } + public String[] getS3FilesToBackup() { + return s3FilesToBackup; + } + public String getS3KeyPattern() { return s3KeyPattern; } @@ -311,8 +326,8 @@ public String toString() { + logrotateConfDirectory + ", logrotateToDirectory=" + logrotateToDirectory + ", logrotateMaxageDays=" + logrotateMaxageDays + ", logrotateCount=" + logrotateCount + ", logrotateDateformat=" + logrotateDateformat + ", logrotateExtrasDateformat=" + logrotateExtrasDateformat + ", logrotateExtrasFiles=" + Arrays.toString(logrotateExtrasFiles) + ", logMetadataDirectory=" + logMetadataDirectory + ", logMetadataSuffix=" + logMetadataSuffix + ", tailLogLinesToSave=" + tailLogLinesToSave + ", serviceFinishedTailLog=" + serviceFinishedTailLog - + ", s3MetadataSuffix=" + s3MetadataSuffix + ", s3MetadataDirectory=" + s3MetadataDirectory + ", s3KeyPattern=" + s3KeyPattern + ", s3Bucket=" + s3Bucket + ", useLocalDownloadService=" - + useLocalDownloadService + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + ", maxTaskThreads=" + maxTaskThreads + "]"; + + ", s3MetadataSuffix=" + s3MetadataSuffix + ", s3MetadataDirectory=" + s3MetadataDirectory + ", s3FilesToBackup=" + Arrays.toString(s3FilesToBackup) + ", s3KeyPattern=" + s3KeyPattern + ", s3Bucket=" + + s3Bucket + ", useLocalDownloadService=" + useLocalDownloadService + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + ", maxTaskThreads=" + maxTaskThreads + "]"; } } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java index 15174b5721..53ede37740 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java @@ -47,6 +47,7 @@ public class SingularityExecutorConfigurationLoader extends SingularityConfigura public static final String TAIL_LOG_LINES_TO_SAVE = "executor.service.log.tail.lines.to.save"; public static final String TAIL_LOG_FILENAME = "executor.service.log.tail.file.name"; + public static final String S3_FILES_TO_BACKUP = "executor.s3.uploader.extra.files"; public static final String S3_UPLOADER_PATTERN = "executor.s3.uploader.pattern"; public static final String S3_UPLOADER_BUCKET = "executor.s3.uploader.bucket"; diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index f3552cb1f2..efc98fe09e 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -4,8 +4,11 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; +import java.util.LinkedList; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import com.google.common.collect.ImmutableList; @@ -102,7 +105,7 @@ private boolean removeLogrotateFile() { /** * Trigger manual logrotate run. - * @return boolean + * @return True on successful run or skip. False on error. */ public boolean manualLogrotate() { if (!Files.exists(getLogrotateConfPath())) { @@ -152,8 +155,17 @@ private boolean writeTailMetadata(boolean finished) { return jsonObjectFileHelper.writeObject(tailMetadata, path, log); } + /** + * Return a String for generating a PathMatcher. + * The matching files are caught by the S3 Uploader and pushed to S3. + * @return file glob String. + */ private String getS3Glob() { - return String.format("%s*.gz*", taskDefinition.getServiceLogOutPath().getFileName()); + List fileNames = new LinkedList<>(); + fileNames.add(taskDefinition.getServiceLogOutPath().getFileName().toString()); + fileNames.addAll(Arrays.asList(configuration.getS3FilesToBackup())); + + return String.format("{%s}*.gz*", StringUtils.join(fileNames, ",")); } private String getS3KeyPattern() { @@ -177,9 +189,9 @@ private boolean writeS3MetadataFile(boolean finished) { S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(logrotateDirectory.toString(), getS3Glob(), configuration.getS3Bucket(), getS3KeyPattern(), finished); - String s3UploadMetadatafilename = String.format("%s%s", taskDefinition.getTaskId(), configuration.getS3MetadataSuffix()); + String s3UploadMetadataFileName = String.format("%s%s", taskDefinition.getTaskId(), configuration.getS3MetadataSuffix()); - Path s3UploadMetadataPath = configuration.getS3MetadataDirectory().resolve(s3UploadMetadatafilename); + Path s3UploadMetadataPath = configuration.getS3MetadataDirectory().resolve(s3UploadMetadataFileName); return jsonObjectFileHelper.writeObject(s3UploadMetadata, s3UploadMetadataPath, log); } From b5b2acac63f88a78f75939c56927478bbf94cc5f Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Fri, 30 Jan 2015 11:01:48 -0500 Subject: [PATCH 20/93] Add S3 docs. --- .../singularity/config/S3Configuration.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java index f20a82ca67..69ecaebbae 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java @@ -9,18 +9,36 @@ public class S3Configuration { @NotNull private int maxS3Threads = 3; + /** + * How long for SingularityService to wait for S3 List + */ @NotNull private int waitForS3ListSeconds = 5; + /** + * How long for SingularityService to wait for generating a link + */ @NotNull private int waitForS3LinksSeconds = 1; + /** + * Links to logs will expire after given number of milliseconds. + * A new link is generated for every /logs API call. + */ @NotNull private long expireS3LinksAfterMillis = TimeUnit.DAYS.toMillis(1); + /** + * S3 Bucket that SingularityS3Uploader puts logs in. + */ @NotNull private String s3Bucket; + /** + * S3 Key format for finding logs. Should be the same as + * configuration set for SingularityS3Uploader + * (e.g. '%requestId/%Y/%m/%taskId_%index-%s%fileext') + */ @NotNull private String s3KeyFormat; From dd8ca7ce83e0e8b8cfa8c5f1b115511164929c10 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Mon, 9 Feb 2015 15:02:25 -0500 Subject: [PATCH 21/93] fix merge --- .../config/SingularityConfiguration.java | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index 1e9fb70b0a..ab019c4f24 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -135,8 +135,6 @@ public class SingularityConfiguration extends Configuration { private long startNewReconcileEverySeconds = TimeUnit.MINUTES.toSeconds(10); - private long threadpoolShutdownDelayInSeconds = 1; - @JsonProperty("ui") @Valid private UIConfiguration uiConfiguration = new UIConfiguration(); @@ -226,6 +224,10 @@ public long getCooldownMinScheduleSeconds() { return cooldownMinScheduleSeconds; } + public int getCoreThreadpoolSize() { + return coreThreadpoolSize; + } + public Optional getDatabaseConfiguration() { return Optional.fromNullable(databaseConfiguration); } @@ -402,14 +404,6 @@ public boolean isSandboxDefaultsToTaskId() { return sandboxDefaultsToTaskId; } - public int getCoreThreadpoolSize() { - return coreThreadpoolSize; - } - - public long getThreadpoolShutdownDelayInSeconds() { - return threadpoolShutdownDelayInSeconds; - } - public boolean isWaitForListeners() { return waitForListeners; } @@ -494,6 +488,10 @@ public void setCooldownMinScheduleSeconds(long cooldownMinScheduleSeconds) { this.cooldownMinScheduleSeconds = cooldownMinScheduleSeconds; } + public void setCoreThreadpoolSize(int coreThreadpoolSize) { + this.coreThreadpoolSize = coreThreadpoolSize; + } + public void setDatabaseConfiguration(DataSourceFactory databaseConfiguration) { this.databaseConfiguration = databaseConfiguration; } @@ -634,10 +632,6 @@ public void setStartNewReconcileEverySeconds(long startNewReconcileEverySeconds) this.startNewReconcileEverySeconds = startNewReconcileEverySeconds; } - public void setThreadpoolShutdownDelayInSeconds(long threadpoolShutdownDelayInSeconds) { - this.threadpoolShutdownDelayInSeconds = threadpoolShutdownDelayInSeconds; - } - public void setUiConfiguration(UIConfiguration uiConfiguration) { this.uiConfiguration = uiConfiguration; } @@ -661,12 +655,5 @@ public void setZookeeperAsyncTimeout(long zookeeperAsyncTimeout) { public void setZooKeeperConfiguration(ZooKeeperConfiguration zooKeeperConfiguration) { this.zooKeeperConfiguration = zooKeeperConfiguration; } - public void setCoreThreadpoolSize(int coreThreadpoolSize) { - this.coreThreadpoolSize = coreThreadpoolSize; - } - - public void setThreadpoolShutdownDelayInSeconds(long threadpoolShutdownDelayInSeconds) { - this.threadpoolShutdownDelayInSeconds = threadpoolShutdownDelayInSeconds; - } } From a7ba65bbee44c777ba7821af971a72b7ee58738a Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Wed, 11 Feb 2015 12:24:59 -0500 Subject: [PATCH 22/93] round humanize file size helper to 2 decimal places --- SingularityUI/app/handlebarsHelpers.coffee | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/SingularityUI/app/handlebarsHelpers.coffee b/SingularityUI/app/handlebarsHelpers.coffee index 5a1d7ef7a0..3e678573d8 100644 --- a/SingularityUI/app/handlebarsHelpers.coffee +++ b/SingularityUI/app/handlebarsHelpers.coffee @@ -76,21 +76,13 @@ Handlebars.registerHelper 'humanizeText', (text) -> text # 2121 => '2 KB' -Handlebars.registerHelper 'humanizeFileSize', (fileSize) -> - kilo = 1024 - mega = 1024 * 1024 - giga = 1024 * 1024 * 1024 - - shorten = (which) -> Math.round fileSize / which - - if fileSize > giga - return "#{ shorten giga } GB" - else if fileSize > mega - return "#{ shorten mega } MB" - else if fileSize > kilo - return "#{ shorten kilo } KB" - else - return "#{ fileSize } B" +Handlebars.registerHelper 'humanizeFileSize', (bytes) -> + k = 1024 + sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] + + return '0 B' if bytes is 0 + i = Math.floor(Math.log(bytes) / Math.log(k)) + return (bytes / Math.pow(k, i)).toPrecision(3) + ' ' + sizes[i] # 'sbacanu@hubspot.com' => 'sbacanu' # 'seb' => 'seb' From 503599c0533aa5e038a95de8c5cf9ee696f96f2a Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Wed, 11 Feb 2015 12:59:53 -0500 Subject: [PATCH 23/93] move dashboard table to new partial and add sorting --- SingularityUI/app/templates/dashboard.hbs | 45 +---------------- .../dashboardTable/dashboardStarred.hbs | 48 +++++++++++++++++++ 2 files changed, 50 insertions(+), 43 deletions(-) create mode 100644 SingularityUI/app/templates/dashboardTable/dashboardStarred.hbs diff --git a/SingularityUI/app/templates/dashboard.hbs b/SingularityUI/app/templates/dashboard.hbs index f5ed017699..ce40befb4e 100644 --- a/SingularityUI/app/templates/dashboard.hbs +++ b/SingularityUI/app/templates/dashboard.hbs @@ -67,52 +67,11 @@ {{/if}}
-
+

Starred requests

- {{#if starredRequests.length}} - - - - - - - - - - - - {{#each starredRequests}} - - - - - - - - {{/each}} - -
RequestDeploy userInstances
- - - - - - {{ request.id }} - - - {{usernameFromEmail requestDeployState.activeDeploy.user}} - - {{ request.instances }} -
- {{else}} -

No starred requests

- {{/if}} + {{> requestsBody }}
{{else}} diff --git a/SingularityUI/app/templates/dashboardTable/dashboardStarred.hbs b/SingularityUI/app/templates/dashboardTable/dashboardStarred.hbs new file mode 100644 index 0000000000..4ba9bf7357 --- /dev/null +++ b/SingularityUI/app/templates/dashboardTable/dashboardStarred.hbs @@ -0,0 +1,48 @@ +{{#if haveStarredRequests}} + + + + + + + + + + + +{{/if}} + {{#each starredRequests}} + + + + + + + + {{/each}} + + {{#if haveStarredRequests}} + +
{{! Star column }}RequestDeploy userInstances
+ + + + + + {{ request.id }} + + + {{usernameFromEmail requestDeployState.activeDeploy.user}} + + {{ request.instances }} +
+{{else}} +

No starred requests

+{{/if}} + + + From 4853d01c23b482a4cfb89bdd2a832b0e23748e67 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Wed, 11 Feb 2015 13:03:56 -0500 Subject: [PATCH 24/93] add sorting methods for dashboard table --- SingularityUI/app/views/dashboard.coffee | 77 +++++++++++++++++++++--- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/SingularityUI/app/views/dashboard.coffee b/SingularityUI/app/views/dashboard.coffee index 1309a58dba..0a5f6929d7 100644 --- a/SingularityUI/app/views/dashboard.coffee +++ b/SingularityUI/app/views/dashboard.coffee @@ -2,12 +2,14 @@ View = require './view' class DashboardView extends View - template: require '../templates/dashboard' + templateBase: require '../templates/dashboard' + templateRequestsTable: require '../templates/dashboardTable/dashboardStarred' events: -> _.extend super, 'click [data-action="unstar"]': 'unstar' 'click [data-action="change-user"]': 'changeUser' + 'click th[data-sort-attribute]': 'sortTable' initialize: => @listenTo app.user, 'change', @render @@ -16,9 +18,9 @@ class DashboardView extends View render: => deployUser = app.user.get 'deployUser' - # Filter starred requests - starredRequests = @collection.getStarredOnly() - starredRequests = _.pluck starredRequests, 'attributes' + partials = + partials: + requestsBody: @templateRequestsTable # Count up the Requests for the clicky boxes userRequests = @collection.filter (model) -> @@ -27,12 +29,12 @@ class DashboardView extends View if not request.owners return false - + for owner in request.owners ownerTrimmed = owner.split("@")[0] if deployUserTrimmed == ownerTrimmed return true - + return false userRequestTotals = all: userRequests.length @@ -49,9 +51,68 @@ class DashboardView extends View deployUser: deployUser collectionSynced: @collection.synced userRequestTotals: userRequestTotals or { } - starredRequests: starredRequests or [] + haveStarredRequests: @collection.getStarredOnly().length + + @$el.html @templateBase context, partials + @renderTable() + + renderTable: => + @sortCollection() + requests = @currentRequests + + $contents = @templateRequestsTable + starredRequests: requests + requests: requests + + $table = @$ ".table-staged table" + $tableBody = $table.find "tbody" + $tableBody.html $contents + + sortCollection: => + requests = _.pluck @collection.getStarredOnly(), "attributes" + + # Sort the table if the user clicked on the table heading things + if @sortAttribute? + requests = _.sortBy requests, (request) => + + # Traverse through the properties to find what we're after + attributes = @sortAttribute.split '.' + value = request + for attribute in attributes + value = value[attribute] + value = '' if not value? + return value + + if not @sortAscending + requests = requests.reverse() + else + requests.reverse() + + @currentRequests = requests + + sortTable: (event) => + @isSorted = true + + $target = $ event.currentTarget + newSortAttribute = $target.attr "data-sort-attribute" + + $currentlySortedHeading = @$ "[data-sorted=true]" + $currentlySortedHeading.removeAttr "data-sorted" + $currentlySortedHeading.find('span').remove() + + + if newSortAttribute is @sortAttribute and @sortAscending? + @sortAscending = not @sortAscending + else + # timestamp should be DESC by default + @sortAscending = if newSortAttribute is "timestamp" then false else true + + @sortAttribute = newSortAttribute + + $target.attr "data-sorted", "true" + $target.append "" - @$el.html @template context + @renderTable() unstar: (e) => $target = $ e.currentTarget From c86a835302cb568e0e7fc467b86c5facb9f67197 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Wed, 11 Feb 2015 15:44:56 -0500 Subject: [PATCH 25/93] update file size rounding calculation --- SingularityUI/app/handlebarsHelpers.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SingularityUI/app/handlebarsHelpers.coffee b/SingularityUI/app/handlebarsHelpers.coffee index 3e678573d8..ead60c16f9 100644 --- a/SingularityUI/app/handlebarsHelpers.coffee +++ b/SingularityUI/app/handlebarsHelpers.coffee @@ -81,8 +81,8 @@ Handlebars.registerHelper 'humanizeFileSize', (bytes) -> sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] return '0 B' if bytes is 0 - i = Math.floor(Math.log(bytes) / Math.log(k)) - return (bytes / Math.pow(k, i)).toPrecision(3) + ' ' + sizes[i] + i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length-1) + return +(bytes / Math.pow(k, i)).toFixed(2) + ' ' + sizes[i] # 'sbacanu@hubspot.com' => 'sbacanu' # 'seb' => 'seb' From d34c5663ccd785f9949d044aafed5cf471994f8e Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 18:29:21 +0000 Subject: [PATCH 26/93] Added support for the sandbox endpoint --- .../singularity/client/SingularityClient.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index d13afbc723..e4589bcd8b 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -5,6 +5,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import javax.inject.Provider; @@ -16,6 +18,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -23,6 +27,7 @@ import com.hubspot.horizon.HttpRequest; import com.hubspot.horizon.HttpRequest.Method; import com.hubspot.horizon.HttpResponse; +import com.hubspot.mesos.json.MesosFileChunkObject; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -35,6 +40,7 @@ import com.hubspot.singularity.SingularityRequestCleanup; import com.hubspot.singularity.SingularityRequestHistory; import com.hubspot.singularity.SingularityRequestParent; +import com.hubspot.singularity.SingularitySandbox; import com.hubspot.singularity.SingularitySlave; import com.hubspot.singularity.SingularityState; import com.hubspot.singularity.SingularityTask; @@ -103,6 +109,10 @@ public class SingularityClient { private static final String WEBHOOKS_GET_QUEUED_REQUEST_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/request/%s"; private static final String WEBHOOKS_GET_QUEUED_TASK_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/task/%s"; + private static final String SANDBOX_FORMAT = "http://%s/%s/sandbox"; + private static final String SANDBOX_BROWSE_FORMAT = SANDBOX_FORMAT + "/%s/browse"; + private static final String SANDBOX_READ_FILE_FORMAT = SANDBOX_FORMAT + "/%s/read"; + private static final TypeReference> REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> PENDING_REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> CLEANUP_REQUESTS_COLLECTION = new TypeReference>() {}; @@ -191,6 +201,44 @@ private Optional getSingle(String uri, String type, String id, Class c return Optional.fromNullable(response.getAs(clazz)); } + private Optional getSingleWithParams(String uri, String type, String id, Map queryParams, Class clazz) { + checkNotNull(id, String.format("Provide a %s id", type)); + + LOG.info("Getting {} {} from {}", type, id, uri); + + final long start = System.currentTimeMillis(); + + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + for (Entry queryParamEntry : queryParams.entrySet()) { + if (queryParamEntry.getValue() instanceof String) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Integer) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Long) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Boolean) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); + } else { + throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", + queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); + } + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); + + if (response.getStatusCode() == 404) { + return Optional.absent(); + } + + checkResponse(type, response); + + LOG.info("Got {} {} in {}ms", type, id, System.currentTimeMillis() - start); + + return Optional.fromNullable(response.getAs(clazz)); + } + private Collection getCollection(String uri, String type, TypeReference> typeReference) { LOG.info("Getting all {} from {}", type, uri); @@ -680,4 +728,60 @@ public Collection getQueuedTaskUpdates(String webh return getCollection(requestUri, "request updates", TASK_UPDATES_COLLECTION); } + // + // SANDBOX + // + + /** + * Retrieve information about a specific task's sandbox + * + * @param taskId + * The task ID to browse + * @param path + * The path to browse from. + * if not specified it will browse from the sandbox root. + * @return + * A {@link SingularitySandbox} object that captures the information for the path to a specific task's Mesos sandbox + */ + public Optional browseTaskSandBox(String taskId, String path) { + final String requestUrl = String.format(SANDBOX_BROWSE_FORMAT, getHost(), contextPath, taskId); + + return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, ImmutableMap.of("path", path), SingularitySandbox.class); + + } + + /** + * Retrieve part of the contents of a file in a specific task's sandbox. + * + * @param taskId + * The task ID of the sandbox to read from + * @param path + * The path to the file to be read. Relative to the sandbox root (without a leading slash) + * @param grep + * Optional string to grep for + * @param offset + * Byte offset to start reading from + * @param length + * Maximum number of bytes to read + * @return + * A {@link MesosFileChunkObject} that contains the requested partial file contents + */ + public Optional read(String taskId, String path, Optional grep, Optional offset, Optional length) { + final String requestUrl = String.format(SANDBOX_READ_FILE_FORMAT, getHost(), contextPath, taskId); + + Builder queryParamBuider = ImmutableMap.builder().put("path", path); + + if (grep.isPresent()) { + queryParamBuider.put("grep", grep); + } + if (offset.isPresent()) { + queryParamBuider.put("offset", offset); + } + if (length.isPresent()) { + queryParamBuider.put("length", length); + } + + return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); + } + } From 54d0c562d9b09dc286f8128fb78a4de36ccd98bd Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 18:29:21 +0000 Subject: [PATCH 27/93] Added support for the sandbox endpoint --- .../singularity/client/SingularityClient.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index d13afbc723..e4589bcd8b 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -5,6 +5,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import javax.inject.Provider; @@ -16,6 +18,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -23,6 +27,7 @@ import com.hubspot.horizon.HttpRequest; import com.hubspot.horizon.HttpRequest.Method; import com.hubspot.horizon.HttpResponse; +import com.hubspot.mesos.json.MesosFileChunkObject; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -35,6 +40,7 @@ import com.hubspot.singularity.SingularityRequestCleanup; import com.hubspot.singularity.SingularityRequestHistory; import com.hubspot.singularity.SingularityRequestParent; +import com.hubspot.singularity.SingularitySandbox; import com.hubspot.singularity.SingularitySlave; import com.hubspot.singularity.SingularityState; import com.hubspot.singularity.SingularityTask; @@ -103,6 +109,10 @@ public class SingularityClient { private static final String WEBHOOKS_GET_QUEUED_REQUEST_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/request/%s"; private static final String WEBHOOKS_GET_QUEUED_TASK_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/task/%s"; + private static final String SANDBOX_FORMAT = "http://%s/%s/sandbox"; + private static final String SANDBOX_BROWSE_FORMAT = SANDBOX_FORMAT + "/%s/browse"; + private static final String SANDBOX_READ_FILE_FORMAT = SANDBOX_FORMAT + "/%s/read"; + private static final TypeReference> REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> PENDING_REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> CLEANUP_REQUESTS_COLLECTION = new TypeReference>() {}; @@ -191,6 +201,44 @@ private Optional getSingle(String uri, String type, String id, Class c return Optional.fromNullable(response.getAs(clazz)); } + private Optional getSingleWithParams(String uri, String type, String id, Map queryParams, Class clazz) { + checkNotNull(id, String.format("Provide a %s id", type)); + + LOG.info("Getting {} {} from {}", type, id, uri); + + final long start = System.currentTimeMillis(); + + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + for (Entry queryParamEntry : queryParams.entrySet()) { + if (queryParamEntry.getValue() instanceof String) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Integer) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Long) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Boolean) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); + } else { + throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", + queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); + } + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); + + if (response.getStatusCode() == 404) { + return Optional.absent(); + } + + checkResponse(type, response); + + LOG.info("Got {} {} in {}ms", type, id, System.currentTimeMillis() - start); + + return Optional.fromNullable(response.getAs(clazz)); + } + private Collection getCollection(String uri, String type, TypeReference> typeReference) { LOG.info("Getting all {} from {}", type, uri); @@ -680,4 +728,60 @@ public Collection getQueuedTaskUpdates(String webh return getCollection(requestUri, "request updates", TASK_UPDATES_COLLECTION); } + // + // SANDBOX + // + + /** + * Retrieve information about a specific task's sandbox + * + * @param taskId + * The task ID to browse + * @param path + * The path to browse from. + * if not specified it will browse from the sandbox root. + * @return + * A {@link SingularitySandbox} object that captures the information for the path to a specific task's Mesos sandbox + */ + public Optional browseTaskSandBox(String taskId, String path) { + final String requestUrl = String.format(SANDBOX_BROWSE_FORMAT, getHost(), contextPath, taskId); + + return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, ImmutableMap.of("path", path), SingularitySandbox.class); + + } + + /** + * Retrieve part of the contents of a file in a specific task's sandbox. + * + * @param taskId + * The task ID of the sandbox to read from + * @param path + * The path to the file to be read. Relative to the sandbox root (without a leading slash) + * @param grep + * Optional string to grep for + * @param offset + * Byte offset to start reading from + * @param length + * Maximum number of bytes to read + * @return + * A {@link MesosFileChunkObject} that contains the requested partial file contents + */ + public Optional read(String taskId, String path, Optional grep, Optional offset, Optional length) { + final String requestUrl = String.format(SANDBOX_READ_FILE_FORMAT, getHost(), contextPath, taskId); + + Builder queryParamBuider = ImmutableMap.builder().put("path", path); + + if (grep.isPresent()) { + queryParamBuider.put("grep", grep); + } + if (offset.isPresent()) { + queryParamBuider.put("offset", offset); + } + if (length.isPresent()) { + queryParamBuider.put("length", length); + } + + return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); + } + } From 91e2c4cfb70c34bb30a976105a32b8848aebf481 Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 19:43:18 +0000 Subject: [PATCH 28/93] A friendlier method name --- .../java/com/hubspot/singularity/client/SingularityClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index e4589bcd8b..94d931309d 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -766,7 +766,7 @@ public Optional browseTaskSandBox(String taskId, String path * @return * A {@link MesosFileChunkObject} that contains the requested partial file contents */ - public Optional read(String taskId, String path, Optional grep, Optional offset, Optional length) { + public Optional readSandBoxFile(String taskId, String path, Optional grep, Optional offset, Optional length) { final String requestUrl = String.format(SANDBOX_READ_FILE_FORMAT, getHost(), contextPath, taskId); Builder queryParamBuider = ImmutableMap.builder().put("path", path); From 9688c9347a684ab60eeec0a6eb7eb411c247fa0b Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 20:02:21 +0000 Subject: [PATCH 29/93] Fixed to get the values from optionals --- .../com/hubspot/singularity/client/SingularityClient.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index 94d931309d..4935217cac 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -772,13 +772,13 @@ public Optional readSandBoxFile(String taskId, String path Builder queryParamBuider = ImmutableMap.builder().put("path", path); if (grep.isPresent()) { - queryParamBuider.put("grep", grep); + queryParamBuider.put("grep", grep.get()); } if (offset.isPresent()) { - queryParamBuider.put("offset", offset); + queryParamBuider.put("offset", offset.get()); } if (length.isPresent()) { - queryParamBuider.put("length", length); + queryParamBuider.put("length", length.get()); } return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); From 8ad6ef21212003e324f40ad5aded513c2421814e Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 19:43:18 +0000 Subject: [PATCH 30/93] A friendlier method name --- .../java/com/hubspot/singularity/client/SingularityClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index e4589bcd8b..94d931309d 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -766,7 +766,7 @@ public Optional browseTaskSandBox(String taskId, String path * @return * A {@link MesosFileChunkObject} that contains the requested partial file contents */ - public Optional read(String taskId, String path, Optional grep, Optional offset, Optional length) { + public Optional readSandBoxFile(String taskId, String path, Optional grep, Optional offset, Optional length) { final String requestUrl = String.format(SANDBOX_READ_FILE_FORMAT, getHost(), contextPath, taskId); Builder queryParamBuider = ImmutableMap.builder().put("path", path); From 661e851ffa0fe48b1fe56ff993d56783c168c0cf Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 20:02:21 +0000 Subject: [PATCH 31/93] Fixed to get the values from optionals --- .../com/hubspot/singularity/client/SingularityClient.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index 94d931309d..4935217cac 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -772,13 +772,13 @@ public Optional readSandBoxFile(String taskId, String path Builder queryParamBuider = ImmutableMap.builder().put("path", path); if (grep.isPresent()) { - queryParamBuider.put("grep", grep); + queryParamBuider.put("grep", grep.get()); } if (offset.isPresent()) { - queryParamBuider.put("offset", offset); + queryParamBuider.put("offset", offset.get()); } if (length.isPresent()) { - queryParamBuider.put("length", length); + queryParamBuider.put("length", length.get()); } return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); From b305113afb3d1da626b2a21141978feb2fdb16fe Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 18:29:21 +0000 Subject: [PATCH 32/93] Added support for the sandbox endpoint --- .../singularity/client/SingularityClient.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index d13afbc723..e4589bcd8b 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -5,6 +5,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import javax.inject.Provider; @@ -16,6 +18,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -23,6 +27,7 @@ import com.hubspot.horizon.HttpRequest; import com.hubspot.horizon.HttpRequest.Method; import com.hubspot.horizon.HttpResponse; +import com.hubspot.mesos.json.MesosFileChunkObject; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -35,6 +40,7 @@ import com.hubspot.singularity.SingularityRequestCleanup; import com.hubspot.singularity.SingularityRequestHistory; import com.hubspot.singularity.SingularityRequestParent; +import com.hubspot.singularity.SingularitySandbox; import com.hubspot.singularity.SingularitySlave; import com.hubspot.singularity.SingularityState; import com.hubspot.singularity.SingularityTask; @@ -103,6 +109,10 @@ public class SingularityClient { private static final String WEBHOOKS_GET_QUEUED_REQUEST_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/request/%s"; private static final String WEBHOOKS_GET_QUEUED_TASK_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/task/%s"; + private static final String SANDBOX_FORMAT = "http://%s/%s/sandbox"; + private static final String SANDBOX_BROWSE_FORMAT = SANDBOX_FORMAT + "/%s/browse"; + private static final String SANDBOX_READ_FILE_FORMAT = SANDBOX_FORMAT + "/%s/read"; + private static final TypeReference> REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> PENDING_REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> CLEANUP_REQUESTS_COLLECTION = new TypeReference>() {}; @@ -191,6 +201,44 @@ private Optional getSingle(String uri, String type, String id, Class c return Optional.fromNullable(response.getAs(clazz)); } + private Optional getSingleWithParams(String uri, String type, String id, Map queryParams, Class clazz) { + checkNotNull(id, String.format("Provide a %s id", type)); + + LOG.info("Getting {} {} from {}", type, id, uri); + + final long start = System.currentTimeMillis(); + + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + for (Entry queryParamEntry : queryParams.entrySet()) { + if (queryParamEntry.getValue() instanceof String) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Integer) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Long) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Boolean) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); + } else { + throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", + queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); + } + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); + + if (response.getStatusCode() == 404) { + return Optional.absent(); + } + + checkResponse(type, response); + + LOG.info("Got {} {} in {}ms", type, id, System.currentTimeMillis() - start); + + return Optional.fromNullable(response.getAs(clazz)); + } + private Collection getCollection(String uri, String type, TypeReference> typeReference) { LOG.info("Getting all {} from {}", type, uri); @@ -680,4 +728,60 @@ public Collection getQueuedTaskUpdates(String webh return getCollection(requestUri, "request updates", TASK_UPDATES_COLLECTION); } + // + // SANDBOX + // + + /** + * Retrieve information about a specific task's sandbox + * + * @param taskId + * The task ID to browse + * @param path + * The path to browse from. + * if not specified it will browse from the sandbox root. + * @return + * A {@link SingularitySandbox} object that captures the information for the path to a specific task's Mesos sandbox + */ + public Optional browseTaskSandBox(String taskId, String path) { + final String requestUrl = String.format(SANDBOX_BROWSE_FORMAT, getHost(), contextPath, taskId); + + return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, ImmutableMap.of("path", path), SingularitySandbox.class); + + } + + /** + * Retrieve part of the contents of a file in a specific task's sandbox. + * + * @param taskId + * The task ID of the sandbox to read from + * @param path + * The path to the file to be read. Relative to the sandbox root (without a leading slash) + * @param grep + * Optional string to grep for + * @param offset + * Byte offset to start reading from + * @param length + * Maximum number of bytes to read + * @return + * A {@link MesosFileChunkObject} that contains the requested partial file contents + */ + public Optional read(String taskId, String path, Optional grep, Optional offset, Optional length) { + final String requestUrl = String.format(SANDBOX_READ_FILE_FORMAT, getHost(), contextPath, taskId); + + Builder queryParamBuider = ImmutableMap.builder().put("path", path); + + if (grep.isPresent()) { + queryParamBuider.put("grep", grep); + } + if (offset.isPresent()) { + queryParamBuider.put("offset", offset); + } + if (length.isPresent()) { + queryParamBuider.put("length", length); + } + + return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); + } + } From 9f32edf26842febc8a9b38c7a6fafb8efda5ca8a Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 19:43:18 +0000 Subject: [PATCH 33/93] A friendlier method name --- .../java/com/hubspot/singularity/client/SingularityClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index e4589bcd8b..94d931309d 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -766,7 +766,7 @@ public Optional browseTaskSandBox(String taskId, String path * @return * A {@link MesosFileChunkObject} that contains the requested partial file contents */ - public Optional read(String taskId, String path, Optional grep, Optional offset, Optional length) { + public Optional readSandBoxFile(String taskId, String path, Optional grep, Optional offset, Optional length) { final String requestUrl = String.format(SANDBOX_READ_FILE_FORMAT, getHost(), contextPath, taskId); Builder queryParamBuider = ImmutableMap.builder().put("path", path); From bb06122526fb7f7a3273e7676fb98d7c20cd052c Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Mon, 16 Feb 2015 20:02:21 +0000 Subject: [PATCH 34/93] Fixed to get the values from optionals --- .../com/hubspot/singularity/client/SingularityClient.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index 94d931309d..4935217cac 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -772,13 +772,13 @@ public Optional readSandBoxFile(String taskId, String path Builder queryParamBuider = ImmutableMap.builder().put("path", path); if (grep.isPresent()) { - queryParamBuider.put("grep", grep); + queryParamBuider.put("grep", grep.get()); } if (offset.isPresent()) { - queryParamBuider.put("offset", offset); + queryParamBuider.put("offset", offset.get()); } if (length.isPresent()) { - queryParamBuider.put("length", length); + queryParamBuider.put("length", length.get()); } return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); From 0be0d3cbb19fc224819b9f59f7ce950ca578d8de Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 00:52:33 +0000 Subject: [PATCH 35/93] Fixed to work with the updated 'slaves' endpoint. Implemented getSingle using getSingeWithParams. Added getCollectionWithParams --- .../singularity/client/SingularityClient.java | 136 +++++++++++------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index 4935217cac..e7a8d9539c 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -9,6 +9,7 @@ import java.util.Map.Entry; import java.util.Random; +import javax.annotation.Nonnull; import javax.inject.Provider; import org.apache.commons.lang3.tuple.Pair; @@ -28,6 +29,7 @@ import com.hubspot.horizon.HttpRequest.Method; import com.hubspot.horizon.HttpResponse; import com.hubspot.mesos.json.MesosFileChunkObject; +import com.hubspot.singularity.MachineState; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -67,12 +69,8 @@ public class SingularityClient { private static final String RACKS_DELETE_DECOMISSIONING_FORMAT = RACKS_FORMAT + "/rack/%s/decomissioning"; private static final String SLAVES_FORMAT = "http://%s/%s/slaves"; - private static final String SLAVES_GET_ACTIVE_FORMAT = SLAVES_FORMAT + "/active"; - private static final String SLAVES_GET_DEAD_FORMAT = SLAVES_FORMAT + "/dead"; - private static final String SLAVES_GET_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/decomissioning"; - private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decomission"; - private static final String SLAVES_DELETE_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; - private static final String SLAVES_DELETE_DEAD_FORMAT = SLAVES_FORMAT + "/slave/%s/dead"; + private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decommission"; + private static final String SLAVES_DELETE_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; private static final String TASKS_FORMAT = "http://%s/%s/tasks"; private static final String TASKS_KILL_TASK_FORMAT = TASKS_FORMAT + "/task/%s"; @@ -182,26 +180,10 @@ private SingularityClientException fail(String type, HttpResponse response) { } private Optional getSingle(String uri, String type, String id, Class clazz) { - checkNotNull(id, String.format("Provide a %s id", type)); - - LOG.info("Getting {} {} from {}", type, id, uri); - - final long start = System.currentTimeMillis(); - - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); - - if (response.getStatusCode() == 404) { - return Optional.absent(); - } - - checkResponse(type, response); - - LOG.info("Got {} {} in {}ms", type, id, System.currentTimeMillis() - start); - - return Optional.fromNullable(response.getAs(clazz)); + return getSingleWithParams(uri, type, id, Optional.>absent(), clazz); } - private Optional getSingleWithParams(String uri, String type, String id, Map queryParams, Class clazz) { + private Optional getSingleWithParams(String uri, String type, String id, Optional> queryParams, Class clazz) { checkNotNull(id, String.format("Provide a %s id", type)); LOG.info("Getting {} {} from {}", type, id, uri); @@ -211,19 +193,8 @@ private Optional getSingleWithParams(String uri, String type, String id, HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() .setUrl(uri); - for (Entry queryParamEntry : queryParams.entrySet()) { - if (queryParamEntry.getValue() instanceof String) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Integer) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Long) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Boolean) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); - } else { - throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", - queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); - } + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); } HttpResponse response = httpClient.execute(requestBuilder.build()); @@ -240,11 +211,22 @@ private Optional getSingleWithParams(String uri, String type, String id, } private Collection getCollection(String uri, String type, TypeReference> typeReference) { + return getCollectionWithParams(uri, type, Optional.>absent(), typeReference); + } + + private Collection getCollectionWithParams(String uri, String type, Optional> queryParams, TypeReference> typeReference) { LOG.info("Getting all {} from {}", type, uri); final long start = System.currentTimeMillis(); - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); if (response.getStatusCode() == 404) { return ImmutableList.of(); @@ -257,6 +239,23 @@ private Collection getCollection(String uri, String type, TypeReference queryParams) { + for (Entry queryParamEntry : queryParams.entrySet()) { + if (queryParamEntry.getValue() instanceof String) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Integer) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Long) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Boolean) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); + } else { + throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", + queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); + } + } + } + private void delete(String uri, String type, String id, Optional user) { delete(uri, type, id, user, Optional.> absent()); } @@ -620,22 +619,55 @@ public void deleteDeadRack(String rackId, Optional user) { // SLAVES // + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getActiveSlaves() { - return getSlaves(SLAVES_GET_ACTIVE_FORMAT, "active"); + return getSlaves(Optional.of(MachineState.ACTIVE)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDeadSlaves() { - return getSlaves(SLAVES_GET_DEAD_FORMAT, "dead"); + return getSlaves(Optional.of(MachineState.DEAD)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDecomissioningSlaves() { - return getSlaves(SLAVES_GET_DECOMISSIONING_FORMAT, "decomissioning"); + return getSlaves(Optional.of(MachineState.DECOMMISSIONING)); } - private Collection getSlaves(String format, String type) { - final String requestUri = String.format(format, getHost(), contextPath); + /** + * Retrieve the list of all known slaves, optionally filtering by a particular slave state + * + * @param slaveState + * Optionally specify a particular state to filter slaves by + * @return + * A collection of {@link SingularitySlave} + */ + public Collection getSlaves(Optional slaveState) { + final String requestUri = String.format(SLAVES_FORMAT, getHost(), contextPath); + + Optional> maybeQueryParams = Optional.>absent(); + + String type = "slaves"; - return getCollection(requestUri, type, SLAVES_COLLECTION); + if (slaveState.isPresent()) { + maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get())); + + type = String.format("%s slaves", slaveState.get().toString()); + } + + return getCollectionWithParams(requestUri, type, maybeQueryParams, SLAVES_COLLECTION); } public void decomissionSlave(String slaveId, Optional user) { @@ -644,16 +676,10 @@ public void decomissionSlave(String slaveId, Optional user) { post(requestUri, String.format("decomission slave %s", slaveId), Optional.absent(), user); } - public void deleteDecomissioningSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DECOMISSIONING_FORMAT, getHost(), contextPath, slaveId); - - delete(requestUri, "decomissioning slave", slaveId, user); - } - - public void deleteDeadSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DEAD_FORMAT, getHost(), contextPath, slaveId); + public void deleteSlave(String slaveId, Optional user) { + final String requestUri = String.format(SLAVES_DELETE_FORMAT, getHost(), contextPath, slaveId); - delete(requestUri, "dead slave", slaveId, user); + delete(requestUri, "deleting slave", slaveId, user); } // @@ -746,7 +772,7 @@ public Collection getQueuedTaskUpdates(String webh public Optional browseTaskSandBox(String taskId, String path) { final String requestUrl = String.format(SANDBOX_BROWSE_FORMAT, getHost(), contextPath, taskId); - return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, ImmutableMap.of("path", path), SingularitySandbox.class); + return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, Optional.>of(ImmutableMap.of("path", path)), SingularitySandbox.class); } @@ -781,7 +807,7 @@ public Optional readSandBoxFile(String taskId, String path queryParamBuider.put("length", length.get()); } - return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); + return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, Optional.>of(queryParamBuider.build()), MesosFileChunkObject.class); } } From 80104f945a80d024ceae0c5f94e3a94dbe6f1708 Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 01:16:05 +0000 Subject: [PATCH 36/93] The Nonnull annotation causes a dependency analysis failure for com.google.code.findbugs:annotations:jar ???? Will resolve at another time... --- .../java/com/hubspot/singularity/client/SingularityClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index e7a8d9539c..b7c0204ea8 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -9,7 +9,6 @@ import java.util.Map.Entry; import java.util.Random; -import javax.annotation.Nonnull; import javax.inject.Provider; import org.apache.commons.lang3.tuple.Pair; @@ -239,7 +238,7 @@ private Collection getCollectionWithParams(String uri, String type, Optio return response.getAs(typeReference); } - private void addQueryParams(@Nonnull HttpRequest.Builder requestBuilder, @Nonnull Map queryParams) { + private void addQueryParams(HttpRequest.Builder requestBuilder, Map queryParams) { for (Entry queryParamEntry : queryParams.entrySet()) { if (queryParamEntry.getValue() instanceof String) { requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); From 143bf2aa3b2c41d57487ccce0613a2cb9c11e3e1 Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 01:34:05 +0000 Subject: [PATCH 37/93] set the string value of the enum in the query params map --- .../java/com/hubspot/singularity/client/SingularityClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index b7c0204ea8..655096a160 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -661,7 +661,7 @@ public Collection getSlaves(Optional slaveState) String type = "slaves"; if (slaveState.isPresent()) { - maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get())); + maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get().toString())); type = String.format("%s slaves", slaveState.get().toString()); } From c2eee75547ca9ab8e22d66eed25e2a97e81b19ce Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 00:52:33 +0000 Subject: [PATCH 38/93] Fixed to work with the updated 'slaves' endpoint. Implemented getSingle using getSingeWithParams. Added getCollectionWithParams --- .../singularity/client/SingularityClient.java | 136 +++++++++++------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index 4935217cac..e7a8d9539c 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -9,6 +9,7 @@ import java.util.Map.Entry; import java.util.Random; +import javax.annotation.Nonnull; import javax.inject.Provider; import org.apache.commons.lang3.tuple.Pair; @@ -28,6 +29,7 @@ import com.hubspot.horizon.HttpRequest.Method; import com.hubspot.horizon.HttpResponse; import com.hubspot.mesos.json.MesosFileChunkObject; +import com.hubspot.singularity.MachineState; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -67,12 +69,8 @@ public class SingularityClient { private static final String RACKS_DELETE_DECOMISSIONING_FORMAT = RACKS_FORMAT + "/rack/%s/decomissioning"; private static final String SLAVES_FORMAT = "http://%s/%s/slaves"; - private static final String SLAVES_GET_ACTIVE_FORMAT = SLAVES_FORMAT + "/active"; - private static final String SLAVES_GET_DEAD_FORMAT = SLAVES_FORMAT + "/dead"; - private static final String SLAVES_GET_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/decomissioning"; - private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decomission"; - private static final String SLAVES_DELETE_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; - private static final String SLAVES_DELETE_DEAD_FORMAT = SLAVES_FORMAT + "/slave/%s/dead"; + private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decommission"; + private static final String SLAVES_DELETE_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; private static final String TASKS_FORMAT = "http://%s/%s/tasks"; private static final String TASKS_KILL_TASK_FORMAT = TASKS_FORMAT + "/task/%s"; @@ -182,26 +180,10 @@ private SingularityClientException fail(String type, HttpResponse response) { } private Optional getSingle(String uri, String type, String id, Class clazz) { - checkNotNull(id, String.format("Provide a %s id", type)); - - LOG.info("Getting {} {} from {}", type, id, uri); - - final long start = System.currentTimeMillis(); - - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); - - if (response.getStatusCode() == 404) { - return Optional.absent(); - } - - checkResponse(type, response); - - LOG.info("Got {} {} in {}ms", type, id, System.currentTimeMillis() - start); - - return Optional.fromNullable(response.getAs(clazz)); + return getSingleWithParams(uri, type, id, Optional.>absent(), clazz); } - private Optional getSingleWithParams(String uri, String type, String id, Map queryParams, Class clazz) { + private Optional getSingleWithParams(String uri, String type, String id, Optional> queryParams, Class clazz) { checkNotNull(id, String.format("Provide a %s id", type)); LOG.info("Getting {} {} from {}", type, id, uri); @@ -211,19 +193,8 @@ private Optional getSingleWithParams(String uri, String type, String id, HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() .setUrl(uri); - for (Entry queryParamEntry : queryParams.entrySet()) { - if (queryParamEntry.getValue() instanceof String) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Integer) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Long) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Boolean) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); - } else { - throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", - queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); - } + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); } HttpResponse response = httpClient.execute(requestBuilder.build()); @@ -240,11 +211,22 @@ private Optional getSingleWithParams(String uri, String type, String id, } private Collection getCollection(String uri, String type, TypeReference> typeReference) { + return getCollectionWithParams(uri, type, Optional.>absent(), typeReference); + } + + private Collection getCollectionWithParams(String uri, String type, Optional> queryParams, TypeReference> typeReference) { LOG.info("Getting all {} from {}", type, uri); final long start = System.currentTimeMillis(); - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); if (response.getStatusCode() == 404) { return ImmutableList.of(); @@ -257,6 +239,23 @@ private Collection getCollection(String uri, String type, TypeReference queryParams) { + for (Entry queryParamEntry : queryParams.entrySet()) { + if (queryParamEntry.getValue() instanceof String) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Integer) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Long) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Boolean) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); + } else { + throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", + queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); + } + } + } + private void delete(String uri, String type, String id, Optional user) { delete(uri, type, id, user, Optional.> absent()); } @@ -620,22 +619,55 @@ public void deleteDeadRack(String rackId, Optional user) { // SLAVES // + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getActiveSlaves() { - return getSlaves(SLAVES_GET_ACTIVE_FORMAT, "active"); + return getSlaves(Optional.of(MachineState.ACTIVE)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDeadSlaves() { - return getSlaves(SLAVES_GET_DEAD_FORMAT, "dead"); + return getSlaves(Optional.of(MachineState.DEAD)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDecomissioningSlaves() { - return getSlaves(SLAVES_GET_DECOMISSIONING_FORMAT, "decomissioning"); + return getSlaves(Optional.of(MachineState.DECOMMISSIONING)); } - private Collection getSlaves(String format, String type) { - final String requestUri = String.format(format, getHost(), contextPath); + /** + * Retrieve the list of all known slaves, optionally filtering by a particular slave state + * + * @param slaveState + * Optionally specify a particular state to filter slaves by + * @return + * A collection of {@link SingularitySlave} + */ + public Collection getSlaves(Optional slaveState) { + final String requestUri = String.format(SLAVES_FORMAT, getHost(), contextPath); + + Optional> maybeQueryParams = Optional.>absent(); + + String type = "slaves"; - return getCollection(requestUri, type, SLAVES_COLLECTION); + if (slaveState.isPresent()) { + maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get())); + + type = String.format("%s slaves", slaveState.get().toString()); + } + + return getCollectionWithParams(requestUri, type, maybeQueryParams, SLAVES_COLLECTION); } public void decomissionSlave(String slaveId, Optional user) { @@ -644,16 +676,10 @@ public void decomissionSlave(String slaveId, Optional user) { post(requestUri, String.format("decomission slave %s", slaveId), Optional.absent(), user); } - public void deleteDecomissioningSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DECOMISSIONING_FORMAT, getHost(), contextPath, slaveId); - - delete(requestUri, "decomissioning slave", slaveId, user); - } - - public void deleteDeadSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DEAD_FORMAT, getHost(), contextPath, slaveId); + public void deleteSlave(String slaveId, Optional user) { + final String requestUri = String.format(SLAVES_DELETE_FORMAT, getHost(), contextPath, slaveId); - delete(requestUri, "dead slave", slaveId, user); + delete(requestUri, "deleting slave", slaveId, user); } // @@ -746,7 +772,7 @@ public Collection getQueuedTaskUpdates(String webh public Optional browseTaskSandBox(String taskId, String path) { final String requestUrl = String.format(SANDBOX_BROWSE_FORMAT, getHost(), contextPath, taskId); - return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, ImmutableMap.of("path", path), SingularitySandbox.class); + return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, Optional.>of(ImmutableMap.of("path", path)), SingularitySandbox.class); } @@ -781,7 +807,7 @@ public Optional readSandBoxFile(String taskId, String path queryParamBuider.put("length", length.get()); } - return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); + return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, Optional.>of(queryParamBuider.build()), MesosFileChunkObject.class); } } From fc6190edbb87a44b22e4850cd4b07c0a11a11716 Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 01:16:05 +0000 Subject: [PATCH 39/93] The Nonnull annotation causes a dependency analysis failure for com.google.code.findbugs:annotations:jar ???? Will resolve at another time... --- .../java/com/hubspot/singularity/client/SingularityClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index e7a8d9539c..b7c0204ea8 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -9,7 +9,6 @@ import java.util.Map.Entry; import java.util.Random; -import javax.annotation.Nonnull; import javax.inject.Provider; import org.apache.commons.lang3.tuple.Pair; @@ -239,7 +238,7 @@ private Collection getCollectionWithParams(String uri, String type, Optio return response.getAs(typeReference); } - private void addQueryParams(@Nonnull HttpRequest.Builder requestBuilder, @Nonnull Map queryParams) { + private void addQueryParams(HttpRequest.Builder requestBuilder, Map queryParams) { for (Entry queryParamEntry : queryParams.entrySet()) { if (queryParamEntry.getValue() instanceof String) { requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); From ad412d0a40dde26d4c6c6bfc27cf9e5db6c2292a Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 01:34:05 +0000 Subject: [PATCH 40/93] set the string value of the enum in the query params map --- .../java/com/hubspot/singularity/client/SingularityClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index b7c0204ea8..655096a160 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -661,7 +661,7 @@ public Collection getSlaves(Optional slaveState) String type = "slaves"; if (slaveState.isPresent()) { - maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get())); + maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get().toString())); type = String.format("%s slaves", slaveState.get().toString()); } From 13c4c706c36ad14c41214b9d4a9e6adbadc1dd39 Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 00:52:33 +0000 Subject: [PATCH 41/93] Fixed to work with the updated 'slaves' endpoint. Implemented getSingle using getSingeWithParams. Added getCollectionWithParams --- .../singularity/client/SingularityClient.java | 136 +++++++++++------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index 4935217cac..e7a8d9539c 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -9,6 +9,7 @@ import java.util.Map.Entry; import java.util.Random; +import javax.annotation.Nonnull; import javax.inject.Provider; import org.apache.commons.lang3.tuple.Pair; @@ -28,6 +29,7 @@ import com.hubspot.horizon.HttpRequest.Method; import com.hubspot.horizon.HttpResponse; import com.hubspot.mesos.json.MesosFileChunkObject; +import com.hubspot.singularity.MachineState; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -67,12 +69,8 @@ public class SingularityClient { private static final String RACKS_DELETE_DECOMISSIONING_FORMAT = RACKS_FORMAT + "/rack/%s/decomissioning"; private static final String SLAVES_FORMAT = "http://%s/%s/slaves"; - private static final String SLAVES_GET_ACTIVE_FORMAT = SLAVES_FORMAT + "/active"; - private static final String SLAVES_GET_DEAD_FORMAT = SLAVES_FORMAT + "/dead"; - private static final String SLAVES_GET_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/decomissioning"; - private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decomission"; - private static final String SLAVES_DELETE_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; - private static final String SLAVES_DELETE_DEAD_FORMAT = SLAVES_FORMAT + "/slave/%s/dead"; + private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decommission"; + private static final String SLAVES_DELETE_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; private static final String TASKS_FORMAT = "http://%s/%s/tasks"; private static final String TASKS_KILL_TASK_FORMAT = TASKS_FORMAT + "/task/%s"; @@ -182,26 +180,10 @@ private SingularityClientException fail(String type, HttpResponse response) { } private Optional getSingle(String uri, String type, String id, Class clazz) { - checkNotNull(id, String.format("Provide a %s id", type)); - - LOG.info("Getting {} {} from {}", type, id, uri); - - final long start = System.currentTimeMillis(); - - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); - - if (response.getStatusCode() == 404) { - return Optional.absent(); - } - - checkResponse(type, response); - - LOG.info("Got {} {} in {}ms", type, id, System.currentTimeMillis() - start); - - return Optional.fromNullable(response.getAs(clazz)); + return getSingleWithParams(uri, type, id, Optional.>absent(), clazz); } - private Optional getSingleWithParams(String uri, String type, String id, Map queryParams, Class clazz) { + private Optional getSingleWithParams(String uri, String type, String id, Optional> queryParams, Class clazz) { checkNotNull(id, String.format("Provide a %s id", type)); LOG.info("Getting {} {} from {}", type, id, uri); @@ -211,19 +193,8 @@ private Optional getSingleWithParams(String uri, String type, String id, HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() .setUrl(uri); - for (Entry queryParamEntry : queryParams.entrySet()) { - if (queryParamEntry.getValue() instanceof String) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Integer) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Long) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); - } else if (queryParamEntry.getValue() instanceof Boolean) { - requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); - } else { - throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", - queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); - } + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); } HttpResponse response = httpClient.execute(requestBuilder.build()); @@ -240,11 +211,22 @@ private Optional getSingleWithParams(String uri, String type, String id, } private Collection getCollection(String uri, String type, TypeReference> typeReference) { + return getCollectionWithParams(uri, type, Optional.>absent(), typeReference); + } + + private Collection getCollectionWithParams(String uri, String type, Optional> queryParams, TypeReference> typeReference) { LOG.info("Getting all {} from {}", type, uri); final long start = System.currentTimeMillis(); - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); if (response.getStatusCode() == 404) { return ImmutableList.of(); @@ -257,6 +239,23 @@ private Collection getCollection(String uri, String type, TypeReference queryParams) { + for (Entry queryParamEntry : queryParams.entrySet()) { + if (queryParamEntry.getValue() instanceof String) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Integer) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Long) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Boolean) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); + } else { + throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", + queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); + } + } + } + private void delete(String uri, String type, String id, Optional user) { delete(uri, type, id, user, Optional.> absent()); } @@ -620,22 +619,55 @@ public void deleteDeadRack(String rackId, Optional user) { // SLAVES // + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getActiveSlaves() { - return getSlaves(SLAVES_GET_ACTIVE_FORMAT, "active"); + return getSlaves(Optional.of(MachineState.ACTIVE)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDeadSlaves() { - return getSlaves(SLAVES_GET_DEAD_FORMAT, "dead"); + return getSlaves(Optional.of(MachineState.DEAD)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDecomissioningSlaves() { - return getSlaves(SLAVES_GET_DECOMISSIONING_FORMAT, "decomissioning"); + return getSlaves(Optional.of(MachineState.DECOMMISSIONING)); } - private Collection getSlaves(String format, String type) { - final String requestUri = String.format(format, getHost(), contextPath); + /** + * Retrieve the list of all known slaves, optionally filtering by a particular slave state + * + * @param slaveState + * Optionally specify a particular state to filter slaves by + * @return + * A collection of {@link SingularitySlave} + */ + public Collection getSlaves(Optional slaveState) { + final String requestUri = String.format(SLAVES_FORMAT, getHost(), contextPath); + + Optional> maybeQueryParams = Optional.>absent(); + + String type = "slaves"; - return getCollection(requestUri, type, SLAVES_COLLECTION); + if (slaveState.isPresent()) { + maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get())); + + type = String.format("%s slaves", slaveState.get().toString()); + } + + return getCollectionWithParams(requestUri, type, maybeQueryParams, SLAVES_COLLECTION); } public void decomissionSlave(String slaveId, Optional user) { @@ -644,16 +676,10 @@ public void decomissionSlave(String slaveId, Optional user) { post(requestUri, String.format("decomission slave %s", slaveId), Optional.absent(), user); } - public void deleteDecomissioningSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DECOMISSIONING_FORMAT, getHost(), contextPath, slaveId); - - delete(requestUri, "decomissioning slave", slaveId, user); - } - - public void deleteDeadSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DEAD_FORMAT, getHost(), contextPath, slaveId); + public void deleteSlave(String slaveId, Optional user) { + final String requestUri = String.format(SLAVES_DELETE_FORMAT, getHost(), contextPath, slaveId); - delete(requestUri, "dead slave", slaveId, user); + delete(requestUri, "deleting slave", slaveId, user); } // @@ -746,7 +772,7 @@ public Collection getQueuedTaskUpdates(String webh public Optional browseTaskSandBox(String taskId, String path) { final String requestUrl = String.format(SANDBOX_BROWSE_FORMAT, getHost(), contextPath, taskId); - return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, ImmutableMap.of("path", path), SingularitySandbox.class); + return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, Optional.>of(ImmutableMap.of("path", path)), SingularitySandbox.class); } @@ -781,7 +807,7 @@ public Optional readSandBoxFile(String taskId, String path queryParamBuider.put("length", length.get()); } - return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, queryParamBuider.build(), MesosFileChunkObject.class); + return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, Optional.>of(queryParamBuider.build()), MesosFileChunkObject.class); } } From 58ad72921ea857d4a86c0179e5e112cd9efaeb6d Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 01:16:05 +0000 Subject: [PATCH 42/93] The Nonnull annotation causes a dependency analysis failure for com.google.code.findbugs:annotations:jar ???? Will resolve at another time... --- .../java/com/hubspot/singularity/client/SingularityClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index e7a8d9539c..b7c0204ea8 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -9,7 +9,6 @@ import java.util.Map.Entry; import java.util.Random; -import javax.annotation.Nonnull; import javax.inject.Provider; import org.apache.commons.lang3.tuple.Pair; @@ -239,7 +238,7 @@ private Collection getCollectionWithParams(String uri, String type, Optio return response.getAs(typeReference); } - private void addQueryParams(@Nonnull HttpRequest.Builder requestBuilder, @Nonnull Map queryParams) { + private void addQueryParams(HttpRequest.Builder requestBuilder, Map queryParams) { for (Entry queryParamEntry : queryParams.entrySet()) { if (queryParamEntry.getValue() instanceof String) { requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); From 6e93dbfeee173d76f47f376c62d3aeb4ffc6e043 Mon Sep 17 00:00:00 2001 From: Grigorios Chomatas Date: Wed, 18 Feb 2015 01:34:05 +0000 Subject: [PATCH 43/93] set the string value of the enum in the query params map --- .../java/com/hubspot/singularity/client/SingularityClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index b7c0204ea8..655096a160 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -661,7 +661,7 @@ public Collection getSlaves(Optional slaveState) String type = "slaves"; if (slaveState.isPresent()) { - maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get())); + maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get().toString())); type = String.format("%s slaves", slaveState.get().toString()); } From 842af60b5dcba53cbf86620b0030cdb0e59e5436 Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 23 Feb 2015 13:28:37 -0500 Subject: [PATCH 44/93] support resources specifically for executors --- .../singularity/SingularityDeploy.java | 10 +++++ .../singularity/SingularityDeployBuilder.java | 15 ++++++- .../config/CustomExecutorConfiguration.java | 27 ++++++++++++ .../config/SingularityConfiguration.java | 11 +++++ .../mesos/SingularityMesosTaskBuilder.java | 41 ++++++++++++++++--- .../SingularityMesosTaskBuilderTest.java | 3 +- 6 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java index bec287502d..0e8bcf0628 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java @@ -29,6 +29,8 @@ public class SingularityDeploy { private final Optional customExecutorCmd; private final Optional customExecutorId; private final Optional customExecutorSource; + private final Optional customExecutorResources; + private final Optional resources; private final Optional command; @@ -63,6 +65,7 @@ public SingularityDeploy(@JsonProperty("requestId") String requestId, @JsonProperty("customExecutorCmd") Optional customExecutorCmd, @JsonProperty("customExecutorId") Optional customExecutorId, @JsonProperty("customExecutorSource") Optional customExecutorSource, + @JsonProperty("customExecutorResources") Optional customExecutorResources, @JsonProperty("resources") Optional resources, @JsonProperty("env") Optional> env, @JsonProperty("uris") Optional> uris, @@ -90,6 +93,7 @@ public SingularityDeploy(@JsonProperty("requestId") String requestId, this.customExecutorCmd = customExecutorCmd; this.customExecutorId = customExecutorId; this.customExecutorSource = customExecutorSource; + this.customExecutorResources = customExecutorResources; this.metadata = metadata; this.version = version; @@ -190,6 +194,11 @@ public Optional getCustomExecutorId() { @ApiModelProperty(required=false, value="Custom Mesos executor source.") public Optional getCustomExecutorSource() { return customExecutorSource; } + @ApiModelProperty(required=false, value="Resources to allocate for custom mesos executor") + public Optional getCustomExecutorResources() { + return customExecutorResources; + } + @ApiModelProperty(required=false, value="Resources required for this deploy.", dataType="com.hubspot.mesos.Resources") public Optional getResources() { return resources; @@ -272,6 +281,7 @@ public String toString() { ", customExecutorCmd=" + customExecutorCmd + ", customExecutorId=" + customExecutorId + ", customExecutorSource=" + customExecutorSource + + ", customExecutorResources=" + customExecutorResources + ", resources=" + resources + ", command=" + command + ", arguments=" + arguments + diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java index 0ab2a9cb84..360f0e26f9 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java @@ -23,6 +23,8 @@ public class SingularityDeployBuilder { private Optional customExecutorCmd; private Optional customExecutorId; private Optional customExecutorSource; + private Optional customExecutorResources; + private Optional resources; private Optional command; @@ -54,6 +56,7 @@ public SingularityDeployBuilder(String requestId, String id) { this.customExecutorCmd = Optional.absent(); this.customExecutorId = Optional.absent(); this.customExecutorSource = Optional.absent(); + this.customExecutorResources = Optional.absent(); this.resources = Optional.absent(); this.command = Optional.absent(); this.arguments = Optional.absent(); @@ -72,7 +75,7 @@ public SingularityDeployBuilder(String requestId, String id) { } public SingularityDeploy build() { - return new SingularityDeploy(requestId, id, command, arguments, containerInfo, customExecutorCmd, customExecutorId, customExecutorSource, resources, env, uris, metadata, executorData, version, timestamp, deployHealthTimeoutSeconds, healthcheckUri, healthcheckIntervalSeconds, + return new SingularityDeploy(requestId, id, command, arguments, containerInfo, customExecutorCmd, customExecutorId, customExecutorSource, customExecutorResources, resources, env, uris, metadata, executorData, version, timestamp, deployHealthTimeoutSeconds, healthcheckUri, healthcheckIntervalSeconds, healthcheckTimeoutSeconds, serviceBasePath, loadBalancerGroups, considerHealthyAfterRunningForSeconds, loadBalancerOptions, skipHealthchecksOnDeploy); } @@ -161,6 +164,15 @@ public SingularityDeployBuilder setCustomExecutorSource(Optional customE return this; } + public Optional getCustomExecutorResources() { + return customExecutorResources; + } + + public SingularityDeployBuilder setCustomExecutorResources(Optional customExecutorResources) { + this.customExecutorResources = customExecutorResources; + return this; + } + public Optional getDeployHealthTimeoutSeconds() { return deployHealthTimeoutSeconds; } @@ -299,6 +311,7 @@ public String toString() { ", customExecutorCmd=" + customExecutorCmd + ", customExecutorId=" + customExecutorId + ", customExecutorSource=" + customExecutorSource + + ", customExecutorResources=" + customExecutorResources + ", resources=" + resources + ", command=" + command + ", arguments=" + arguments + diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java new file mode 100644 index 0000000000..c656986874 --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java @@ -0,0 +1,27 @@ +package com.hubspot.singularity.config; + +import javax.validation.constraints.Min; + +public class CustomExecutorConfiguration { + @Min(0) + private double numCpus = 0; + + @Min(0) + private int memoryMb = 0; + + public double getNumCpus() { + return numCpus; + } + + public void setNumCpus(double numCpus) { + this.numCpus = numCpus; + } + + public int getMemoryMb() { + return memoryMb; + } + + public void setMemoryMb(int memoryMb) { + this.memoryMb = memoryMb; + } +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index ab019c4f24..7d5b3bcb92 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -152,6 +152,10 @@ public class SingularityConfiguration extends Configuration { private long threadpoolShutdownDelayInSeconds = 1; + @Valid + @JsonProperty("customExecutor") + private CustomExecutorConfiguration customExecutorConfiguration; + @JsonProperty("zookeeper") @Valid private ZooKeeperConfiguration zooKeeperConfiguration; @@ -656,4 +660,11 @@ public void setZooKeeperConfiguration(ZooKeeperConfiguration zooKeeperConfigurat this.zooKeeperConfiguration = zooKeeperConfiguration; } + public CustomExecutorConfiguration getCustomExecutorConfiguration() { + return customExecutorConfiguration; + } + + public void setCustomExecutorConfiguration(CustomExecutorConfiguration customExecutorConfiguration) { + this.customExecutorConfiguration = customExecutorConfiguration; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java index 30cb756d5a..4f071d36e2 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java @@ -37,9 +37,11 @@ import com.hubspot.mesos.SingularityDockerInfo; import com.hubspot.mesos.SingularityDockerPortMapping; import com.hubspot.mesos.SingularityVolume; +import com.hubspot.singularity.SingularityDeploy; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskRequest; +import com.hubspot.singularity.config.SingularityConfiguration; import com.hubspot.singularity.data.ExecutorIdGenerator; @Singleton @@ -50,12 +52,14 @@ class SingularityMesosTaskBuilder { private final ObjectMapper objectMapper; private final SingularitySlaveAndRackManager slaveAndRackManager; private final ExecutorIdGenerator idGenerator; + private final SingularityConfiguration configuration; @Inject - SingularityMesosTaskBuilder(ObjectMapper objectMapper, SingularitySlaveAndRackManager slaveAndRackManager, ExecutorIdGenerator idGenerator) { + SingularityMesosTaskBuilder(ObjectMapper objectMapper, SingularitySlaveAndRackManager slaveAndRackManager, ExecutorIdGenerator idGenerator, SingularityConfiguration configuration) { this.objectMapper = objectMapper; this.slaveAndRackManager = slaveAndRackManager; this.idGenerator = idGenerator; + this.configuration = configuration; } public SingularityTask buildTask(Protos.Offer offer, List availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources) { @@ -209,16 +213,41 @@ private void prepareContainerInfo(final SingularityTaskId taskId, final TaskInfo bldr.setContainer(containerBuilder); } + private List buildCustomExecutorResources(final SingularityTaskRequest task) { + ImmutableList.Builder builder = ImmutableList.builder(); + + if (task.getDeploy().getCustomExecutorResources().isPresent()) { + if (task.getDeploy().getCustomExecutorResources().get().getCpus() > 0) { + builder.add(MesosUtils.getCpuResource(task.getDeploy().getCustomExecutorResources().get().getCpus())); + } + + if (task.getDeploy().getCustomExecutorResources().get().getMemoryMb() > 0) { + builder.add(MesosUtils.getMemoryResource(task.getDeploy().getCustomExecutorResources().get().getMemoryMb())); + } + } else { + if (configuration.getCustomExecutorConfiguration().getNumCpus() > 0) { + builder.add(MesosUtils.getCpuResource(configuration.getCustomExecutorConfiguration().getNumCpus())); + } + + if (configuration.getCustomExecutorConfiguration().getMemoryMb() > 0) { + builder.add(MesosUtils.getMemoryResource(configuration.getCustomExecutorConfiguration().getMemoryMb())); + } + } + + return builder.build(); + } + private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final Optional ports) { CommandInfo.Builder commandBuilder = CommandInfo.newBuilder().setValue(task.getDeploy().getCustomExecutorCmd().get()); prepareEnvironment(task, taskId, commandBuilder, ports); - bldr.setExecutor( - ExecutorInfo.newBuilder() - .setCommand(commandBuilder.build()) - .setExecutorId(ExecutorID.newBuilder().setValue(task.getDeploy().getCustomExecutorId().or(idGenerator.getNextExecutorId()))) - .setSource(task.getDeploy().getCustomExecutorSource().or(task.getPendingTask().getPendingTaskId().getId())) + bldr.setExecutor(ExecutorInfo.newBuilder() + .setCommand(commandBuilder.build()) + .setExecutorId(ExecutorID.newBuilder().setValue(task.getDeploy().getCustomExecutorId().or(idGenerator.getNextExecutorId()))) + .setSource(task.getDeploy().getCustomExecutorSource().or(task.getPendingTask().getPendingTaskId().getId())) + .addAllResources(buildCustomExecutorResources(task)) + .build() ); if (task.getDeploy().getExecutorData().isPresent()) { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java index 1b5afc64e1..9cf8f14086 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java @@ -40,6 +40,7 @@ import com.hubspot.singularity.SingularityRequestBuilder; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskRequest; +import com.hubspot.singularity.config.SingularityConfiguration; import com.hubspot.singularity.data.ExecutorIdGenerator; public class SingularityMesosTaskBuilderTest { @@ -57,7 +58,7 @@ public void createMocks() { when(idGenerator.getNextExecutorId()).then(new CreateFakeId()); - builder = new SingularityMesosTaskBuilder(new ObjectMapper(), rackManager, idGenerator); + builder = new SingularityMesosTaskBuilder(new ObjectMapper(), rackManager, idGenerator, new SingularityConfiguration()); resources = new Resources(1, 1, 0); offer = Offer.newBuilder() From 96ac2eac69c9adfa26f6746e7d25108100af69bc Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 23 Feb 2015 13:45:43 -0500 Subject: [PATCH 45/93] factor in custom executor resources during allocation --- .../src/main/java/com/hubspot/mesos/Resources.java | 8 ++++++++ .../com/hubspot/singularity/SingularityMainModule.java | 7 +++++++ .../singularity/mesos/SingularityMesosScheduler.java | 10 +++++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java b/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java index 7b9521a7bc..9554415ab8 100644 --- a/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java +++ b/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java @@ -3,7 +3,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import static com.google.common.base.Preconditions.checkNotNull; + public class Resources { + public static Resources add(Resources a, Resources b) { + checkNotNull(a, "first argument of Resources.add() is null"); + checkNotNull(b, "second argument of Resources.add() is null"); + + return new Resources(a.getCpus() + b.getCpus(), a.getMemoryMb() + b.getMemoryMb(), a.getNumPorts() + b.getNumPorts()); + } private final double cpus; private final double memoryMb; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java index fb5f8579f8..0b1d835c21 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java @@ -3,6 +3,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.inject.name.Names.named; +import com.hubspot.singularity.config.CustomExecutorConfiguration; import com.hubspot.singularity.smtp.JadeTemplateLoader; import com.hubspot.singularity.smtp.MailTemplateHelpers; import com.hubspot.singularity.smtp.SingularityMailRecordCleaner; @@ -207,6 +208,12 @@ public MesosConfiguration mesosConfiguration(final SingularityConfiguration conf return config.getMesosConfiguration(); } + @Provides + @Singleton + public CustomExecutorConfiguration customExecutorConfiguration(final SingularityConfiguration config) { + return config.getCustomExecutorConfiguration(); + } + @Provides @Singleton public Optional smtpConfiguration(final SingularityConfiguration config) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java index cde37d7129..667b731122 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java @@ -34,6 +34,7 @@ import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskRequest; import com.hubspot.singularity.SingularityTaskStatusHolder; +import com.hubspot.singularity.config.CustomExecutorConfiguration; import com.hubspot.singularity.config.MesosConfiguration; import com.hubspot.singularity.data.DeployManager; import com.hubspot.singularity.data.TaskManager; @@ -51,6 +52,7 @@ public class SingularityMesosScheduler implements Scheduler { private static final Logger LOG = LoggerFactory.getLogger(SingularityMesosScheduler.class); private final Resources defaultResources; + private final Resources defaultCustomExecutorResources; private final TaskManager taskManager; private final DeployManager deployManager; private final SingularityScheduler scheduler; @@ -71,8 +73,9 @@ public class SingularityMesosScheduler implements Scheduler { SingularityMesosScheduler(MesosConfiguration mesosConfiguration, TaskManager taskManager, SingularityScheduler scheduler, SingularitySlaveAndRackManager slaveAndRackManager, SingularitySchedulerPriority schedulerPriority, SingularityNewTaskChecker newTaskChecker, SingularityMesosTaskBuilder mesosTaskBuilder, SingularityLogSupport logSupport, Provider stateCacheProvider, SingularityHealthchecker healthchecker, DeployManager deployManager, - @Named(SingularityMainModule.SERVER_ID_PROPERTY) String serverId, SchedulerDriverSupplier schedulerDriverSupplier, final IdTranscoder taskIdTranscoder) { + @Named(SingularityMainModule.SERVER_ID_PROPERTY) String serverId, SchedulerDriverSupplier schedulerDriverSupplier, final IdTranscoder taskIdTranscoder, CustomExecutorConfiguration customExecutorConfiguration) { this.defaultResources = new Resources(mesosConfiguration.getDefaultCpus(), mesosConfiguration.getDefaultMemory(), 0); + this.defaultCustomExecutorResources = new Resources(customExecutorConfiguration.getNumCpus(), customExecutorConfiguration.getMemoryMb(), 0); this.taskManager = taskManager; this.deployManager = deployManager; this.schedulerPriority = schedulerPriority; @@ -197,6 +200,11 @@ private Optional match(Collection taskR taskResources = taskRequest.getDeploy().getResources().get(); } + // if using a custom executor, factor in custom executor resources too + if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) { + taskResources = Resources.add(taskResources, taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources)); + } + LOG.trace("Attempting to match task {} resources {} with remaining offer resources {}", taskRequest.getPendingTask().getPendingTaskId(), taskResources, offerHolder.getCurrentResources()); final boolean matchesResources = MesosUtils.doesOfferMatchResources(taskResources, offerHolder.getCurrentResources()); From 8ee2cf205bcadb94a7c58113bd3563fc790af4be Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 23 Feb 2015 14:14:19 -0500 Subject: [PATCH 46/93] fix null issue --- .../hubspot/singularity/config/SingularityConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index 7d5b3bcb92..2509f3606b 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -154,7 +154,8 @@ public class SingularityConfiguration extends Configuration { @Valid @JsonProperty("customExecutor") - private CustomExecutorConfiguration customExecutorConfiguration; + @NotNull + private CustomExecutorConfiguration customExecutorConfiguration = new CustomExecutorConfiguration(); @JsonProperty("zookeeper") @Valid From 70f12a0a5505862ce602f8d97ec81128477d8a16 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Mon, 23 Feb 2015 14:25:10 -0500 Subject: [PATCH 47/93] per-uploader credentials - s3uploader can have its own credentials - s3uploader can watch a PID to expire itself --- .../SingularityExecutorTaskLogManager.java | 3 +- .../runner/base/shared/ProcessUtils.java | 77 +++++++++++++++++-- .../runner/base/shared/S3UploadMetadata.java | 39 +++++++--- .../base/shared/SafeProcessManager.java | 8 +- .../runner/base/shared/Signal.java | 2 +- .../s3uploader/SingularityS3Uploader.java | 31 +++++++- .../SingularityS3UploaderDriver.java | 31 +++++--- 7 files changed, 157 insertions(+), 34 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index aed305a579..11424837a5 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -8,6 +8,7 @@ import org.slf4j.Logger; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.hubspot.singularity.SingularityS3FormatHelper; import com.hubspot.singularity.SingularityTaskId; @@ -171,7 +172,7 @@ public Path getLogrotateConfPath() { private boolean writeS3MetadataFile(boolean finished) { Path logrotateDirectory = taskDefinition.getServiceLogOutPath().getParent().resolve(configuration.getLogrotateToDirectory()); - S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(logrotateDirectory.toString(), getS3Glob(), configuration.getS3Bucket(), getS3KeyPattern(), finished); + S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(logrotateDirectory.toString(), getS3Glob(), configuration.getS3Bucket(), getS3KeyPattern(), finished, Optional. absent(), Optional. absent(), Optional. absent()); String s3UploadMetadatafilename = String.format("%s%s", taskDefinition.getTaskId(), configuration.getS3MetadataSuffix()); diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java index b8778c47bf..ad59c252ac 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java @@ -1,33 +1,88 @@ package com.hubspot.singularity.runner.base.shared; import java.io.IOException; +import java.io.InputStreamReader; import java.lang.reflect.Field; +import javax.annotation.Nullable; + import org.slf4j.Logger; +import com.google.common.base.Charsets; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import com.google.common.io.CharStreams; +import com.google.common.io.Closeables; import com.hubspot.mesos.JavaUtils; public class ProcessUtils { - public static void sendSignal(Signal signal, Logger log, int pid) { - final long start = System.currentTimeMillis(); + private final Optional log; + + public ProcessUtils() { + this(null); + } + + public ProcessUtils(@Nullable Logger log) { + this.log = Optional.fromNullable(log); + } + + public static class ProcessResult { + + private final int exitCode; + private final String output; + + public ProcessResult(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + + public int getExitCode() { + return exitCode; + } + + public String getOutput() { + return output; + } - log.info("Signaling {} ({}) to process {}", signal, signal.getCode(), pid); + @Override + public String toString() { + return "ProcessResult [exitCode=" + exitCode + ", output=" + output + "]"; + } + + } - final String killCmd = String.format("kill -%s %s", signal.getCode(), pid); + public ProcessResult sendSignal(Signal signal, int pid) { + final long start = System.currentTimeMillis(); + + if (log.isPresent()) { + log.get().info("Signaling {} ({}) to process {}", signal, signal.getCode(), pid); + } try { - int signalCode = Runtime.getRuntime().exec(killCmd).waitFor(); + final ProcessBuilder pb = new ProcessBuilder("kill", String.format("-%s", signal.getCode()), Integer.toString(pid)); + pb.redirectErrorStream(true); + + final Process p = pb.start(); - log.trace("Kill signal process for {} got exit code {} after {}", pid, signalCode, JavaUtils.duration(start)); + final int exitCode = p.waitFor(); + + final String output = CharStreams.toString(new InputStreamReader(p.getInputStream(), Charsets.UTF_8)); + + Closeables.closeQuietly(p.getInputStream()); + + if (log.isPresent()) { + log.get().trace("Kill signal process for {} got exit code {} after {}", pid, exitCode, JavaUtils.duration(start)); + } + + return new ProcessResult(exitCode, output.trim()); } catch (InterruptedException | IOException e) { throw Throwables.propagate(e); } } - public static int getUnixPID(Process process) { + public int getUnixPID(Process process) { Preconditions.checkArgument(process.getClass().getName().equals("java.lang.UNIXProcess")); Class clazz = process.getClass(); @@ -42,5 +97,13 @@ public static int getUnixPID(Process process) { } } + public boolean doesProcessExist(int pid) { + ProcessResult processResult = sendSignal(Signal.CHECK, pid); + if (processResult.getExitCode() != 0 && processResult.output.contains("No such process")) { + return false; + } + return true; + } + } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java index 34b43274b5..876d9bde3f 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; /** @@ -24,9 +25,13 @@ public class S3UploadMetadata { private final String s3Bucket; private final String s3KeyFormat; private final boolean finished; + private final Optional pid; + private final Optional s3AccessKey; + private final Optional s3Secret; @JsonCreator - public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, @JsonProperty("finished") boolean finished) { + public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, + @JsonProperty("finished") boolean finished, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, @JsonProperty("directory") Optional s3Secret) { Preconditions.checkNotNull(directory); Preconditions.checkNotNull(fileGlob); Preconditions.checkNotNull(s3Bucket); @@ -37,6 +42,9 @@ public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProper this.s3Bucket = s3Bucket; this.s3KeyFormat = s3KeyFormat; this.finished = finished; + this.pid = pid; + this.s3AccessKey = s3AccessKey; + this.s3Secret = s3Secret; } @Override @@ -51,28 +59,28 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) { - return true; + return true; } if (obj == null) { - return false; + return false; } if (getClass() != obj.getClass()) { - return false; + return false; } S3UploadMetadata other = (S3UploadMetadata) obj; if (directory == null) { if (other.directory != null) { return false; - } + } } else if (!directory.equals(other.directory)) { - return false; + return false; } if (fileGlob == null) { if (other.fileGlob != null) { return false; - } + } } else if (!fileGlob.equals(other.fileGlob)) { - return false; + return false; } return true; } @@ -97,9 +105,22 @@ public boolean isFinished() { return finished; } + public Optional getPid() { + return pid; + } + + public Optional getS3AccessKey() { + return s3AccessKey; + } + + public Optional getS3Secret() { + return s3Secret; + } + @Override public String toString() { - return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + "]"; + return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + ", pid=" + pid + + ", s3AccessKey=" + s3AccessKey + ", s3Secret=" + s3Secret + "]"; } } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java index 4a2d063195..4bf77efe79 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java @@ -29,9 +29,11 @@ public abstract class SafeProcessManager { private volatile Optional currentProcessStart; private final AtomicBoolean killed; + private final ProcessUtils processUtils; public SafeProcessManager(Logger log) { this.log = log; + this.processUtils = new ProcessUtils(log); this.currentProcessCmd = Optional.absent(); this.currentProcess = Optional.absent(); @@ -85,7 +87,7 @@ public Process startProcess(ProcessBuilder builder) { process = builder.start(); - currentProcessPid = Optional.of(ProcessUtils.getUnixPID(process)); + currentProcessPid = Optional.of(processUtils.getUnixPID(process)); currentProcess = Optional.of(process); currentProcessCmd = Optional.of(cmd); @@ -140,7 +142,7 @@ public void signalTermToProcessIfActive() { try { if (currentProcessPid.isPresent()) { - ProcessUtils.sendSignal(Signal.SIGTERM, log, currentProcessPid.get()); + processUtils.sendSignal(Signal.SIGTERM, currentProcessPid.get()); } } finally { this.processLock.unlock(); @@ -152,7 +154,7 @@ public void signalKillToProcessIfActive() { try { if (currentProcess.isPresent()) { - ProcessUtils.sendSignal(Signal.SIGKILL, log, currentProcessPid.get()); + processUtils.sendSignal(Signal.SIGKILL, currentProcessPid.get()); } } finally { this.processLock.unlock(); diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java index bd64242eb5..783f4e56ce 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java @@ -1,7 +1,7 @@ package com.hubspot.singularity.runner.base.shared; public enum Signal { - SIGTERM(15), SIGKILL(9); + SIGTERM(15), SIGKILL(9), CHECK(0); private final int code; diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index 22ce4dfcd8..9284873853 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -1,5 +1,6 @@ package com.hubspot.singularity.s3uploader; +import java.io.Closeable; import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -11,18 +12,22 @@ import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; +import org.jets3t.service.ServiceException; +import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; +import org.jets3t.service.security.AWSCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.codahale.metrics.Timer.Context; +import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.SingularityS3FormatHelper; import com.hubspot.singularity.runner.base.shared.S3UploadMetadata; -public class SingularityS3Uploader { +public class SingularityS3Uploader implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(SingularityS3Uploader.class); @@ -35,8 +40,19 @@ public class SingularityS3Uploader { private final SingularityS3UploaderMetrics metrics; private final String logIdentifier; - public SingularityS3Uploader(S3Service s3Service, S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath) { - this.s3Service = s3Service; + public SingularityS3Uploader(AWSCredentials defaultCredentials, S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath) { + AWSCredentials credentials = defaultCredentials; + + if (uploadMetadata.getS3Secret().isPresent() && uploadMetadata.getS3AccessKey().isPresent()) { + credentials = new AWSCredentials(uploadMetadata.getS3AccessKey().get(), uploadMetadata.getS3Secret().get()); + } + + try { + this.s3Service = new RestS3Service(credentials); + } catch (S3ServiceException e) { + throw Throwables.propagate(e); + } + this.metrics = metrics; this.uploadMetadata = uploadMetadata; this.fileDirectory = uploadMetadata.getDirectory(); @@ -54,6 +70,15 @@ public S3UploadMetadata getUploadMetadata() { return uploadMetadata; } + @Override + public void close() throws IOException { + try { + s3Service.shutdown(); + } catch (ServiceException e) { + throw new IOException(e); + } + } + @Override public String toString() { return "SingularityS3Uploader [uploadMetadata=" + uploadMetadata + ", metadataPath=" + metadataPath + "]"; diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java index 836d104a52..600b8fc6ff 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java @@ -22,8 +22,6 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.jets3t.service.S3Service; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.security.AWSCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,10 +33,12 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.io.Closeables; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.runner.base.shared.JsonObjectFileHelper; +import com.hubspot.singularity.runner.base.shared.ProcessUtils; import com.hubspot.singularity.runner.base.shared.S3UploadMetadata; import com.hubspot.singularity.runner.base.shared.SingularityDriver; import com.hubspot.singularity.runner.base.shared.WatchServiceHelper; @@ -56,10 +56,11 @@ public class SingularityS3UploaderDriver extends WatchServiceHelper implements S private final Lock runLock; private final ExecutorService executorService; private final FileSystem fileSystem; - private final S3Service s3Service; private final Set expiring; private final SingularityS3UploaderMetrics metrics; private final JsonObjectFileHelper jsonObjectFileHelper; + private final ProcessUtils processUtils; + private final AWSCredentials defaultCredentials; private ScheduledFuture future; @@ -68,13 +69,9 @@ public SingularityS3UploaderDriver(SingularityS3UploaderConfiguration configurat super(configuration.getPollForShutDownMillis(), configuration.getS3MetadataDirectory(), ImmutableList.of(StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY, StandardWatchEventKinds.ENTRY_DELETE)); this.metrics = metrics; + this.defaultCredentials = new AWSCredentials(s3Configuration.getS3AccessKey(), s3Configuration.getS3AccessKey()); this.fileSystem = FileSystems.getDefault(); - try { - this.s3Service = new RestS3Service(new AWSCredentials(s3Configuration.getS3AccessKey(), s3Configuration.getS3SecretKey())); - } catch (Throwable t) { - throw Throwables.propagate(t); - } this.jsonObjectFileHelper = jsonObjectFileHelper; this.configuration = configuration; @@ -87,6 +84,8 @@ public SingularityS3UploaderDriver(SingularityS3UploaderConfiguration configurat this.runLock = new ReentrantLock(); + this.processUtils = new ProcessUtils(LOG); + this.executorService = JavaUtils.newFixedTimingOutThreadPool(configuration.getExecutorMaxUploadThreads(), TimeUnit.SECONDS.toMillis(30), "SingularityS3Uploader-%d"); this.scheduler = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("SingularityS3Driver-%d").build()); } @@ -245,6 +244,8 @@ public Integer call() { expiring.remove(expiredUploader); try { + Closeables.close(expiredUploader, true); + Files.delete(expiredUploader.getMetadataPath()); } catch (IOException e) { LOG.warn("Couldn't delete {}", expiredUploader.getMetadataPath(), e); @@ -255,7 +256,17 @@ public Integer call() { } private boolean isFinished(SingularityS3Uploader uploader) { - return expiring.contains(uploader); + if (expiring.contains(uploader)) { + return true; + } + + if (uploader.getUploadMetadata().getPid().isPresent()) { + if (!processUtils.doesProcessExist(uploader.getUploadMetadata().getPid().get())) { + return true; + } + } + + return false; } private boolean handleNewOrModifiedS3Metadata(Path filename) throws IOException { @@ -289,7 +300,7 @@ private boolean handleNewOrModifiedS3Metadata(Path filename) throws IOException try { metrics.getUploaderCounter().inc(); - SingularityS3Uploader uploader = new SingularityS3Uploader(s3Service, metadata, fileSystem, metrics, filename); + SingularityS3Uploader uploader = new SingularityS3Uploader(defaultCredentials, metadata, fileSystem, metrics, filename); if (metadata.isFinished()) { expiring.add(uploader); From 13b93245f5d2e0e88847b57caaee16573cd18708 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Tue, 24 Feb 2015 08:15:01 -0500 Subject: [PATCH 48/93] add cpu usage calculation --- .../app/controllers/TaskDetail.coffee | 24 +++++++++++++++---- .../app/models/TaskResourceUsage.coffee | 17 ++++++++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/SingularityUI/app/controllers/TaskDetail.coffee b/SingularityUI/app/controllers/TaskDetail.coffee index 47f27ef780..e599a175a4 100644 --- a/SingularityUI/app/controllers/TaskDetail.coffee +++ b/SingularityUI/app/controllers/TaskDetail.coffee @@ -82,6 +82,15 @@ class TaskDetailController extends Controller app.showView @view + # Refresh once again on page load to calculate CPU Usage + @initialPageLoad = true + + if @initialPageLoad + setTimeout (=> + @refresh() + @initialPageLoad = false + ), 2000 + refresh: -> @models.task.fetch error: => @@ -89,10 +98,17 @@ class TaskDetailController extends Controller app.caughtError() app.router.notFound() - @models.resourceUsage?.fetch().error => - # If this 404s there's nothing to get so don't bother - app.caughtError() - delete @models.resourceUsage + + @models.resourceUsage?.fetch() + .done => + # Store current resource usage to compare against future resource usage + @models.resourceUsage.setCpuUsage() if @models.resourceUsage.get('previousUsage') + @models.resourceUsage.set('previousUsage', @models.resourceUsage.toJSON()) + + .error => + # If this 404s there's nothing to get so don't bother + app.caughtError() + delete @models.resourceUsage if @collections.s3Logs?.currentPage is 1 @collections.s3Logs.fetch().error => diff --git a/SingularityUI/app/models/TaskResourceUsage.coffee b/SingularityUI/app/models/TaskResourceUsage.coffee index 6a202623ef..366e799973 100644 --- a/SingularityUI/app/models/TaskResourceUsage.coffee +++ b/SingularityUI/app/models/TaskResourceUsage.coffee @@ -6,4 +6,19 @@ class TaskResourceUsage extends Model initialize: ({ @taskId }) => -module.exports = TaskResourceUsage + # Calculate CPU Usage by comparing previous usage to current usage + setCpuUsage: -> + previous = @get('previousUsage') + currentTime = @get('cpusSystemTimeSecs') + @get('cpusUserTimeSecs') + previousTime = previous.cpusSystemTimeSecs + previous.cpusUserTimeSecs + timestampDiff = @get('timestamp') - previous.timestamp + cpus_used_raw = (currentTime - previousTime) / timestampDiff + cpus_used = Math.round(cpus_used_raw * 100) / 100 + cpuUsageExceeding = (cpus_used / @get('cpusLimit')) > 1.10 + + if cpuUsageExceeding + @set 'cpuUsageExceeding', cpuUsageExceeding + @set 'cpuUsageClassStatus', 'danger' + @set 'cpuUsage', cpus_used + +module.exports = TaskResourceUsage \ No newline at end of file From 7bde938c55a39f35570b6abd9d1bc7686b9c7eeb Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Tue, 24 Feb 2015 08:15:08 -0500 Subject: [PATCH 49/93] add style and markup for cpu usage --- SingularityUI/app/styles/loader.styl | 6 ++ .../taskDetail/taskResourceUsage.hbs | 59 +++++++++++-------- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/SingularityUI/app/styles/loader.styl b/SingularityUI/app/styles/loader.styl index 70a5f27cc3..a395fb19d7 100644 --- a/SingularityUI/app/styles/loader.styl +++ b/SingularityUI/app/styles/loader.styl @@ -31,6 +31,12 @@ &.cushy margin-top 3em margin-bottom 3em + + &.inline-left + display inline-block + margin-right 5px + + .page-loader-with-message text-align center diff --git a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs index c110d1a534..fedbdb8110 100644 --- a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs +++ b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs @@ -11,17 +11,41 @@
- {{#ifAll data.memRssBytes data.memLimitBytes}} -
-
-

Memory (rss vs limit)

-
-
+
+ {{#ifAll data.memRssBytes data.memLimitBytes}} +
+
+

Memory (rss vs limit)

+
+
+
+
+
+ {{humanizeFileSize data.memRssBytes}} / {{humanizeFileSize data.memLimitBytes}} +
- {{humanizeFileSize data.memRssBytes}} / {{humanizeFileSize data.memLimitBytes}} -
+ {{/ifAll}} + {{#ifAll data.cpusUserTimeSecs data.cpusSystemTimeSecs }} +
+
+

CPU Usage

+
+
+ {{#ifAll data.cpuUsage}} + {{#if data.cpuUsageExceeding }} +

CPU usage > 110% allocated

+ {{/if}} +
+
+
+ {{data.cpuUsage}} used / {{data.cpusLimit}} allocated CPUs + {{else}} + Calculating... + {{/ifAll}} +
+
+ {{/ifAll}}
- {{/ifAll}}
    {{#if data.cpusNrPeriods }} @@ -40,14 +64,7 @@
{{/if}} - {{#if data.cpusSystemTimeSecs }} -
  • -
    -

    System time (sec)

    -

    {{ data.cpusSystemTimeSecs }}

    -
    -
    • - {{/if}} + {{#if data.cpusThrottledTimeSecs }}
  • @@ -56,14 +73,6 @@
    • {{/if}} - {{#if data.cpusUserTimeSecs }} -
  • -
    -

    User time (sec)

    -

    {{ data.cpusUserTimeSecs }}

    -
    -
    • - {{/if}} {{#if data.memAnonBytes }}
  • From e6f1a08fe0241d0b98208e14eb140ec3de5976cc Mon Sep 17 00:00:00 2001 From: tpetr Date: Tue, 24 Feb 2015 11:27:38 -0500 Subject: [PATCH 50/93] upgrade to Baragon 0.1.4 + handle INVALID_REQUEST_NOOP --- .../com/hubspot/singularity/scheduler/SingularityCleaner.java | 3 +++ .../singularity/scheduler/SingularityNewTaskChecker.java | 1 + pom.xml | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java index 44127d404f..cb72474a8d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java @@ -383,6 +383,7 @@ private boolean shouldRemoveLbState(SingularityTaskId taskId, SingularityLoadBal case UNKNOWN: case WAITING: case SUCCESS: + case INVALID_REQUEST_NOOP: return true; default: LOG.trace("Task {} had abnormal LB state {}", taskId, loadBalancerUpdate); @@ -417,6 +418,7 @@ private boolean shouldEnqueueLbRequest(Optional m case CANCELING: case SUCCESS: case WAITING: + case INVALID_REQUEST_NOOP: } return false; @@ -459,6 +461,7 @@ private CheckLBState checkLbState(SingularityTaskId taskId) { switch (lbRemoveUpdate.getLoadBalancerState()) { case SUCCESS: + case INVALID_REQUEST_NOOP: return CheckLBState.DONE; case FAILED: case CANCELED: diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java index 4b1c558719..1166db48d0 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java @@ -300,6 +300,7 @@ private CheckTaskState getTaskState(SingularityTask task) { private Optional checkLbState(BaragonRequestState lbState) { switch (lbState) { case SUCCESS: + case INVALID_REQUEST_NOOP: return Optional.of(CheckTaskState.HEALTHY); case CANCELED: case FAILED: diff --git a/pom.xml b/pom.xml index 962ecf0a05..8496f77a0c 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ 1.3.12 2.3.2 - 0.1.2 + 0.1.4 2.4.2 0.7.1 0.9.0 From 0ce0dff2d9d0379d07d3be015e4415d8ad088fa0 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Tue, 24 Feb 2015 13:26:49 -0500 Subject: [PATCH 51/93] move rounding of cpus too at page render --- SingularityUI/app/handlebarsHelpers.coffee | 3 +++ SingularityUI/app/models/TaskResourceUsage.coffee | 3 +-- SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/SingularityUI/app/handlebarsHelpers.coffee b/SingularityUI/app/handlebarsHelpers.coffee index 5a1d7ef7a0..ecac1f1121 100644 --- a/SingularityUI/app/handlebarsHelpers.coffee +++ b/SingularityUI/app/handlebarsHelpers.coffee @@ -21,6 +21,9 @@ Handlebars.registerHelper "ifAll", (conditions..., options)-> Handlebars.registerHelper 'percentageOf', (v1, v2) -> (v1/v2) * 100 +Handlebars.registerHelper 'round', (value, place) -> + +(value).toFixed(place) + Handlebars.registerHelper 'ifInSubFilter', (needle, haystack, options) -> return options.fn @ if haystack is 'all' if haystack.indexOf(needle) isnt -1 diff --git a/SingularityUI/app/models/TaskResourceUsage.coffee b/SingularityUI/app/models/TaskResourceUsage.coffee index 366e799973..0ff4447556 100644 --- a/SingularityUI/app/models/TaskResourceUsage.coffee +++ b/SingularityUI/app/models/TaskResourceUsage.coffee @@ -12,8 +12,7 @@ class TaskResourceUsage extends Model currentTime = @get('cpusSystemTimeSecs') + @get('cpusUserTimeSecs') previousTime = previous.cpusSystemTimeSecs + previous.cpusUserTimeSecs timestampDiff = @get('timestamp') - previous.timestamp - cpus_used_raw = (currentTime - previousTime) / timestampDiff - cpus_used = Math.round(cpus_used_raw * 100) / 100 + cpus_used = (currentTime - previousTime) / timestampDiff cpuUsageExceeding = (cpus_used / @get('cpusLimit')) > 1.10 if cpuUsageExceeding diff --git a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs index fedbdb8110..a2dccd3321 100644 --- a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs +++ b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs @@ -38,7 +38,7 @@
    - {{data.cpuUsage}} used / {{data.cpusLimit}} allocated CPUs + {{round data.cpuUsage 2}} used / {{data.cpusLimit}} allocated CPUs {{else}} Calculating... {{/ifAll}} From 1182eb98f121784a75ebe3267b1d5237288c38d6 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Tue, 24 Feb 2015 13:27:54 -0500 Subject: [PATCH 52/93] only fetch resource usage if task is still running --- .../app/controllers/TaskDetail.coffee | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/SingularityUI/app/controllers/TaskDetail.coffee b/SingularityUI/app/controllers/TaskDetail.coffee index e599a175a4..87b8bc2ab9 100644 --- a/SingularityUI/app/controllers/TaskDetail.coffee +++ b/SingularityUI/app/controllers/TaskDetail.coffee @@ -82,23 +82,15 @@ class TaskDetailController extends Controller app.showView @view - # Refresh once again on page load to calculate CPU Usage + # Refresh once again on page load to calculate initial CPU Usage value @initialPageLoad = true - if @initialPageLoad setTimeout (=> @refresh() @initialPageLoad = false ), 2000 - refresh: -> - @models.task.fetch - error: => - # If this 404s the task doesn't exist - app.caughtError() - app.router.notFound() - - + fetchResourceUsage: -> @models.resourceUsage?.fetch() .done => # Store current resource usage to compare against future resource usage @@ -110,6 +102,17 @@ class TaskDetailController extends Controller app.caughtError() delete @models.resourceUsage + refresh: -> + @models.task.fetch() + .done => + @fetchResourceUsage() if @models.task.get('isStillRunning') + + .error => + # If this 404s the task doesn't exist + app.caughtError() + app.router.notFound() + + if @collections.s3Logs?.currentPage is 1 @collections.s3Logs.fetch().error => # It probably means S3 logs haven't been configured From 70ef585797afa6ab3330f87a8b22681f78f15717 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Feb 2015 14:34:36 -0500 Subject: [PATCH 53/93] handle edge cases for log rotate / s3uploader --- .../SingularityExecutorTaskBuilder.java | 6 +- .../task/SingularityExecutorTaskCleanup.java | 3 +- .../SingularityExecutorTaskDefinition.java | 22 ++++- .../SingularityExecutorTaskLogManager.java | 2 +- .../cleanup/SingularityExecutorCleanup.java | 80 ++++++++++++++++++- .../s3uploader/SingularityS3Uploader.java | 7 +- 6 files changed, 110 insertions(+), 10 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java index 8461cc53ab..cc131c85a5 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java @@ -2,7 +2,6 @@ import java.nio.file.Path; -import com.hubspot.mesos.MesosUtils; import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos; import org.apache.mesos.Protos.TaskInfo; @@ -16,6 +15,7 @@ import com.google.inject.Singleton; import com.google.inject.name.Named; import com.hubspot.deploy.ExecutorData; +import com.hubspot.mesos.MesosUtils; import com.hubspot.singularity.executor.TemplateManager; import com.hubspot.singularity.executor.task.SingularityExecutorArtifactFetcher; import com.hubspot.singularity.executor.task.SingularityExecutorTask; @@ -64,8 +64,8 @@ public Logger buildTaskLogger(String taskId) { public SingularityExecutorTask buildTask(String taskId, ExecutorDriver driver, TaskInfo taskInfo, Logger log) { ExecutorData executorData = readExecutorData(jsonObjectMapper, taskInfo); - SingularityExecutorTaskDefinition taskDefinition = new SingularityExecutorTaskDefinition(taskId, executorData, MesosUtils.getTaskDirectoryPath(taskId).toString(), configuration.getServiceLog(), - configuration.getTaskAppDirectory(), configuration.getExecutorBashLog(), configuration.getLogrotateStateFile()); + SingularityExecutorTaskDefinition taskDefinition = new SingularityExecutorTaskDefinition(taskId, executorData, MesosUtils.getTaskDirectoryPath(taskId).toString(), executorPid, + configuration.getServiceLog(), configuration.getTaskAppDirectory(), configuration.getExecutorBashLog(), configuration.getLogrotateStateFile()); jsonObjectFileHelper.writeObject(taskDefinition, configuration.getTaskDefinitionPath(taskId), log); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java index 7175241467..07fca79860 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java @@ -31,6 +31,7 @@ public boolean cleanup(boolean cleanupTaskAppDirectory) { if (!Files.exists(taskDirectory)) { log.info("Directory {} didn't exist for cleanup", taskDirectory); + taskLogManager.removeLogrotateFile(); return cleanTaskDefinitionFile(); } @@ -77,7 +78,7 @@ private boolean cleanupTaskAppDirectory() { "rm", "-rf", pathToDelete - ); + ); new SimpleProcessManager(log).runCommand(cmd); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java index 354e21f10e..022e474e09 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; import com.hubspot.deploy.ExecutorData; public class SingularityExecutorTaskDefinition { @@ -17,14 +18,16 @@ public class SingularityExecutorTaskDefinition { private final String serviceLogOut; private final String taskAppDirectory; private final String logrotateStateFile; + private final String executorPid; @JsonCreator - public SingularityExecutorTaskDefinition(@JsonProperty("taskId") String taskId, @JsonProperty("executorData") ExecutorData executorData, @JsonProperty("taskDirectory") String taskDirectory, + public SingularityExecutorTaskDefinition(@JsonProperty("taskId") String taskId, @JsonProperty("executorData") ExecutorData executorData, @JsonProperty("taskDirectory") String taskDirectory, @JsonProperty("executorPid") String executorPid, @JsonProperty("serviceLogOut") String serviceLogOut, @JsonProperty("taskAppDirectory") String taskAppDirectory, @JsonProperty("executorBashOut") String executorBashOut, @JsonProperty("logrotateStateFilePath") String logrotateStateFile) { this.executorData = executorData; this.taskId = taskId; this.taskDirectoryPath = Paths.get(taskDirectory); + this.executorPid = executorPid; this.executorBashOut = executorBashOut; this.serviceLogOut = serviceLogOut; this.taskAppDirectory = taskAppDirectory; @@ -84,10 +87,23 @@ public String getTaskId() { return taskId; } + public String getExecutorPid() { + return executorPid; + } + + @JsonIgnore + public Optional getExecutorPidSafe() { + try { + return Optional.of(Integer.parseInt(executorPid)); + } catch (NumberFormatException nfe) { + return Optional. absent(); + } + } + @Override public String toString() { - return "SingularityExecutorTaskDefinition [taskId=" + taskId + "]"; + return "SingularityExecutorTaskDefinition [executorData=" + executorData + ", taskId=" + taskId + ", taskDirectoryPath=" + taskDirectoryPath + ", executorBashOut=" + executorBashOut + + ", serviceLogOut=" + serviceLogOut + ", taskAppDirectory=" + taskAppDirectory + ", logrotateStateFile=" + logrotateStateFile + ", executorPid=" + executorPid + "]"; } - } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index 11424837a5..fac0e59c3e 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -89,7 +89,7 @@ private void copyLogTail() { } } - private boolean removeLogrotateFile() { + public boolean removeLogrotateFile() { boolean deleted = false; try { deleted = Files.deleteIfExists(getLogrotateConfPath()); diff --git a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java index 9a02673a37..d1f1ac45c6 100644 --- a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java +++ b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java @@ -2,9 +2,15 @@ import java.io.IOException; import java.net.SocketException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; import java.util.Set; import org.slf4j.Logger; @@ -12,6 +18,7 @@ import com.google.common.base.Optional; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.hubspot.mesos.JavaUtils; @@ -31,6 +38,9 @@ import com.hubspot.singularity.executor.task.SingularityExecutorTaskDefinition; import com.hubspot.singularity.executor.task.SingularityExecutorTaskLogManager; import com.hubspot.singularity.runner.base.shared.JsonObjectFileHelper; +import com.hubspot.singularity.runner.base.shared.ProcessFailedException; +import com.hubspot.singularity.runner.base.shared.ProcessUtils; +import com.hubspot.singularity.runner.base.shared.SimpleProcessManager; public class SingularityExecutorCleanup { @@ -42,6 +52,7 @@ public class SingularityExecutorCleanup { private final TemplateManager templateManager; private final SingularityExecutorCleanupConfiguration cleanupConfiguration; private final MesosClient mesosClient; + private final ProcessUtils processUtils; @Inject public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjectFileHelper jsonObjectFileHelper, SingularityExecutorConfiguration executorConfiguration, SingularityExecutorCleanupConfiguration cleanupConfiguration, TemplateManager templateManager, MesosClient mesosClient) { @@ -51,6 +62,7 @@ public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjec this.singularityClient = singularityClient; this.templateManager = templateManager; this.mesosClient = mesosClient; + this.processUtils = new ProcessUtils(LOG); } public SingularityExecutorCleanupStatistics clean() { @@ -101,7 +113,7 @@ public SingularityExecutorCleanupStatistics clean() { final String taskId = taskDefinition.get().getTaskId(); - if (runningTaskIds.contains(taskId)) { + if (runningTaskIds.contains(taskId) || executorStillRunning(taskDefinition.get())) { statisticsBldr.incrRunningTasksIgnored(); continue; } @@ -150,6 +162,16 @@ private Set getRunningTaskIds() { } } + private boolean executorStillRunning(SingularityExecutorTaskDefinition taskDefinition) { + Optional executorPidSafe = taskDefinition.getExecutorPidSafe(); + + if (!executorPidSafe.isPresent()) { + return false; + } + + return processUtils.doesProcessExist(executorPidSafe.get()); + } + private boolean cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional taskHistory) { SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, executorConfiguration, LOG, jsonObjectFileHelper); @@ -171,7 +193,63 @@ private boolean cleanTask(SingularityExecutorTaskDefinition taskDefinition, Opti } } + checkForUncompressedLogrotatedFile(taskDefinition); + return taskCleanup.cleanup(cleanupTaskAppDirectory); } + private Iterator getUncompressedLogrotatedFileIterator(SingularityExecutorTaskDefinition taskDefinition) { + final Path serviceLogOutPath = taskDefinition.getServiceLogOutPath(); + final Path logrotateToPath = taskDefinition.getServiceLogOutPath().getParent().resolve(executorConfiguration.getLogrotateToDirectory()); + + try { + DirectoryStream dirStream = Files.newDirectoryStream(logrotateToPath, String.format("%s-*", serviceLogOutPath.getFileName())); + return dirStream.iterator(); + } catch (IOException e) { + throw Throwables.propagate(e); + } + } + + private void checkForUncompressedLogrotatedFile(SingularityExecutorTaskDefinition taskDefinition) { + final Iterator iterator = getUncompressedLogrotatedFileIterator(taskDefinition); + final Set emptyPaths = new HashSet<>(); + final List ungzippedFiles = new ArrayList<>(); + + // check for matched 0 byte gz files.. and delete/gzip them + + while (iterator.hasNext()) { + Path path = iterator.next(); + + if (path.getFileName().toString().endsWith(".gz")) { + try { + if (Files.size(path) == 0) { + Files.deleteIfExists(path); + + String pathString = path.getFileName().toString(); + + emptyPaths.add(pathString.substring(0, pathString.length() - 3)); // removing .gz + } + } catch (IOException ioe) { + LOG.error("Failed to handle empty gz file {}", path, ioe); + } + } else { + ungzippedFiles.add(path); + } + } + + for (Path path : ungzippedFiles) { + if (emptyPaths.contains(path.getFileName().toString())) { + LOG.info("Gzipping abandoned file {}", path); + try { + new SimpleProcessManager(LOG).runCommand(ImmutableList. of("gzip", path.toString())); + } catch (InterruptedException | ProcessFailedException e) { + LOG.error("Failed to gzip {}", path, e); + } + } else { + LOG.debug("Didn't find matched empty gz file for {}", path); + } + } + } + + } diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index 9284873853..5610a092be 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -97,7 +97,12 @@ public int upload(Set synchronizedToUpload) throws IOException { for (Path file : JavaUtils.iterable(directory)) { if (!pathMatcher.matches(file.getFileName())) { - LOG.trace("{} Skipping {} because it didn't match {}", logIdentifier, file, uploadMetadata.getFileGlob()); + LOG.trace("{} Skipping {} because it doesn't match {}", logIdentifier, file, uploadMetadata.getFileGlob()); + continue; + } + + if (Files.size(file) == 0) { + LOG.trace("{} Skipping {} because its size is 0", logIdentifier, file); continue; } From 8a677518223052b85553f0fbf0fd761c3c26dc41 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Feb 2015 15:15:14 -0500 Subject: [PATCH 54/93] fix bug --- .../singularity/runner/base/shared/S3UploadMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java index 876d9bde3f..b75edeaae0 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java @@ -31,7 +31,7 @@ public class S3UploadMetadata { @JsonCreator public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, - @JsonProperty("finished") boolean finished, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, @JsonProperty("directory") Optional s3Secret) { + @JsonProperty("finished") boolean finished, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, @JsonProperty("s3Secret") Optional s3Secret) { Preconditions.checkNotNull(directory); Preconditions.checkNotNull(fileGlob); Preconditions.checkNotNull(s3Bucket); From a87defe7f41513b8ed4c46cd4cfe21d00b283442 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Feb 2015 15:59:10 -0500 Subject: [PATCH 55/93] another dumb bug --- .../singularity/s3uploader/SingularityS3UploaderDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java index 600b8fc6ff..61860edb2f 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java @@ -69,7 +69,7 @@ public SingularityS3UploaderDriver(SingularityS3UploaderConfiguration configurat super(configuration.getPollForShutDownMillis(), configuration.getS3MetadataDirectory(), ImmutableList.of(StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY, StandardWatchEventKinds.ENTRY_DELETE)); this.metrics = metrics; - this.defaultCredentials = new AWSCredentials(s3Configuration.getS3AccessKey(), s3Configuration.getS3AccessKey()); + this.defaultCredentials = new AWSCredentials(s3Configuration.getS3AccessKey(), s3Configuration.getS3SecretKey()); this.fileSystem = FileSystems.getDefault(); From a85f1f15135459f7bb04e976127cd7bc3dbe41ab Mon Sep 17 00:00:00 2001 From: tpetr Date: Wed, 25 Feb 2015 09:57:23 -0500 Subject: [PATCH 56/93] logic fixes --- .../com/hubspot/singularity/scheduler/SingularityCleaner.java | 3 ++- .../singularity/scheduler/SingularityNewTaskChecker.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java index cb72474a8d..8ce951f5bf 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java @@ -383,8 +383,9 @@ private boolean shouldRemoveLbState(SingularityTaskId taskId, SingularityLoadBal case UNKNOWN: case WAITING: case SUCCESS: - case INVALID_REQUEST_NOOP: return true; + case INVALID_REQUEST_NOOP: + return false; // don't need to remove because Baragon doesnt know about it default: LOG.trace("Task {} had abnormal LB state {}", taskId, loadBalancerUpdate); return false; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java index 1166db48d0..861f1b51d9 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java @@ -300,10 +300,10 @@ private CheckTaskState getTaskState(SingularityTask task) { private Optional checkLbState(BaragonRequestState lbState) { switch (lbState) { case SUCCESS: - case INVALID_REQUEST_NOOP: return Optional.of(CheckTaskState.HEALTHY); case CANCELED: case FAILED: + case INVALID_REQUEST_NOOP: return Optional.of(CheckTaskState.UNHEALTHY_KILL_TASK); case CANCELING: case UNKNOWN: From 65350a5b433dae2f0bc62876644666c27306d5c6 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Wed, 25 Feb 2015 13:59:15 -0500 Subject: [PATCH 57/93] rename rounding helper and default to 2 decimals --- SingularityUI/app/handlebarsHelpers.coffee | 5 ++++- SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/SingularityUI/app/handlebarsHelpers.coffee b/SingularityUI/app/handlebarsHelpers.coffee index ecac1f1121..453940a628 100644 --- a/SingularityUI/app/handlebarsHelpers.coffee +++ b/SingularityUI/app/handlebarsHelpers.coffee @@ -21,9 +21,12 @@ Handlebars.registerHelper "ifAll", (conditions..., options)-> Handlebars.registerHelper 'percentageOf', (v1, v2) -> (v1/v2) * 100 -Handlebars.registerHelper 'round', (value, place) -> +# Override decimal rounding: {{fixedDecimal data.cpuUsage place="4"}} +Handlebars.registerHelper 'fixedDecimal', (value, options) -> + if options.hash.place then place = options.hash.place else place = 2 +(value).toFixed(place) + Handlebars.registerHelper 'ifInSubFilter', (needle, haystack, options) -> return options.fn @ if haystack is 'all' if haystack.indexOf(needle) isnt -1 diff --git a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs index a2dccd3321..38a2ce361a 100644 --- a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs +++ b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs @@ -38,7 +38,7 @@
    - {{round data.cpuUsage 2}} used / {{data.cpusLimit}} allocated CPUs + {{fixedDecimal data.cpuUsage}} used / {{data.cpusLimit}} allocated CPUs {{else}} Calculating... {{/ifAll}} From 94580309b71099aebbcc06b2c0d1edae21b23a99 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Thu, 26 Feb 2015 09:44:54 -0500 Subject: [PATCH 58/93] fix global suppress refresh bug the global refresh check was only looking if the suppressRefresh key existed, not its value --- SingularityUI/app/application.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityUI/app/application.coffee b/SingularityUI/app/application.coffee index 3a61fc6608..53de3aee48 100644 --- a/SingularityUI/app/application.coffee +++ b/SingularityUI/app/application.coffee @@ -66,7 +66,7 @@ class Application setInterval @globalRefresh, @globalRefreshTime globalRefresh: => - return if localStorage.getItem 'suppressRefresh' + return if localStorage.getItem('suppressRefresh') is 'true' if @blurred clearInterval @globalRefreshInterval return From d1a8935d5c808c27ac105083d602076bd65343c4 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Mon, 9 Feb 2015 16:20:55 -0500 Subject: [PATCH 59/93] use hostname + web_url in framework info --- .../singularity/SingularityMainModule.java | 14 ++++++-------- .../config/SingularityConfiguration.java | 7 ++++--- .../singularity/mesos/SingularityDriver.java | 19 +++++++++++++++---- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java index 88510abee0..6c4f0269e4 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java @@ -2,12 +2,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.inject.name.Names.named; - -import com.hubspot.singularity.smtp.JadeTemplateLoader; -import com.hubspot.singularity.smtp.MailTemplateHelpers; -import com.hubspot.singularity.smtp.SingularityMailRecordCleaner; -import com.hubspot.singularity.smtp.SingularityMailer; -import com.hubspot.singularity.smtp.SingularitySmtpSender; import io.dropwizard.jetty.HttpConnectorFactory; import io.dropwizard.server.SimpleServerFactory; @@ -29,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; -import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.net.HostAndPort; import com.google.inject.Binder; @@ -56,6 +49,11 @@ import com.hubspot.singularity.sentry.NotifyingExceptionMapper; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; import com.hubspot.singularity.sentry.SingularityExceptionNotifierManaged; +import com.hubspot.singularity.smtp.JadeTemplateLoader; +import com.hubspot.singularity.smtp.MailTemplateHelpers; +import com.hubspot.singularity.smtp.SingularityMailRecordCleaner; +import com.hubspot.singularity.smtp.SingularityMailer; +import com.hubspot.singularity.smtp.SingularitySmtpSender; import com.ning.http.client.AsyncHttpClient; import de.neuland.jade4j.parser.Parser; @@ -136,7 +134,7 @@ public static class SingularityHostAndPortProvider implements Provider getCommonHostnameSuffixToOmit() { - return Optional.fromNullable(commonHostnameSuffixToOmit); + return Optional.fromNullable(Strings.emptyToNull(commonHostnameSuffixToOmit)); } public long getConsiderTaskHealthyAfterRunningForSeconds() { @@ -266,8 +267,8 @@ public long getHealthcheckTimeoutSeconds() { return healthcheckTimeoutSeconds; } - public String getHostname() { - return hostname; + public Optional getHostname() { + return Optional.fromNullable(Strings.emptyToNull(hostname)); } public long getKillAfterTasksDoNotRunDefaultSeconds() { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java index 50fdcc5a39..26a7675ea8 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java @@ -7,6 +7,7 @@ import org.apache.mesos.MesosSchedulerDriver; import org.apache.mesos.Protos; import org.apache.mesos.Protos.FrameworkID; +import org.apache.mesos.Protos.FrameworkInfo; import org.apache.mesos.Protos.MasterInfo; import org.apache.mesos.Protos.TaskID; import org.apache.mesos.Scheduler; @@ -20,6 +21,7 @@ import com.groupon.mesos.JesosSchedulerDriver; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.config.MesosConfiguration; +import com.hubspot.singularity.config.SingularityConfiguration; @Singleton public class SingularityDriver { @@ -31,14 +33,23 @@ public class SingularityDriver { private final SchedulerDriver driver; @Inject - SingularityDriver(final SingularityMesosSchedulerDelegator scheduler, final MesosConfiguration configuration) throws IOException { - this.frameworkInfo = Protos.FrameworkInfo.newBuilder() + SingularityDriver(final SingularityMesosSchedulerDelegator scheduler, final SingularityConfiguration singularityConfiguration, final MesosConfiguration configuration) throws IOException { + FrameworkInfo.Builder frameworkInfoBuilder = Protos.FrameworkInfo.newBuilder() .setCheckpoint(configuration.getCheckpoint()) .setFailoverTimeout(configuration.getFrameworkFailoverTimeout()) .setName(configuration.getFrameworkName()) .setId(FrameworkID.newBuilder().setValue(configuration.getFrameworkId())) - .setUser("") // let mesos assign - .build(); + .setUser(""); // let mesos assign + + if (singularityConfiguration.getHostname().isPresent()) { + frameworkInfoBuilder.setHostname(singularityConfiguration.getHostname().get()); + } + + if (singularityConfiguration.getUiConfiguration().getBaseUrl().isPresent()) { + frameworkInfoBuilder.setWebuiUrl(singularityConfiguration.getUiConfiguration().getBaseUrl().get()); + } + + this.frameworkInfo = frameworkInfoBuilder.build(); this.scheduler = scheduler; From 313183b954537487aca025a024e0b1642f28eeb9 Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 9 Feb 2015 18:46:01 -0500 Subject: [PATCH 60/93] generate proper webuiUrl based on ui config --- .../singularity/mesos/SingularityDriver.java | 18 ++++++++++++++---- .../singularity/resources/UiResource.java | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java index 26a7675ea8..1cbf141a9d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java @@ -18,10 +18,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.inject.Inject; +import com.google.inject.name.Named; import com.groupon.mesos.JesosSchedulerDriver; +import com.hubspot.singularity.SingularityMainModule; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.config.MesosConfiguration; import com.hubspot.singularity.config.SingularityConfiguration; +import com.hubspot.singularity.config.UIConfiguration; +import com.hubspot.singularity.resources.UiResource; @Singleton public class SingularityDriver { @@ -33,8 +37,9 @@ public class SingularityDriver { private final SchedulerDriver driver; @Inject - SingularityDriver(final SingularityMesosSchedulerDelegator scheduler, final SingularityConfiguration singularityConfiguration, final MesosConfiguration configuration) throws IOException { - FrameworkInfo.Builder frameworkInfoBuilder = Protos.FrameworkInfo.newBuilder() + SingularityDriver(final SingularityMesosSchedulerDelegator scheduler, final SingularityConfiguration singularityConfiguration, final MesosConfiguration configuration, + @Named(SingularityMainModule.SINGULARITY_URI_BASE) final String singularityUriBase) throws IOException { + final FrameworkInfo.Builder frameworkInfoBuilder = Protos.FrameworkInfo.newBuilder() .setCheckpoint(configuration.getCheckpoint()) .setFailoverTimeout(configuration.getFrameworkFailoverTimeout()) .setName(configuration.getFrameworkName()) @@ -45,8 +50,13 @@ public class SingularityDriver { frameworkInfoBuilder.setHostname(singularityConfiguration.getHostname().get()); } - if (singularityConfiguration.getUiConfiguration().getBaseUrl().isPresent()) { - frameworkInfoBuilder.setWebuiUrl(singularityConfiguration.getUiConfiguration().getBaseUrl().get()); + // only set the web UI URL if it's fully qualified + if (singularityUriBase.startsWith("http://") || singularityUriBase.startsWith("https://")) { + if (singularityConfiguration.getUiConfiguration().getRootUrlMode() == UIConfiguration.RootUrlMode.INDEX_CATCHALL) { + frameworkInfoBuilder.setWebuiUrl(singularityUriBase); + } else { + frameworkInfoBuilder.setWebuiUrl(singularityUriBase + UiResource.UI_RESOURCE_LOCATION); + } } this.frameworkInfo = frameworkInfoBuilder.build(); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java index b3ee39e797..3d74df4754 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java @@ -20,7 +20,7 @@ @Path(UiResource.UI_RESOURCE_LOCATION + "{uiPath:.*}") public class UiResource { - static final String UI_RESOURCE_LOCATION = "/ui/"; + public static final String UI_RESOURCE_LOCATION = "/ui/"; private final SingularityConfiguration configuration; private final String singularityUriBase; From 9672e9e0053af2c8e27c63a73813e1db91934afd Mon Sep 17 00:00:00 2001 From: tpetr Date: Thu, 26 Feb 2015 17:26:32 -0500 Subject: [PATCH 61/93] fix bug in original custom executor resources implementation --- .../java/com/hubspot/mesos/Resources.java | 2 ++ .../mesos/SingularityMesosScheduler.java | 18 +++++------- .../mesos/SingularityMesosTaskBuilder.java | 28 +++++++------------ .../SingularityMesosTaskBuilderTest.java | 6 ++-- 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java b/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java index 9554415ab8..a4e2306d7f 100644 --- a/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java +++ b/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java @@ -13,6 +13,8 @@ public static Resources add(Resources a, Resources b) { return new Resources(a.getCpus() + b.getCpus(), a.getMemoryMb() + b.getMemoryMb(), a.getNumPorts() + b.getNumPorts()); } + public static final Resources EMPTY_RESOURCES = new Resources(0, 0, 0); + private final double cpus; private final double memoryMb; private final int numPorts; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java index 667b731122..e6339bc4c2 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java @@ -194,24 +194,20 @@ public void resourceOffers(SchedulerDriver driver, List offers) { private Optional match(Collection taskRequests, SingularitySchedulerStateCache stateCache, SingularityOfferHolder offerHolder) { for (SingularityTaskRequest taskRequest : taskRequests) { - Resources taskResources = defaultResources; + final Resources taskResources = taskRequest.getDeploy().getResources().or(defaultResources); - if (taskRequest.getDeploy().getResources().isPresent()) { - taskResources = taskRequest.getDeploy().getResources().get(); - } + // only factor in executor resources if we're running a custom executor + final Optional executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(Optional.of(defaultCustomExecutorResources)) : Optional.absent(); - // if using a custom executor, factor in custom executor resources too - if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) { - taskResources = Resources.add(taskResources, taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources)); - } + final Resources totalResources = Resources.add(taskResources, executorResources.or(Resources.EMPTY_RESOURCES)); - LOG.trace("Attempting to match task {} resources {} with remaining offer resources {}", taskRequest.getPendingTask().getPendingTaskId(), taskResources, offerHolder.getCurrentResources()); + LOG.trace("Attempting to match task {} resources {} ({} for task + {} for executor) with remaining offer resources {}", taskRequest.getPendingTask().getPendingTaskId(), totalResources, taskResources, executorResources, offerHolder.getCurrentResources()); - final boolean matchesResources = MesosUtils.doesOfferMatchResources(taskResources, offerHolder.getCurrentResources()); + final boolean matchesResources = MesosUtils.doesOfferMatchResources(totalResources, offerHolder.getCurrentResources()); final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder.getOffer(), taskRequest, stateCache); if (matchesResources && slaveMatchState.isMatchAllowed()) { - final SingularityTask task = mesosTaskBuilder.buildTask(offerHolder.getOffer(), offerHolder.getCurrentResources(), taskRequest, taskResources); + final SingularityTask task = mesosTaskBuilder.buildTask(offerHolder.getOffer(), offerHolder.getCurrentResources(), taskRequest, taskResources, executorResources); LOG.trace("Accepted and built task {}", task); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java index 4f071d36e2..93284fddc5 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java @@ -62,7 +62,7 @@ class SingularityMesosTaskBuilder { this.configuration = configuration; } - public SingularityTask buildTask(Protos.Offer offer, List availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources) { + public SingularityTask buildTask(Protos.Offer offer, List availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources, Optional desiredExecutorResources) { final String rackId = slaveAndRackManager.getRackId(offer); final String host = slaveAndRackManager.getSlaveHost(offer); @@ -85,7 +85,7 @@ public SingularityTask buildTask(Protos.Offer offer, List availableRes } if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) { - prepareCustomExecutor(bldr, taskId, taskRequest, ports); + prepareCustomExecutor(bldr, taskId, taskRequest, ports, desiredExecutorResources); } else { prepareCommand(bldr, taskId, taskRequest, ports); } @@ -213,31 +213,23 @@ private void prepareContainerInfo(final SingularityTaskId taskId, final TaskInfo bldr.setContainer(containerBuilder); } - private List buildCustomExecutorResources(final SingularityTaskRequest task) { + private List buildMesosResources(final Optional resources) { ImmutableList.Builder builder = ImmutableList.builder(); - if (task.getDeploy().getCustomExecutorResources().isPresent()) { - if (task.getDeploy().getCustomExecutorResources().get().getCpus() > 0) { - builder.add(MesosUtils.getCpuResource(task.getDeploy().getCustomExecutorResources().get().getCpus())); + if (resources.isPresent()) { + if (resources.get().getCpus() > 0) { + builder.add(MesosUtils.getCpuResource(resources.get().getCpus())); } - if (task.getDeploy().getCustomExecutorResources().get().getMemoryMb() > 0) { - builder.add(MesosUtils.getMemoryResource(task.getDeploy().getCustomExecutorResources().get().getMemoryMb())); - } - } else { - if (configuration.getCustomExecutorConfiguration().getNumCpus() > 0) { - builder.add(MesosUtils.getCpuResource(configuration.getCustomExecutorConfiguration().getNumCpus())); - } - - if (configuration.getCustomExecutorConfiguration().getMemoryMb() > 0) { - builder.add(MesosUtils.getMemoryResource(configuration.getCustomExecutorConfiguration().getMemoryMb())); + if (resources.get().getMemoryMb() > 0) { + builder.add(MesosUtils.getMemoryResource(resources.get().getMemoryMb())); } } return builder.build(); } - private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final Optional ports) { + private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final Optional ports, final Optional desiredExecutorResources) { CommandInfo.Builder commandBuilder = CommandInfo.newBuilder().setValue(task.getDeploy().getCustomExecutorCmd().get()); prepareEnvironment(task, taskId, commandBuilder, ports); @@ -246,7 +238,7 @@ private void prepareCustomExecutor(final TaskInfo.Builder bldr, final Singularit .setCommand(commandBuilder.build()) .setExecutorId(ExecutorID.newBuilder().setValue(task.getDeploy().getCustomExecutorId().or(idGenerator.getNextExecutorId()))) .setSource(task.getDeploy().getCustomExecutorSource().or(task.getPendingTask().getPendingTaskId().getId())) - .addAllResources(buildCustomExecutorResources(task)) + .addAllResources(buildMesosResources(desiredExecutorResources)) .build() ); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java index 9cf8f14086..d78c40a75f 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java @@ -76,7 +76,7 @@ public void testShellCommand() { .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources); + final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources, Optional.absent()); assertEquals("/bin/echo hi", task.getMesosTask().getCommand().getValue()); assertEquals(0, task.getMesosTask().getCommand().getArgumentsCount()); @@ -91,7 +91,7 @@ public void testArgumentCommand() { .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources); + final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources, Optional.absent()); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); @@ -125,7 +125,7 @@ public void testDockerTask() { .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, Collections.singletonList(portsResource), taskRequest, resources); + final SingularityTask task = builder.buildTask(offer, Collections.singletonList(portsResource), taskRequest, resources, Optional.absent()); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); From b0f6287d8ed77e8237ba20c03f1b5c2f187bb206 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Fri, 27 Feb 2015 12:02:12 -0500 Subject: [PATCH 62/93] fetch resources for cpu usage calculation after fetch should call the second resource fetch after initial resourced are fetched, where then the state is set so resources are only fetched again during future refresh calls --- SingularityUI/app/controllers/TaskDetail.coffee | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/SingularityUI/app/controllers/TaskDetail.coffee b/SingularityUI/app/controllers/TaskDetail.coffee index 87b8bc2ab9..ffa90cd973 100644 --- a/SingularityUI/app/controllers/TaskDetail.coffee +++ b/SingularityUI/app/controllers/TaskDetail.coffee @@ -82,13 +82,6 @@ class TaskDetailController extends Controller app.showView @view - # Refresh once again on page load to calculate initial CPU Usage value - @initialPageLoad = true - if @initialPageLoad - setTimeout (=> - @refresh() - @initialPageLoad = false - ), 2000 fetchResourceUsage: -> @models.resourceUsage?.fetch() @@ -96,6 +89,10 @@ class TaskDetailController extends Controller # Store current resource usage to compare against future resource usage @models.resourceUsage.setCpuUsage() if @models.resourceUsage.get('previousUsage') @models.resourceUsage.set('previousUsage', @models.resourceUsage.toJSON()) + + if not @resourcesFetched + setTimeout (=> @fetchResourceUsage() ), 2000 + @resourcesFetched = true .error => # If this 404s there's nothing to get so don't bother @@ -103,6 +100,8 @@ class TaskDetailController extends Controller delete @models.resourceUsage refresh: -> + @resourcesFetched = false + @models.task.fetch() .done => @fetchResourceUsage() if @models.task.get('isStillRunning') From 1c635c985864b1cd417561b2cf629f617bd94ae7 Mon Sep 17 00:00:00 2001 From: tpetr Date: Sat, 28 Feb 2015 21:13:28 -0500 Subject: [PATCH 63/93] executor resources doesn't need to be Optional<> --- .../mesos/SingularityMesosScheduler.java | 4 ++-- .../mesos/SingularityMesosTaskBuilder.java | 18 ++++++++---------- .../mesos/SingularityMesosTaskBuilderTest.java | 15 +++++++++------ 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java index e6339bc4c2..3a38527b11 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java @@ -197,9 +197,9 @@ private Optional match(Collection taskR final Resources taskResources = taskRequest.getDeploy().getResources().or(defaultResources); // only factor in executor resources if we're running a custom executor - final Optional executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(Optional.of(defaultCustomExecutorResources)) : Optional.absent(); + final Resources executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources) : Resources.EMPTY_RESOURCES; - final Resources totalResources = Resources.add(taskResources, executorResources.or(Resources.EMPTY_RESOURCES)); + final Resources totalResources = Resources.add(taskResources, executorResources); LOG.trace("Attempting to match task {} resources {} ({} for task + {} for executor) with remaining offer resources {}", taskRequest.getPendingTask().getPendingTaskId(), totalResources, taskResources, executorResources, offerHolder.getCurrentResources()); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java index 93284fddc5..2dbe0672bb 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java @@ -62,7 +62,7 @@ class SingularityMesosTaskBuilder { this.configuration = configuration; } - public SingularityTask buildTask(Protos.Offer offer, List availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources, Optional desiredExecutorResources) { + public SingularityTask buildTask(Protos.Offer offer, List availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources, Resources desiredExecutorResources) { final String rackId = slaveAndRackManager.getRackId(offer); final String host = slaveAndRackManager.getSlaveHost(offer); @@ -213,23 +213,21 @@ private void prepareContainerInfo(final SingularityTaskId taskId, final TaskInfo bldr.setContainer(containerBuilder); } - private List buildMesosResources(final Optional resources) { + private List buildMesosResources(final Resources resources) { ImmutableList.Builder builder = ImmutableList.builder(); - if (resources.isPresent()) { - if (resources.get().getCpus() > 0) { - builder.add(MesosUtils.getCpuResource(resources.get().getCpus())); - } + if (resources.getCpus() > 0) { + builder.add(MesosUtils.getCpuResource(resources.getCpus())); + } - if (resources.get().getMemoryMb() > 0) { - builder.add(MesosUtils.getMemoryResource(resources.get().getMemoryMb())); - } + if (resources.getMemoryMb() > 0) { + builder.add(MesosUtils.getMemoryResource(resources.getMemoryMb())); } return builder.build(); } - private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final Optional ports, final Optional desiredExecutorResources) { + private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final Optional ports, final Resources desiredExecutorResources) { CommandInfo.Builder commandBuilder = CommandInfo.newBuilder().setValue(task.getDeploy().getCustomExecutorCmd().get()); prepareEnvironment(task, taskId, commandBuilder, ports); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java index d78c40a75f..544a41a4a2 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java @@ -45,7 +45,8 @@ public class SingularityMesosTaskBuilderTest { private SingularityMesosTaskBuilder builder; - private Resources resources; + private Resources taskResources; + private Resources executorResources; private Offer offer; private SingularityPendingTask pendingTask; @@ -60,7 +61,9 @@ public void createMocks() { builder = new SingularityMesosTaskBuilder(new ObjectMapper(), rackManager, idGenerator, new SingularityConfiguration()); - resources = new Resources(1, 1, 0); + taskResources = new Resources(1, 1, 0); + executorResources = new Resources(0.1, 1, 0); + offer = Offer.newBuilder() .setSlaveId(SlaveID.newBuilder().setValue("1")) .setId(OfferID.newBuilder().setValue("1")) @@ -76,7 +79,7 @@ public void testShellCommand() { .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources, Optional.absent()); + final SingularityTask task = builder.buildTask(offer, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo hi", task.getMesosTask().getCommand().getValue()); assertEquals(0, task.getMesosTask().getCommand().getArgumentsCount()); @@ -91,7 +94,7 @@ public void testArgumentCommand() { .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources, Optional.absent()); + final SingularityTask task = builder.buildTask(offer, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); @@ -101,7 +104,7 @@ public void testArgumentCommand() { @Test public void testDockerTask() { - resources = new Resources(1, 1, 1); + taskResources = new Resources(1, 1, 1); final Protos.Resource portsResource = Protos.Resource.newBuilder() .setName("ports") @@ -125,7 +128,7 @@ public void testDockerTask() { .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, Collections.singletonList(portsResource), taskRequest, resources, Optional.absent()); + final SingularityTask task = builder.buildTask(offer, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); From b30edac59b877bdd9697cbfa878313ebb30a42b7 Mon Sep 17 00:00:00 2001 From: tpetr Date: Sat, 28 Feb 2015 21:14:43 -0500 Subject: [PATCH 64/93] also need to subtract executor resources in SingularityOfferHolder.addMatchedTask() --- .../hubspot/singularity/mesos/SingularityOfferHolder.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java index 11ceca8240..42a9d3342b 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java @@ -32,7 +32,14 @@ public SingularityOfferHolder(Protos.Offer offer, int taskSizeHint) { public void addMatchedTask(SingularityTask task) { acceptedTasks.add(task); + + // subtract task resources from offer currentResources = MesosUtils.subtractResources(currentResources, task.getMesosTask().getResourcesList()); + + // subtract executor resources from offer, if any are defined + if (task.getMesosTask().hasExecutor() && task.getMesosTask().getExecutor().getResourcesCount() > 0) { + currentResources = MesosUtils.subtractResources(currentResources, task.getMesosTask().getExecutor().getResourcesList()); + } } public void launchTasks(SchedulerDriver driver) { From ecfd677d02dfc890827a5e5641e30140f352cf31 Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 2 Mar 2015 09:45:28 -0500 Subject: [PATCH 65/93] optionally include stacktrace for abort email --- .../hubspot/singularity/SingularityAbort.java | 16 +++++++++------- .../singularity/SingularityLeaderController.java | 6 +++--- .../SingularityMesosSchedulerDelegator.java | 4 ++-- .../singularity/resources/TestResource.java | 3 ++- .../SingularityHealthcheckAsyncHandler.java | 2 +- .../scheduler/SingularityLeaderOnlyPoller.java | 2 +- .../scheduler/SingularityNewTaskChecker.java | 2 +- .../scheduler/SingularityTaskReconciliation.java | 2 +- 8 files changed, 20 insertions(+), 17 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java index 7df061e82c..e3cb97d78e 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java @@ -54,7 +54,7 @@ public SingularityAbort(SingularitySmtpSender smtpSender, ServerProvider serverP public void stateChanged(CuratorFramework client, ConnectionState newState) { if (newState == ConnectionState.LOST) { LOG.error("Aborting due to new connection state received from ZooKeeper: {}", newState); - abort(AbortReason.LOST_ZK_CONNECTION); + abort(AbortReason.LOST_ZK_CONNECTION, Optional.absent()); } } @@ -62,10 +62,10 @@ public enum AbortReason { LOST_ZK_CONNECTION, LOST_LEADERSHIP, UNRECOVERABLE_ERROR, TEST_ABORT, MESOS_ERROR; } - public void abort(AbortReason abortReason) { + public void abort(AbortReason abortReason, Optional throwable) { if (!aborting.getAndSet(true)) { try { - sendAbortNotification(abortReason); + sendAbortNotification(abortReason, throwable); flushLogs(); } finally { exit(); @@ -89,17 +89,17 @@ private void exit() { } } - private void sendAbortNotification(AbortReason abortReason) { + private void sendAbortNotification(AbortReason abortReason, Optional throwable) { final String message = String.format("Singularity on %s is aborting due to %s", hostAndPort.getHostText(), abortReason); LOG.error(message); - sendAbortMail(message); + sendAbortMail(message, throwable); exceptionNotifier.notify(message); } - private void sendAbortMail(final String message) { + private void sendAbortMail(final String message, final Optional throwable) { if (!maybeSmtpConfiguration.isPresent()) { LOG.warn("Couldn't send abort mail because no SMTP configuration is present"); return; @@ -112,7 +112,9 @@ private void sendAbortMail(final String message) { return; } - smtpSender.queueMail(maybeSmtpConfiguration.get().getAdmins(), ImmutableList. of(), message, ""); + final String body = throwable.isPresent() ? throwable.get().toString() : "(no stack trace)"; + + smtpSender.queueMail(maybeSmtpConfiguration.get().getAdmins(), ImmutableList. of(), message, body); } private void flushLogs() { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java index 5ec0a539dd..63615870ef 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java @@ -84,11 +84,11 @@ public void isLeader() { } catch (Throwable t) { LOG.error("While starting driver", t); exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } if (driverManager.getCurrentStatus() != Protos.Status.DRIVER_RUNNING) { - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.absent()); } } } @@ -124,7 +124,7 @@ public void notLeader() { LOG.error("While stopping driver", t); exceptionNotifier.notify(t); } finally { - abort.abort(AbortReason.LOST_LEADERSHIP); + abort.abort(AbortReason.LOST_LEADERSHIP, Optional.absent()); } } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java index c9e76e8e52..017ea25a9f 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java @@ -94,7 +94,7 @@ private void handleUncaughtSchedulerException(Throwable t) { exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } private void startup(SchedulerDriver driver, MasterInfo masterInfo) throws Exception { @@ -308,7 +308,7 @@ public void error(SchedulerDriver driver, String message) { LOG.error("Aborting due to error: {}", message); - abort.abort(AbortReason.MESOS_ERROR); + abort.abort(AbortReason.MESOS_ERROR, Optional.absent()); } catch (Throwable t) { handleUncaughtSchedulerException(t); } finally { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java index ca5d4b9a04..11a8686ba5 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java @@ -6,6 +6,7 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import com.google.common.base.Optional; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import org.apache.mesos.Protos.TaskID; @@ -83,7 +84,7 @@ public void stop() throws Exception { public void abort() { checkForbidden(configuration.isAllowTestResourceCalls(), "Test resource calls are disabled (set isAllowTestResourceCalls to true in configuration)"); - abort.abort(AbortReason.TEST_ABORT); + abort.abort(AbortReason.TEST_ABORT, Optional.absent()); } @POST diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java index 6067e846c3..70a94322ca 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java @@ -93,7 +93,7 @@ private void reEnqueueOrAbort(SingularityTask task) { LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java index ea28fdded3..1a504c222a 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java @@ -111,7 +111,7 @@ private void runActionIfLeaderAndMesosIsRunning() { LOG.error("Caught an exception while running {}", getClass().getSimpleName(), t); exceptionNotifier.notify(t); if (abortsOnError()) { - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } } finally { if (lockHolder.isPresent()) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java index 4b1c558719..6d5a0ed912 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java @@ -183,7 +183,7 @@ private void reEnqueueCheckOrAbort(SingularityTask task) { } catch (Throwable t) { LOG.error("Uncaught throwable re-enqueuing task check for task {}, aborting", task, t); exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java index ab396bf871..6342db0bc2 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java @@ -128,7 +128,7 @@ public void run() { } catch (Throwable t) { LOG.error("While checking for reconciliation tasks", t); exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } } }, configuration.getCheckReconcileWhenRunningEveryMillis(), TimeUnit.MILLISECONDS); From 51b588cc61cf8e5ea7f136d7729aeb98c8172a9b Mon Sep 17 00:00:00 2001 From: tpetr Date: Mon, 2 Mar 2015 10:50:13 -0500 Subject: [PATCH 66/93] fix SingularityExceptionNotifier usage --- .../singularity/mesos/SingularitySlaveAndRackManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java index da876b331d..19d14e9d9c 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.inject.Inject; import com.hubspot.mesos.json.MesosMasterSlaveObject; @@ -321,7 +322,7 @@ public void checkStateAfterFinishedTask(SingularityTaskId taskId, String slaveId if (!slave.isPresent()) { final String message = String.format("Couldn't find slave with id %s for task %s", slaveId, taskId); LOG.warn(message); - exceptionNotifier.notify(message); + exceptionNotifier.notify(message, ImmutableMap.of("slaveId", slaveId, "taskId", taskId.toString())); return; } @@ -336,7 +337,7 @@ public void checkStateAfterFinishedTask(SingularityTaskId taskId, String slaveId if (!rack.isPresent()) { final String message = String.format("Couldn't find rack with id %s for task %s", taskId.getRackId(), taskId); LOG.warn(message); - exceptionNotifier.notify(message); + exceptionNotifier.notify(message, ImmutableMap.of("rackId", taskId.getRackId(), "taskId", taskId.toString())); return; } From a508a99d8a9bdb35a2b8c46904a83a5ba303387d Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Mon, 2 Mar 2015 16:49:54 -0500 Subject: [PATCH 67/93] Clean up SingularityExecutor. --- .../executor/models/RunnerContext.java | 44 ++++++++++++------- ...SingularityExecutorTaskProcessBuilder.java | 12 ++++- .../src/main/resources/runner.sh.hbs | 11 ++++- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java index 43ca2a03b5..1dba3bcd6c 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java @@ -2,26 +2,29 @@ import com.google.common.base.Optional; +/** + * Handlebars context for generating the runner.sh file. + */ public class RunnerContext { private final String cmd; + private final String taskAppDirectory; + private final String logDir; private final String user; - private final String logfile; + private final String logFile; private final String taskId; - private final String taskAppDirectory; + private final Optional maxTaskThreads; - public RunnerContext(String cmd, String taskAppDirectory, String user, String logfile, String taskId, Optional maxTaskThreads) { + public RunnerContext(String cmd, String taskAppDirectory, String logDir, String user, String logFile, String taskId, Optional maxTaskThreads) { this.cmd = cmd; + this.taskAppDirectory = taskAppDirectory; + this.logDir = logDir; this.user = user; - this.logfile = logfile; + this.logFile = logFile; this.taskId = taskId; - this.taskAppDirectory = taskAppDirectory; - this.maxTaskThreads = maxTaskThreads; - } - public String getTaskId() { - return taskId; + this.maxTaskThreads = maxTaskThreads; } public String getCmd() { @@ -32,12 +35,20 @@ public String getTaskAppDirectory() { return taskAppDirectory; } + public String getLogDir() { + return logDir; + } + public String getUser() { return user; } - public String getLogfile() { - return logfile; + public String getLogFile() { + return logFile; + } + + public String getTaskId() { + return taskId; } public Optional getMaxTaskThreads() { @@ -47,11 +58,12 @@ public Optional getMaxTaskThreads() { @Override public String toString() { return "RunnerContext [" + - "cmd='" + cmd + '\'' + - ", user='" + user + '\'' + - ", logfile='" + logfile + '\'' + - ", taskId='" + taskId + '\'' + - ", taskAppDirectory='" + taskAppDirectory + '\'' + + "cmd='" + cmd + "'" + + ", taskAppDirectory='" + taskAppDirectory + "'" + + ", logDir='" + logDir + "'" + + ", user='" + user + "'" + + ", logFile='" + logFile + "'" + + ", taskId='" + taskId + "'" + ", maxTaskThreads=" + maxTaskThreads + ']'; } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java index 39ff92cdc6..3f56fb34ba 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java @@ -34,7 +34,8 @@ public class SingularityExecutorTaskProcessBuilder implements Callable taskArtifactFetcher; - public SingularityExecutorTaskProcessBuilder(SingularityExecutorTask task, ExecutorUtils executorUtils, SingularityExecutorArtifactFetcher artifactFetcher, TemplateManager templateManager, SingularityExecutorConfiguration configuration, ExecutorData executorData, String executorPid) { + public SingularityExecutorTaskProcessBuilder(SingularityExecutorTask task, ExecutorUtils executorUtils, SingularityExecutorArtifactFetcher artifactFetcher, TemplateManager templateManager, SingularityExecutorConfiguration configuration, + ExecutorData executorData, String executorPid) { this.executorData = executorData; this.task = task; this.executorUtils = executorUtils; @@ -85,7 +86,14 @@ private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData execu task.getLog().info("Writing a runner script to execute {}", cmd); - templateManager.writeRunnerScript(getPath("runner.sh"), new RunnerContext(cmd, configuration.getTaskAppDirectory(), executorData.getUser().or(configuration.getDefaultRunAsUser()), configuration.getServiceLog(), task.getTaskId(), executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()))); + templateManager.writeRunnerScript(getPath("runner.sh"), new RunnerContext( + cmd, // cmd + configuration.getTaskAppDirectory(), // taskAppDirectory + configuration.getLogrotateToDirectory(), // logDir + executorData.getUser().or(configuration.getDefaultRunAsUser()), // user + configuration.getServiceLog(), // logFile + task.getTaskId(), // taskId + executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()))); // maxTaskThreads List command = Lists.newArrayList(); command.add("bash"); diff --git a/SingularityExecutor/src/main/resources/runner.sh.hbs b/SingularityExecutor/src/main/resources/runner.sh.hbs index ec3afaa8e8..206b14e98c 100644 --- a/SingularityExecutor/src/main/resources/runner.sh.hbs +++ b/SingularityExecutor/src/main/resources/runner.sh.hbs @@ -32,6 +32,13 @@ if [[ ! -d ./tmp ]]; then sudo chown -R {{{ user }}} ./tmp fi +# Create log directory for logrotate runs +if [[ ! -d {{{ logDir }}} ]]; then + echo "Creating log directory ({{{ logDir }}})" + mkdir {{{ logDir }}} + sudo chown -R {{{ user }}} {{{ logDir }}} +fi + echo "Ensuring {{{ taskAppDirectory }}} is owned by {{{ user }}}" sudo chown -R {{{ user }}} {{{ taskAppDirectory }}} @@ -48,5 +55,5 @@ else fi # execute command -echo "Executing: sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1" -exec sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1 +echo "Executing: sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logFile }}} 2>&1" +exec sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logFile }}} 2>&1 From 1ce328367c1ec34cce11ed3dadce1550a6778cf8 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 3 Mar 2015 11:55:28 -0500 Subject: [PATCH 68/93] add transaction for task creation avoids race condition with history persister --- .../hubspot/singularity/data/TaskManager.java | 36 ++++++++++++------- .../history/SingularityHistoryPersister.java | 4 +-- .../SingularityTaskHistoryPersister.java | 3 +- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java index 44a3e5af49..91c6087f51 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java @@ -1,16 +1,5 @@ package com.hubspot.singularity.data; -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.utils.ZKPaths; -import org.apache.mesos.Protos.TaskStatus; - import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; @@ -44,10 +33,26 @@ import com.hubspot.singularity.data.transcoders.StringTranscoder; import com.hubspot.singularity.data.transcoders.Transcoder; import com.hubspot.singularity.event.SingularityEventListener; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; +import org.apache.curator.utils.ZKPaths; +import org.apache.mesos.Protos.TaskStatus; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; @Singleton public class TaskManager extends CuratorAsyncManager { + private static final Logger LOG = LoggerFactory.getLogger(CuratorAsyncManager.class); + private static final String TASKS_ROOT = "/tasks"; private static final String ACTIVE_PATH_ROOT = TASKS_ROOT + "/active"; @@ -464,8 +469,13 @@ private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getOffer().getSlaveId().getValue()))); - create(getTaskPath(task.getTaskId()), task, taskTranscoder); - create(getActivePath(task.getTaskId().getId())); + try { + CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(getTaskPath(task.getTaskId()), taskTranscoder.toBytes(task)).and(); + + transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); + } catch (KeeperException.NodeExistsException nee) { + LOG.error("Task or active path already existed for {}", task.getTaskId()); + } } public List getLBCleanupTasks() { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java index 5621cf90c8..0942446481 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java @@ -48,7 +48,7 @@ protected boolean moveToHistoryOrCheckForPurge(T object) { if (moveToHistoryOrCheckForPurgeAndShouldDelete(object)) { SingularityDeleteResult deleteResult = purgeFromZk(object); - LOG.debug("%s %s (deleted: %s) in %s", persistsHistoryInsteadOfPurging() ? "Persisted" : "Purged", object, deleteResult, JavaUtils.duration(start)); + LOG.debug("{} {} (deleted: {}) in {}", persistsHistoryInsteadOfPurging() ? "Persisted" : "Purged", object, deleteResult, JavaUtils.duration(start)); return true; } @@ -63,7 +63,7 @@ private boolean moveToHistoryOrCheckForPurgeAndShouldDelete(T object) { final long age = System.currentTimeMillis() - object.getCreateTimestampForCalculatingHistoryAge(); if (age > getMaxAgeInMillisOfItem()) { - LOG.trace("Deleting %s because it is %s old (max : %s)", object, JavaUtils.durationFromMillis(age), JavaUtils.durationFromMillis(getMaxAgeInMillisOfItem())); + LOG.trace("Deleting {} because it is {} old (max : {})", object, JavaUtils.durationFromMillis(age), JavaUtils.durationFromMillis(getMaxAgeInMillisOfItem())); return true; } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java index 7dab662fd4..5baad8e761 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java @@ -40,7 +40,7 @@ public SingularityTaskHistoryPersister(SingularityConfiguration configuration, T @Override public void runActionOnPoll() { - LOG.info("Checking inactive task ids for task history persistance"); + LOG.info("Checking inactive task ids for task history persistence"); final long start = System.currentTimeMillis(); @@ -86,6 +86,7 @@ protected boolean moveToHistory(SingularityTaskId object) { final Optional taskHistory = taskManager.getTaskHistory(object); if (taskHistory.isPresent()) { + LOG.debug("Moving {} to history", object); try { historyManager.saveTaskHistory(taskHistory.get()); } catch (Throwable t) { From d1183381cfead8b92dc2870db4046e22b628c197 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 3 Mar 2015 13:45:12 -0500 Subject: [PATCH 69/93] add migration + to fix test --- .../hubspot/singularity/data/TaskManager.java | 6 ++++ .../SingularityZkMigrationsModule.java | 1 + ...quiredParentsForTransactionsMigration.java | 35 +++++++++++++++++++ .../SingularitySchedulerTestBase.java | 5 +++ .../singularity/data/BlendedHistoryTest.java | 1 - .../data/zkmigrations/ZkMigrationTest.java | 4 +-- .../scheduler/SingularitySchedulerTest.java | 2 +- 7 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/TaskManagerRequiredParentsForTransactionsMigration.java diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java index 91c6087f51..130e161361 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java @@ -112,6 +112,12 @@ public TaskManager(SingularityConfiguration configuration, CuratorFramework cura this.serverId = serverId; } + // since we can't use creatingParentsIfNeeded in transactions + public void createRequiredParents() { + create(HISTORY_PATH_ROOT); + create(ACTIVE_PATH_ROOT); + } + private String getLastHealthcheckPath(SingularityTaskId taskId) { return ZKPaths.makePath(getHistoryPath(taskId), LAST_HEALTHCHECK_KEY); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java index 3744acaef6..fa07b2b851 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java @@ -16,5 +16,6 @@ public void configure(Binder binder) { dataMigrations.addBinding().to(SingularityPendingTaskIdMigration.class); dataMigrations.addBinding().to(SlaveAndRackMigration.class); dataMigrations.addBinding().to(SingularityCmdLineArgsMigration.class); + dataMigrations.addBinding().to(TaskManagerRequiredParentsForTransactionsMigration.class); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/TaskManagerRequiredParentsForTransactionsMigration.java b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/TaskManagerRequiredParentsForTransactionsMigration.java new file mode 100644 index 0000000000..cb111754cf --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/TaskManagerRequiredParentsForTransactionsMigration.java @@ -0,0 +1,35 @@ +package com.hubspot.singularity.data.zkmigrations; + +import com.google.common.base.Optional; +import com.google.common.collect.Lists; +import com.google.inject.Inject; +import com.google.inject.name.Named; +import com.hubspot.singularity.SingularityMainModule; +import com.hubspot.singularity.SingularityTask; +import com.hubspot.singularity.SingularityTaskHistoryUpdate; +import com.hubspot.singularity.SingularityTaskId; +import com.hubspot.singularity.SingularityTaskStatusHolder; +import com.hubspot.singularity.data.TaskManager; +import org.apache.mesos.Protos.TaskID; +import org.apache.mesos.Protos.TaskStatus; + +import javax.inject.Singleton; +import java.util.List; + +@Singleton +public class TaskManagerRequiredParentsForTransactionsMigration extends ZkDataMigration { + + private final TaskManager taskManager; + + @Inject + public TaskManagerRequiredParentsForTransactionsMigration(TaskManager taskManager) { + super(5); + this.taskManager = taskManager; + } + + @Override + public void applyMigration() { + taskManager.createRequiredParents(); + } + +} diff --git a/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java b/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java index 81e7d31588..6af481a341 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java @@ -5,6 +5,7 @@ import java.util.Random; import java.util.concurrent.TimeUnit; +import com.hubspot.singularity.data.zkmigrations.ZkDataMigrationRunner; import org.apache.mesos.Protos.Attribute; import org.apache.mesos.Protos.FrameworkID; import org.apache.mesos.Protos.Offer; @@ -107,6 +108,8 @@ public class SingularitySchedulerTestBase extends SingularityCuratorTestBase { protected SingularityMailer mailer; @Inject protected SingularityScheduledJobPoller scheduledJobPoller; + @Inject + protected ZkDataMigrationRunner migrationRunner; @Inject @Named(SingularityMainModule.SERVER_ID_PROPERTY) @@ -135,6 +138,8 @@ public void teardown() throws Exception { @Before public final void setupDriver() throws Exception { driver = driverSupplier.get().get(); + + migrationRunner.checkMigrations(); } protected Offer createOffer(double cpus, double memory) { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java b/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java index cccfc9bba3..3cf353e197 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java @@ -42,7 +42,6 @@ private SingularityRequestHistory makeHistory(long createdAt, RequestHistoryType return new SingularityRequestHistory(createdAt, Optional. absent(), type, request); } - // DESCENDING @Test public void testBlendedRequestHistory() { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java b/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java index c677972371..fd6b2e1f67 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java @@ -37,9 +37,9 @@ public class ZkMigrationTest extends SingularityCuratorTestBase { @Test public void testMigrationRunner() { - Assert.assertTrue(migrationRunner.checkMigrations() == 4); + Assert.assertTrue(migrationRunner.checkMigrations() == 5); - Assert.assertTrue(metadataManager.getZkDataVersion().isPresent() && metadataManager.getZkDataVersion().get().equals("4")); + Assert.assertTrue(metadataManager.getZkDataVersion().isPresent() && metadataManager.getZkDataVersion().get().equals("5")); Assert.assertTrue(migrationRunner.checkMigrations() == 0); } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java index 1d5d1947ea..02079606a4 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java @@ -896,7 +896,7 @@ public void testScheduledNotification() { saveRequest(request.toBuilder().setScheduledExpectedRuntimeMillis(Optional.of(1L)).build()); - SingularityTask thirdTask = launchTask(request, firstDeploy, now - 500, 1, TaskState.TASK_RUNNING); + SingularityTask thirdTask = launchTask(request, firstDeploy, now - 502, 1, TaskState.TASK_RUNNING); scheduledJobPoller.runActionOnPoll(); From 94f8dd86c2cec0b646be0bdd2be83630e6e3b40d Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Thu, 5 Mar 2015 16:20:06 -0500 Subject: [PATCH 70/93] if health check already worked, don't do it re-enqueue it also be more careful about re-enqueuing. --- .../SingularityHealthcheckAsyncHandler.java | 22 +++++------------- .../scheduler/SingularityHealthchecker.java | 23 ++++++++++++++++++- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java index 6067e846c3..7d61c5e63f 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java @@ -60,12 +60,12 @@ public void onThrowable(Throwable t) { } public void saveResult(Optional statusCode, Optional responseBody, Optional errorMessage) { - SingularityTaskHealthcheckResult result = new SingularityTaskHealthcheckResult(statusCode, Optional.of(System.currentTimeMillis() - startTime), startTime, responseBody, - errorMessage, task.getTaskId()); + try { + SingularityTaskHealthcheckResult result = new SingularityTaskHealthcheckResult(statusCode, Optional.of(System.currentTimeMillis() - startTime), startTime, responseBody, + errorMessage, task.getTaskId()); - LOG.trace("Saving healthcheck result {}", result); + LOG.trace("Saving healthcheck result {}", result); - try { taskManager.saveHealthcheckResult(result); if (result.isFailed()) { @@ -79,22 +79,12 @@ public void saveResult(Optional statusCode, Optional responseBo newTaskChecker.runNewTaskCheckImmediately(task); } } catch (Throwable t) { - LOG.error("Caught throwable while saving health check result {}, will re-enqueue", result, t); + LOG.error("Caught throwable while saving health check result for {}, will re-enqueue", task.getTaskId(), t); exceptionNotifier.notify(t); - reEnqueueOrAbort(task); + healthchecker.reEnqueueOrAbort(task); } } - private void reEnqueueOrAbort(SingularityTask task) { - try { - healthchecker.enqueueHealthcheck(task); - } catch (Throwable t) { - LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); - exceptionNotifier.notify(t); - - abort.abort(AbortReason.UNRECOVERABLE_ERROR); - } - } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java index 6008a5056d..8c54178f4c 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java @@ -10,6 +10,7 @@ import javax.inject.Singleton; +import com.hubspot.singularity.SingularityTaskHealthcheckResult; import org.apache.commons.lang3.time.DurationFormatUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,7 +105,7 @@ public void cancelHealthcheck(String taskId) { } private ScheduledFuture enqueueHealthcheckWithDelay(final SingularityTask task, long delaySeconds) { - LOG.trace("Enqueing a healthcheck for task {} with delay {}", task.getTaskId(), DurationFormatUtils.formatDurationHMS(TimeUnit.SECONDS.toMillis(delaySeconds))); + LOG.trace("En-queuing a healthcheck for task {} with delay {}", task.getTaskId(), DurationFormatUtils.formatDurationHMS(TimeUnit.SECONDS.toMillis(delaySeconds))); return executorService.schedule(new Runnable() { @@ -117,12 +118,25 @@ public void run() { } catch (Throwable t) { LOG.error("Uncaught throwable in async healthcheck", t); exceptionNotifier.notify(t); + + reEnqueueOrAbort(task); } } }, delaySeconds, TimeUnit.SECONDS); } + public void reEnqueueOrAbort(SingularityTask task) { + try { + enqueueHealthcheck(task); + } catch (Throwable t) { + LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); + exceptionNotifier.notify(t); + + abort.abort(SingularityAbort.AbortReason.UNRECOVERABLE_ERROR); + } + } + private Optional getHealthcheckUri(SingularityTask task) { if (task.getTaskRequest().getDeploy().getHealthcheckUri() == null) { return Optional.absent(); @@ -159,6 +173,13 @@ private boolean shouldHealthcheck(final SingularityTask task, Optional lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); + + if (lastHealthcheck.isPresent() && !lastHealthcheck.get().isFailed()) { + LOG.debug("Not submitting a new healthcheck for {} because it already passed a healthcheck", task.getTaskId()); + return false; + } + return true; } From 7dd17d259bd2c0e6eb23c6f50358dd91c6db3eb0 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Thu, 5 Mar 2015 16:23:15 -0500 Subject: [PATCH 71/93] resolving merge stuff --- .../singularity/scheduler/SingularityHealthchecker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java index 94ac539c1c..056d6e5d6d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java @@ -110,19 +110,19 @@ public void run() { LOG.error("Uncaught throwable in async healthcheck", t); exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); - reEnqueueOrAbort(task, t); + reEnqueueOrAbort(task); } } }, delaySeconds, TimeUnit.SECONDS); } - public void reEnqueueOrAbort(SingularityTask task, Throwable t) { + public void reEnqueueOrAbort(SingularityTask task) { try { enqueueHealthcheck(task); } catch (Throwable t) { LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); abort.abort(SingularityAbort.AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } From 31a81010960fe98a6ccf5995c71f12d831529370 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Thu, 5 Mar 2015 21:26:15 -0500 Subject: [PATCH 72/93] base64 encode embedded artifact content field before POST --- SingularityUI/app/views/formBaseView.coffee | 4 ++++ SingularityUI/app/views/newDeploy.coffee | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/SingularityUI/app/views/formBaseView.coffee b/SingularityUI/app/views/formBaseView.coffee index 3271aa3c85..dde2473e97 100644 --- a/SingularityUI/app/views/formBaseView.coffee +++ b/SingularityUI/app/views/formBaseView.coffee @@ -96,6 +96,10 @@ class FormBaseView extends View val = $element.val() return val if val return if $element.parents('.required').length then "" else undefined + + base64Encode: (content) -> + return if not content? + btoa content multiMap: (selector) => $elements = @$ selector diff --git a/SingularityUI/app/views/newDeploy.coffee b/SingularityUI/app/views/newDeploy.coffee index 072f2f6ae6..9ebae1d6f8 100644 --- a/SingularityUI/app/views/newDeploy.coffee +++ b/SingularityUI/app/views/newDeploy.coffee @@ -139,7 +139,7 @@ class NewDeployView extends FormBaseView name: @valOrNothing '.name', $artifact filename: @valOrNothing '.filename', $artifact md5sum: @valOrNothing '.md5', $artifact - content: @valOrNothing '.content', $artifact + content: @base64Encode @valOrNothing '.content', $artifact else if type is 'external' deployObject.executorData.externalArtifacts = [] unless deployObject.executorData.externalArtifacts deployObject.executorData.externalArtifacts.push From 45a3d0dbc4d40efc055ee98e733183721639f88b Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Mon, 9 Mar 2015 10:41:21 -0400 Subject: [PATCH 73/93] remove unecessary parse --- SingularityUI/app/collections/Slaves.coffee | 4 ---- 1 file changed, 4 deletions(-) diff --git a/SingularityUI/app/collections/Slaves.coffee b/SingularityUI/app/collections/Slaves.coffee index 52c2c683a6..41dcc48279 100644 --- a/SingularityUI/app/collections/Slaves.coffee +++ b/SingularityUI/app/collections/Slaves.coffee @@ -9,8 +9,4 @@ class Slaves extends Collection initialize: (models) => - parse: (slaves) -> - _.map slaves, (slave) => - slave - module.exports = Slaves From 10d42354d11893fe8fd206b6aab6f5681e8aacbb Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Mon, 9 Mar 2015 10:43:09 -0400 Subject: [PATCH 74/93] reset collection on initial fetch to prevent add event causing unneeded rendering --- SingularityUI/app/controllers/Slaves.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityUI/app/controllers/Slaves.coffee b/SingularityUI/app/controllers/Slaves.coffee index 00b567bda7..07fa021de4 100644 --- a/SingularityUI/app/controllers/Slaves.coffee +++ b/SingularityUI/app/controllers/Slaves.coffee @@ -18,6 +18,6 @@ class SlavesController extends Controller @refresh() refresh: -> - @collections.slaves.fetch() + @collections.slaves.fetch reset: true module.exports = SlavesController From a808829defe3542895bc269b06ff5ae20d6ea262 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Wed, 11 Mar 2015 10:59:35 -0400 Subject: [PATCH 75/93] stop tailing when task has finished --- SingularityUI/app/controllers/Tail.coffee | 6 +++++- SingularityUI/app/views/tail.coffee | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/SingularityUI/app/controllers/Tail.coffee b/SingularityUI/app/controllers/Tail.coffee index 5a30dd2d2a..b3ea80295c 100644 --- a/SingularityUI/app/controllers/Tail.coffee +++ b/SingularityUI/app/controllers/Tail.coffee @@ -1,6 +1,7 @@ Controller = require './Controller' LogLines = require '../collections/LogLines' +TaskHistory = require '../models/TaskHistory' TailView = require '../views/tail' @@ -8,12 +9,15 @@ class TailController extends Controller initialize: ({@taskId, @path}) -> @collections.logLines = new LogLines [], {@taskId, @path} - + @models.taskHistory = new TaskHistory {@taskId} + @setView new TailView _.extend {@taskId, @path}, collection: @collections.logLines + model: @models.taskHistory app.showView @view @collections.logLines.fetchInitialData() + @models.taskHistory.fetch() module.exports = TailController diff --git a/SingularityUI/app/views/tail.coffee b/SingularityUI/app/views/tail.coffee index a2e5f163c9..11af73beb6 100644 --- a/SingularityUI/app/views/tail.coffee +++ b/SingularityUI/app/views/tail.coffee @@ -29,6 +29,12 @@ class TailView extends View @listenTo @collection, 'sync', => @$el.removeClass 'fetching-data' + @listenTo @model, 'change', @setTaskHistory + + + setTaskHistory: -> + @isTaskRunning = @model.get 'isStillRunning' + handleAjaxError: (response) => # ATM we get 404s if we request dirs and 500s if the file doesn't exist if response.status in [404, 500] @@ -142,16 +148,20 @@ class TailView extends View @startTailing() startTailing: => - return if @isTailing is true + return if @isTailing or not @isTaskRunning @isTailing = true @scrollToBottom() clearInterval @tailInterval @tailInterval = setInterval => + @stopTailing() if not @isTaskRunning + @collection.fetchNext().done => # Only show the newly tail-ed lines if we are still tailing @scrollToBottom() if @isTailing + @model.fetch() + , @pollingTimeout # The class is for CSS stylin' of certain stuff From cb97c25cfb3b663973456435a11c795c47dd62cd Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Wed, 11 Mar 2015 12:32:11 -0400 Subject: [PATCH 76/93] use global refresh for task history model --- SingularityUI/app/controllers/Tail.coffee | 3 +++ SingularityUI/app/views/tail.coffee | 16 +++++----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/SingularityUI/app/controllers/Tail.coffee b/SingularityUI/app/controllers/Tail.coffee index b3ea80295c..40f32e5476 100644 --- a/SingularityUI/app/controllers/Tail.coffee +++ b/SingularityUI/app/controllers/Tail.coffee @@ -16,8 +16,11 @@ class TailController extends Controller model: @models.taskHistory app.showView @view + @refresh() + refresh: -> @collections.logLines.fetchInitialData() @models.taskHistory.fetch() + module.exports = TailController diff --git a/SingularityUI/app/views/tail.coffee b/SingularityUI/app/views/tail.coffee index 11af73beb6..507f48fddb 100644 --- a/SingularityUI/app/views/tail.coffee +++ b/SingularityUI/app/views/tail.coffee @@ -28,12 +28,8 @@ class TailView extends View @$el.addClass 'fetching-data' @listenTo @collection, 'sync', => @$el.removeClass 'fetching-data' - - @listenTo @model, 'change', @setTaskHistory - - - setTaskHistory: -> - @isTaskRunning = @model.get 'isStillRunning' + + @listenTo @model, 'change:isStillRunning', => @stopTailing() unless @model.get 'isStillRunning' handleAjaxError: (response) => # ATM we get 404s if we request dirs and 500s if the file doesn't exist @@ -148,20 +144,18 @@ class TailView extends View @startTailing() startTailing: => - return if @isTailing or not @isTaskRunning - + return if @isTailing or not @model.get 'isStillRunning' + @isTailing = true @scrollToBottom() clearInterval @tailInterval @tailInterval = setInterval => - @stopTailing() if not @isTaskRunning + @stopTailing() if not @model.get 'isStillRunning' @collection.fetchNext().done => # Only show the newly tail-ed lines if we are still tailing @scrollToBottom() if @isTailing - @model.fetch() - , @pollingTimeout # The class is for CSS stylin' of certain stuff From 9ecef165db306e936b786ab9958539fe6498a10c Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Wed, 11 Mar 2015 13:36:39 -0400 Subject: [PATCH 77/93] add case for INVALID_REQUEST_NOOP in deploy checker --- .../hubspot/singularity/scheduler/SingularityDeployChecker.java | 1 + 1 file changed, 1 insertion(+) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java index 9271c21e0c..a081d8d201 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java @@ -307,6 +307,7 @@ private DeployState interpretLoadBalancerState(SingularityLoadBalancerUpdate lbU case SUCCESS: return DeployState.SUCCEEDED; case FAILED: + case INVALID_REQUEST_NOOP: return DeployState.FAILED; case CANCELING: return DeployState.CANCELING; From b0234d1c623bc5573fcbca79091d9052d3a07289 Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Fri, 13 Mar 2015 16:11:26 -0400 Subject: [PATCH 78/93] Change config option name to align with logrotate option. Logrotate to olddir as well. --- .../config/SingularityExecutorConfigurationLoader.java | 4 +++- .../executor/task/SingularityExecutorTaskCleanup.java | 2 +- SingularityExecutor/src/main/resources/logrotate.conf.hbs | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java index 53ede37740..904b2a8572 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java @@ -47,7 +47,7 @@ public class SingularityExecutorConfigurationLoader extends SingularityConfigura public static final String TAIL_LOG_LINES_TO_SAVE = "executor.service.log.tail.lines.to.save"; public static final String TAIL_LOG_FILENAME = "executor.service.log.tail.file.name"; - public static final String S3_FILES_TO_BACKUP = "executor.s3.uploader.extra.files"; + public static final String S3_FILES_TO_BACKUP = "executor.s3.uploader.extras.files"; public static final String S3_UPLOADER_PATTERN = "executor.s3.uploader.pattern"; public static final String S3_UPLOADER_BUCKET = "executor.s3.uploader.bucket"; @@ -91,6 +91,8 @@ protected void bindDefaults(Properties properties) { properties.put(LOGROTATE_EXTRAS_FILES, ""); properties.put(LOGROTATE_EXTRAS_DATEFORMAT, "-%Y%m%d"); + properties.put(S3_FILES_TO_BACKUP, ""); + properties.put(USE_LOCAL_DOWNLOAD_SERVICE, Boolean.toString(false)); properties.put(LOCAL_DOWNLOAD_SERVICE_TIMEOUT_MILLIS, Long.toString(TimeUnit.MINUTES.toMillis(3))); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java index 7175241467..f0fb30bb23 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java @@ -53,7 +53,7 @@ public boolean cleanup(boolean cleanupTaskAppDirectory) { public boolean cleanTaskDefinitionFile() { Path taskDefinitionPath = configuration.getTaskDefinitionPath(taskDefinition.getTaskId()); - log.info("Successfull cleanup, deleting file {}", taskDefinitionPath); + log.info("Successful cleanup, deleting file {}", taskDefinitionPath); try { boolean deleted = Files.deleteIfExists(taskDefinitionPath); diff --git a/SingularityExecutor/src/main/resources/logrotate.conf.hbs b/SingularityExecutor/src/main/resources/logrotate.conf.hbs index fd703882f3..0b0d9c7a82 100644 --- a/SingularityExecutor/src/main/resources/logrotate.conf.hbs +++ b/SingularityExecutor/src/main/resources/logrotate.conf.hbs @@ -16,6 +16,7 @@ notifempty {{#if extrasFiles}} {{#each extrasFiles}}{{{this}}} {{/each}} { dateformat {{{ extrasDateformat }}} + olddir {{{ rotateDirectory }}} missingok } {{/if}} From 3d3bdf1f48357b7d77e1d01e34ae6d8f390cec00 Mon Sep 17 00:00:00 2001 From: Ben Lodge Date: Fri, 13 Mar 2015 17:28:14 -0400 Subject: [PATCH 79/93] remove invalid auto value that may be the root of issues on qa --- SingularityUI/app/styles/table.styl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/SingularityUI/app/styles/table.styl b/SingularityUI/app/styles/table.styl index a7881bb86a..8f2142f153 100644 --- a/SingularityUI/app/styles/table.styl +++ b/SingularityUI/app/styles/table.styl @@ -39,9 +39,7 @@ a position relative overflow hidden - padding auto - padding-left 12px - padding-right 12px + padding 0 12px display inline-block background transparent text-decoration none From e07632c18763f78264ae5d9dae7df7a27557894c Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Mon, 16 Mar 2015 11:06:49 -0400 Subject: [PATCH 80/93] Use guava's Joiner instead of StringUtils. Add -p flag to mkdir for logs. --- .../executor/task/SingularityExecutorTaskLogManager.java | 4 ++-- SingularityExecutor/src/main/resources/runner.sh.hbs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index efc98fe09e..0a68b69f4d 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -8,7 +8,7 @@ import java.util.LinkedList; import java.util.List; -import org.apache.commons.lang3.StringUtils; +import com.google.common.base.Joiner; import org.slf4j.Logger; import com.google.common.collect.ImmutableList; @@ -165,7 +165,7 @@ private String getS3Glob() { fileNames.add(taskDefinition.getServiceLogOutPath().getFileName().toString()); fileNames.addAll(Arrays.asList(configuration.getS3FilesToBackup())); - return String.format("{%s}*.gz*", StringUtils.join(fileNames, ",")); + return String.format("{%s}*.gz*", Joiner.on(",").join(fileNames)); } private String getS3KeyPattern() { diff --git a/SingularityExecutor/src/main/resources/runner.sh.hbs b/SingularityExecutor/src/main/resources/runner.sh.hbs index 206b14e98c..8df707d56e 100644 --- a/SingularityExecutor/src/main/resources/runner.sh.hbs +++ b/SingularityExecutor/src/main/resources/runner.sh.hbs @@ -35,7 +35,7 @@ fi # Create log directory for logrotate runs if [[ ! -d {{{ logDir }}} ]]; then echo "Creating log directory ({{{ logDir }}})" - mkdir {{{ logDir }}} + mkdir -p {{{ logDir }}} sudo chown -R {{{ user }}} {{{ logDir }}} fi From cf7722bb4a6f5b22795b56116d39435aff0b63a7 Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Mon, 16 Mar 2015 13:39:32 -0400 Subject: [PATCH 81/93] Remove trivial documentation that is likely to become outdated. --- .../models/LogrotateTemplateContext.java | 24 +------------------ .../SingularityExecutorTaskLogManager.java | 4 ---- ...SingularityExecutorTaskProcessBuilder.java | 14 +++++------ .../singularity/config/S3Configuration.java | 10 -------- 4 files changed, 8 insertions(+), 44 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java index 3717643449..692bc82fa9 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java @@ -12,10 +12,6 @@ public class LogrotateTemplateContext { private final SingularityExecutorTaskDefinition taskDefinition; private final SingularityExecutorConfiguration configuration; - /** - * @param configuration configuration to pull from. - * @param taskDefinition information about the task we're writing the logrotate configs for. - */ public LogrotateTemplateContext(SingularityExecutorConfiguration configuration, SingularityExecutorTaskDefinition taskDefinition) { this.configuration = configuration; this.taskDefinition = taskDefinition; @@ -25,33 +21,20 @@ public String getRotateDateformat() { return configuration.getLogrotateDateformat(); } - /** - * Log files are rotated count times before being removed. - * @return count. - */ public String getRotateCount() { return configuration.getLogrotateCount(); } - /** - * Remove rotated logs older than $count days. - * The age is only checked if the logfile is to be rotated. - * @return days. - */ public String getMaxageDays() { return configuration.getLogrotateMaxageDays(); } - /** - * Logs are moved into $directory for rotation. - * @return directory. - */ public String getRotateDirectory() { return configuration.getLogrotateToDirectory(); } /** - * Extra files for logrotate to rotate. + * Extra files for logrotate to rotate. If these do not exist logrotate will continue without error. * @return filenames to rotate. */ public String[] getExtrasFiles() { @@ -65,11 +48,6 @@ public String[] getExtrasFiles() { return transformed; } - /** - * dateformat for extra files. - * Only %Y %m %d and %s specifiers are allowed. - * @return dateformat (e.g. "-%Y%m%d%s"). - */ public String getExtrasDateformat() { return configuration.getLogrotateExtrasDateformat(); } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index 0a68b69f4d..760da646f0 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -103,10 +103,6 @@ private boolean removeLogrotateFile() { return true; } - /** - * Trigger manual logrotate run. - * @return True on successful run or skip. False on error. - */ public boolean manualLogrotate() { if (!Files.exists(getLogrotateConfPath())) { log.info("{} did not exist, skipping manual logrotation", getLogrotateConfPath()); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java index 3f56fb34ba..a846ba7642 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java @@ -87,13 +87,13 @@ private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData execu task.getLog().info("Writing a runner script to execute {}", cmd); templateManager.writeRunnerScript(getPath("runner.sh"), new RunnerContext( - cmd, // cmd - configuration.getTaskAppDirectory(), // taskAppDirectory - configuration.getLogrotateToDirectory(), // logDir - executorData.getUser().or(configuration.getDefaultRunAsUser()), // user - configuration.getServiceLog(), // logFile - task.getTaskId(), // taskId - executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()))); // maxTaskThreads + cmd, + configuration.getTaskAppDirectory(), + configuration.getLogrotateToDirectory(), + executorData.getUser().or(configuration.getDefaultRunAsUser()), + configuration.getServiceLog(), + task.getTaskId(), + executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()))); List command = Lists.newArrayList(); command.add("bash"); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java index 69ecaebbae..7c7d6c5216 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java @@ -9,15 +9,9 @@ public class S3Configuration { @NotNull private int maxS3Threads = 3; - /** - * How long for SingularityService to wait for S3 List - */ @NotNull private int waitForS3ListSeconds = 5; - /** - * How long for SingularityService to wait for generating a link - */ @NotNull private int waitForS3LinksSeconds = 1; @@ -28,9 +22,6 @@ public class S3Configuration { @NotNull private long expireS3LinksAfterMillis = TimeUnit.DAYS.toMillis(1); - /** - * S3 Bucket that SingularityS3Uploader puts logs in. - */ @NotNull private String s3Bucket; @@ -112,5 +103,4 @@ public void setS3SecretKey(String s3SecretKey) { this.s3SecretKey = s3SecretKey; } - } From e9230bb1cb69d0911353962769fd0c148db0e96a Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Mon, 16 Mar 2015 14:41:03 -0400 Subject: [PATCH 82/93] Ignore 'may expose internal representation' error. --- .../executor/config/SingularityExecutorConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java index c7aceaae0f..5bbddf80d8 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java @@ -277,6 +277,7 @@ public Path getS3MetadataDirectory() { return s3MetadataDirectory; } + @SuppressFBWarnings("EI_EXPOSE_REP") public String[] getS3FilesToBackup() { return s3FilesToBackup; } From 1d3670bc075095fa5cd16a137337c0321219f713 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Mon, 16 Mar 2015 16:30:13 -0400 Subject: [PATCH 83/93] allows tasks to have a grace period after they finish in case they were holding onto some sort of shared resources that need to be freed --- .../hubspot/singularity/SingularityRequest.java | 15 ++++++++++++--- .../singularity/SingularityRequestBuilder.java | 16 +++++++++++++--- .../scheduler/SingularityScheduler.java | 6 ++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java index d7da9b9ac2..bb9e181871 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java @@ -24,6 +24,8 @@ public class SingularityRequest { private final Optional killOldNonLongRunningTasksAfterMillis; private final Optional scheduledExpectedRuntimeMillis; + private final Optional waitAtLeastMillisAfterTaskFinishesForReschedule; + //"use requestType instead" @Deprecated private final Optional daemon; @@ -42,7 +44,8 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT @JsonProperty("rackSensitive") Optional rackSensitive, @JsonProperty("loadBalanced") Optional loadBalanced, @JsonProperty("killOldNonLongRunningTasksAfterMillis") Optional killOldNonLongRunningTasksAfterMillis, @JsonProperty("scheduleType") Optional scheduleType, @JsonProperty("quartzSchedule") Optional quartzSchedule, @JsonProperty("rackAffinity") Optional> rackAffinity, - @JsonProperty("slavePlacement") Optional slavePlacement, @JsonProperty("scheduledExpectedRuntimeMillis") Optional scheduledExpectedRuntimeMillis) { + @JsonProperty("slavePlacement") Optional slavePlacement, @JsonProperty("scheduledExpectedRuntimeMillis") Optional scheduledExpectedRuntimeMillis, + @JsonProperty("waitAtLeastMillisAfterTaskFinishesForReschedule") Optional waitAtLeastMillisAfterTaskFinishesForReschedule) { this.id = id; this.owners = owners; this.numRetriesOnFailure = numRetriesOnFailure; @@ -57,6 +60,7 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT this.rackAffinity = rackAffinity; this.slavePlacement = slavePlacement; this.scheduledExpectedRuntimeMillis = scheduledExpectedRuntimeMillis; + this.waitAtLeastMillisAfterTaskFinishesForReschedule = waitAtLeastMillisAfterTaskFinishesForReschedule; if (requestType == null) { this.requestType = RequestType.fromDaemonAndScheduleAndLoadBalanced(schedule, daemon, loadBalanced); @@ -77,6 +81,7 @@ public SingularityRequestBuilder toBuilder() { .setScheduleType(scheduleType) .setQuartzSchedule(quartzSchedule) .setRackAffinity(copyOfList(rackAffinity)) + .setWaitAtLeastMillisAfterTaskFinishesForReschedule(waitAtLeastMillisAfterTaskFinishesForReschedule) .setSlavePlacement(slavePlacement) .setScheduledExpectedRuntimeMillis(scheduledExpectedRuntimeMillis); } @@ -202,12 +207,16 @@ public ScheduleType getScheduleTypeSafe() { return scheduleType.or(ScheduleType.CRON); } + public Optional getWaitAtLeastMillisAfterTaskFinishesForReschedule() { + return waitAtLeastMillisAfterTaskFinishesForReschedule; + } + @Override public String toString() { return "SingularityRequest [id=" + id + ", requestType=" + requestType + ", owners=" + owners + ", numRetriesOnFailure=" + numRetriesOnFailure + ", schedule=" + schedule + ", quartzSchedule=" + quartzSchedule + ", scheduleType=" + scheduleType + ", killOldNonLongRunningTasksAfterMillis=" + killOldNonLongRunningTasksAfterMillis + ", scheduledExpectedRuntimeMillis=" - + scheduledExpectedRuntimeMillis + ", daemon=" + daemon + ", instances=" + instances + ", rackSensitive=" + rackSensitive + ", rackAffinity=" + rackAffinity + ", slavePlacement=" - + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; + + scheduledExpectedRuntimeMillis + ", waitAtLeastMillisAfterTaskFinishesForReschedule=" + waitAtLeastMillisAfterTaskFinishesForReschedule + ", daemon=" + daemon + ", instances=" + instances + + ", rackSensitive=" + rackSensitive + ", rackAffinity=" + rackAffinity + ", slavePlacement=" + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java index 3a22dcb5ce..0fb0028df0 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java @@ -19,6 +19,7 @@ public class SingularityRequestBuilder { private Optional killOldNonLongRunningTasksAfterMillis; private Optional scheduledExpectedRuntimeMillis; + private Optional waitAtLeastMillisAfterTaskFinishesForReschedule; @Deprecated // use requestType @@ -52,7 +53,7 @@ public SingularityRequestBuilder(String id, RequestType requestType) { public SingularityRequest build() { return new SingularityRequest(id, requestType, owners, numRetriesOnFailure, schedule, daemon, instances, rackSensitive, loadBalanced, killOldNonLongRunningTasksAfterMillis, scheduleType, quartzSchedule, - rackAffinity, slavePlacement, scheduledExpectedRuntimeMillis); + rackAffinity, slavePlacement, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule); } public Optional getLoadBalanced() { @@ -182,12 +183,21 @@ public RequestType getRequestType() { return requestType; } + public Optional getWaitAtLeastMillisAfterTaskFinishesForReschedule() { + return waitAtLeastMillisAfterTaskFinishesForReschedule; + } + + public SingularityRequestBuilder setWaitAtLeastMillisAfterTaskFinishesForReschedule(Optional waitAtLeastMillisAfterTaskFinishesForReschedule) { + this.waitAtLeastMillisAfterTaskFinishesForReschedule = waitAtLeastMillisAfterTaskFinishesForReschedule; + return this; + } + @Override public String toString() { return "SingularityRequestBuilder [id=" + id + ", requestType=" + requestType + ", owners=" + owners + ", numRetriesOnFailure=" + numRetriesOnFailure + ", schedule=" + schedule + ", quartzSchedule=" + quartzSchedule + ", scheduleType=" + scheduleType + ", killOldNonLongRunningTasksAfterMillis=" + killOldNonLongRunningTasksAfterMillis - + ", scheduledExpectedRuntimeMillis=" + scheduledExpectedRuntimeMillis + ", daemon=" + daemon + ", instances=" + instances + ", rackSensitive=" + rackSensitive + ", rackAffinity=" - + rackAffinity + ", slavePlacement=" + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; + + ", scheduledExpectedRuntimeMillis=" + scheduledExpectedRuntimeMillis + ", waitAtLeastMillisAfterTaskFinishesForReschedule=" + waitAtLeastMillisAfterTaskFinishesForReschedule + ", daemon=" + + daemon + ", instances=" + instances + ", rackSensitive=" + rackSensitive + ", rackAffinity=" + rackAffinity + ", slavePlacement=" + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java index 1c054fe3e9..7128f12583 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java @@ -624,6 +624,12 @@ private Optional getNextRunAt(SingularityRequest request, RequestState sta } } + if (pendingType == PendingType.TASK_DONE && request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().or(0L) > 0) { + nextRunAt = Math.max(nextRunAt, now + request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().get()); + + LOG.trace("Adjusted next run of {} to {} (by {}) due to waitAtLeastMillisAfterTaskFinishesForReschedule", request.getId(), nextRunAt, JavaUtils.durationFromMillis(request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().get())); + } + if (state == RequestState.SYSTEM_COOLDOWN && pendingType != PendingType.NEW_DEPLOY) { final long prevNextRunAt = nextRunAt; nextRunAt = Math.max(nextRunAt, now + TimeUnit.SECONDS.toMillis(configuration.getCooldownMinScheduleSeconds())); From d6229bf6ea6096ecb66b526c58e7b43c8d161166 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Mon, 16 Mar 2015 16:41:07 -0400 Subject: [PATCH 84/93] add test --- .../scheduler/SingularitySchedulerTest.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java index 1d5d1947ea..2f6e1469a3 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java @@ -1139,4 +1139,28 @@ public void testScaleDownTakesHighestInstances() { } + @Test + public void testWaitAfterTaskWorks() { + initRequest(); + initFirstDeploy(); + + SingularityTask task = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); + + statusUpdate(task, TaskState.TASK_FAILED); + + Assert.assertTrue(taskManager.getPendingTaskIds().get(0).getNextRunAt() - System.currentTimeMillis() < 1000L); + + resourceOffers(); + + long extraWait = 100000L; + + saveAndSchedule(request.toBuilder().setWaitAtLeastMillisAfterTaskFinishesForReschedule(Optional.of(extraWait)).setInstances(Optional.of(2))); + resourceOffers(); + + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FAILED); + + Assert.assertTrue(taskManager.getPendingTaskIds().get(0).getNextRunAt() - System.currentTimeMillis() > 1000L); + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + } + } From 2ec6219209cba824b3b837d52084385d15579dc2 Mon Sep 17 00:00:00 2001 From: tpetr Date: Fri, 20 Mar 2015 15:14:00 -0400 Subject: [PATCH 85/93] remove olddir for extra files to rotate -- cwd is already correct --- SingularityExecutor/src/main/resources/logrotate.conf.hbs | 1 - 1 file changed, 1 deletion(-) diff --git a/SingularityExecutor/src/main/resources/logrotate.conf.hbs b/SingularityExecutor/src/main/resources/logrotate.conf.hbs index 0b0d9c7a82..fd703882f3 100644 --- a/SingularityExecutor/src/main/resources/logrotate.conf.hbs +++ b/SingularityExecutor/src/main/resources/logrotate.conf.hbs @@ -16,7 +16,6 @@ notifempty {{#if extrasFiles}} {{#each extrasFiles}}{{{this}}} {{/each}} { dateformat {{{ extrasDateformat }}} - olddir {{{ rotateDirectory }}} missingok } {{/if}} From 04fed47085dd49687cfdfde613a28b0d2e55bcd0 Mon Sep 17 00:00:00 2001 From: tpetr Date: Fri, 20 Mar 2015 15:14:00 -0400 Subject: [PATCH 86/93] remove olddir for extra files to rotate -- cwd is already correct --- SingularityExecutor/src/main/resources/logrotate.conf.hbs | 1 - 1 file changed, 1 deletion(-) diff --git a/SingularityExecutor/src/main/resources/logrotate.conf.hbs b/SingularityExecutor/src/main/resources/logrotate.conf.hbs index 0b0d9c7a82..fd703882f3 100644 --- a/SingularityExecutor/src/main/resources/logrotate.conf.hbs +++ b/SingularityExecutor/src/main/resources/logrotate.conf.hbs @@ -16,7 +16,6 @@ notifempty {{#if extrasFiles}} {{#each extrasFiles}}{{{this}}} {{/each}} { dateformat {{{ extrasDateformat }}} - olddir {{{ rotateDirectory }}} missingok } {{/if}} From fc0b2afe059acbe77900d2e2e95f98b935481262 Mon Sep 17 00:00:00 2001 From: Danny Wolf Date: Fri, 20 Mar 2015 17:35:06 -0400 Subject: [PATCH 87/93] Change String[] to List and clean up how configuration is parsed. --- .../SingularityExecutorConfiguration.java | 33 +++++++------------ .../models/LogrotateTemplateContext.java | 13 +++++--- .../SingularityExecutorTaskLogManager.java | 6 ++-- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java index 5bbddf80d8..3c83322ad4 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java @@ -2,9 +2,10 @@ import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; +import java.util.List; import com.google.common.base.Optional; +import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -13,8 +14,6 @@ import com.hubspot.mesos.MesosUtils; import com.hubspot.singularity.runner.base.config.SingularityRunnerBaseConfigurationLoader; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - @Singleton public class SingularityExecutorConfiguration { @@ -47,12 +46,12 @@ public class SingularityExecutorConfiguration { private final String logrotateDateformat; private final String logrotateExtrasDateformat; - private final String[] logrotateExtrasFiles; + private final List logrotateExtrasFiles; /** * Extra files to backup to S3 besides the service log. */ - private final String[] s3FilesToBackup; + private final List additionalS3FilesToBackup; private final Path logMetadataDirectory; private final String logMetadataSuffix; @@ -133,11 +132,7 @@ public SingularityExecutorConfiguration( this.logrotateCount = logrotateCount; this.logrotateMaxageDays = logrotateMaxageDays; this.logrotateDateformat = logrotateDateformat; - if ((s3FilesToBackup != null) && (s3FilesToBackup.trim().length() > 0)) { - this.s3FilesToBackup = s3FilesToBackup.split(","); - } else { - this.s3FilesToBackup = new String[0]; - } + this.additionalS3FilesToBackup = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(s3FilesToBackup); this.s3Bucket = s3Bucket; this.s3KeyPattern = s3KeyPattern; this.s3MetadataSuffix = s3MetadataSuffix; @@ -145,11 +140,7 @@ public SingularityExecutorConfiguration( this.tailLogLinesToSave = Integer.parseInt(tailLogLinesToSave); this.serviceFinishedTailLog = serviceFinishedTailLog; this.logrotateExtrasDateformat = logrotateExtrasDateformat; - if ((logrotateExtrasFiles != null) && (logrotateExtrasFiles.trim().length() > 0)) { - this.logrotateExtrasFiles = logrotateExtrasFiles.split(","); - } else { - this.logrotateExtrasFiles = new String[0]; - } + this.logrotateExtrasFiles = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(logrotateExtrasFiles); this.useLocalDownloadService = Boolean.parseBoolean(useLocalDownloadService); this.localDownloadServiceTimeoutMillis = Long.parseLong(localDownloadServiceTimeoutMillis); @@ -196,8 +187,7 @@ public String getLogrotateExtrasDateformat() { return logrotateExtrasDateformat; } - @SuppressFBWarnings("EI_EXPOSE_REP") - public String[] getLogrotateExtrasFiles() { + public List getLogrotateExtrasFiles() { return logrotateExtrasFiles; } @@ -277,9 +267,8 @@ public Path getS3MetadataDirectory() { return s3MetadataDirectory; } - @SuppressFBWarnings("EI_EXPOSE_REP") - public String[] getS3FilesToBackup() { - return s3FilesToBackup; + public List getAdditionalS3FilesToBackup() { + return additionalS3FilesToBackup; } public String getS3KeyPattern() { @@ -318,9 +307,9 @@ public String toString() { + ", hardKillAfterMillis=" + hardKillAfterMillis + ", killThreads=" + killThreads + ", threadCheckThreads=" + threadCheckThreads + ", checkThreadsEveryMillis=" + checkThreadsEveryMillis + ", maxTaskMessageLength=" + maxTaskMessageLength + ", logrotateCommand=" + logrotateCommand + ", logrotateStateFile=" + logrotateStateFile + ", logrotateConfDirectory=" + logrotateConfDirectory + ", logrotateToDirectory=" + logrotateToDirectory + ", logrotateMaxageDays=" + logrotateMaxageDays + ", logrotateCount=" + logrotateCount + ", logrotateDateformat=" - + logrotateDateformat + ", logrotateExtrasDateformat=" + logrotateExtrasDateformat + ", logrotateExtrasFiles=" + Arrays.toString(logrotateExtrasFiles) + ", logMetadataDirectory=" + + logrotateDateformat + ", logrotateExtrasDateformat=" + logrotateExtrasDateformat + ", logrotateExtrasFiles=" + logrotateExtrasFiles + ", logMetadataDirectory=" + logMetadataDirectory + ", logMetadataSuffix=" + logMetadataSuffix + ", tailLogLinesToSave=" + tailLogLinesToSave + ", serviceFinishedTailLog=" + serviceFinishedTailLog - + ", s3MetadataSuffix=" + s3MetadataSuffix + ", s3MetadataDirectory=" + s3MetadataDirectory + ", s3FilesToBackup=" + Arrays.toString(s3FilesToBackup) + ", s3KeyPattern=" + s3KeyPattern + ", s3Bucket=" + + ", s3MetadataSuffix=" + s3MetadataSuffix + ", s3MetadataDirectory=" + s3MetadataDirectory + ", additionalS3FilesToBackup=" + additionalS3FilesToBackup + ", s3KeyPattern=" + s3KeyPattern + ", s3Bucket=" + s3Bucket + ", useLocalDownloadService=" + useLocalDownloadService + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + ", maxTaskThreads=" + maxTaskThreads + "]"; } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java index 692bc82fa9..aa967d3d4e 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java @@ -3,6 +3,9 @@ import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; import com.hubspot.singularity.executor.task.SingularityExecutorTaskDefinition; +import java.util.ArrayList; +import java.util.List; + /** * Handlebars context for generating logrotate.conf files. * Check `man logrotate` for more information. @@ -37,12 +40,12 @@ public String getRotateDirectory() { * Extra files for logrotate to rotate. If these do not exist logrotate will continue without error. * @return filenames to rotate. */ - public String[] getExtrasFiles() { - final String[] original = configuration.getLogrotateExtrasFiles(); - final String[] transformed = new String[original.length]; + public List getExtrasFiles() { + final List original = configuration.getLogrotateExtrasFiles(); + final List transformed = new ArrayList<>(original.size()); - for (int i = 0; i < original.length; i++) { - transformed[i] = taskDefinition.getTaskDirectoryPath().resolve(original[i]).toString(); + for (String filename : original) { + transformed.add(taskDefinition.getTaskDirectoryPath().resolve(filename).toString()); } return transformed; diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index 760da646f0..b3417d69b8 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -4,8 +4,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import com.google.common.base.Joiner; @@ -157,9 +156,8 @@ private boolean writeTailMetadata(boolean finished) { * @return file glob String. */ private String getS3Glob() { - List fileNames = new LinkedList<>(); + List fileNames = new ArrayList<>(configuration.getAdditionalS3FilesToBackup()); fileNames.add(taskDefinition.getServiceLogOutPath().getFileName().toString()); - fileNames.addAll(Arrays.asList(configuration.getS3FilesToBackup())); return String.format("{%s}*.gz*", Joiner.on(",").join(fileNames)); } From 2f02b621e2304de1b57c4809b477bdb3a3c6f776 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Mar 2015 11:25:59 -0400 Subject: [PATCH 88/93] Enforce task history updates by taskState type first. this trusts the lifecycle of TaskState over the ordering of clocks. --- .../com/hubspot/singularity/SingularityTaskHistoryUpdate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java index 35a82394f4..1a7d698b95 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java @@ -61,8 +61,8 @@ public SingularityTaskHistoryUpdate(@JsonProperty("taskId") SingularityTaskId ta @Override public int compareTo(SingularityTaskHistoryUpdate o) { return ComparisonChain.start() - .compare(timestamp, o.getTimestamp()) .compare(taskState.ordinal(), o.getTaskState().ordinal()) + .compare(timestamp, o.getTimestamp()) .compare(o.getTaskId().getId(), getTaskId().getId()) .result(); } From 23002a00014ba0788798b2d074e9442b12d73672 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Mar 2015 14:08:10 -0400 Subject: [PATCH 89/93] add better doc --- .../runner/base/shared/S3UploadMetadata.java | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java index b75edeaae0..db349f7748 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java @@ -6,9 +6,14 @@ import com.google.common.base.Preconditions; /** - * s3KeyFormat is the format for the S3 file. * - * It can contain the following: + * directory - the directory to watch for files inside of + * fileGlob - only files matching this glob will be uploaded to S3. + * s3Bucket - the name of the bucket to upload to in S3 + * + * s3KeyFormat - the format for the actual key name for the object in S3 corresponding to each file uploaded. This can be dynamically + * formatted with the following variables: + * * %filename - adds the original file's filename * %fileext - adds the original file's file ext * %Y - adds year @@ -17,6 +22,14 @@ * %s - adds milliseconds * %index - adds the index of the file uploaded at this moment (to preserve uniqueness) * + * For example, if the s3KeyFormat was: %filename-%Y and the file name on local disk was "file1.txt" the S3 key would be : s3Bucket/file1.txt-2015 (assuming current year is 2015) + * + * finished - set this to true if you wish *this s3 upload metadata configuration* file to be deleted after the last matching file is uploaded to S3 successfully (think of it as safe delete.) + * onFinishGlob - a glob to match files which should be uploaded *only* after finished is set to true OR the pid is no longer active + * pid - the pid of the process to watch, such that when that pid is no longer running, finished is set to true (stop uploading files / watching directory once all files are successfully uploaded.) + * s3AccessKey - the access key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) + * s3Secret - the secret key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) + * */ public class S3UploadMetadata { @@ -25,13 +38,15 @@ public class S3UploadMetadata { private final String s3Bucket; private final String s3KeyFormat; private final boolean finished; + private final Optional onFinishGlob; private final Optional pid; private final Optional s3AccessKey; private final Optional s3Secret; @JsonCreator public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, - @JsonProperty("finished") boolean finished, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, @JsonProperty("s3Secret") Optional s3Secret) { + @JsonProperty("finished") boolean finished, @JsonProperty("onFinishGlob") Optional onFinishGlob, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, + @JsonProperty("s3Secret") Optional s3Secret) { Preconditions.checkNotNull(directory); Preconditions.checkNotNull(fileGlob); Preconditions.checkNotNull(s3Bucket); @@ -45,6 +60,7 @@ public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProper this.pid = pid; this.s3AccessKey = s3AccessKey; this.s3Secret = s3Secret; + this.onFinishGlob = onFinishGlob; } @Override @@ -117,10 +133,15 @@ public Optional getS3Secret() { return s3Secret; } + public Optional getOnFinishGlob() { + return onFinishGlob; + } + @Override public String toString() { - return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + ", pid=" + pid - + ", s3AccessKey=" + s3AccessKey + ", s3Secret=" + s3Secret + "]"; + return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + ", onFinishGlob=" + + onFinishGlob + ", pid=" + pid + ", s3AccessKey=" + s3AccessKey + ", s3Secret=" + s3Secret + "]"; } + } From 3831791668cffb0b289e45f98533fe3f3de86490 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Mar 2015 15:03:07 -0400 Subject: [PATCH 90/93] add expire configuration --- .../SingularityExecutorTaskLogManager.java | 3 +- .../runner/base/shared/S3UploadMetadata.java | 15 +++++-- .../s3uploader/SingularityS3Uploader.java | 2 +- .../SingularityS3UploaderDriver.java | 45 +++++++++++++++---- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index 11424837a5..9b4e0a2f7c 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -172,7 +172,8 @@ public Path getLogrotateConfPath() { private boolean writeS3MetadataFile(boolean finished) { Path logrotateDirectory = taskDefinition.getServiceLogOutPath().getParent().resolve(configuration.getLogrotateToDirectory()); - S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(logrotateDirectory.toString(), getS3Glob(), configuration.getS3Bucket(), getS3KeyPattern(), finished, Optional. absent(), Optional. absent(), Optional. absent()); + S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(logrotateDirectory.toString(), getS3Glob(), configuration.getS3Bucket(), getS3KeyPattern(), finished, Optional. absent(), Optional. absent(), Optional. absent(), + Optional. absent(), Optional. absent()); String s3UploadMetadatafilename = String.format("%s%s", taskDefinition.getTaskId(), configuration.getS3MetadataSuffix()); diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java index db349f7748..77d9d3c26b 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java @@ -24,12 +24,14 @@ * * For example, if the s3KeyFormat was: %filename-%Y and the file name on local disk was "file1.txt" the S3 key would be : s3Bucket/file1.txt-2015 (assuming current year is 2015) * - * finished - set this to true if you wish *this s3 upload metadata configuration* file to be deleted after the last matching file is uploaded to S3 successfully (think of it as safe delete.) + * finished - set this to true if you wish *this s3 upload metadata configuration* file to be deleted and no more files uploaded after the last matching file is uploaded to S3 successfully (think of it as safe delete.) * onFinishGlob - a glob to match files which should be uploaded *only* after finished is set to true OR the pid is no longer active * pid - the pid of the process to watch, such that when that pid is no longer running, finished is set to true (stop uploading files / watching directory once all files are successfully uploaded.) * s3AccessKey - the access key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) * s3Secret - the secret key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) * + * finishedAfterMillisWithoutNewFile - after millis without a new file, set finished to true (see above for result.) - (-1 never expire) - absent - uses system default. + * */ public class S3UploadMetadata { @@ -42,11 +44,12 @@ public class S3UploadMetadata { private final Optional pid; private final Optional s3AccessKey; private final Optional s3Secret; + private final Optional finishedAfterMillisWithoutNewFile; @JsonCreator public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, @JsonProperty("finished") boolean finished, @JsonProperty("onFinishGlob") Optional onFinishGlob, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, - @JsonProperty("s3Secret") Optional s3Secret) { + @JsonProperty("s3Secret") Optional s3Secret, @JsonProperty("finishedAfterMillisWithoutNewFile") Optional finishedAfterMillisWithoutNewFile) { Preconditions.checkNotNull(directory); Preconditions.checkNotNull(fileGlob); Preconditions.checkNotNull(s3Bucket); @@ -61,6 +64,7 @@ public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProper this.s3AccessKey = s3AccessKey; this.s3Secret = s3Secret; this.onFinishGlob = onFinishGlob; + this.finishedAfterMillisWithoutNewFile = finishedAfterMillisWithoutNewFile; } @Override @@ -137,11 +141,14 @@ public Optional getOnFinishGlob() { return onFinishGlob; } + public Optional getFinishedAfterMillisWithoutNewFile() { + return finishedAfterMillisWithoutNewFile; + } + @Override public String toString() { return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + ", onFinishGlob=" - + onFinishGlob + ", pid=" + pid + ", s3AccessKey=" + s3AccessKey + ", s3Secret=" + s3Secret + "]"; + + onFinishGlob + ", pid=" + pid + ", s3AccessKey=" + s3AccessKey + ", s3Secret=" + s3Secret + ", finishedAfterMillisWithoutNewFile=" + finishedAfterMillisWithoutNewFile + "]"; } - } diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index 9284873853..030b992c56 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -84,7 +84,7 @@ public String toString() { return "SingularityS3Uploader [uploadMetadata=" + uploadMetadata + ", metadataPath=" + metadataPath + "]"; } - public int upload(Set synchronizedToUpload) throws IOException { + public int upload(Set synchronizedToUpload, boolean isFinished) throws IOException { final List toUpload = Lists.newArrayList(); int found = 0; diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java index 61860edb2f..74e89afbed 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java @@ -183,8 +183,13 @@ private int checkUploads() { final Set filesToUpload = Collections.newSetFromMap(new ConcurrentHashMap(metadataToUploader.size() * 2, 0.75f, metadataToUploader.size())); final Map> futures = Maps.newHashMapWithExpectedSize(metadataToUploader.size()); + final Map finishing = Maps.newHashMapWithExpectedSize(metadataToUploader.size()); for (final SingularityS3Uploader uploader : metadataToUploader.values()) { + final boolean isFinished = isFinished(uploader); + // do this here so we run at least once with isFinished = true + finishing.put(uploader, isFinished); + futures.put(uploader, executorService.submit(new Callable() { @Override @@ -192,7 +197,7 @@ public Integer call() { Integer returnValue = 0; try { - returnValue = uploader.upload(filesToUpload); + returnValue = uploader.upload(filesToUpload, isFinished); } catch (Throwable t) { metrics.error(); LOG.error("Error while processing uploader {}", uploader, t); @@ -208,21 +213,16 @@ public Integer call() { final long now = System.currentTimeMillis(); final Set expiredUploaders = Sets.newHashSetWithExpectedSize(metadataToUploader.size()); - // TODO cancel/timeouts? for (Entry> uploaderToFuture : futures.entrySet()) { final SingularityS3Uploader uploader = uploaderToFuture.getKey(); try { final int foundFiles = uploaderToFuture.getValue().get(); + final boolean isFinished = finishing.get(uploader); if (foundFiles == 0) { - final long durationSinceLastFile = now - uploaderLastHadFilesAt.get(uploader); - final boolean isFinished = isFinished(uploader); - - if ((durationSinceLastFile > configuration.getStopCheckingAfterMillisWithoutNewFile()) || isFinished) { - LOG.info("Expiring uploader {}", uploader); + if (shouldExpire(uploader, isFinished)) { + LOG.info("Expiring {}", uploader); expiredUploaders.add(uploader); - } else { - LOG.trace("Not expiring uploader {}, duration {} (max {}), isFinished: {})", uploader, durationSinceLastFile, configuration.getStopCheckingAfterMillisWithoutNewFile(), isFinished); } } else { LOG.trace("Updating uploader {} last expire time", uploader); @@ -255,6 +255,31 @@ public Integer call() { return totesUploads; } + private boolean shouldExpire(SingularityS3Uploader uploader, boolean isFinished) { + if (isFinished) { + return true; + } + + if (uploader.getUploadMetadata().getFinishedAfterMillisWithoutNewFile().isPresent()) { + if (uploader.getUploadMetadata().getFinishedAfterMillisWithoutNewFile().get() < 0) { + LOG.trace("{} never expires", uploader); + return false; + } + } + + final long durationSinceLastFile = System.currentTimeMillis() - uploaderLastHadFilesAt.get(uploader); + + final long expireAfterMillis = uploader.getUploadMetadata().getFinishedAfterMillisWithoutNewFile().or(configuration.getStopCheckingAfterMillisWithoutNewFile()); + + if (durationSinceLastFile > expireAfterMillis) { + return true; + } else { + LOG.trace("Not expiring uploader {}, duration {} (max {}), isFinished: {})", uploader, JavaUtils.durationFromMillis(durationSinceLastFile), JavaUtils.durationFromMillis(expireAfterMillis), isFinished); + } + + return false; + } + private boolean isFinished(SingularityS3Uploader uploader) { if (expiring.contains(uploader)) { return true; @@ -262,6 +287,8 @@ private boolean isFinished(SingularityS3Uploader uploader) { if (uploader.getUploadMetadata().getPid().isPresent()) { if (!processUtils.doesProcessExist(uploader.getUploadMetadata().getPid().get())) { + LOG.info("Pid {} not present - expiring uploader {}", uploader.getUploadMetadata().getPid().get(), uploader); + expiring.add(uploader); return true; } } From b21ad1d45e5ce9e3db8448e64d13836906129ed8 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Mar 2015 15:08:36 -0400 Subject: [PATCH 91/93] actually implement finish file matching --- .../s3uploader/SingularityS3Uploader.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index 030b992c56..d8ca05a7a0 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import com.codahale.metrics.Timer.Context; +import com.google.common.base.Optional; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.hubspot.mesos.JavaUtils; @@ -33,6 +34,7 @@ public class SingularityS3Uploader implements Closeable { private final S3UploadMetadata uploadMetadata; private final PathMatcher pathMatcher; + private final Optional finishedPathMatcher; private final String fileDirectory; private final S3Service s3Service; private final S3Bucket s3Bucket; @@ -57,6 +59,13 @@ public SingularityS3Uploader(AWSCredentials defaultCredentials, S3UploadMetadata this.uploadMetadata = uploadMetadata; this.fileDirectory = uploadMetadata.getDirectory(); this.pathMatcher = fileSystem.getPathMatcher("glob:" + uploadMetadata.getFileGlob()); + + if (uploadMetadata.getOnFinishGlob().isPresent()) { + finishedPathMatcher = Optional.of(fileSystem.getPathMatcher("glob:" + uploadMetadata.getOnFinishGlob().get())); + } else { + finishedPathMatcher = Optional. absent(); + } + this.s3Bucket = new S3Bucket(uploadMetadata.getS3Bucket()); this.metadataPath = metadataPath; this.logIdentifier = String.format("[%s]", metadataPath.getFileName()); @@ -97,8 +106,12 @@ public int upload(Set synchronizedToUpload, boolean isFinished) throws IOE for (Path file : JavaUtils.iterable(directory)) { if (!pathMatcher.matches(file.getFileName())) { - LOG.trace("{} Skipping {} because it didn't match {}", logIdentifier, file, uploadMetadata.getFileGlob()); - continue; + if (!isFinished || !finishedPathMatcher.isPresent() || !finishedPathMatcher.get().matches(file.getFileName())) { + LOG.trace("{} Skipping {} because it didn't match {}", logIdentifier, file, uploadMetadata.getFileGlob()); + continue; + } else { + LOG.trace("Not skipping file {} because it matched finish glob {}", file, uploadMetadata.getOnFinishGlob().get()); + } } found++; From 0b97719496a42cc6c4430c07fc91efa0778c00ed Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Tue, 24 Mar 2015 16:33:36 -0400 Subject: [PATCH 92/93] logging improvements --- .../config/SingularityRunnerBaseLogging.java | 2 +- .../runner/base/shared/ProcessUtils.java | 7 +++++- .../runner/base/shared/S3UploadMetadata.java | 23 +++++++++++++------ .../s3uploader/SingularityS3Uploader.java | 6 ++--- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java index 93cffd92cb..c90a467552 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java @@ -73,7 +73,7 @@ private boolean shouldObfuscateValue(String key) { return false; } - private static String obfuscateValue(String value) { + public static String obfuscateValue(String value) { if (value == null) { return value; } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java index ad59c252ac..6464772a8e 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java @@ -57,7 +57,12 @@ public ProcessResult sendSignal(Signal signal, int pid) { final long start = System.currentTimeMillis(); if (log.isPresent()) { - log.get().info("Signaling {} ({}) to process {}", signal, signal.getCode(), pid); + final String logLine = String.format("Signaling %s (%s) to process %s", signal, signal.getCode(), pid); + if (signal == Signal.CHECK) { + log.get().trace(logLine); + } else { + log.get().info(logLine); + } } try { diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java index 77d9d3c26b..0e1862bc26 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.hubspot.singularity.runner.base.config.SingularityRunnerBaseLogging; /** * @@ -28,7 +29,7 @@ * onFinishGlob - a glob to match files which should be uploaded *only* after finished is set to true OR the pid is no longer active * pid - the pid of the process to watch, such that when that pid is no longer running, finished is set to true (stop uploading files / watching directory once all files are successfully uploaded.) * s3AccessKey - the access key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) - * s3Secret - the secret key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) + * s3SecretKey - the secret key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) * * finishedAfterMillisWithoutNewFile - after millis without a new file, set finished to true (see above for result.) - (-1 never expire) - absent - uses system default. * @@ -43,13 +44,13 @@ public class S3UploadMetadata { private final Optional onFinishGlob; private final Optional pid; private final Optional s3AccessKey; - private final Optional s3Secret; + private final Optional s3SecretKey; private final Optional finishedAfterMillisWithoutNewFile; @JsonCreator public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, @JsonProperty("finished") boolean finished, @JsonProperty("onFinishGlob") Optional onFinishGlob, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, - @JsonProperty("s3Secret") Optional s3Secret, @JsonProperty("finishedAfterMillisWithoutNewFile") Optional finishedAfterMillisWithoutNewFile) { + @JsonProperty("s3SecretKey") Optional s3SecretKey, @JsonProperty("finishedAfterMillisWithoutNewFile") Optional finishedAfterMillisWithoutNewFile) { Preconditions.checkNotNull(directory); Preconditions.checkNotNull(fileGlob); Preconditions.checkNotNull(s3Bucket); @@ -62,7 +63,7 @@ public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProper this.finished = finished; this.pid = pid; this.s3AccessKey = s3AccessKey; - this.s3Secret = s3Secret; + this.s3SecretKey = s3SecretKey; this.onFinishGlob = onFinishGlob; this.finishedAfterMillisWithoutNewFile = finishedAfterMillisWithoutNewFile; } @@ -133,8 +134,8 @@ public Optional getS3AccessKey() { return s3AccessKey; } - public Optional getS3Secret() { - return s3Secret; + public Optional getS3SecretKey() { + return s3SecretKey; } public Optional getOnFinishGlob() { @@ -145,10 +146,18 @@ public Optional getFinishedAfterMillisWithoutNewFile() { return finishedAfterMillisWithoutNewFile; } + private String obfuscateValue(Optional optional) { + if (!optional.isPresent()) { + return optional.toString(); + } + + return SingularityRunnerBaseLogging.obfuscateValue(optional.get()); + } + @Override public String toString() { return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + ", onFinishGlob=" - + onFinishGlob + ", pid=" + pid + ", s3AccessKey=" + s3AccessKey + ", s3Secret=" + s3Secret + ", finishedAfterMillisWithoutNewFile=" + finishedAfterMillisWithoutNewFile + "]"; + + onFinishGlob + ", pid=" + pid + ", s3AccessKey=" + obfuscateValue(s3AccessKey) + ", s3Secret=" + obfuscateValue(s3SecretKey) + ", finishedAfterMillisWithoutNewFile=" + finishedAfterMillisWithoutNewFile + "]"; } } diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index d8ca05a7a0..3edaf603e3 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -45,8 +45,8 @@ public class SingularityS3Uploader implements Closeable { public SingularityS3Uploader(AWSCredentials defaultCredentials, S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath) { AWSCredentials credentials = defaultCredentials; - if (uploadMetadata.getS3Secret().isPresent() && uploadMetadata.getS3AccessKey().isPresent()) { - credentials = new AWSCredentials(uploadMetadata.getS3AccessKey().get(), uploadMetadata.getS3Secret().get()); + if (uploadMetadata.getS3SecretKey().isPresent() && uploadMetadata.getS3AccessKey().isPresent()) { + credentials = new AWSCredentials(uploadMetadata.getS3AccessKey().get(), uploadMetadata.getS3SecretKey().get()); } try { @@ -148,7 +148,7 @@ private void uploadBatch(List toUpload) { Files.delete(file); } catch (S3ServiceException se) { metrics.error(); - LOG.warn("{} Couldn't upload due to {} ({}) - {}", logIdentifier, se.getErrorCode(), se.getResponseCode(), se.getErrorMessage()); + LOG.warn("{} Couldn't upload {} due to {} ({}) - {}", logIdentifier, file, se.getErrorCode(), se.getResponseCode(), se.getErrorMessage(), se); } catch (Exception e) { metrics.error(); LOG.warn("{} Couldn't upload or delete {}", logIdentifier, file, e); From e7ac4c566b4a8b7f640d42e81f4d3dd01e49b652 Mon Sep 17 00:00:00 2001 From: Whitney Sorenson Date: Thu, 26 Mar 2015 16:33:05 -0400 Subject: [PATCH 93/93] fix decommission for on_demand and run_once - add tests --- .../scheduler/SingularityScheduler.java | 2 +- .../scheduler/SingularitySchedulerTest.java | 77 ++++++++++++++++++- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java index 1c054fe3e9..4450d1f453 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java @@ -331,7 +331,7 @@ private void deleteScheduledTasks(final Collection sched } private List getMatchingTaskIds(SingularitySchedulerStateCache stateCache, SingularityRequest request, SingularityPendingRequest pendingRequest) { - if (!request.isScheduled()) { + if (request.isLongRunning()) { return SingularityTaskId.matchingAndNotIn(stateCache.getActiveTaskIds(), request.getId(), pendingRequest.getDeployId(), stateCache.getCleaningTasks()); } else { return Lists.newArrayList(Iterables.filter(stateCache.getActiveTaskIds(), SingularityTaskId.matchingRequest(request.getId()))); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java index 1d5d1947ea..68e993c335 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java @@ -363,7 +363,6 @@ public void testScheduledJobLivesThroughDeploy() { @Test public void testOneOffsDontRunByThemselves() { - requestId = "oneoffRequest"; SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.submit(bldr.build(), Optional. absent()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); @@ -375,6 +374,65 @@ public void testOneOffsDontRunByThemselves() { Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); + requestResource.scheduleImmediately(requestId, user, Collections. emptyList()); + + resourceOffers(); + + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FINISHED); + + resourceOffers(); + Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); + Assert.assertEquals(0, taskManager.getPendingTaskIds().size()); + + requestResource.scheduleImmediately(requestId, user, Collections. emptyList()); + + resourceOffers(); + + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); + + resourceOffers(); + Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); + Assert.assertEquals(0, taskManager.getPendingTaskIds().size()); + } + + @Test + public void testOneOffsDontMoveDuringDecomission() { + SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); + requestResource.submit(bldr.build(), Optional. absent()); + deploy("d2"); + + requestResource.scheduleImmediately(requestId, user, Collections. emptyList()); + + validateTaskDoesntMoveDuringDecommission(); + } + + private void validateTaskDoesntMoveDuringDecommission() { + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave1", "host1", Optional.of("rack1")))); + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave2", "host2", Optional.of("rack1")))); + + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + + Assert.assertEquals("host1", taskManager.getActiveTaskIds().get(0).getHost()); + + Assert.assertEquals(StateChangeResult.SUCCESS, slaveManager.changeState("slave1", MachineState.STARTING_DECOMMISSION, Optional.of("user1"))); + + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave2", "host2", Optional.of("rack1")))); + + cleaner.drainCleanupQueue(); + + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave2", "host2", Optional.of("rack1")))); + + cleaner.drainCleanupQueue(); + + // task should not move! + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + Assert.assertEquals("host1", taskManager.getActiveTaskIds().get(0).getHost()); + Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); + Assert.assertTrue(taskManager.getCleanupTaskIds().size() == 1); } @Test @@ -396,7 +454,7 @@ public void testRunOnceRunOnlyOnce() { Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); - statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FINISHED); + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); resourceOffers(); @@ -422,6 +480,21 @@ public void testRunOnceRunOnlyOnce() { Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); } + @Test + public void testRunOnceDontMoveDuringDecomission() { + SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); + request = bldr.build(); + saveRequest(request); + + deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional. absent(), Optional. absent())); + + scheduler.drainPendingQueue(stateCacheProvider.get()); + + deployChecker.checkDeploys(); + + validateTaskDoesntMoveDuringDecommission(); + } + @Test public void testRetries() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE);