diff --git a/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java b/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java index ef93873dfe..477bab269f 100644 --- a/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java +++ b/SingularityBase/src/main/java/com/hubspot/mesos/Resources.java @@ -3,7 +3,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import static com.google.common.base.Preconditions.checkNotNull; + public class Resources { + public static Resources add(Resources a, Resources b) { + checkNotNull(a, "first argument of Resources.add() is null"); + checkNotNull(b, "second argument of Resources.add() is null"); + + return new Resources(a.getCpus() + b.getCpus(), a.getMemoryMb() + b.getMemoryMb(), a.getNumPorts() + b.getNumPorts()); + } + + public static final Resources EMPTY_RESOURCES = new Resources(0, 0, 0); private final double cpus; private final double memoryMb; diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java index bec287502d..0e8bcf0628 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java @@ -29,6 +29,8 @@ public class SingularityDeploy { private final Optional customExecutorCmd; private final Optional customExecutorId; private final Optional customExecutorSource; + private final Optional customExecutorResources; + private final Optional resources; private final Optional command; @@ -63,6 +65,7 @@ public SingularityDeploy(@JsonProperty("requestId") String requestId, @JsonProperty("customExecutorCmd") Optional customExecutorCmd, @JsonProperty("customExecutorId") Optional customExecutorId, @JsonProperty("customExecutorSource") Optional customExecutorSource, + @JsonProperty("customExecutorResources") Optional customExecutorResources, @JsonProperty("resources") Optional resources, @JsonProperty("env") Optional> env, @JsonProperty("uris") Optional> uris, @@ -90,6 +93,7 @@ public SingularityDeploy(@JsonProperty("requestId") String requestId, this.customExecutorCmd = customExecutorCmd; this.customExecutorId = customExecutorId; this.customExecutorSource = customExecutorSource; + this.customExecutorResources = customExecutorResources; this.metadata = metadata; this.version = version; @@ -190,6 +194,11 @@ public Optional getCustomExecutorId() { @ApiModelProperty(required=false, value="Custom Mesos executor source.") public Optional getCustomExecutorSource() { return customExecutorSource; } + @ApiModelProperty(required=false, value="Resources to allocate for custom mesos executor") + public Optional getCustomExecutorResources() { + return customExecutorResources; + } + @ApiModelProperty(required=false, value="Resources required for this deploy.", dataType="com.hubspot.mesos.Resources") public Optional getResources() { return resources; @@ -272,6 +281,7 @@ public String toString() { ", customExecutorCmd=" + customExecutorCmd + ", customExecutorId=" + customExecutorId + ", customExecutorSource=" + customExecutorSource + + ", customExecutorResources=" + customExecutorResources + ", resources=" + resources + ", command=" + command + ", arguments=" + arguments + diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java index 0ab2a9cb84..360f0e26f9 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployBuilder.java @@ -23,6 +23,8 @@ public class SingularityDeployBuilder { private Optional customExecutorCmd; private Optional customExecutorId; private Optional customExecutorSource; + private Optional customExecutorResources; + private Optional resources; private Optional command; @@ -54,6 +56,7 @@ public SingularityDeployBuilder(String requestId, String id) { this.customExecutorCmd = Optional.absent(); this.customExecutorId = Optional.absent(); this.customExecutorSource = Optional.absent(); + this.customExecutorResources = Optional.absent(); this.resources = Optional.absent(); this.command = Optional.absent(); this.arguments = Optional.absent(); @@ -72,7 +75,7 @@ public SingularityDeployBuilder(String requestId, String id) { } public SingularityDeploy build() { - return new SingularityDeploy(requestId, id, command, arguments, containerInfo, customExecutorCmd, customExecutorId, customExecutorSource, resources, env, uris, metadata, executorData, version, timestamp, deployHealthTimeoutSeconds, healthcheckUri, healthcheckIntervalSeconds, + return new SingularityDeploy(requestId, id, command, arguments, containerInfo, customExecutorCmd, customExecutorId, customExecutorSource, customExecutorResources, resources, env, uris, metadata, executorData, version, timestamp, deployHealthTimeoutSeconds, healthcheckUri, healthcheckIntervalSeconds, healthcheckTimeoutSeconds, serviceBasePath, loadBalancerGroups, considerHealthyAfterRunningForSeconds, loadBalancerOptions, skipHealthchecksOnDeploy); } @@ -161,6 +164,15 @@ public SingularityDeployBuilder setCustomExecutorSource(Optional customE return this; } + public Optional getCustomExecutorResources() { + return customExecutorResources; + } + + public SingularityDeployBuilder setCustomExecutorResources(Optional customExecutorResources) { + this.customExecutorResources = customExecutorResources; + return this; + } + public Optional getDeployHealthTimeoutSeconds() { return deployHealthTimeoutSeconds; } @@ -299,6 +311,7 @@ public String toString() { ", customExecutorCmd=" + customExecutorCmd + ", customExecutorId=" + customExecutorId + ", customExecutorSource=" + customExecutorSource + + ", customExecutorResources=" + customExecutorResources + ", resources=" + resources + ", command=" + command + ", arguments=" + arguments + diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java index d7da9b9ac2..bb9e181871 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java @@ -24,6 +24,8 @@ public class SingularityRequest { private final Optional killOldNonLongRunningTasksAfterMillis; private final Optional scheduledExpectedRuntimeMillis; + private final Optional waitAtLeastMillisAfterTaskFinishesForReschedule; + //"use requestType instead" @Deprecated private final Optional daemon; @@ -42,7 +44,8 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT @JsonProperty("rackSensitive") Optional rackSensitive, @JsonProperty("loadBalanced") Optional loadBalanced, @JsonProperty("killOldNonLongRunningTasksAfterMillis") Optional killOldNonLongRunningTasksAfterMillis, @JsonProperty("scheduleType") Optional scheduleType, @JsonProperty("quartzSchedule") Optional quartzSchedule, @JsonProperty("rackAffinity") Optional> rackAffinity, - @JsonProperty("slavePlacement") Optional slavePlacement, @JsonProperty("scheduledExpectedRuntimeMillis") Optional scheduledExpectedRuntimeMillis) { + @JsonProperty("slavePlacement") Optional slavePlacement, @JsonProperty("scheduledExpectedRuntimeMillis") Optional scheduledExpectedRuntimeMillis, + @JsonProperty("waitAtLeastMillisAfterTaskFinishesForReschedule") Optional waitAtLeastMillisAfterTaskFinishesForReschedule) { this.id = id; this.owners = owners; this.numRetriesOnFailure = numRetriesOnFailure; @@ -57,6 +60,7 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT this.rackAffinity = rackAffinity; this.slavePlacement = slavePlacement; this.scheduledExpectedRuntimeMillis = scheduledExpectedRuntimeMillis; + this.waitAtLeastMillisAfterTaskFinishesForReschedule = waitAtLeastMillisAfterTaskFinishesForReschedule; if (requestType == null) { this.requestType = RequestType.fromDaemonAndScheduleAndLoadBalanced(schedule, daemon, loadBalanced); @@ -77,6 +81,7 @@ public SingularityRequestBuilder toBuilder() { .setScheduleType(scheduleType) .setQuartzSchedule(quartzSchedule) .setRackAffinity(copyOfList(rackAffinity)) + .setWaitAtLeastMillisAfterTaskFinishesForReschedule(waitAtLeastMillisAfterTaskFinishesForReschedule) .setSlavePlacement(slavePlacement) .setScheduledExpectedRuntimeMillis(scheduledExpectedRuntimeMillis); } @@ -202,12 +207,16 @@ public ScheduleType getScheduleTypeSafe() { return scheduleType.or(ScheduleType.CRON); } + public Optional getWaitAtLeastMillisAfterTaskFinishesForReschedule() { + return waitAtLeastMillisAfterTaskFinishesForReschedule; + } + @Override public String toString() { return "SingularityRequest [id=" + id + ", requestType=" + requestType + ", owners=" + owners + ", numRetriesOnFailure=" + numRetriesOnFailure + ", schedule=" + schedule + ", quartzSchedule=" + quartzSchedule + ", scheduleType=" + scheduleType + ", killOldNonLongRunningTasksAfterMillis=" + killOldNonLongRunningTasksAfterMillis + ", scheduledExpectedRuntimeMillis=" - + scheduledExpectedRuntimeMillis + ", daemon=" + daemon + ", instances=" + instances + ", rackSensitive=" + rackSensitive + ", rackAffinity=" + rackAffinity + ", slavePlacement=" - + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; + + scheduledExpectedRuntimeMillis + ", waitAtLeastMillisAfterTaskFinishesForReschedule=" + waitAtLeastMillisAfterTaskFinishesForReschedule + ", daemon=" + daemon + ", instances=" + instances + + ", rackSensitive=" + rackSensitive + ", rackAffinity=" + rackAffinity + ", slavePlacement=" + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java index 3a22dcb5ce..0fb0028df0 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java @@ -19,6 +19,7 @@ public class SingularityRequestBuilder { private Optional killOldNonLongRunningTasksAfterMillis; private Optional scheduledExpectedRuntimeMillis; + private Optional waitAtLeastMillisAfterTaskFinishesForReschedule; @Deprecated // use requestType @@ -52,7 +53,7 @@ public SingularityRequestBuilder(String id, RequestType requestType) { public SingularityRequest build() { return new SingularityRequest(id, requestType, owners, numRetriesOnFailure, schedule, daemon, instances, rackSensitive, loadBalanced, killOldNonLongRunningTasksAfterMillis, scheduleType, quartzSchedule, - rackAffinity, slavePlacement, scheduledExpectedRuntimeMillis); + rackAffinity, slavePlacement, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule); } public Optional getLoadBalanced() { @@ -182,12 +183,21 @@ public RequestType getRequestType() { return requestType; } + public Optional getWaitAtLeastMillisAfterTaskFinishesForReschedule() { + return waitAtLeastMillisAfterTaskFinishesForReschedule; + } + + public SingularityRequestBuilder setWaitAtLeastMillisAfterTaskFinishesForReschedule(Optional waitAtLeastMillisAfterTaskFinishesForReschedule) { + this.waitAtLeastMillisAfterTaskFinishesForReschedule = waitAtLeastMillisAfterTaskFinishesForReschedule; + return this; + } + @Override public String toString() { return "SingularityRequestBuilder [id=" + id + ", requestType=" + requestType + ", owners=" + owners + ", numRetriesOnFailure=" + numRetriesOnFailure + ", schedule=" + schedule + ", quartzSchedule=" + quartzSchedule + ", scheduleType=" + scheduleType + ", killOldNonLongRunningTasksAfterMillis=" + killOldNonLongRunningTasksAfterMillis - + ", scheduledExpectedRuntimeMillis=" + scheduledExpectedRuntimeMillis + ", daemon=" + daemon + ", instances=" + instances + ", rackSensitive=" + rackSensitive + ", rackAffinity=" - + rackAffinity + ", slavePlacement=" + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; + + ", scheduledExpectedRuntimeMillis=" + scheduledExpectedRuntimeMillis + ", waitAtLeastMillisAfterTaskFinishesForReschedule=" + waitAtLeastMillisAfterTaskFinishesForReschedule + ", daemon=" + + daemon + ", instances=" + instances + ", rackSensitive=" + rackSensitive + ", rackAffinity=" + rackAffinity + ", slavePlacement=" + slavePlacement + ", loadBalanced=" + loadBalanced + "]"; } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java index 35a82394f4..1a7d698b95 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistoryUpdate.java @@ -61,8 +61,8 @@ public SingularityTaskHistoryUpdate(@JsonProperty("taskId") SingularityTaskId ta @Override public int compareTo(SingularityTaskHistoryUpdate o) { return ComparisonChain.start() - .compare(timestamp, o.getTimestamp()) .compare(taskState.ordinal(), o.getTaskState().ordinal()) + .compare(timestamp, o.getTimestamp()) .compare(o.getTaskId().getId(), getTaskId().getId()) .result(); } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java index 942f2e9ab4..48fb079505 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityDeployRequest.java @@ -1,6 +1,7 @@ package com.hubspot.singularity.api; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Optional; import com.hubspot.singularity.SingularityDeploy; @@ -32,6 +33,11 @@ public Optional getUnpauseOnSuccessfulDeploy() { return unpauseOnSuccessfulDeploy; } + @JsonIgnore + public boolean isUnpauseOnSuccessfulDeploy() { + return unpauseOnSuccessfulDeploy.or(Boolean.FALSE); + } + @ApiModelProperty(required=true, value="The Singularity deploy object") public SingularityDeploy getDeploy() { return deploy; diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index d13afbc723..655096a160 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -5,6 +5,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import javax.inject.Provider; @@ -16,6 +18,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -23,6 +27,8 @@ import com.hubspot.horizon.HttpRequest; import com.hubspot.horizon.HttpRequest.Method; import com.hubspot.horizon.HttpResponse; +import com.hubspot.mesos.json.MesosFileChunkObject; +import com.hubspot.singularity.MachineState; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -35,6 +41,7 @@ import com.hubspot.singularity.SingularityRequestCleanup; import com.hubspot.singularity.SingularityRequestHistory; import com.hubspot.singularity.SingularityRequestParent; +import com.hubspot.singularity.SingularitySandbox; import com.hubspot.singularity.SingularitySlave; import com.hubspot.singularity.SingularityState; import com.hubspot.singularity.SingularityTask; @@ -61,12 +68,8 @@ public class SingularityClient { private static final String RACKS_DELETE_DECOMISSIONING_FORMAT = RACKS_FORMAT + "/rack/%s/decomissioning"; private static final String SLAVES_FORMAT = "http://%s/%s/slaves"; - private static final String SLAVES_GET_ACTIVE_FORMAT = SLAVES_FORMAT + "/active"; - private static final String SLAVES_GET_DEAD_FORMAT = SLAVES_FORMAT + "/dead"; - private static final String SLAVES_GET_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/decomissioning"; - private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decomission"; - private static final String SLAVES_DELETE_DECOMISSIONING_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; - private static final String SLAVES_DELETE_DEAD_FORMAT = SLAVES_FORMAT + "/slave/%s/dead"; + private static final String SLAVES_DECOMISSION_FORMAT = SLAVES_FORMAT + "/slave/%s/decommission"; + private static final String SLAVES_DELETE_FORMAT = SLAVES_FORMAT + "/slave/%s/decomissioning"; private static final String TASKS_FORMAT = "http://%s/%s/tasks"; private static final String TASKS_KILL_TASK_FORMAT = TASKS_FORMAT + "/task/%s"; @@ -103,6 +106,10 @@ public class SingularityClient { private static final String WEBHOOKS_GET_QUEUED_REQUEST_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/request/%s"; private static final String WEBHOOKS_GET_QUEUED_TASK_UPDATES_FORMAT = WEBHOOKS_FORMAT + "/task/%s"; + private static final String SANDBOX_FORMAT = "http://%s/%s/sandbox"; + private static final String SANDBOX_BROWSE_FORMAT = SANDBOX_FORMAT + "/%s/browse"; + private static final String SANDBOX_READ_FILE_FORMAT = SANDBOX_FORMAT + "/%s/read"; + private static final TypeReference> REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> PENDING_REQUESTS_COLLECTION = new TypeReference>() {}; private static final TypeReference> CLEANUP_REQUESTS_COLLECTION = new TypeReference>() {}; @@ -172,13 +179,24 @@ private SingularityClientException fail(String type, HttpResponse response) { } private Optional getSingle(String uri, String type, String id, Class clazz) { + return getSingleWithParams(uri, type, id, Optional.>absent(), clazz); + } + + private Optional getSingleWithParams(String uri, String type, String id, Optional> queryParams, Class clazz) { checkNotNull(id, String.format("Provide a %s id", type)); LOG.info("Getting {} {} from {}", type, id, uri); final long start = System.currentTimeMillis(); - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); if (response.getStatusCode() == 404) { return Optional.absent(); @@ -192,11 +210,22 @@ private Optional getSingle(String uri, String type, String id, Class c } private Collection getCollection(String uri, String type, TypeReference> typeReference) { + return getCollectionWithParams(uri, type, Optional.>absent(), typeReference); + } + + private Collection getCollectionWithParams(String uri, String type, Optional> queryParams, TypeReference> typeReference) { LOG.info("Getting all {} from {}", type, uri); final long start = System.currentTimeMillis(); - HttpResponse response = httpClient.execute(HttpRequest.newBuilder().setUrl(uri).build()); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .setUrl(uri); + + if (queryParams.isPresent()) { + addQueryParams(requestBuilder, queryParams.get()); + } + + HttpResponse response = httpClient.execute(requestBuilder.build()); if (response.getStatusCode() == 404) { return ImmutableList.of(); @@ -209,6 +238,23 @@ private Collection getCollection(String uri, String type, TypeReference queryParams) { + for (Entry queryParamEntry : queryParams.entrySet()) { + if (queryParamEntry.getValue() instanceof String) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (String) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Integer) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Integer) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Long) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Long) queryParamEntry.getValue()); + } else if (queryParamEntry.getValue() instanceof Boolean) { + requestBuilder.addQueryParam(queryParamEntry.getKey(), (Boolean) queryParamEntry.getValue()); + } else { + throw new RuntimeException(String.format("The type '%s' of query param %s is not supported. Only String, long, int and boolean values are supported", + queryParamEntry.getValue().getClass().getName(), queryParamEntry.getKey())); + } + } + } + private void delete(String uri, String type, String id, Optional user) { delete(uri, type, id, user, Optional.> absent()); } @@ -572,22 +618,55 @@ public void deleteDeadRack(String rackId, Optional user) { // SLAVES // + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getActiveSlaves() { - return getSlaves(SLAVES_GET_ACTIVE_FORMAT, "active"); + return getSlaves(Optional.of(MachineState.ACTIVE)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDeadSlaves() { - return getSlaves(SLAVES_GET_DEAD_FORMAT, "dead"); + return getSlaves(Optional.of(MachineState.DEAD)); } + /** + * Use {@link getSlaves} specifying the desired slave state to filter by + * + */ + @Deprecated public Collection getDecomissioningSlaves() { - return getSlaves(SLAVES_GET_DECOMISSIONING_FORMAT, "decomissioning"); + return getSlaves(Optional.of(MachineState.DECOMMISSIONING)); } - private Collection getSlaves(String format, String type) { - final String requestUri = String.format(format, getHost(), contextPath); + /** + * Retrieve the list of all known slaves, optionally filtering by a particular slave state + * + * @param slaveState + * Optionally specify a particular state to filter slaves by + * @return + * A collection of {@link SingularitySlave} + */ + public Collection getSlaves(Optional slaveState) { + final String requestUri = String.format(SLAVES_FORMAT, getHost(), contextPath); - return getCollection(requestUri, type, SLAVES_COLLECTION); + Optional> maybeQueryParams = Optional.>absent(); + + String type = "slaves"; + + if (slaveState.isPresent()) { + maybeQueryParams = Optional.>of(ImmutableMap.of("state", slaveState.get().toString())); + + type = String.format("%s slaves", slaveState.get().toString()); + } + + return getCollectionWithParams(requestUri, type, maybeQueryParams, SLAVES_COLLECTION); } public void decomissionSlave(String slaveId, Optional user) { @@ -596,16 +675,10 @@ public void decomissionSlave(String slaveId, Optional user) { post(requestUri, String.format("decomission slave %s", slaveId), Optional.absent(), user); } - public void deleteDecomissioningSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DECOMISSIONING_FORMAT, getHost(), contextPath, slaveId); - - delete(requestUri, "decomissioning slave", slaveId, user); - } - - public void deleteDeadSlave(String slaveId, Optional user) { - final String requestUri = String.format(SLAVES_DELETE_DEAD_FORMAT, getHost(), contextPath, slaveId); + public void deleteSlave(String slaveId, Optional user) { + final String requestUri = String.format(SLAVES_DELETE_FORMAT, getHost(), contextPath, slaveId); - delete(requestUri, "dead slave", slaveId, user); + delete(requestUri, "deleting slave", slaveId, user); } // @@ -680,4 +753,60 @@ public Collection getQueuedTaskUpdates(String webh return getCollection(requestUri, "request updates", TASK_UPDATES_COLLECTION); } + // + // SANDBOX + // + + /** + * Retrieve information about a specific task's sandbox + * + * @param taskId + * The task ID to browse + * @param path + * The path to browse from. + * if not specified it will browse from the sandbox root. + * @return + * A {@link SingularitySandbox} object that captures the information for the path to a specific task's Mesos sandbox + */ + public Optional browseTaskSandBox(String taskId, String path) { + final String requestUrl = String.format(SANDBOX_BROWSE_FORMAT, getHost(), contextPath, taskId); + + return getSingleWithParams(requestUrl, "browse sandbox for task", taskId, Optional.>of(ImmutableMap.of("path", path)), SingularitySandbox.class); + + } + + /** + * Retrieve part of the contents of a file in a specific task's sandbox. + * + * @param taskId + * The task ID of the sandbox to read from + * @param path + * The path to the file to be read. Relative to the sandbox root (without a leading slash) + * @param grep + * Optional string to grep for + * @param offset + * Byte offset to start reading from + * @param length + * Maximum number of bytes to read + * @return + * A {@link MesosFileChunkObject} that contains the requested partial file contents + */ + public Optional readSandBoxFile(String taskId, String path, Optional grep, Optional offset, Optional length) { + final String requestUrl = String.format(SANDBOX_READ_FILE_FORMAT, getHost(), contextPath, taskId); + + Builder queryParamBuider = ImmutableMap.builder().put("path", path); + + if (grep.isPresent()) { + queryParamBuider.put("grep", grep.get()); + } + if (offset.isPresent()) { + queryParamBuider.put("offset", offset.get()); + } + if (length.isPresent()) { + queryParamBuider.put("length", length.get()); + } + + return getSingleWithParams(requestUrl, "Read sandbox file for task", taskId, Optional.>of(queryParamBuider.build()), MesosFileChunkObject.class); + } + } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java index f7868f865b..3c83322ad4 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java @@ -2,9 +2,10 @@ import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; +import java.util.List; import com.google.common.base.Optional; +import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -13,8 +14,6 @@ import com.hubspot.mesos.MesosUtils; import com.hubspot.singularity.runner.base.config.SingularityRunnerBaseConfigurationLoader; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - @Singleton public class SingularityExecutorConfiguration { @@ -47,7 +46,12 @@ public class SingularityExecutorConfiguration { private final String logrotateDateformat; private final String logrotateExtrasDateformat; - private final String[] logrotateExtrasFiles; + private final List logrotateExtrasFiles; + + /** + * Extra files to backup to S3 besides the service log. + */ + private final List additionalS3FilesToBackup; private final Path logMetadataDirectory; private final String logMetadataSuffix; @@ -85,6 +89,7 @@ public SingularityExecutorConfiguration( @Named(SingularityExecutorConfigurationLoader.MAX_TASK_MESSAGE_LENGTH) String maxTaskMessageLength, @Named(SingularityRunnerBaseConfigurationLoader.LOG_METADATA_DIRECTORY) String logMetadataDirectory, @Named(SingularityRunnerBaseConfigurationLoader.LOG_METADATA_SUFFIX) String logMetadataSuffix, + @Named(SingularityExecutorConfigurationLoader.S3_FILES_TO_BACKUP) String s3FilesToBackup, @Named(SingularityExecutorConfigurationLoader.S3_UPLOADER_BUCKET) String s3Bucket, @Named(SingularityExecutorConfigurationLoader.S3_UPLOADER_PATTERN) String s3KeyPattern, @Named(SingularityRunnerBaseConfigurationLoader.S3_METADATA_DIRECTORY) String s3MetadataDirectory, @@ -127,6 +132,7 @@ public SingularityExecutorConfiguration( this.logrotateCount = logrotateCount; this.logrotateMaxageDays = logrotateMaxageDays; this.logrotateDateformat = logrotateDateformat; + this.additionalS3FilesToBackup = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(s3FilesToBackup); this.s3Bucket = s3Bucket; this.s3KeyPattern = s3KeyPattern; this.s3MetadataSuffix = s3MetadataSuffix; @@ -134,11 +140,7 @@ public SingularityExecutorConfiguration( this.tailLogLinesToSave = Integer.parseInt(tailLogLinesToSave); this.serviceFinishedTailLog = serviceFinishedTailLog; this.logrotateExtrasDateformat = logrotateExtrasDateformat; - if ((logrotateExtrasFiles != null) && (logrotateExtrasFiles.trim().length() > 0)) { - this.logrotateExtrasFiles = logrotateExtrasFiles.split(","); - } else { - this.logrotateExtrasFiles = new String[0]; - } + this.logrotateExtrasFiles = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(logrotateExtrasFiles); this.useLocalDownloadService = Boolean.parseBoolean(useLocalDownloadService); this.localDownloadServiceTimeoutMillis = Long.parseLong(localDownloadServiceTimeoutMillis); @@ -185,8 +187,7 @@ public String getLogrotateExtrasDateformat() { return logrotateExtrasDateformat; } - @SuppressFBWarnings("EI_EXPOSE_REP") - public String[] getLogrotateExtrasFiles() { + public List getLogrotateExtrasFiles() { return logrotateExtrasFiles; } @@ -266,6 +267,10 @@ public Path getS3MetadataDirectory() { return s3MetadataDirectory; } + public List getAdditionalS3FilesToBackup() { + return additionalS3FilesToBackup; + } + public String getS3KeyPattern() { return s3KeyPattern; } @@ -302,10 +307,10 @@ public String toString() { + ", hardKillAfterMillis=" + hardKillAfterMillis + ", killThreads=" + killThreads + ", threadCheckThreads=" + threadCheckThreads + ", checkThreadsEveryMillis=" + checkThreadsEveryMillis + ", maxTaskMessageLength=" + maxTaskMessageLength + ", logrotateCommand=" + logrotateCommand + ", logrotateStateFile=" + logrotateStateFile + ", logrotateConfDirectory=" + logrotateConfDirectory + ", logrotateToDirectory=" + logrotateToDirectory + ", logrotateMaxageDays=" + logrotateMaxageDays + ", logrotateCount=" + logrotateCount + ", logrotateDateformat=" - + logrotateDateformat + ", logrotateExtrasDateformat=" + logrotateExtrasDateformat + ", logrotateExtrasFiles=" + Arrays.toString(logrotateExtrasFiles) + ", logMetadataDirectory=" + + logrotateDateformat + ", logrotateExtrasDateformat=" + logrotateExtrasDateformat + ", logrotateExtrasFiles=" + logrotateExtrasFiles + ", logMetadataDirectory=" + logMetadataDirectory + ", logMetadataSuffix=" + logMetadataSuffix + ", tailLogLinesToSave=" + tailLogLinesToSave + ", serviceFinishedTailLog=" + serviceFinishedTailLog - + ", s3MetadataSuffix=" + s3MetadataSuffix + ", s3MetadataDirectory=" + s3MetadataDirectory + ", s3KeyPattern=" + s3KeyPattern + ", s3Bucket=" + s3Bucket + ", useLocalDownloadService=" - + useLocalDownloadService + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + ", maxTaskThreads=" + maxTaskThreads + "]"; + + ", s3MetadataSuffix=" + s3MetadataSuffix + ", s3MetadataDirectory=" + s3MetadataDirectory + ", additionalS3FilesToBackup=" + additionalS3FilesToBackup + ", s3KeyPattern=" + s3KeyPattern + ", s3Bucket=" + + s3Bucket + ", useLocalDownloadService=" + useLocalDownloadService + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + ", maxTaskThreads=" + maxTaskThreads + "]"; } } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java index 15174b5721..904b2a8572 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationLoader.java @@ -47,6 +47,7 @@ public class SingularityExecutorConfigurationLoader extends SingularityConfigura public static final String TAIL_LOG_LINES_TO_SAVE = "executor.service.log.tail.lines.to.save"; public static final String TAIL_LOG_FILENAME = "executor.service.log.tail.file.name"; + public static final String S3_FILES_TO_BACKUP = "executor.s3.uploader.extras.files"; public static final String S3_UPLOADER_PATTERN = "executor.s3.uploader.pattern"; public static final String S3_UPLOADER_BUCKET = "executor.s3.uploader.bucket"; @@ -90,6 +91,8 @@ protected void bindDefaults(Properties properties) { properties.put(LOGROTATE_EXTRAS_FILES, ""); properties.put(LOGROTATE_EXTRAS_DATEFORMAT, "-%Y%m%d"); + properties.put(S3_FILES_TO_BACKUP, ""); + properties.put(USE_LOCAL_DOWNLOAD_SERVICE, Boolean.toString(false)); properties.put(LOCAL_DOWNLOAD_SERVICE_TIMEOUT_MILLIS, Long.toString(TimeUnit.MINUTES.toMillis(3))); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java index 8461cc53ab..cc131c85a5 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java @@ -2,7 +2,6 @@ import java.nio.file.Path; -import com.hubspot.mesos.MesosUtils; import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos; import org.apache.mesos.Protos.TaskInfo; @@ -16,6 +15,7 @@ import com.google.inject.Singleton; import com.google.inject.name.Named; import com.hubspot.deploy.ExecutorData; +import com.hubspot.mesos.MesosUtils; import com.hubspot.singularity.executor.TemplateManager; import com.hubspot.singularity.executor.task.SingularityExecutorArtifactFetcher; import com.hubspot.singularity.executor.task.SingularityExecutorTask; @@ -64,8 +64,8 @@ public Logger buildTaskLogger(String taskId) { public SingularityExecutorTask buildTask(String taskId, ExecutorDriver driver, TaskInfo taskInfo, Logger log) { ExecutorData executorData = readExecutorData(jsonObjectMapper, taskInfo); - SingularityExecutorTaskDefinition taskDefinition = new SingularityExecutorTaskDefinition(taskId, executorData, MesosUtils.getTaskDirectoryPath(taskId).toString(), configuration.getServiceLog(), - configuration.getTaskAppDirectory(), configuration.getExecutorBashLog(), configuration.getLogrotateStateFile()); + SingularityExecutorTaskDefinition taskDefinition = new SingularityExecutorTaskDefinition(taskId, executorData, MesosUtils.getTaskDirectoryPath(taskId).toString(), executorPid, + configuration.getServiceLog(), configuration.getTaskAppDirectory(), configuration.getExecutorBashLog(), configuration.getLogrotateStateFile()); jsonObjectFileHelper.writeObject(taskDefinition, configuration.getTaskDefinitionPath(taskId), log); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java index c90ec910aa..aa967d3d4e 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java @@ -3,6 +3,13 @@ import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; import com.hubspot.singularity.executor.task.SingularityExecutorTaskDefinition; +import java.util.ArrayList; +import java.util.List; + +/** + * Handlebars context for generating logrotate.conf files. + * Check `man logrotate` for more information. + */ public class LogrotateTemplateContext { private final SingularityExecutorTaskDefinition taskDefinition; @@ -29,12 +36,16 @@ public String getRotateDirectory() { return configuration.getLogrotateToDirectory(); } - public String[] getExtrasFiles() { - final String[] original = configuration.getLogrotateExtrasFiles(); - final String[] transformed = new String[original.length]; + /** + * Extra files for logrotate to rotate. If these do not exist logrotate will continue without error. + * @return filenames to rotate. + */ + public List getExtrasFiles() { + final List original = configuration.getLogrotateExtrasFiles(); + final List transformed = new ArrayList<>(original.size()); - for (int i = 0; i < original.length; i++) { - transformed[i] = taskDefinition.getTaskDirectoryPath().resolve(original[i]).toString(); + for (String filename : original) { + transformed.add(taskDefinition.getTaskDirectoryPath().resolve(filename).toString()); } return transformed; @@ -44,6 +55,11 @@ public String getExtrasDateformat() { return configuration.getLogrotateExtrasDateformat(); } + /** + * Default log to logrotate, defaults to service.log. + * This if this log doesn't exist, logrotate will return an error message. + * @return filename to rotate. + */ public String getLogfile() { return taskDefinition.getServiceLogOut(); } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java index 43ca2a03b5..1dba3bcd6c 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/RunnerContext.java @@ -2,26 +2,29 @@ import com.google.common.base.Optional; +/** + * Handlebars context for generating the runner.sh file. + */ public class RunnerContext { private final String cmd; + private final String taskAppDirectory; + private final String logDir; private final String user; - private final String logfile; + private final String logFile; private final String taskId; - private final String taskAppDirectory; + private final Optional maxTaskThreads; - public RunnerContext(String cmd, String taskAppDirectory, String user, String logfile, String taskId, Optional maxTaskThreads) { + public RunnerContext(String cmd, String taskAppDirectory, String logDir, String user, String logFile, String taskId, Optional maxTaskThreads) { this.cmd = cmd; + this.taskAppDirectory = taskAppDirectory; + this.logDir = logDir; this.user = user; - this.logfile = logfile; + this.logFile = logFile; this.taskId = taskId; - this.taskAppDirectory = taskAppDirectory; - this.maxTaskThreads = maxTaskThreads; - } - public String getTaskId() { - return taskId; + this.maxTaskThreads = maxTaskThreads; } public String getCmd() { @@ -32,12 +35,20 @@ public String getTaskAppDirectory() { return taskAppDirectory; } + public String getLogDir() { + return logDir; + } + public String getUser() { return user; } - public String getLogfile() { - return logfile; + public String getLogFile() { + return logFile; + } + + public String getTaskId() { + return taskId; } public Optional getMaxTaskThreads() { @@ -47,11 +58,12 @@ public Optional getMaxTaskThreads() { @Override public String toString() { return "RunnerContext [" + - "cmd='" + cmd + '\'' + - ", user='" + user + '\'' + - ", logfile='" + logfile + '\'' + - ", taskId='" + taskId + '\'' + - ", taskAppDirectory='" + taskAppDirectory + '\'' + + "cmd='" + cmd + "'" + + ", taskAppDirectory='" + taskAppDirectory + "'" + + ", logDir='" + logDir + "'" + + ", user='" + user + "'" + + ", logFile='" + logFile + "'" + + ", taskId='" + taskId + "'" + ", maxTaskThreads=" + maxTaskThreads + ']'; } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java index 7175241467..ff1c510e07 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskCleanup.java @@ -31,6 +31,7 @@ public boolean cleanup(boolean cleanupTaskAppDirectory) { if (!Files.exists(taskDirectory)) { log.info("Directory {} didn't exist for cleanup", taskDirectory); + taskLogManager.removeLogrotateFile(); return cleanTaskDefinitionFile(); } @@ -53,7 +54,7 @@ public boolean cleanup(boolean cleanupTaskAppDirectory) { public boolean cleanTaskDefinitionFile() { Path taskDefinitionPath = configuration.getTaskDefinitionPath(taskDefinition.getTaskId()); - log.info("Successfull cleanup, deleting file {}", taskDefinitionPath); + log.info("Successful cleanup, deleting file {}", taskDefinitionPath); try { boolean deleted = Files.deleteIfExists(taskDefinitionPath); @@ -77,7 +78,7 @@ private boolean cleanupTaskAppDirectory() { "rm", "-rf", pathToDelete - ); + ); new SimpleProcessManager(log).runCommand(cmd); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java index 354e21f10e..022e474e09 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; import com.hubspot.deploy.ExecutorData; public class SingularityExecutorTaskDefinition { @@ -17,14 +18,16 @@ public class SingularityExecutorTaskDefinition { private final String serviceLogOut; private final String taskAppDirectory; private final String logrotateStateFile; + private final String executorPid; @JsonCreator - public SingularityExecutorTaskDefinition(@JsonProperty("taskId") String taskId, @JsonProperty("executorData") ExecutorData executorData, @JsonProperty("taskDirectory") String taskDirectory, + public SingularityExecutorTaskDefinition(@JsonProperty("taskId") String taskId, @JsonProperty("executorData") ExecutorData executorData, @JsonProperty("taskDirectory") String taskDirectory, @JsonProperty("executorPid") String executorPid, @JsonProperty("serviceLogOut") String serviceLogOut, @JsonProperty("taskAppDirectory") String taskAppDirectory, @JsonProperty("executorBashOut") String executorBashOut, @JsonProperty("logrotateStateFilePath") String logrotateStateFile) { this.executorData = executorData; this.taskId = taskId; this.taskDirectoryPath = Paths.get(taskDirectory); + this.executorPid = executorPid; this.executorBashOut = executorBashOut; this.serviceLogOut = serviceLogOut; this.taskAppDirectory = taskAppDirectory; @@ -84,10 +87,23 @@ public String getTaskId() { return taskId; } + public String getExecutorPid() { + return executorPid; + } + + @JsonIgnore + public Optional getExecutorPidSafe() { + try { + return Optional.of(Integer.parseInt(executorPid)); + } catch (NumberFormatException nfe) { + return Optional. absent(); + } + } + @Override public String toString() { - return "SingularityExecutorTaskDefinition [taskId=" + taskId + "]"; + return "SingularityExecutorTaskDefinition [executorData=" + executorData + ", taskId=" + taskId + ", taskDirectoryPath=" + taskDirectoryPath + ", executorBashOut=" + executorBashOut + + ", serviceLogOut=" + serviceLogOut + ", taskAppDirectory=" + taskAppDirectory + ", logrotateStateFile=" + logrotateStateFile + ", executorPid=" + executorPid + "]"; } - } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index aed305a579..1ad49da894 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -4,10 +4,13 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; +import com.google.common.base.Joiner; import org.slf4j.Logger; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.hubspot.singularity.SingularityS3FormatHelper; import com.hubspot.singularity.SingularityTaskId; @@ -88,7 +91,7 @@ private void copyLogTail() { } } - private boolean removeLogrotateFile() { + public boolean removeLogrotateFile() { boolean deleted = false; try { deleted = Files.deleteIfExists(getLogrotateConfPath()); @@ -148,8 +151,16 @@ private boolean writeTailMetadata(boolean finished) { return jsonObjectFileHelper.writeObject(tailMetadata, path, log); } + /** + * Return a String for generating a PathMatcher. + * The matching files are caught by the S3 Uploader and pushed to S3. + * @return file glob String. + */ private String getS3Glob() { - return String.format("%s*.gz*", taskDefinition.getServiceLogOutPath().getFileName()); + List fileNames = new ArrayList<>(configuration.getAdditionalS3FilesToBackup()); + fileNames.add(taskDefinition.getServiceLogOutPath().getFileName().toString()); + + return String.format("{%s}*.gz*", Joiner.on(",").join(fileNames)); } private String getS3KeyPattern() { @@ -171,11 +182,12 @@ public Path getLogrotateConfPath() { private boolean writeS3MetadataFile(boolean finished) { Path logrotateDirectory = taskDefinition.getServiceLogOutPath().getParent().resolve(configuration.getLogrotateToDirectory()); - S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(logrotateDirectory.toString(), getS3Glob(), configuration.getS3Bucket(), getS3KeyPattern(), finished); + S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(logrotateDirectory.toString(), getS3Glob(), configuration.getS3Bucket(), getS3KeyPattern(), finished, Optional. absent(), Optional. absent(), Optional. absent(), + Optional. absent(), Optional. absent()); - String s3UploadMetadatafilename = String.format("%s%s", taskDefinition.getTaskId(), configuration.getS3MetadataSuffix()); + String s3UploadMetadataFileName = String.format("%s%s", taskDefinition.getTaskId(), configuration.getS3MetadataSuffix()); - Path s3UploadMetadataPath = configuration.getS3MetadataDirectory().resolve(s3UploadMetadatafilename); + Path s3UploadMetadataPath = configuration.getS3MetadataDirectory().resolve(s3UploadMetadataFileName); return jsonObjectFileHelper.writeObject(s3UploadMetadata, s3UploadMetadataPath, log); } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java index 39ff92cdc6..a846ba7642 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java @@ -34,7 +34,8 @@ public class SingularityExecutorTaskProcessBuilder implements Callable taskArtifactFetcher; - public SingularityExecutorTaskProcessBuilder(SingularityExecutorTask task, ExecutorUtils executorUtils, SingularityExecutorArtifactFetcher artifactFetcher, TemplateManager templateManager, SingularityExecutorConfiguration configuration, ExecutorData executorData, String executorPid) { + public SingularityExecutorTaskProcessBuilder(SingularityExecutorTask task, ExecutorUtils executorUtils, SingularityExecutorArtifactFetcher artifactFetcher, TemplateManager templateManager, SingularityExecutorConfiguration configuration, + ExecutorData executorData, String executorPid) { this.executorData = executorData; this.task = task; this.executorUtils = executorUtils; @@ -85,7 +86,14 @@ private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData execu task.getLog().info("Writing a runner script to execute {}", cmd); - templateManager.writeRunnerScript(getPath("runner.sh"), new RunnerContext(cmd, configuration.getTaskAppDirectory(), executorData.getUser().or(configuration.getDefaultRunAsUser()), configuration.getServiceLog(), task.getTaskId(), executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()))); + templateManager.writeRunnerScript(getPath("runner.sh"), new RunnerContext( + cmd, + configuration.getTaskAppDirectory(), + configuration.getLogrotateToDirectory(), + executorData.getUser().or(configuration.getDefaultRunAsUser()), + configuration.getServiceLog(), + task.getTaskId(), + executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()))); List command = Lists.newArrayList(); command.add("bash"); diff --git a/SingularityExecutor/src/main/resources/runner.sh.hbs b/SingularityExecutor/src/main/resources/runner.sh.hbs index ec3afaa8e8..8df707d56e 100644 --- a/SingularityExecutor/src/main/resources/runner.sh.hbs +++ b/SingularityExecutor/src/main/resources/runner.sh.hbs @@ -32,6 +32,13 @@ if [[ ! -d ./tmp ]]; then sudo chown -R {{{ user }}} ./tmp fi +# Create log directory for logrotate runs +if [[ ! -d {{{ logDir }}} ]]; then + echo "Creating log directory ({{{ logDir }}})" + mkdir -p {{{ logDir }}} + sudo chown -R {{{ user }}} {{{ logDir }}} +fi + echo "Ensuring {{{ taskAppDirectory }}} is owned by {{{ user }}}" sudo chown -R {{{ user }}} {{{ taskAppDirectory }}} @@ -48,5 +55,5 @@ else fi # execute command -echo "Executing: sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1" -exec sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logfile }}} 2>&1 +echo "Executing: sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logFile }}} 2>&1" +exec sudo -E -u {{{ user }}} {{{ cmd }}} >> ../{{{ logFile }}} 2>&1 diff --git a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java index 9a02673a37..d1f1ac45c6 100644 --- a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java +++ b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java @@ -2,9 +2,15 @@ import java.io.IOException; import java.net.SocketException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; import java.util.Set; import org.slf4j.Logger; @@ -12,6 +18,7 @@ import com.google.common.base.Optional; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.hubspot.mesos.JavaUtils; @@ -31,6 +38,9 @@ import com.hubspot.singularity.executor.task.SingularityExecutorTaskDefinition; import com.hubspot.singularity.executor.task.SingularityExecutorTaskLogManager; import com.hubspot.singularity.runner.base.shared.JsonObjectFileHelper; +import com.hubspot.singularity.runner.base.shared.ProcessFailedException; +import com.hubspot.singularity.runner.base.shared.ProcessUtils; +import com.hubspot.singularity.runner.base.shared.SimpleProcessManager; public class SingularityExecutorCleanup { @@ -42,6 +52,7 @@ public class SingularityExecutorCleanup { private final TemplateManager templateManager; private final SingularityExecutorCleanupConfiguration cleanupConfiguration; private final MesosClient mesosClient; + private final ProcessUtils processUtils; @Inject public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjectFileHelper jsonObjectFileHelper, SingularityExecutorConfiguration executorConfiguration, SingularityExecutorCleanupConfiguration cleanupConfiguration, TemplateManager templateManager, MesosClient mesosClient) { @@ -51,6 +62,7 @@ public SingularityExecutorCleanup(SingularityClient singularityClient, JsonObjec this.singularityClient = singularityClient; this.templateManager = templateManager; this.mesosClient = mesosClient; + this.processUtils = new ProcessUtils(LOG); } public SingularityExecutorCleanupStatistics clean() { @@ -101,7 +113,7 @@ public SingularityExecutorCleanupStatistics clean() { final String taskId = taskDefinition.get().getTaskId(); - if (runningTaskIds.contains(taskId)) { + if (runningTaskIds.contains(taskId) || executorStillRunning(taskDefinition.get())) { statisticsBldr.incrRunningTasksIgnored(); continue; } @@ -150,6 +162,16 @@ private Set getRunningTaskIds() { } } + private boolean executorStillRunning(SingularityExecutorTaskDefinition taskDefinition) { + Optional executorPidSafe = taskDefinition.getExecutorPidSafe(); + + if (!executorPidSafe.isPresent()) { + return false; + } + + return processUtils.doesProcessExist(executorPidSafe.get()); + } + private boolean cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional taskHistory) { SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, executorConfiguration, LOG, jsonObjectFileHelper); @@ -171,7 +193,63 @@ private boolean cleanTask(SingularityExecutorTaskDefinition taskDefinition, Opti } } + checkForUncompressedLogrotatedFile(taskDefinition); + return taskCleanup.cleanup(cleanupTaskAppDirectory); } + private Iterator getUncompressedLogrotatedFileIterator(SingularityExecutorTaskDefinition taskDefinition) { + final Path serviceLogOutPath = taskDefinition.getServiceLogOutPath(); + final Path logrotateToPath = taskDefinition.getServiceLogOutPath().getParent().resolve(executorConfiguration.getLogrotateToDirectory()); + + try { + DirectoryStream dirStream = Files.newDirectoryStream(logrotateToPath, String.format("%s-*", serviceLogOutPath.getFileName())); + return dirStream.iterator(); + } catch (IOException e) { + throw Throwables.propagate(e); + } + } + + private void checkForUncompressedLogrotatedFile(SingularityExecutorTaskDefinition taskDefinition) { + final Iterator iterator = getUncompressedLogrotatedFileIterator(taskDefinition); + final Set emptyPaths = new HashSet<>(); + final List ungzippedFiles = new ArrayList<>(); + + // check for matched 0 byte gz files.. and delete/gzip them + + while (iterator.hasNext()) { + Path path = iterator.next(); + + if (path.getFileName().toString().endsWith(".gz")) { + try { + if (Files.size(path) == 0) { + Files.deleteIfExists(path); + + String pathString = path.getFileName().toString(); + + emptyPaths.add(pathString.substring(0, pathString.length() - 3)); // removing .gz + } + } catch (IOException ioe) { + LOG.error("Failed to handle empty gz file {}", path, ioe); + } + } else { + ungzippedFiles.add(path); + } + } + + for (Path path : ungzippedFiles) { + if (emptyPaths.contains(path.getFileName().toString())) { + LOG.info("Gzipping abandoned file {}", path); + try { + new SimpleProcessManager(LOG).runCommand(ImmutableList. of("gzip", path.toString())); + } catch (InterruptedException | ProcessFailedException e) { + LOG.error("Failed to gzip {}", path, e); + } + } else { + LOG.debug("Didn't find matched empty gz file for {}", path); + } + } + } + + } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java index 93cffd92cb..c90a467552 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/config/SingularityRunnerBaseLogging.java @@ -73,7 +73,7 @@ private boolean shouldObfuscateValue(String key) { return false; } - private static String obfuscateValue(String value) { + public static String obfuscateValue(String value) { if (value == null) { return value; } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java index b8778c47bf..6464772a8e 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/ProcessUtils.java @@ -1,33 +1,93 @@ package com.hubspot.singularity.runner.base.shared; import java.io.IOException; +import java.io.InputStreamReader; import java.lang.reflect.Field; +import javax.annotation.Nullable; + import org.slf4j.Logger; +import com.google.common.base.Charsets; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import com.google.common.io.CharStreams; +import com.google.common.io.Closeables; import com.hubspot.mesos.JavaUtils; public class ProcessUtils { - public static void sendSignal(Signal signal, Logger log, int pid) { - final long start = System.currentTimeMillis(); + private final Optional log; + + public ProcessUtils() { + this(null); + } + + public ProcessUtils(@Nullable Logger log) { + this.log = Optional.fromNullable(log); + } + + public static class ProcessResult { + + private final int exitCode; + private final String output; + + public ProcessResult(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + + public int getExitCode() { + return exitCode; + } + + public String getOutput() { + return output; + } - log.info("Signaling {} ({}) to process {}", signal, signal.getCode(), pid); + @Override + public String toString() { + return "ProcessResult [exitCode=" + exitCode + ", output=" + output + "]"; + } + + } - final String killCmd = String.format("kill -%s %s", signal.getCode(), pid); + public ProcessResult sendSignal(Signal signal, int pid) { + final long start = System.currentTimeMillis(); + + if (log.isPresent()) { + final String logLine = String.format("Signaling %s (%s) to process %s", signal, signal.getCode(), pid); + if (signal == Signal.CHECK) { + log.get().trace(logLine); + } else { + log.get().info(logLine); + } + } try { - int signalCode = Runtime.getRuntime().exec(killCmd).waitFor(); + final ProcessBuilder pb = new ProcessBuilder("kill", String.format("-%s", signal.getCode()), Integer.toString(pid)); + pb.redirectErrorStream(true); + + final Process p = pb.start(); - log.trace("Kill signal process for {} got exit code {} after {}", pid, signalCode, JavaUtils.duration(start)); + final int exitCode = p.waitFor(); + + final String output = CharStreams.toString(new InputStreamReader(p.getInputStream(), Charsets.UTF_8)); + + Closeables.closeQuietly(p.getInputStream()); + + if (log.isPresent()) { + log.get().trace("Kill signal process for {} got exit code {} after {}", pid, exitCode, JavaUtils.duration(start)); + } + + return new ProcessResult(exitCode, output.trim()); } catch (InterruptedException | IOException e) { throw Throwables.propagate(e); } } - public static int getUnixPID(Process process) { + public int getUnixPID(Process process) { Preconditions.checkArgument(process.getClass().getName().equals("java.lang.UNIXProcess")); Class clazz = process.getClass(); @@ -42,5 +102,13 @@ public static int getUnixPID(Process process) { } } + public boolean doesProcessExist(int pid) { + ProcessResult processResult = sendSignal(Signal.CHECK, pid); + if (processResult.getExitCode() != 0 && processResult.output.contains("No such process")) { + return false; + } + return true; + } + } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java index 34b43274b5..0e1862bc26 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java @@ -2,12 +2,19 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.hubspot.singularity.runner.base.config.SingularityRunnerBaseLogging; /** - * s3KeyFormat is the format for the S3 file. * - * It can contain the following: + * directory - the directory to watch for files inside of + * fileGlob - only files matching this glob will be uploaded to S3. + * s3Bucket - the name of the bucket to upload to in S3 + * + * s3KeyFormat - the format for the actual key name for the object in S3 corresponding to each file uploaded. This can be dynamically + * formatted with the following variables: + * * %filename - adds the original file's filename * %fileext - adds the original file's file ext * %Y - adds year @@ -16,6 +23,16 @@ * %s - adds milliseconds * %index - adds the index of the file uploaded at this moment (to preserve uniqueness) * + * For example, if the s3KeyFormat was: %filename-%Y and the file name on local disk was "file1.txt" the S3 key would be : s3Bucket/file1.txt-2015 (assuming current year is 2015) + * + * finished - set this to true if you wish *this s3 upload metadata configuration* file to be deleted and no more files uploaded after the last matching file is uploaded to S3 successfully (think of it as safe delete.) + * onFinishGlob - a glob to match files which should be uploaded *only* after finished is set to true OR the pid is no longer active + * pid - the pid of the process to watch, such that when that pid is no longer running, finished is set to true (stop uploading files / watching directory once all files are successfully uploaded.) + * s3AccessKey - the access key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) + * s3SecretKey - the secret key to use to talk to s3 (optional in case you want to re-use the default Singularity configuration's key) + * + * finishedAfterMillisWithoutNewFile - after millis without a new file, set finished to true (see above for result.) - (-1 never expire) - absent - uses system default. + * */ public class S3UploadMetadata { @@ -24,9 +41,16 @@ public class S3UploadMetadata { private final String s3Bucket; private final String s3KeyFormat; private final boolean finished; + private final Optional onFinishGlob; + private final Optional pid; + private final Optional s3AccessKey; + private final Optional s3SecretKey; + private final Optional finishedAfterMillisWithoutNewFile; @JsonCreator - public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, @JsonProperty("finished") boolean finished) { + public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProperty("fileGlob") String fileGlob, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3KeyFormat") String s3KeyFormat, + @JsonProperty("finished") boolean finished, @JsonProperty("onFinishGlob") Optional onFinishGlob, @JsonProperty("pid") Optional pid, @JsonProperty("s3AccessKey") Optional s3AccessKey, + @JsonProperty("s3SecretKey") Optional s3SecretKey, @JsonProperty("finishedAfterMillisWithoutNewFile") Optional finishedAfterMillisWithoutNewFile) { Preconditions.checkNotNull(directory); Preconditions.checkNotNull(fileGlob); Preconditions.checkNotNull(s3Bucket); @@ -37,6 +61,11 @@ public S3UploadMetadata(@JsonProperty("directory") String directory, @JsonProper this.s3Bucket = s3Bucket; this.s3KeyFormat = s3KeyFormat; this.finished = finished; + this.pid = pid; + this.s3AccessKey = s3AccessKey; + this.s3SecretKey = s3SecretKey; + this.onFinishGlob = onFinishGlob; + this.finishedAfterMillisWithoutNewFile = finishedAfterMillisWithoutNewFile; } @Override @@ -51,28 +80,28 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) { - return true; + return true; } if (obj == null) { - return false; + return false; } if (getClass() != obj.getClass()) { - return false; + return false; } S3UploadMetadata other = (S3UploadMetadata) obj; if (directory == null) { if (other.directory != null) { return false; - } + } } else if (!directory.equals(other.directory)) { - return false; + return false; } if (fileGlob == null) { if (other.fileGlob != null) { return false; - } + } } else if (!fileGlob.equals(other.fileGlob)) { - return false; + return false; } return true; } @@ -97,9 +126,38 @@ public boolean isFinished() { return finished; } + public Optional getPid() { + return pid; + } + + public Optional getS3AccessKey() { + return s3AccessKey; + } + + public Optional getS3SecretKey() { + return s3SecretKey; + } + + public Optional getOnFinishGlob() { + return onFinishGlob; + } + + public Optional getFinishedAfterMillisWithoutNewFile() { + return finishedAfterMillisWithoutNewFile; + } + + private String obfuscateValue(Optional optional) { + if (!optional.isPresent()) { + return optional.toString(); + } + + return SingularityRunnerBaseLogging.obfuscateValue(optional.get()); + } + @Override public String toString() { - return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + "]"; + return "S3UploadMetadata [directory=" + directory + ", fileGlob=" + fileGlob + ", s3Bucket=" + s3Bucket + ", s3KeyFormat=" + s3KeyFormat + ", finished=" + finished + ", onFinishGlob=" + + onFinishGlob + ", pid=" + pid + ", s3AccessKey=" + obfuscateValue(s3AccessKey) + ", s3Secret=" + obfuscateValue(s3SecretKey) + ", finishedAfterMillisWithoutNewFile=" + finishedAfterMillisWithoutNewFile + "]"; } } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java index 4a2d063195..4bf77efe79 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SafeProcessManager.java @@ -29,9 +29,11 @@ public abstract class SafeProcessManager { private volatile Optional currentProcessStart; private final AtomicBoolean killed; + private final ProcessUtils processUtils; public SafeProcessManager(Logger log) { this.log = log; + this.processUtils = new ProcessUtils(log); this.currentProcessCmd = Optional.absent(); this.currentProcess = Optional.absent(); @@ -85,7 +87,7 @@ public Process startProcess(ProcessBuilder builder) { process = builder.start(); - currentProcessPid = Optional.of(ProcessUtils.getUnixPID(process)); + currentProcessPid = Optional.of(processUtils.getUnixPID(process)); currentProcess = Optional.of(process); currentProcessCmd = Optional.of(cmd); @@ -140,7 +142,7 @@ public void signalTermToProcessIfActive() { try { if (currentProcessPid.isPresent()) { - ProcessUtils.sendSignal(Signal.SIGTERM, log, currentProcessPid.get()); + processUtils.sendSignal(Signal.SIGTERM, currentProcessPid.get()); } } finally { this.processLock.unlock(); @@ -152,7 +154,7 @@ public void signalKillToProcessIfActive() { try { if (currentProcess.isPresent()) { - ProcessUtils.sendSignal(Signal.SIGKILL, log, currentProcessPid.get()); + processUtils.sendSignal(Signal.SIGKILL, currentProcessPid.get()); } } finally { this.processLock.unlock(); diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java index bd64242eb5..783f4e56ce 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/Signal.java @@ -1,7 +1,7 @@ package com.hubspot.singularity.runner.base.shared; public enum Signal { - SIGTERM(15), SIGKILL(9); + SIGTERM(15), SIGKILL(9), CHECK(0); private final int code; diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index 22ce4dfcd8..3c484ec056 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -1,5 +1,6 @@ package com.hubspot.singularity.s3uploader; +import java.io.Closeable; import java.io.IOException; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -11,23 +12,29 @@ import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; +import org.jets3t.service.ServiceException; +import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; +import org.jets3t.service.security.AWSCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.codahale.metrics.Timer.Context; +import com.google.common.base.Optional; +import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.SingularityS3FormatHelper; import com.hubspot.singularity.runner.base.shared.S3UploadMetadata; -public class SingularityS3Uploader { +public class SingularityS3Uploader implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(SingularityS3Uploader.class); private final S3UploadMetadata uploadMetadata; private final PathMatcher pathMatcher; + private final Optional finishedPathMatcher; private final String fileDirectory; private final S3Service s3Service; private final S3Bucket s3Bucket; @@ -35,12 +42,30 @@ public class SingularityS3Uploader { private final SingularityS3UploaderMetrics metrics; private final String logIdentifier; - public SingularityS3Uploader(S3Service s3Service, S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath) { - this.s3Service = s3Service; + public SingularityS3Uploader(AWSCredentials defaultCredentials, S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath) { + AWSCredentials credentials = defaultCredentials; + + if (uploadMetadata.getS3SecretKey().isPresent() && uploadMetadata.getS3AccessKey().isPresent()) { + credentials = new AWSCredentials(uploadMetadata.getS3AccessKey().get(), uploadMetadata.getS3SecretKey().get()); + } + + try { + this.s3Service = new RestS3Service(credentials); + } catch (S3ServiceException e) { + throw Throwables.propagate(e); + } + this.metrics = metrics; this.uploadMetadata = uploadMetadata; this.fileDirectory = uploadMetadata.getDirectory(); this.pathMatcher = fileSystem.getPathMatcher("glob:" + uploadMetadata.getFileGlob()); + + if (uploadMetadata.getOnFinishGlob().isPresent()) { + finishedPathMatcher = Optional.of(fileSystem.getPathMatcher("glob:" + uploadMetadata.getOnFinishGlob().get())); + } else { + finishedPathMatcher = Optional. absent(); + } + this.s3Bucket = new S3Bucket(uploadMetadata.getS3Bucket()); this.metadataPath = metadataPath; this.logIdentifier = String.format("[%s]", metadataPath.getFileName()); @@ -54,12 +79,21 @@ public S3UploadMetadata getUploadMetadata() { return uploadMetadata; } + @Override + public void close() throws IOException { + try { + s3Service.shutdown(); + } catch (ServiceException e) { + throw new IOException(e); + } + } + @Override public String toString() { return "SingularityS3Uploader [uploadMetadata=" + uploadMetadata + ", metadataPath=" + metadataPath + "]"; } - public int upload(Set synchronizedToUpload) throws IOException { + public int upload(Set synchronizedToUpload, boolean isFinished) throws IOException { final List toUpload = Lists.newArrayList(); int found = 0; @@ -72,7 +106,16 @@ public int upload(Set synchronizedToUpload) throws IOException { for (Path file : JavaUtils.iterable(directory)) { if (!pathMatcher.matches(file.getFileName())) { - LOG.trace("{} Skipping {} because it didn't match {}", logIdentifier, file, uploadMetadata.getFileGlob()); + if (!isFinished || !finishedPathMatcher.isPresent() || !finishedPathMatcher.get().matches(file.getFileName())) { + LOG.trace("{} Skipping {} because it doesn't match {}", logIdentifier, file, uploadMetadata.getFileGlob()); + continue; + } else { + LOG.trace("Not skipping file {} because it matched finish glob {}", file, uploadMetadata.getOnFinishGlob().get()); + } + } + + if (Files.size(file) == 0) { + LOG.trace("{} Skipping {} because its size is 0", logIdentifier, file); continue; } @@ -110,7 +153,7 @@ private void uploadBatch(List toUpload) { Files.delete(file); } catch (S3ServiceException se) { metrics.error(); - LOG.warn("{} Couldn't upload due to {} ({}) - {}", logIdentifier, se.getErrorCode(), se.getResponseCode(), se.getErrorMessage()); + LOG.warn("{} Couldn't upload {} due to {} ({}) - {}", logIdentifier, file, se.getErrorCode(), se.getResponseCode(), se.getErrorMessage(), se); } catch (Exception e) { metrics.error(); LOG.warn("{} Couldn't upload or delete {}", logIdentifier, file, e); diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java index 836d104a52..74e89afbed 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java @@ -22,8 +22,6 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.jets3t.service.S3Service; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.security.AWSCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,10 +33,12 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.io.Closeables; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.runner.base.shared.JsonObjectFileHelper; +import com.hubspot.singularity.runner.base.shared.ProcessUtils; import com.hubspot.singularity.runner.base.shared.S3UploadMetadata; import com.hubspot.singularity.runner.base.shared.SingularityDriver; import com.hubspot.singularity.runner.base.shared.WatchServiceHelper; @@ -56,10 +56,11 @@ public class SingularityS3UploaderDriver extends WatchServiceHelper implements S private final Lock runLock; private final ExecutorService executorService; private final FileSystem fileSystem; - private final S3Service s3Service; private final Set expiring; private final SingularityS3UploaderMetrics metrics; private final JsonObjectFileHelper jsonObjectFileHelper; + private final ProcessUtils processUtils; + private final AWSCredentials defaultCredentials; private ScheduledFuture future; @@ -68,13 +69,9 @@ public SingularityS3UploaderDriver(SingularityS3UploaderConfiguration configurat super(configuration.getPollForShutDownMillis(), configuration.getS3MetadataDirectory(), ImmutableList.of(StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY, StandardWatchEventKinds.ENTRY_DELETE)); this.metrics = metrics; + this.defaultCredentials = new AWSCredentials(s3Configuration.getS3AccessKey(), s3Configuration.getS3SecretKey()); this.fileSystem = FileSystems.getDefault(); - try { - this.s3Service = new RestS3Service(new AWSCredentials(s3Configuration.getS3AccessKey(), s3Configuration.getS3SecretKey())); - } catch (Throwable t) { - throw Throwables.propagate(t); - } this.jsonObjectFileHelper = jsonObjectFileHelper; this.configuration = configuration; @@ -87,6 +84,8 @@ public SingularityS3UploaderDriver(SingularityS3UploaderConfiguration configurat this.runLock = new ReentrantLock(); + this.processUtils = new ProcessUtils(LOG); + this.executorService = JavaUtils.newFixedTimingOutThreadPool(configuration.getExecutorMaxUploadThreads(), TimeUnit.SECONDS.toMillis(30), "SingularityS3Uploader-%d"); this.scheduler = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("SingularityS3Driver-%d").build()); } @@ -184,8 +183,13 @@ private int checkUploads() { final Set filesToUpload = Collections.newSetFromMap(new ConcurrentHashMap(metadataToUploader.size() * 2, 0.75f, metadataToUploader.size())); final Map> futures = Maps.newHashMapWithExpectedSize(metadataToUploader.size()); + final Map finishing = Maps.newHashMapWithExpectedSize(metadataToUploader.size()); for (final SingularityS3Uploader uploader : metadataToUploader.values()) { + final boolean isFinished = isFinished(uploader); + // do this here so we run at least once with isFinished = true + finishing.put(uploader, isFinished); + futures.put(uploader, executorService.submit(new Callable() { @Override @@ -193,7 +197,7 @@ public Integer call() { Integer returnValue = 0; try { - returnValue = uploader.upload(filesToUpload); + returnValue = uploader.upload(filesToUpload, isFinished); } catch (Throwable t) { metrics.error(); LOG.error("Error while processing uploader {}", uploader, t); @@ -209,21 +213,16 @@ public Integer call() { final long now = System.currentTimeMillis(); final Set expiredUploaders = Sets.newHashSetWithExpectedSize(metadataToUploader.size()); - // TODO cancel/timeouts? for (Entry> uploaderToFuture : futures.entrySet()) { final SingularityS3Uploader uploader = uploaderToFuture.getKey(); try { final int foundFiles = uploaderToFuture.getValue().get(); + final boolean isFinished = finishing.get(uploader); if (foundFiles == 0) { - final long durationSinceLastFile = now - uploaderLastHadFilesAt.get(uploader); - final boolean isFinished = isFinished(uploader); - - if ((durationSinceLastFile > configuration.getStopCheckingAfterMillisWithoutNewFile()) || isFinished) { - LOG.info("Expiring uploader {}", uploader); + if (shouldExpire(uploader, isFinished)) { + LOG.info("Expiring {}", uploader); expiredUploaders.add(uploader); - } else { - LOG.trace("Not expiring uploader {}, duration {} (max {}), isFinished: {})", uploader, durationSinceLastFile, configuration.getStopCheckingAfterMillisWithoutNewFile(), isFinished); } } else { LOG.trace("Updating uploader {} last expire time", uploader); @@ -245,6 +244,8 @@ public Integer call() { expiring.remove(expiredUploader); try { + Closeables.close(expiredUploader, true); + Files.delete(expiredUploader.getMetadataPath()); } catch (IOException e) { LOG.warn("Couldn't delete {}", expiredUploader.getMetadataPath(), e); @@ -254,8 +255,45 @@ public Integer call() { return totesUploads; } + private boolean shouldExpire(SingularityS3Uploader uploader, boolean isFinished) { + if (isFinished) { + return true; + } + + if (uploader.getUploadMetadata().getFinishedAfterMillisWithoutNewFile().isPresent()) { + if (uploader.getUploadMetadata().getFinishedAfterMillisWithoutNewFile().get() < 0) { + LOG.trace("{} never expires", uploader); + return false; + } + } + + final long durationSinceLastFile = System.currentTimeMillis() - uploaderLastHadFilesAt.get(uploader); + + final long expireAfterMillis = uploader.getUploadMetadata().getFinishedAfterMillisWithoutNewFile().or(configuration.getStopCheckingAfterMillisWithoutNewFile()); + + if (durationSinceLastFile > expireAfterMillis) { + return true; + } else { + LOG.trace("Not expiring uploader {}, duration {} (max {}), isFinished: {})", uploader, JavaUtils.durationFromMillis(durationSinceLastFile), JavaUtils.durationFromMillis(expireAfterMillis), isFinished); + } + + return false; + } + private boolean isFinished(SingularityS3Uploader uploader) { - return expiring.contains(uploader); + if (expiring.contains(uploader)) { + return true; + } + + if (uploader.getUploadMetadata().getPid().isPresent()) { + if (!processUtils.doesProcessExist(uploader.getUploadMetadata().getPid().get())) { + LOG.info("Pid {} not present - expiring uploader {}", uploader.getUploadMetadata().getPid().get(), uploader); + expiring.add(uploader); + return true; + } + } + + return false; } private boolean handleNewOrModifiedS3Metadata(Path filename) throws IOException { @@ -289,7 +327,7 @@ private boolean handleNewOrModifiedS3Metadata(Path filename) throws IOException try { metrics.getUploaderCounter().inc(); - SingularityS3Uploader uploader = new SingularityS3Uploader(s3Service, metadata, fileSystem, metrics, filename); + SingularityS3Uploader uploader = new SingularityS3Uploader(defaultCredentials, metadata, fileSystem, metrics, filename); if (metadata.isFinished()) { expiring.add(uploader); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java index 7df061e82c..f9870f402e 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityAbort.java @@ -18,6 +18,7 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.net.HostAndPort; import com.google.inject.Inject; import com.hubspot.mesos.JavaUtils; @@ -54,7 +55,7 @@ public SingularityAbort(SingularitySmtpSender smtpSender, ServerProvider serverP public void stateChanged(CuratorFramework client, ConnectionState newState) { if (newState == ConnectionState.LOST) { LOG.error("Aborting due to new connection state received from ZooKeeper: {}", newState); - abort(AbortReason.LOST_ZK_CONNECTION); + abort(AbortReason.LOST_ZK_CONNECTION, Optional.absent()); } } @@ -62,10 +63,10 @@ public enum AbortReason { LOST_ZK_CONNECTION, LOST_LEADERSHIP, UNRECOVERABLE_ERROR, TEST_ABORT, MESOS_ERROR; } - public void abort(AbortReason abortReason) { + public void abort(AbortReason abortReason, Optional throwable) { if (!aborting.getAndSet(true)) { try { - sendAbortNotification(abortReason); + sendAbortNotification(abortReason, throwable); flushLogs(); } finally { exit(); @@ -89,17 +90,17 @@ private void exit() { } } - private void sendAbortNotification(AbortReason abortReason) { + private void sendAbortNotification(AbortReason abortReason, Optional throwable) { final String message = String.format("Singularity on %s is aborting due to %s", hostAndPort.getHostText(), abortReason); LOG.error(message); - sendAbortMail(message); + sendAbortMail(message, throwable); - exceptionNotifier.notify(message); + exceptionNotifier.notify(message, ImmutableMap.of("abortReason", abortReason.name())); } - private void sendAbortMail(final String message) { + private void sendAbortMail(final String message, final Optional throwable) { if (!maybeSmtpConfiguration.isPresent()) { LOG.warn("Couldn't send abort mail because no SMTP configuration is present"); return; @@ -112,7 +113,9 @@ private void sendAbortMail(final String message) { return; } - smtpSender.queueMail(maybeSmtpConfiguration.get().getAdmins(), ImmutableList. of(), message, ""); + final String body = throwable.isPresent() ? throwable.get().toString() : "(no stack trace)"; + + smtpSender.queueMail(maybeSmtpConfiguration.get().getAdmins(), ImmutableList. of(), message, body); } private void flushLogs() { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java index 5ec0a539dd..a6c4697035 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityLeaderController.java @@ -4,6 +4,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; +import java.util.Collections; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; @@ -83,12 +84,12 @@ public void isLeader() { statePoller.wake(); } catch (Throwable t) { LOG.error("While starting driver", t); - exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + exceptionNotifier.notify(t, Collections.emptyMap()); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } if (driverManager.getCurrentStatus() != Protos.Status.DRIVER_RUNNING) { - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.absent()); } } } @@ -122,9 +123,9 @@ public void notLeader() { statePoller.wake(); } catch (Throwable t) { LOG.error("While stopping driver", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, Collections.emptyMap()); } finally { - abort.abort(AbortReason.LOST_LEADERSHIP); + abort.abort(AbortReason.LOST_LEADERSHIP, Optional.absent()); } } } @@ -196,7 +197,7 @@ public void run() { LOG.trace("Caught interrupted exception, running the loop"); } catch (Throwable t) { LOG.error("Caught exception while saving state", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, Collections.emptyMap()); } finally { lock.unlock(); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java index 88510abee0..3b7e984804 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java @@ -3,17 +3,13 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.inject.name.Names.named; -import com.hubspot.singularity.smtp.JadeTemplateLoader; -import com.hubspot.singularity.smtp.MailTemplateHelpers; -import com.hubspot.singularity.smtp.SingularityMailRecordCleaner; -import com.hubspot.singularity.smtp.SingularityMailer; -import com.hubspot.singularity.smtp.SingularitySmtpSender; import io.dropwizard.jetty.HttpConnectorFactory; import io.dropwizard.server.SimpleServerFactory; import java.io.IOException; import java.net.SocketException; import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; import javax.inject.Inject; import javax.inject.Provider; @@ -29,7 +25,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; -import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.net.HostAndPort; import com.google.inject.Binder; @@ -42,6 +37,7 @@ import com.google.inject.name.Names; import com.hubspot.mesos.JavaUtils; import com.hubspot.mesos.client.MesosClient; +import com.hubspot.singularity.config.CustomExecutorConfiguration; import com.hubspot.singularity.config.MesosConfiguration; import com.hubspot.singularity.config.S3Configuration; import com.hubspot.singularity.config.SMTPConfiguration; @@ -56,6 +52,11 @@ import com.hubspot.singularity.sentry.NotifyingExceptionMapper; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; import com.hubspot.singularity.sentry.SingularityExceptionNotifierManaged; +import com.hubspot.singularity.smtp.JadeTemplateLoader; +import com.hubspot.singularity.smtp.MailTemplateHelpers; +import com.hubspot.singularity.smtp.SingularityMailRecordCleaner; +import com.hubspot.singularity.smtp.SingularityMailer; +import com.hubspot.singularity.smtp.SingularitySmtpSender; import com.ning.http.client.AsyncHttpClient; import de.neuland.jade4j.parser.Parser; @@ -80,6 +81,18 @@ public class SingularityMainModule implements Module { public static final String SINGULARITY_URI_BASE = "_singularity_uri_base"; + public static final String HEALTHCHECK_THREADPOOL_NAME = "_healthcheck_threadpool"; + public static final Named HEALTHCHECK_THREADPOOL_NAMED = Names.named(HEALTHCHECK_THREADPOOL_NAME); + + public static final String NEW_TASK_THREADPOOL_NAME = "_new_task_threadpool"; + public static final Named NEW_TASK_THREADPOOL_NAMED = Names.named(NEW_TASK_THREADPOOL_NAME); + + private final SingularityConfiguration configuration; + + public SingularityMainModule(final SingularityConfiguration configuration) { + this.configuration = configuration; + } + @Override public void configure(Binder binder) { binder.bind(HostAndPort.class).annotatedWith(named(HTTP_HOST_AND_PORT)).toProvider(SingularityHostAndPortProvider.class).in(Scopes.SINGLETON); @@ -115,12 +128,21 @@ public void configure(Binder binder) { binder.bind(ObjectMapper.class).toProvider(DropwizardObjectMapperProvider.class).in(Scopes.SINGLETON); binder.bind(AsyncHttpClient.class).to(SingularityHttpClient.class).in(Scopes.SINGLETON); - binder.bind(ServerProvider.class).in(Scopes.SINGLETON); binder.bind(SingularityDropwizardHealthcheck.class).in(Scopes.SINGLETON); binder.bindConstant().annotatedWith(Names.named(SERVER_ID_PROPERTY)).to(UUID.randomUUID().toString()); + binder.bind(SingularityManagedScheduledExecutorServiceFactory.class).in(Scopes.SINGLETON); + + binder.bind(ScheduledExecutorService.class).annotatedWith(HEALTHCHECK_THREADPOOL_NAMED).toProvider(new SingularityManagedScheduledExecutorServiceProvider(configuration.getHealthcheckStartThreads(), + configuration.getThreadpoolShutdownDelayInSeconds(), + "healthcheck")).in(Scopes.SINGLETON); + + binder.bind(ScheduledExecutorService.class).annotatedWith(NEW_TASK_THREADPOOL_NAMED).toProvider(new SingularityManagedScheduledExecutorServiceProvider(configuration.getCheckNewTasksScheduledThreads(), + configuration.getThreadpoolShutdownDelayInSeconds(), + "check-new-task")).in(Scopes.SINGLETON); + try { binder.bindConstant().annotatedWith(Names.named(HOST_ADDRESS_PROPERTY)).to(JavaUtils.getHostAddress()); } catch (SocketException e) { @@ -136,7 +158,7 @@ public static class SingularityHostAndPortProvider implements Provider smtpConfiguration(final SingularityConfiguration config) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceFactory.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceFactory.java new file mode 100644 index 0000000000..317355d812 --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityManagedScheduledExecutorServiceFactory.java @@ -0,0 +1,63 @@ +package com.hubspot.singularity; + +import static com.google.common.base.Preconditions.checkState; +import io.dropwizard.lifecycle.Managed; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.hubspot.singularity.config.SingularityConfiguration; + +@Singleton +public class SingularityManagedScheduledExecutorServiceFactory implements Managed { + + private final AtomicBoolean stopped = new AtomicBoolean(); + private final List executorPools = new ArrayList<>(); + + private final long timeoutInMillis; + + @Inject + public SingularityManagedScheduledExecutorServiceFactory(final SingularityConfiguration configuration) { + this.timeoutInMillis = TimeUnit.SECONDS.toMillis(configuration.getThreadpoolShutdownDelayInSeconds()); + } + + public synchronized ScheduledExecutorService get(String name) { + checkState(!stopped.get(), "already stopped"); + ScheduledExecutorService service = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(name + "-%d").setDaemon(true).build()); + executorPools.add(service); + return service; + } + + @Override + public void start() throws Exception { + // Ignored + } + + @Override + public void stop() throws Exception { + if (!stopped.getAndSet(true)) { + for (ScheduledExecutorService service : executorPools) { + service.shutdown(); + } + + long timeoutLeftInMillis = timeoutInMillis; + + for (ScheduledExecutorService service : executorPools) { + final long start = System.currentTimeMillis(); + + if (!service.awaitTermination(timeoutLeftInMillis, TimeUnit.MILLISECONDS)) { + return; + } + + timeoutLeftInMillis -= (System.currentTimeMillis() - start); + } + } + } +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java index 4d14e2ce09..fe404dec92 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityServiceModule.java @@ -18,7 +18,7 @@ public class SingularityServiceModule extends ConfigurationAwareModule getCommonHostnameSuffixToOmit() { - return Optional.fromNullable(commonHostnameSuffixToOmit); + return Optional.fromNullable(Strings.emptyToNull(commonHostnameSuffixToOmit)); } public long getConsiderTaskHealthyAfterRunningForSeconds() { @@ -222,6 +238,10 @@ public long getCooldownMinScheduleSeconds() { return cooldownMinScheduleSeconds; } + public int getCoreThreadpoolSize() { + return coreThreadpoolSize; + } + public Optional getDatabaseConfiguration() { return Optional.fromNullable(databaseConfiguration); } @@ -266,8 +286,8 @@ public long getHealthcheckTimeoutSeconds() { return healthcheckTimeoutSeconds; } - public String getHostname() { - return hostname; + public Optional getHostname() { + return Optional.fromNullable(Strings.emptyToNull(hostname)); } public long getKillAfterTasksDoNotRunDefaultSeconds() { @@ -482,6 +502,10 @@ public void setCooldownMinScheduleSeconds(long cooldownMinScheduleSeconds) { this.cooldownMinScheduleSeconds = cooldownMinScheduleSeconds; } + public void setCoreThreadpoolSize(int coreThreadpoolSize) { + this.coreThreadpoolSize = coreThreadpoolSize; + } + public void setDatabaseConfiguration(DataSourceFactory databaseConfiguration) { this.databaseConfiguration = databaseConfiguration; } @@ -622,10 +646,6 @@ public void setStartNewReconcileEverySeconds(long startNewReconcileEverySeconds) this.startNewReconcileEverySeconds = startNewReconcileEverySeconds; } - public void setThreadpoolShutdownDelayInSeconds(long threadpoolShutdownDelayInSeconds) { - this.threadpoolShutdownDelayInSeconds = threadpoolShutdownDelayInSeconds; - } - public void setUiConfiguration(UIConfiguration uiConfiguration) { this.uiConfiguration = uiConfiguration; } @@ -649,4 +669,28 @@ public void setZookeeperAsyncTimeout(long zookeeperAsyncTimeout) { public void setZooKeeperConfiguration(ZooKeeperConfiguration zooKeeperConfiguration) { this.zooKeeperConfiguration = zooKeeperConfiguration; } + + public CustomExecutorConfiguration getCustomExecutorConfiguration() { + return customExecutorConfiguration; + } + + public void setCustomExecutorConfiguration(CustomExecutorConfiguration customExecutorConfiguration) { + this.customExecutorConfiguration = customExecutorConfiguration; + } + + public boolean isCreateDeployIds() { + return createDeployIds; + } + + public void setCreateDeployIds(boolean createDeployIds) { + this.createDeployIds = createDeployIds; + } + + public int getDeployIdLength() { + return deployIdLength; + } + + public void setDeployIdLength(int deployIdLength) { + this.deployIdLength = deployIdLength; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java b/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java index 9bb315babb..c6d07856f7 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java @@ -4,6 +4,7 @@ import static com.hubspot.singularity.WebExceptions.checkBadRequest; import java.util.List; +import java.util.UUID; import javax.inject.Singleton; @@ -13,6 +14,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.collect.Lists; +import com.google.common.hash.Hashing; import com.google.inject.Inject; import com.hubspot.mesos.Resources; import com.hubspot.mesos.SingularityDockerInfo; @@ -20,6 +22,7 @@ import com.hubspot.mesos.SingularityPortMappingType; import com.hubspot.singularity.ScheduleType; import com.hubspot.singularity.SingularityDeploy; +import com.hubspot.singularity.SingularityDeployBuilder; import com.hubspot.singularity.SingularityRequest; import com.hubspot.singularity.config.SingularityConfiguration; import com.hubspot.singularity.data.history.DeployHistoryHelper; @@ -39,6 +42,8 @@ public class SingularityValidator { private final int defaultMemoryMb; private final int maxMemoryMbPerInstance; private final boolean allowRequestsWithoutOwners; + private final boolean createDeployIds; + private final int deployIdLength; private final DeployHistoryHelper deployHistoryHelper; private final Resources defaultResources; @@ -47,6 +52,8 @@ public SingularityValidator(SingularityConfiguration configuration, DeployHistor this.maxDeployIdSize = configuration.getMaxDeployIdSize(); this.maxRequestIdSize = configuration.getMaxRequestIdSize(); this.allowRequestsWithoutOwners = configuration.isAllowRequestsWithoutOwners(); + this.createDeployIds = configuration.isCreateDeployIds(); + this.deployIdLength = configuration.getDeployIdLength(); this.deployHistoryHelper = deployHistoryHelper; this.defaultCpus = configuration.getMesosConfiguration().getDefaultCpus(); @@ -153,14 +160,20 @@ public SingularityRequest checkSingularityRequest(SingularityRequest request, Op return request.toBuilder().setQuartzSchedule(Optional.fromNullable(quartzSchedule)).build(); } - public void checkDeploy(SingularityRequest request, SingularityDeploy deploy) { - + public SingularityDeploy checkDeploy(SingularityRequest request, SingularityDeploy deploy) { checkNotNull(request, "request is null"); checkNotNull(deploy, "deploy is null"); String deployId = deploy.getId(); - checkBadRequest(deployId != null, "Id must not be null"); + if (deployId == null) { + checkBadRequest(createDeployIds, "Id must not be null"); + SingularityDeployBuilder builder = deploy.toBuilder(); + builder.setId(createUniqueDeployId()); + deploy = builder.build(); + deployId = deploy.getId(); + } + checkBadRequest(!deployId.contains("/") && !deployId.contains("-"), "Id must not be null and can not contain / or - characters"); checkBadRequest(deployId.length() < maxDeployIdSize, "Deploy id must be less than %s characters, it is %s (%s)", maxDeployIdSize, deployId.length(), deployId); checkBadRequest(deploy.getRequestId() != null && deploy.getRequestId().equals(request.getId()), "Deploy id must match request id"); @@ -183,6 +196,14 @@ public void checkDeploy(SingularityRequest request, SingularityDeploy deploy) { } checkBadRequest(deployHistoryHelper.isDeployIdAvailable(request.getId(), deployId), "Can not deploy a deploy that has already been deployed"); + + return deploy; + } + + private String createUniqueDeployId() { + UUID id = UUID.randomUUID(); + String result = Hashing.sha256().newHasher().putLong(id.getLeastSignificantBits()).putLong(id.getMostSignificantBits()).hash().toString(); + return result.substring(0, deployIdLength); } private void checkDocker(SingularityDeploy deploy) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java index 44a3e5af49..130e161361 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java @@ -1,16 +1,5 @@ package com.hubspot.singularity.data; -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.utils.ZKPaths; -import org.apache.mesos.Protos.TaskStatus; - import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; @@ -44,10 +33,26 @@ import com.hubspot.singularity.data.transcoders.StringTranscoder; import com.hubspot.singularity.data.transcoders.Transcoder; import com.hubspot.singularity.event.SingularityEventListener; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; +import org.apache.curator.utils.ZKPaths; +import org.apache.mesos.Protos.TaskStatus; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; @Singleton public class TaskManager extends CuratorAsyncManager { + private static final Logger LOG = LoggerFactory.getLogger(CuratorAsyncManager.class); + private static final String TASKS_ROOT = "/tasks"; private static final String ACTIVE_PATH_ROOT = TASKS_ROOT + "/active"; @@ -107,6 +112,12 @@ public TaskManager(SingularityConfiguration configuration, CuratorFramework cura this.serverId = serverId; } + // since we can't use creatingParentsIfNeeded in transactions + public void createRequiredParents() { + create(HISTORY_PATH_ROOT); + create(ACTIVE_PATH_ROOT); + } + private String getLastHealthcheckPath(SingularityTaskId taskId) { return ZKPaths.makePath(getHistoryPath(taskId), LAST_HEALTHCHECK_KEY); } @@ -464,8 +475,13 @@ private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getOffer().getSlaveId().getValue()))); - create(getTaskPath(task.getTaskId()), task, taskTranscoder); - create(getActivePath(task.getTaskId().getId())); + try { + CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(getTaskPath(task.getTaskId()), taskTranscoder.toBytes(task)).and(); + + transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); + } catch (KeeperException.NodeExistsException nee) { + LOG.error("Task or active path already existed for {}", task.getTaskId()); + } } public List getLBCleanupTasks() { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java index 5621cf90c8..0942446481 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityHistoryPersister.java @@ -48,7 +48,7 @@ protected boolean moveToHistoryOrCheckForPurge(T object) { if (moveToHistoryOrCheckForPurgeAndShouldDelete(object)) { SingularityDeleteResult deleteResult = purgeFromZk(object); - LOG.debug("%s %s (deleted: %s) in %s", persistsHistoryInsteadOfPurging() ? "Persisted" : "Purged", object, deleteResult, JavaUtils.duration(start)); + LOG.debug("{} {} (deleted: {}) in {}", persistsHistoryInsteadOfPurging() ? "Persisted" : "Purged", object, deleteResult, JavaUtils.duration(start)); return true; } @@ -63,7 +63,7 @@ private boolean moveToHistoryOrCheckForPurgeAndShouldDelete(T object) { final long age = System.currentTimeMillis() - object.getCreateTimestampForCalculatingHistoryAge(); if (age > getMaxAgeInMillisOfItem()) { - LOG.trace("Deleting %s because it is %s old (max : %s)", object, JavaUtils.durationFromMillis(age), JavaUtils.durationFromMillis(getMaxAgeInMillisOfItem())); + LOG.trace("Deleting {} because it is {} old (max : {})", object, JavaUtils.durationFromMillis(age), JavaUtils.durationFromMillis(getMaxAgeInMillisOfItem())); return true; } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java index 7dab662fd4..5baad8e761 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityTaskHistoryPersister.java @@ -40,7 +40,7 @@ public SingularityTaskHistoryPersister(SingularityConfiguration configuration, T @Override public void runActionOnPoll() { - LOG.info("Checking inactive task ids for task history persistance"); + LOG.info("Checking inactive task ids for task history persistence"); final long start = System.currentTimeMillis(); @@ -86,6 +86,7 @@ protected boolean moveToHistory(SingularityTaskId object) { final Optional taskHistory = taskManager.getTaskHistory(object); if (taskHistory.isPresent()) { + LOG.debug("Moving {} to history", object); try { historyManager.saveTaskHistory(taskHistory.get()); } catch (Throwable t) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java index 3744acaef6..fa07b2b851 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityZkMigrationsModule.java @@ -16,5 +16,6 @@ public void configure(Binder binder) { dataMigrations.addBinding().to(SingularityPendingTaskIdMigration.class); dataMigrations.addBinding().to(SlaveAndRackMigration.class); dataMigrations.addBinding().to(SingularityCmdLineArgsMigration.class); + dataMigrations.addBinding().to(TaskManagerRequiredParentsForTransactionsMigration.class); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/TaskManagerRequiredParentsForTransactionsMigration.java b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/TaskManagerRequiredParentsForTransactionsMigration.java new file mode 100644 index 0000000000..cb111754cf --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/TaskManagerRequiredParentsForTransactionsMigration.java @@ -0,0 +1,35 @@ +package com.hubspot.singularity.data.zkmigrations; + +import com.google.common.base.Optional; +import com.google.common.collect.Lists; +import com.google.inject.Inject; +import com.google.inject.name.Named; +import com.hubspot.singularity.SingularityMainModule; +import com.hubspot.singularity.SingularityTask; +import com.hubspot.singularity.SingularityTaskHistoryUpdate; +import com.hubspot.singularity.SingularityTaskId; +import com.hubspot.singularity.SingularityTaskStatusHolder; +import com.hubspot.singularity.data.TaskManager; +import org.apache.mesos.Protos.TaskID; +import org.apache.mesos.Protos.TaskStatus; + +import javax.inject.Singleton; +import java.util.List; + +@Singleton +public class TaskManagerRequiredParentsForTransactionsMigration extends ZkDataMigration { + + private final TaskManager taskManager; + + @Inject + public TaskManagerRequiredParentsForTransactionsMigration(TaskManager taskManager) { + super(5); + this.taskManager = taskManager; + } + + @Override + public void applyMigration() { + taskManager.createRequiredParents(); + } + +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java index 50fdcc5a39..1cbf141a9d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityDriver.java @@ -7,6 +7,7 @@ import org.apache.mesos.MesosSchedulerDriver; import org.apache.mesos.Protos; import org.apache.mesos.Protos.FrameworkID; +import org.apache.mesos.Protos.FrameworkInfo; import org.apache.mesos.Protos.MasterInfo; import org.apache.mesos.Protos.TaskID; import org.apache.mesos.Scheduler; @@ -17,9 +18,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.inject.Inject; +import com.google.inject.name.Named; import com.groupon.mesos.JesosSchedulerDriver; +import com.hubspot.singularity.SingularityMainModule; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.config.MesosConfiguration; +import com.hubspot.singularity.config.SingularityConfiguration; +import com.hubspot.singularity.config.UIConfiguration; +import com.hubspot.singularity.resources.UiResource; @Singleton public class SingularityDriver { @@ -31,14 +37,29 @@ public class SingularityDriver { private final SchedulerDriver driver; @Inject - SingularityDriver(final SingularityMesosSchedulerDelegator scheduler, final MesosConfiguration configuration) throws IOException { - this.frameworkInfo = Protos.FrameworkInfo.newBuilder() + SingularityDriver(final SingularityMesosSchedulerDelegator scheduler, final SingularityConfiguration singularityConfiguration, final MesosConfiguration configuration, + @Named(SingularityMainModule.SINGULARITY_URI_BASE) final String singularityUriBase) throws IOException { + final FrameworkInfo.Builder frameworkInfoBuilder = Protos.FrameworkInfo.newBuilder() .setCheckpoint(configuration.getCheckpoint()) .setFailoverTimeout(configuration.getFrameworkFailoverTimeout()) .setName(configuration.getFrameworkName()) .setId(FrameworkID.newBuilder().setValue(configuration.getFrameworkId())) - .setUser("") // let mesos assign - .build(); + .setUser(""); // let mesos assign + + if (singularityConfiguration.getHostname().isPresent()) { + frameworkInfoBuilder.setHostname(singularityConfiguration.getHostname().get()); + } + + // only set the web UI URL if it's fully qualified + if (singularityUriBase.startsWith("http://") || singularityUriBase.startsWith("https://")) { + if (singularityConfiguration.getUiConfiguration().getRootUrlMode() == UIConfiguration.RootUrlMode.INDEX_CATCHALL) { + frameworkInfoBuilder.setWebuiUrl(singularityUriBase); + } else { + frameworkInfoBuilder.setWebuiUrl(singularityUriBase + UiResource.UI_RESOURCE_LOCATION); + } + } + + this.frameworkInfo = frameworkInfoBuilder.build(); this.scheduler = scheduler; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java index cde37d7129..3a38527b11 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosScheduler.java @@ -34,6 +34,7 @@ import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskRequest; import com.hubspot.singularity.SingularityTaskStatusHolder; +import com.hubspot.singularity.config.CustomExecutorConfiguration; import com.hubspot.singularity.config.MesosConfiguration; import com.hubspot.singularity.data.DeployManager; import com.hubspot.singularity.data.TaskManager; @@ -51,6 +52,7 @@ public class SingularityMesosScheduler implements Scheduler { private static final Logger LOG = LoggerFactory.getLogger(SingularityMesosScheduler.class); private final Resources defaultResources; + private final Resources defaultCustomExecutorResources; private final TaskManager taskManager; private final DeployManager deployManager; private final SingularityScheduler scheduler; @@ -71,8 +73,9 @@ public class SingularityMesosScheduler implements Scheduler { SingularityMesosScheduler(MesosConfiguration mesosConfiguration, TaskManager taskManager, SingularityScheduler scheduler, SingularitySlaveAndRackManager slaveAndRackManager, SingularitySchedulerPriority schedulerPriority, SingularityNewTaskChecker newTaskChecker, SingularityMesosTaskBuilder mesosTaskBuilder, SingularityLogSupport logSupport, Provider stateCacheProvider, SingularityHealthchecker healthchecker, DeployManager deployManager, - @Named(SingularityMainModule.SERVER_ID_PROPERTY) String serverId, SchedulerDriverSupplier schedulerDriverSupplier, final IdTranscoder taskIdTranscoder) { + @Named(SingularityMainModule.SERVER_ID_PROPERTY) String serverId, SchedulerDriverSupplier schedulerDriverSupplier, final IdTranscoder taskIdTranscoder, CustomExecutorConfiguration customExecutorConfiguration) { this.defaultResources = new Resources(mesosConfiguration.getDefaultCpus(), mesosConfiguration.getDefaultMemory(), 0); + this.defaultCustomExecutorResources = new Resources(customExecutorConfiguration.getNumCpus(), customExecutorConfiguration.getMemoryMb(), 0); this.taskManager = taskManager; this.deployManager = deployManager; this.schedulerPriority = schedulerPriority; @@ -191,19 +194,20 @@ public void resourceOffers(SchedulerDriver driver, List offers) { private Optional match(Collection taskRequests, SingularitySchedulerStateCache stateCache, SingularityOfferHolder offerHolder) { for (SingularityTaskRequest taskRequest : taskRequests) { - Resources taskResources = defaultResources; + final Resources taskResources = taskRequest.getDeploy().getResources().or(defaultResources); - if (taskRequest.getDeploy().getResources().isPresent()) { - taskResources = taskRequest.getDeploy().getResources().get(); - } + // only factor in executor resources if we're running a custom executor + final Resources executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources) : Resources.EMPTY_RESOURCES; + + final Resources totalResources = Resources.add(taskResources, executorResources); - LOG.trace("Attempting to match task {} resources {} with remaining offer resources {}", taskRequest.getPendingTask().getPendingTaskId(), taskResources, offerHolder.getCurrentResources()); + LOG.trace("Attempting to match task {} resources {} ({} for task + {} for executor) with remaining offer resources {}", taskRequest.getPendingTask().getPendingTaskId(), totalResources, taskResources, executorResources, offerHolder.getCurrentResources()); - final boolean matchesResources = MesosUtils.doesOfferMatchResources(taskResources, offerHolder.getCurrentResources()); + final boolean matchesResources = MesosUtils.doesOfferMatchResources(totalResources, offerHolder.getCurrentResources()); final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder.getOffer(), taskRequest, stateCache); if (matchesResources && slaveMatchState.isMatchAllowed()) { - final SingularityTask task = mesosTaskBuilder.buildTask(offerHolder.getOffer(), offerHolder.getCurrentResources(), taskRequest, taskResources); + final SingularityTask task = mesosTaskBuilder.buildTask(offerHolder.getOffer(), offerHolder.getCurrentResources(), taskRequest, taskResources, executorResources); LOG.trace("Accepted and built task {}", task); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java index c9e76e8e52..e9a3ded8c0 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerDelegator.java @@ -1,5 +1,6 @@ package com.hubspot.singularity.mesos; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -92,9 +93,9 @@ public void notifyStopping() { private void handleUncaughtSchedulerException(Throwable t) { LOG.error("Scheduler threw an uncaught exception - exiting", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, Collections.emptyMap()); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } private void startup(SchedulerDriver driver, MasterInfo masterInfo) throws Exception { @@ -308,7 +309,7 @@ public void error(SchedulerDriver driver, String message) { LOG.error("Aborting due to error: {}", message); - abort.abort(AbortReason.MESOS_ERROR); + abort.abort(AbortReason.MESOS_ERROR, Optional.absent()); } catch (Throwable t) { handleUncaughtSchedulerException(t); } finally { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java index 30cb756d5a..2dbe0672bb 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java @@ -37,9 +37,11 @@ import com.hubspot.mesos.SingularityDockerInfo; import com.hubspot.mesos.SingularityDockerPortMapping; import com.hubspot.mesos.SingularityVolume; +import com.hubspot.singularity.SingularityDeploy; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskRequest; +import com.hubspot.singularity.config.SingularityConfiguration; import com.hubspot.singularity.data.ExecutorIdGenerator; @Singleton @@ -50,15 +52,17 @@ class SingularityMesosTaskBuilder { private final ObjectMapper objectMapper; private final SingularitySlaveAndRackManager slaveAndRackManager; private final ExecutorIdGenerator idGenerator; + private final SingularityConfiguration configuration; @Inject - SingularityMesosTaskBuilder(ObjectMapper objectMapper, SingularitySlaveAndRackManager slaveAndRackManager, ExecutorIdGenerator idGenerator) { + SingularityMesosTaskBuilder(ObjectMapper objectMapper, SingularitySlaveAndRackManager slaveAndRackManager, ExecutorIdGenerator idGenerator, SingularityConfiguration configuration) { this.objectMapper = objectMapper; this.slaveAndRackManager = slaveAndRackManager; this.idGenerator = idGenerator; + this.configuration = configuration; } - public SingularityTask buildTask(Protos.Offer offer, List availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources) { + public SingularityTask buildTask(Protos.Offer offer, List availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources, Resources desiredExecutorResources) { final String rackId = slaveAndRackManager.getRackId(offer); final String host = slaveAndRackManager.getSlaveHost(offer); @@ -81,7 +85,7 @@ public SingularityTask buildTask(Protos.Offer offer, List availableRes } if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) { - prepareCustomExecutor(bldr, taskId, taskRequest, ports); + prepareCustomExecutor(bldr, taskId, taskRequest, ports, desiredExecutorResources); } else { prepareCommand(bldr, taskId, taskRequest, ports); } @@ -209,16 +213,31 @@ private void prepareContainerInfo(final SingularityTaskId taskId, final TaskInfo bldr.setContainer(containerBuilder); } - private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final Optional ports) { + private List buildMesosResources(final Resources resources) { + ImmutableList.Builder builder = ImmutableList.builder(); + + if (resources.getCpus() > 0) { + builder.add(MesosUtils.getCpuResource(resources.getCpus())); + } + + if (resources.getMemoryMb() > 0) { + builder.add(MesosUtils.getMemoryResource(resources.getMemoryMb())); + } + + return builder.build(); + } + + private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final Optional ports, final Resources desiredExecutorResources) { CommandInfo.Builder commandBuilder = CommandInfo.newBuilder().setValue(task.getDeploy().getCustomExecutorCmd().get()); prepareEnvironment(task, taskId, commandBuilder, ports); - bldr.setExecutor( - ExecutorInfo.newBuilder() - .setCommand(commandBuilder.build()) - .setExecutorId(ExecutorID.newBuilder().setValue(task.getDeploy().getCustomExecutorId().or(idGenerator.getNextExecutorId()))) - .setSource(task.getDeploy().getCustomExecutorSource().or(task.getPendingTask().getPendingTaskId().getId())) + bldr.setExecutor(ExecutorInfo.newBuilder() + .setCommand(commandBuilder.build()) + .setExecutorId(ExecutorID.newBuilder().setValue(task.getDeploy().getCustomExecutorId().or(idGenerator.getNextExecutorId()))) + .setSource(task.getDeploy().getCustomExecutorSource().or(task.getPendingTask().getPendingTaskId().getId())) + .addAllResources(buildMesosResources(desiredExecutorResources)) + .build() ); if (task.getDeploy().getExecutorData().isPresent()) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java index 11ceca8240..42a9d3342b 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferHolder.java @@ -32,7 +32,14 @@ public SingularityOfferHolder(Protos.Offer offer, int taskSizeHint) { public void addMatchedTask(SingularityTask task) { acceptedTasks.add(task); + + // subtract task resources from offer currentResources = MesosUtils.subtractResources(currentResources, task.getMesosTask().getResourcesList()); + + // subtract executor resources from offer, if any are defined + if (task.getMesosTask().hasExecutor() && task.getMesosTask().getExecutor().getResourcesCount() > 0) { + currentResources = MesosUtils.subtractResources(currentResources, task.getMesosTask().getExecutor().getResourcesList()); + } } public void launchTasks(SchedulerDriver driver) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java index da876b331d..19d14e9d9c 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.inject.Inject; import com.hubspot.mesos.json.MesosMasterSlaveObject; @@ -321,7 +322,7 @@ public void checkStateAfterFinishedTask(SingularityTaskId taskId, String slaveId if (!slave.isPresent()) { final String message = String.format("Couldn't find slave with id %s for task %s", slaveId, taskId); LOG.warn(message); - exceptionNotifier.notify(message); + exceptionNotifier.notify(message, ImmutableMap.of("slaveId", slaveId, "taskId", taskId.toString())); return; } @@ -336,7 +337,7 @@ public void checkStateAfterFinishedTask(SingularityTaskId taskId, String slaveId if (!rack.isPresent()) { final String message = String.format("Couldn't find rack with id %s for task %s", taskId.getRackId(), taskId); LOG.warn(message); - exceptionNotifier.notify(message); + exceptionNotifier.notify(message, ImmutableMap.of("rackId", taskId.getRackId(), "taskId", taskId.toString())); return; } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java index 9c93db3737..cdc5fc4144 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/DeployResource.java @@ -90,11 +90,11 @@ public SingularityRequestParent deploy(@ApiParam(required=true) SingularityDeplo SingularityRequestWithState requestWithState = fetchRequestWithState(requestId); SingularityRequest request = requestWithState.getRequest(); - if (!deployRequest.getUnpauseOnSuccessfulDeploy().isPresent() || !deployRequest.getUnpauseOnSuccessfulDeploy().get().booleanValue()) { + if (!deployRequest.isUnpauseOnSuccessfulDeploy()) { checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to deploy (it must be manually unpaused first)", requestWithState.getRequest().getId()); } - validator.checkDeploy(request, deploy); + deploy = validator.checkDeploy(request, deploy); final long now = System.currentTimeMillis(); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java index ca5d4b9a04..11a8686ba5 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TestResource.java @@ -6,6 +6,7 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import com.google.common.base.Optional; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import org.apache.mesos.Protos.TaskID; @@ -83,7 +84,7 @@ public void stop() throws Exception { public void abort() { checkForbidden(configuration.isAllowTestResourceCalls(), "Test resource calls are disabled (set isAllowTestResourceCalls to true in configuration)"); - abort.abort(AbortReason.TEST_ABORT); + abort.abort(AbortReason.TEST_ABORT, Optional.absent()); } @POST diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java index b3ee39e797..3d74df4754 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/UiResource.java @@ -20,7 +20,7 @@ @Path(UiResource.UI_RESOURCE_LOCATION + "{uiPath:.*}") public class UiResource { - static final String UI_RESOURCE_LOCATION = "/ui/"; + public static final String UI_RESOURCE_LOCATION = "/ui/"; private final SingularityConfiguration configuration; private final String singularityUriBase; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java index 44127d404f..9883651c54 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java @@ -384,6 +384,8 @@ private boolean shouldRemoveLbState(SingularityTaskId taskId, SingularityLoadBal case WAITING: case SUCCESS: return true; + case INVALID_REQUEST_NOOP: + return false; // don't need to remove because Baragon doesnt know about it default: LOG.trace("Task {} had abnormal LB state {}", taskId, loadBalancerUpdate); return false; @@ -417,6 +419,7 @@ private boolean shouldEnqueueLbRequest(Optional m case CANCELING: case SUCCESS: case WAITING: + case INVALID_REQUEST_NOOP: } return false; @@ -459,12 +462,13 @@ private CheckLBState checkLbState(SingularityTaskId taskId) { switch (lbRemoveUpdate.getLoadBalancerState()) { case SUCCESS: + case INVALID_REQUEST_NOOP: return CheckLBState.DONE; case FAILED: case CANCELED: LOG.error("LB removal request {} ({}) got unexpected response {}", lbAddUpdate.get(), loadBalancerRequestId, lbRemoveUpdate.getLoadBalancerState()); exceptionNotifier.notify(String.format("LB removal failed for %s", lbAddUpdate.get().getLoadBalancerRequestId().toString()), - ImmutableMap. of("state", lbRemoveUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbAddUpdate.get().toString())); + ImmutableMap.of("state", lbRemoveUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbAddUpdate.get().toString())); return CheckLBState.RETRY; case UNKNOWN: case CANCELING: diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java index 9271c21e0c..a081d8d201 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployChecker.java @@ -307,6 +307,7 @@ private DeployState interpretLoadBalancerState(SingularityLoadBalancerUpdate lbU case SUCCESS: return DeployState.SUCCEEDED; case FAILED: + case INVALID_REQUEST_NOOP: return DeployState.FAILED; case CANCELING: return DeployState.CANCELING; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java index 6067e846c3..d6ca39ad52 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthcheckAsyncHandler.java @@ -4,6 +4,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.hubspot.singularity.SingularityAbort; import com.hubspot.singularity.SingularityAbort.AbortReason; import com.hubspot.singularity.SingularityTask; @@ -60,12 +61,12 @@ public void onThrowable(Throwable t) { } public void saveResult(Optional statusCode, Optional responseBody, Optional errorMessage) { - SingularityTaskHealthcheckResult result = new SingularityTaskHealthcheckResult(statusCode, Optional.of(System.currentTimeMillis() - startTime), startTime, responseBody, - errorMessage, task.getTaskId()); + try { + SingularityTaskHealthcheckResult result = new SingularityTaskHealthcheckResult(statusCode, Optional.of(System.currentTimeMillis() - startTime), startTime, responseBody, + errorMessage, task.getTaskId()); - LOG.trace("Saving healthcheck result {}", result); + LOG.trace("Saving healthcheck result {}", result); - try { taskManager.saveHealthcheckResult(result); if (result.isFailed()) { @@ -79,22 +80,12 @@ public void saveResult(Optional statusCode, Optional responseBo newTaskChecker.runNewTaskCheckImmediately(task); } } catch (Throwable t) { - LOG.error("Caught throwable while saving health check result {}, will re-enqueue", result, t); - exceptionNotifier.notify(t); + LOG.error("Caught throwable while saving health check result for {}, will re-enqueue", task.getTaskId(), t); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); - reEnqueueOrAbort(task); + healthchecker.reEnqueueOrAbort(task); } } - private void reEnqueueOrAbort(SingularityTask task) { - try { - healthchecker.enqueueHealthcheck(task); - } catch (Throwable t) { - LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); - exceptionNotifier.notify(t); - - abort.abort(AbortReason.UNRECOVERABLE_ERROR); - } - } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java index 6008a5056d..056d6e5d6d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityHealthchecker.java @@ -1,25 +1,24 @@ package com.hubspot.singularity.scheduler; -import io.dropwizard.lifecycle.Managed; - import java.util.Map; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import javax.inject.Singleton; +import com.hubspot.singularity.SingularityTaskHealthcheckResult; import org.apache.commons.lang3.time.DurationFormatUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; +import com.google.inject.name.Named; import com.hubspot.singularity.SingularityAbort; +import com.hubspot.singularity.SingularityMainModule; import com.hubspot.singularity.SingularityPendingDeploy; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.config.SingularityConfiguration; @@ -31,7 +30,7 @@ @SuppressWarnings("deprecation") @Singleton -public class SingularityHealthchecker implements Managed { +public class SingularityHealthchecker { private static final Logger LOG = LoggerFactory.getLogger(SingularityHealthchecker.class); @@ -48,7 +47,9 @@ public class SingularityHealthchecker implements Managed { private final SingularityExceptionNotifier exceptionNotifier; @Inject - public SingularityHealthchecker(AsyncHttpClient http, SingularityConfiguration configuration, SingularityNewTaskChecker newTaskChecker, TaskManager taskManager, SingularityAbort abort, SingularityExceptionNotifier exceptionNotifier) { + public SingularityHealthchecker(@Named(SingularityMainModule.HEALTHCHECK_THREADPOOL_NAME) ScheduledExecutorService executorService, + AsyncHttpClient http, SingularityConfiguration configuration, SingularityNewTaskChecker newTaskChecker, + TaskManager taskManager, SingularityAbort abort, SingularityExceptionNotifier exceptionNotifier) { this.http = http; this.configuration = configuration; this.newTaskChecker = newTaskChecker; @@ -58,16 +59,7 @@ public SingularityHealthchecker(AsyncHttpClient http, SingularityConfiguration c this.taskIdToHealthcheck = Maps.newConcurrentMap(); - this.executorService = Executors.newScheduledThreadPool(configuration.getHealthcheckStartThreads(), new ThreadFactoryBuilder().setNameFormat("SingularityHealthchecker-%d").build()); - } - - @Override - public void start() { - } - - @Override - public void stop() { - MoreExecutors.shutdownAndAwaitTermination(executorService, 1, TimeUnit.SECONDS); + this.executorService = executorService; } public void enqueueHealthcheck(SingularityTask task) { @@ -104,7 +96,7 @@ public void cancelHealthcheck(String taskId) { } private ScheduledFuture enqueueHealthcheckWithDelay(final SingularityTask task, long delaySeconds) { - LOG.trace("Enqueing a healthcheck for task {} with delay {}", task.getTaskId(), DurationFormatUtils.formatDurationHMS(TimeUnit.SECONDS.toMillis(delaySeconds))); + LOG.trace("En-queuing a healthcheck for task {} with delay {}", task.getTaskId(), DurationFormatUtils.formatDurationHMS(TimeUnit.SECONDS.toMillis(delaySeconds))); return executorService.schedule(new Runnable() { @@ -116,13 +108,26 @@ public void run() { asyncHealthcheck(task); } catch (Throwable t) { LOG.error("Uncaught throwable in async healthcheck", t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); + + reEnqueueOrAbort(task); } } }, delaySeconds, TimeUnit.SECONDS); } + public void reEnqueueOrAbort(SingularityTask task) { + try { + enqueueHealthcheck(task); + } catch (Throwable t) { + LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); + + abort.abort(SingularityAbort.AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); + } + } + private Optional getHealthcheckUri(SingularityTask task) { if (task.getTaskRequest().getDeploy().getHealthcheckUri() == null) { return Optional.absent(); @@ -159,6 +164,13 @@ private boolean shouldHealthcheck(final SingularityTask task, Optional lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); + + if (lastHealthcheck.isPresent() && !lastHealthcheck.get().isFailed()) { + LOG.debug("Not submitting a new healthcheck for {} because it already passed a healthcheck", task.getTaskId()); + return false; + } + return true; } @@ -187,7 +199,7 @@ private void asyncHealthcheck(final SingularityTask task) { http.prepareRequest(builder.build()).execute(handler); } catch (Throwable t) { LOG.debug("Exception while preparing healthcheck ({}) for task ({})", uri, task.getTaskId(), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); saveFailure(handler, String.format("Healthcheck failed due to exception: %s", t.getMessage())); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java index ea28fdded3..44fc268091 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityLeaderOnlyPoller.java @@ -3,7 +3,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import io.dropwizard.lifecycle.Managed; -import java.util.concurrent.Executors; +import java.util.Collections; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; @@ -13,12 +13,11 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.SingularityAbort; import com.hubspot.singularity.SingularityAbort.AbortReason; +import com.hubspot.singularity.SingularityManagedScheduledExecutorServiceFactory; import com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; @@ -26,11 +25,11 @@ public abstract class SingularityLeaderOnlyPoller implements Managed { private static final Logger LOG = LoggerFactory.getLogger(SingularityLeaderOnlyPoller.class); - private final ScheduledExecutorService executorService; private final long pollDelay; private final TimeUnit pollTimeUnit; private final Optional lockHolder; + private ScheduledExecutorService executorService; private LeaderLatch leaderLatch; private SingularityExceptionNotifier exceptionNotifier; private SingularityAbort abort; @@ -48,15 +47,15 @@ private SingularityLeaderOnlyPoller(long pollDelay, TimeUnit pollTimeUnit, Optio this.pollDelay = pollDelay; this.pollTimeUnit = pollTimeUnit; this.lockHolder = lockHolder; - - this.executorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat(getClass().getSimpleName() + "-%d").build()); } @Inject - void injectPollerDependencies(LeaderLatch leaderLatch, + void injectPollerDependencies(SingularityManagedScheduledExecutorServiceFactory executorServiceFactory, + LeaderLatch leaderLatch, SingularityExceptionNotifier exceptionNotifier, SingularityAbort abort, SingularityMesosSchedulerDelegator mesosScheduler) { + this.executorService = executorServiceFactory.get(getClass().getSimpleName()); this.leaderLatch = checkNotNull(leaderLatch, "leaderLatch is null"); this.exceptionNotifier = checkNotNull(exceptionNotifier, "exceptionNotifier is null"); this.abort = checkNotNull(abort, "abort is null"); @@ -109,9 +108,9 @@ private void runActionIfLeaderAndMesosIsRunning() { runActionOnPoll(); } catch (Throwable t) { LOG.error("Caught an exception while running {}", getClass().getSimpleName(), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, Collections.emptyMap()); if (abortsOnError()) { - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } } finally { if (lockHolder.isPresent()) { @@ -134,6 +133,5 @@ protected boolean abortsOnError() { @Override public void stop() { - MoreExecutors.shutdownAndAwaitTermination(executorService, 1, TimeUnit.SECONDS); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java index 4b1c558719..c643651f40 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityNewTaskChecker.java @@ -1,10 +1,7 @@ package com.hubspot.singularity.scheduler; -import io.dropwizard.lifecycle.Managed; - import java.util.Collections; import java.util.Map; -import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -17,10 +14,10 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; +import com.google.inject.name.Named; import com.hubspot.baragon.models.BaragonRequestState; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.LoadBalancerRequestType; @@ -29,6 +26,7 @@ import com.hubspot.singularity.SingularityAbort.AbortReason; import com.hubspot.singularity.SingularityLoadBalancerUpdate; import com.hubspot.singularity.SingularityLoadBalancerUpdate.LoadBalancerMethod; +import com.hubspot.singularity.SingularityMainModule; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskCleanup; import com.hubspot.singularity.SingularityTaskCleanup.TaskCleanupType; @@ -46,7 +44,7 @@ * b/c we will use a queue to kill them. */ @Singleton -public class SingularityNewTaskChecker implements Managed { +public class SingularityNewTaskChecker { private static final Logger LOG = LoggerFactory.getLogger(SingularityNewTaskChecker.class); @@ -63,7 +61,8 @@ public class SingularityNewTaskChecker implements Managed { private final SingularityExceptionNotifier exceptionNotifier; @Inject - public SingularityNewTaskChecker(SingularityConfiguration configuration, LoadBalancerClient lbClient, TaskManager taskManager, SingularityExceptionNotifier exceptionNotifier, SingularityAbort abort) { + public SingularityNewTaskChecker(@Named(SingularityMainModule.NEW_TASK_THREADPOOL_NAME) ScheduledExecutorService executorService, + SingularityConfiguration configuration, LoadBalancerClient lbClient, TaskManager taskManager, SingularityExceptionNotifier exceptionNotifier, SingularityAbort abort) { this.configuration = configuration; this.taskManager = taskManager; this.lbClient = lbClient; @@ -72,20 +71,11 @@ public SingularityNewTaskChecker(SingularityConfiguration configuration, LoadBal this.taskIdToCheck = Maps.newConcurrentMap(); this.killAfterUnhealthyMillis = TimeUnit.SECONDS.toMillis(configuration.getKillAfterTasksDoNotRunDefaultSeconds()); - this.executorService = Executors.newScheduledThreadPool(configuration.getCheckNewTasksScheduledThreads(), new ThreadFactoryBuilder().setNameFormat("SingularityNewTaskChecker-%d").build()); + this.executorService = executorService; this.exceptionNotifier = exceptionNotifier; } - @Override - public void start() { - } - - @Override - public void stop() { - MoreExecutors.shutdownAndAwaitTermination(executorService, 1, TimeUnit.SECONDS); - } - private boolean hasHealthcheck(SingularityTask task) { return task.getTaskRequest().getDeploy().getHealthcheckUri().isPresent(); } @@ -169,7 +159,7 @@ public void run() { checkTask(task); } catch (Throwable t) { LOG.error("Uncaught throwable in task check for task {}, re-enqueing", task, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); reEnqueueCheckOrAbort(task); } @@ -182,8 +172,8 @@ private void reEnqueueCheckOrAbort(SingularityTask task) { reEnqueueCheck(task); } catch (Throwable t) { LOG.error("Uncaught throwable re-enqueuing task check for task {}, aborting", task, t); - exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", task.getTaskId().toString())); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } } @@ -303,6 +293,7 @@ private Optional checkLbState(BaragonRequestState lbState) { return Optional.of(CheckTaskState.HEALTHY); case CANCELED: case FAILED: + case INVALID_REQUEST_NOOP: return Optional.of(CheckTaskState.UNHEALTHY_KILL_TASK); case CANCELING: case UNKNOWN: diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java index 54b164be27..f5b7646d15 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduledJobPoller.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.inject.Inject; @@ -115,7 +116,7 @@ private Optional getExpectedRuntime(SingularityRequestWithState request, S cronExpression = new CronExpression(request.getRequest().getQuartzScheduleSafe()); } catch (ParseException e) { LOG.warn("Unable to parse cron for {} ({})", taskId, request.getRequest().getQuartzScheduleSafe(), e); - exceptionNotifier.notify(e); + exceptionNotifier.notify(e, ImmutableMap.of("taskId", taskId.toString())); return Optional.absent(); } @@ -125,7 +126,7 @@ private Optional getExpectedRuntime(SingularityRequestWithState request, S if (nextRunAtDate == null) { String msg = String.format("No next run date found for %s (%s)", taskId, request.getRequest().getQuartzScheduleSafe()); LOG.warn(msg); - exceptionNotifier.notify(msg); + exceptionNotifier.notify(msg, ImmutableMap.of("taskId", taskId.toString())); return Optional.absent(); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java index 1c054fe3e9..9508a7913b 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java @@ -331,7 +331,7 @@ private void deleteScheduledTasks(final Collection sched } private List getMatchingTaskIds(SingularitySchedulerStateCache stateCache, SingularityRequest request, SingularityPendingRequest pendingRequest) { - if (!request.isScheduled()) { + if (request.isLongRunning()) { return SingularityTaskId.matchingAndNotIn(stateCache.getActiveTaskIds(), request.getId(), pendingRequest.getDeployId(), stateCache.getCleaningTasks()); } else { return Lists.newArrayList(Iterables.filter(stateCache.getActiveTaskIds(), SingularityTaskId.matchingRequest(request.getId()))); @@ -624,6 +624,12 @@ private Optional getNextRunAt(SingularityRequest request, RequestState sta } } + if (pendingType == PendingType.TASK_DONE && request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().or(0L) > 0) { + nextRunAt = Math.max(nextRunAt, now + request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().get()); + + LOG.trace("Adjusted next run of {} to {} (by {}) due to waitAtLeastMillisAfterTaskFinishesForReschedule", request.getId(), nextRunAt, JavaUtils.durationFromMillis(request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().get())); + } + if (state == RequestState.SYSTEM_COOLDOWN && pendingType != PendingType.NEW_DEPLOY) { final long prevNextRunAt = nextRunAt; nextRunAt = Math.max(nextRunAt, now + TimeUnit.SECONDS.toMillis(configuration.getCooldownMinScheduleSeconds())); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java index ab396bf871..abf69b35f7 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityTaskReconciliation.java @@ -1,11 +1,8 @@ package com.hubspot.singularity.scheduler; -import io.dropwizard.lifecycle.Managed; - import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -23,14 +20,13 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import com.google.inject.name.Named; import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.SingularityAbort; import com.hubspot.singularity.SingularityAbort.AbortReason; import com.hubspot.singularity.SingularityMainModule; +import com.hubspot.singularity.SingularityManagedScheduledExecutorServiceFactory; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskStatusHolder; import com.hubspot.singularity.config.SingularityConfiguration; @@ -39,7 +35,7 @@ import com.hubspot.singularity.sentry.SingularityExceptionNotifier; @Singleton -public class SingularityTaskReconciliation implements Managed { +public class SingularityTaskReconciliation { private static final Logger LOG = LoggerFactory.getLogger(SingularityTaskReconciliation.class); @@ -53,7 +49,8 @@ public class SingularityTaskReconciliation implements Managed { private final SchedulerDriverSupplier schedulerDriverSupplier; @Inject - public SingularityTaskReconciliation(SingularityExceptionNotifier exceptionNotifier, + public SingularityTaskReconciliation(SingularityManagedScheduledExecutorServiceFactory executorServiceFactory, + SingularityExceptionNotifier exceptionNotifier, TaskManager taskManager, SingularityConfiguration configuration, @Named(SingularityMainModule.SERVER_ID_PROPERTY) String serverId, @@ -68,16 +65,7 @@ public SingularityTaskReconciliation(SingularityExceptionNotifier exceptionNotif this.schedulerDriverSupplier = schedulerDriverSupplier; this.isRunningReconciliation = new AtomicBoolean(false); - this.executorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("SingularityTaskReconciliation-%d").build()); - } - - @Override - public void start() { - } - - @Override - public void stop() { - MoreExecutors.shutdownAndAwaitTermination(executorService, 1, TimeUnit.SECONDS); + this.executorService = executorServiceFactory.get(getClass().getSimpleName()); } enum ReconciliationState { @@ -127,8 +115,8 @@ public void run() { checkReconciliation(driver, reconciliationStart, remainingTaskIds, numTimes + 1); } catch (Throwable t) { LOG.error("While checking for reconciliation tasks", t); - exceptionNotifier.notify(t); - abort.abort(AbortReason.UNRECOVERABLE_ERROR); + exceptionNotifier.notify(t, Collections.emptyMap()); + abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } } }, configuration.getCheckReconcileWhenRunningEveryMillis(), TimeUnit.MILLISECONDS); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java index 33853538d1..c742cbd402 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingExceptionMapper.java @@ -2,6 +2,8 @@ import io.dropwizard.jersey.errors.LoggingExceptionMapper; +import java.util.Collections; + import javax.ws.rs.core.Response; import com.google.inject.Inject; @@ -20,7 +22,7 @@ public Response toResponse(final Exception e) { final Response response = super.toResponse(e); if (response.getStatus() >= 500) { - notifier.notify(e); + notifier.notify(e, Collections.emptyMap()); } return response; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java index 2b803d7fec..ddef1b32ee 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/sentry/NotifyingUncaughtExceptionManager.java @@ -1,6 +1,7 @@ package com.hubspot.singularity.sentry; import java.lang.Thread.UncaughtExceptionHandler; +import java.util.Collections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,6 +18,6 @@ public NotifyingUncaughtExceptionManager(SingularityExceptionNotifier notifier) @Override public void uncaughtException(Thread t, Throwable e) { LOG.error("Uncaught exception!", e); - notifier.notify(e); + notifier.notify(e, Collections.emptyMap()); } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java b/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java index 9ded1ab42a..3ac54fb591 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/sentry/SingularityExceptionNotifier.java @@ -15,8 +15,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; import com.google.inject.Inject; import com.hubspot.singularity.config.SentryConfiguration; @@ -45,59 +52,61 @@ private String getPrefix() { return sentryConfiguration.get().getPrefix() + " "; } - public void notify(Throwable t) { - if (!raven.isPresent()) { - return; - } - - try { - notify(raven.get(), t); - } catch (Throwable e) { - LOG.error("Caught exception while trying to report {} to Sentry", t.getMessage(), e); + private String getCallingClassName(StackTraceElement[] stackTrace) { + if (stackTrace != null && stackTrace.length > 2) { + return stackTrace[2].getClassName(); + } else { + return "(unknown)"; } } - public void notify(String subject) { - notify(subject, Collections. emptyMap()); + private void sendEvent(Raven raven, final EventBuilder eventBuilder) { + raven.runBuilderHelpers(eventBuilder); + + raven.sendEvent(eventBuilder.build()); } - public void notify(String subject, Map extraData) { + public void notify(Throwable t, Map extraData) { if (!raven.isPresent()) { return; } - try { - notify(raven.get(), subject, extraData); - } catch (Throwable e) { - LOG.error("Caught exception while trying to report {} ({}) to Sentry", subject, extraData, e); - } - } + final StackTraceElement[] currentThreadStackTrace = Thread.currentThread().getStackTrace(); - private void notify(Raven raven, String subject, Map extraData) { final EventBuilder eventBuilder = new EventBuilder() - .setMessage(getPrefix() + subject) - .setLevel(Event.Level.ERROR); - - for (Entry extraDataEntry : extraData.entrySet()) { - eventBuilder.addExtra(extraDataEntry.getKey(), extraDataEntry.getValue()); + .setCulprit(getPrefix() + t.getMessage()) + .setMessage(Strings.nullToEmpty(t.getMessage())) + .setLevel(Event.Level.ERROR) + .setLogger(getCallingClassName(currentThreadStackTrace)) + .addSentryInterface(new ExceptionInterface(t)); + + if (extraData != null && !extraData.isEmpty()) { + for (Map.Entry entry : extraData.entrySet()) { + eventBuilder.addExtra(entry.getKey(), entry.getValue()); + } } - sendEvent(raven, eventBuilder); + sendEvent(raven.get(), eventBuilder); } - private void notify(Raven raven, Throwable t) { - final EventBuilder eventBuilder = new EventBuilder() - .setMessage(getPrefix() + t.getMessage()) - .setLevel(Event.Level.ERROR) - .addSentryInterface(new ExceptionInterface(t)); + public void notify(String subject, Map extraData) { + if (!raven.isPresent()) { + return; + } - sendEvent(raven, eventBuilder); - } + final StackTraceElement[] currentThreadStackTrace = Thread.currentThread().getStackTrace(); - private void sendEvent(Raven raven, final EventBuilder eventBuilder) { - raven.runBuilderHelpers(eventBuilder); + final EventBuilder eventBuilder = new EventBuilder() + .setMessage(getPrefix() + subject) + .setLevel(Event.Level.ERROR) + .setLogger(getCallingClassName(currentThreadStackTrace)); + + if (extraData != null && !extraData.isEmpty()) { + for (Map.Entry entry : extraData.entrySet()) { + eventBuilder.addExtra(entry.getKey(), entry.getValue()); + } + } - raven.sendEvent(eventBuilder.build()); + sendEvent(raven.get(), eventBuilder); } - } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java index 521e231811..409afc7241 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularityMailer.java @@ -235,7 +235,7 @@ public void run() { prepareTaskCompletedMail(task, taskId, request, taskState); } catch (Throwable t) { LOG.error("While preparing task completed mail for {}", taskId, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("taskId", taskId.toString())); } } }); @@ -318,7 +318,7 @@ public void run() { prepareRequestMail(request, type, user); } catch (Throwable t) { LOG.error("While preparing request mail for {} / {}", request, type, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("requestId", request.getId())); } } }); @@ -376,7 +376,7 @@ public void run() { prepareRequestInCooldownMail(request); } catch (Throwable t) { LOG.error("While preparing request in cooldown mail for {}", request, t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("requestId", request.getId())); } } }); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java index 686cdf9e0f..7f0c2b7274 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/smtp/SingularitySmtpSender.java @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; @@ -136,7 +137,7 @@ private void sendMail(List toList, List ccList, String subject, Transport.send(message); } catch (Throwable t) { LOG.warn("Unable to send message {}", getEmailLogFormat(toList, subject), t); - exceptionNotifier.notify(t); + exceptionNotifier.notify(t, ImmutableMap.of("subject", subject)); } } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java b/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java index 81e7d31588..6af481a341 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/SingularitySchedulerTestBase.java @@ -5,6 +5,7 @@ import java.util.Random; import java.util.concurrent.TimeUnit; +import com.hubspot.singularity.data.zkmigrations.ZkDataMigrationRunner; import org.apache.mesos.Protos.Attribute; import org.apache.mesos.Protos.FrameworkID; import org.apache.mesos.Protos.Offer; @@ -107,6 +108,8 @@ public class SingularitySchedulerTestBase extends SingularityCuratorTestBase { protected SingularityMailer mailer; @Inject protected SingularityScheduledJobPoller scheduledJobPoller; + @Inject + protected ZkDataMigrationRunner migrationRunner; @Inject @Named(SingularityMainModule.SERVER_ID_PROPERTY) @@ -135,6 +138,8 @@ public void teardown() throws Exception { @Before public final void setupDriver() throws Exception { driver = driverSupplier.get().get(); + + migrationRunner.checkMigrations(); } protected Offer createOffer(double cpus, double memory) { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java b/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java index cccfc9bba3..3cf353e197 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/data/BlendedHistoryTest.java @@ -42,7 +42,6 @@ private SingularityRequestHistory makeHistory(long createdAt, RequestHistoryType return new SingularityRequestHistory(createdAt, Optional. absent(), type, request); } - // DESCENDING @Test public void testBlendedRequestHistory() { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java b/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java index c677972371..fd6b2e1f67 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/data/zkmigrations/ZkMigrationTest.java @@ -37,9 +37,9 @@ public class ZkMigrationTest extends SingularityCuratorTestBase { @Test public void testMigrationRunner() { - Assert.assertTrue(migrationRunner.checkMigrations() == 4); + Assert.assertTrue(migrationRunner.checkMigrations() == 5); - Assert.assertTrue(metadataManager.getZkDataVersion().isPresent() && metadataManager.getZkDataVersion().get().equals("4")); + Assert.assertTrue(metadataManager.getZkDataVersion().isPresent() && metadataManager.getZkDataVersion().get().equals("5")); Assert.assertTrue(migrationRunner.checkMigrations() == 0); } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java index 1b5afc64e1..544a41a4a2 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java @@ -40,11 +40,13 @@ import com.hubspot.singularity.SingularityRequestBuilder; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskRequest; +import com.hubspot.singularity.config.SingularityConfiguration; import com.hubspot.singularity.data.ExecutorIdGenerator; public class SingularityMesosTaskBuilderTest { private SingularityMesosTaskBuilder builder; - private Resources resources; + private Resources taskResources; + private Resources executorResources; private Offer offer; private SingularityPendingTask pendingTask; @@ -57,9 +59,11 @@ public void createMocks() { when(idGenerator.getNextExecutorId()).then(new CreateFakeId()); - builder = new SingularityMesosTaskBuilder(new ObjectMapper(), rackManager, idGenerator); + builder = new SingularityMesosTaskBuilder(new ObjectMapper(), rackManager, idGenerator, new SingularityConfiguration()); + + taskResources = new Resources(1, 1, 0); + executorResources = new Resources(0.1, 1, 0); - resources = new Resources(1, 1, 0); offer = Offer.newBuilder() .setSlaveId(SlaveID.newBuilder().setValue("1")) .setId(OfferID.newBuilder().setValue("1")) @@ -75,7 +79,7 @@ public void testShellCommand() { .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources); + final SingularityTask task = builder.buildTask(offer, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo hi", task.getMesosTask().getCommand().getValue()); assertEquals(0, task.getMesosTask().getCommand().getArgumentsCount()); @@ -90,7 +94,7 @@ public void testArgumentCommand() { .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, null, taskRequest, resources); + final SingularityTask task = builder.buildTask(offer, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); @@ -100,7 +104,7 @@ public void testArgumentCommand() { @Test public void testDockerTask() { - resources = new Resources(1, 1, 1); + taskResources = new Resources(1, 1, 1); final Protos.Resource portsResource = Protos.Resource.newBuilder() .setName("ports") @@ -124,7 +128,7 @@ public void testDockerTask() { .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); - final SingularityTask task = builder.buildTask(offer, Collections.singletonList(portsResource), taskRequest, resources); + final SingularityTask task = builder.buildTask(offer, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java index 1d5d1947ea..3f327891ca 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java @@ -363,7 +363,6 @@ public void testScheduledJobLivesThroughDeploy() { @Test public void testOneOffsDontRunByThemselves() { - requestId = "oneoffRequest"; SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.submit(bldr.build(), Optional. absent()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); @@ -375,6 +374,65 @@ public void testOneOffsDontRunByThemselves() { Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); + requestResource.scheduleImmediately(requestId, user, Collections. emptyList()); + + resourceOffers(); + + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FINISHED); + + resourceOffers(); + Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); + Assert.assertEquals(0, taskManager.getPendingTaskIds().size()); + + requestResource.scheduleImmediately(requestId, user, Collections. emptyList()); + + resourceOffers(); + + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); + + resourceOffers(); + Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); + Assert.assertEquals(0, taskManager.getPendingTaskIds().size()); + } + + @Test + public void testOneOffsDontMoveDuringDecomission() { + SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); + requestResource.submit(bldr.build(), Optional. absent()); + deploy("d2"); + + requestResource.scheduleImmediately(requestId, user, Collections. emptyList()); + + validateTaskDoesntMoveDuringDecommission(); + } + + private void validateTaskDoesntMoveDuringDecommission() { + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave1", "host1", Optional.of("rack1")))); + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave2", "host2", Optional.of("rack1")))); + + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + + Assert.assertEquals("host1", taskManager.getActiveTaskIds().get(0).getHost()); + + Assert.assertEquals(StateChangeResult.SUCCESS, slaveManager.changeState("slave1", MachineState.STARTING_DECOMMISSION, Optional.of("user1"))); + + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave2", "host2", Optional.of("rack1")))); + + cleaner.drainCleanupQueue(); + + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 129, "slave2", "host2", Optional.of("rack1")))); + + cleaner.drainCleanupQueue(); + + // task should not move! + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + Assert.assertEquals("host1", taskManager.getActiveTaskIds().get(0).getHost()); + Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); + Assert.assertTrue(taskManager.getCleanupTaskIds().size() == 1); } @Test @@ -396,7 +454,7 @@ public void testRunOnceRunOnlyOnce() { Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); - statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FINISHED); + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); resourceOffers(); @@ -422,6 +480,21 @@ public void testRunOnceRunOnlyOnce() { Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); } + @Test + public void testRunOnceDontMoveDuringDecomission() { + SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); + request = bldr.build(); + saveRequest(request); + + deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional. absent(), Optional. absent())); + + scheduler.drainPendingQueue(stateCacheProvider.get()); + + deployChecker.checkDeploys(); + + validateTaskDoesntMoveDuringDecommission(); + } + @Test public void testRetries() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); @@ -896,7 +969,7 @@ public void testScheduledNotification() { saveRequest(request.toBuilder().setScheduledExpectedRuntimeMillis(Optional.of(1L)).build()); - SingularityTask thirdTask = launchTask(request, firstDeploy, now - 500, 1, TaskState.TASK_RUNNING); + SingularityTask thirdTask = launchTask(request, firstDeploy, now - 502, 1, TaskState.TASK_RUNNING); scheduledJobPoller.runActionOnPoll(); @@ -1139,4 +1212,28 @@ public void testScaleDownTakesHighestInstances() { } + @Test + public void testWaitAfterTaskWorks() { + initRequest(); + initFirstDeploy(); + + SingularityTask task = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); + + statusUpdate(task, TaskState.TASK_FAILED); + + Assert.assertTrue(taskManager.getPendingTaskIds().get(0).getNextRunAt() - System.currentTimeMillis() < 1000L); + + resourceOffers(); + + long extraWait = 100000L; + + saveAndSchedule(request.toBuilder().setWaitAtLeastMillisAfterTaskFinishesForReschedule(Optional.of(extraWait)).setInstances(Optional.of(2))); + resourceOffers(); + + statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FAILED); + + Assert.assertTrue(taskManager.getPendingTaskIds().get(0).getNextRunAt() - System.currentTimeMillis() > 1000L); + Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); + } + } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java index 7fbbd82f7c..7adc07ea4b 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java @@ -109,7 +109,7 @@ public void configure(Binder mainBinder) { final SingularityConfiguration configuration = getSingularityConfigurationForTestingServer(ts); mainBinder.bind(SingularityConfiguration.class).toInstance(configuration); - mainBinder.install(Modules.override(new SingularityMainModule()) + mainBinder.install(Modules.override(new SingularityMainModule(configuration)) .with(new Module() { @Override diff --git a/SingularityUI/app/application.coffee b/SingularityUI/app/application.coffee index 3a61fc6608..53de3aee48 100644 --- a/SingularityUI/app/application.coffee +++ b/SingularityUI/app/application.coffee @@ -66,7 +66,7 @@ class Application setInterval @globalRefresh, @globalRefreshTime globalRefresh: => - return if localStorage.getItem 'suppressRefresh' + return if localStorage.getItem('suppressRefresh') is 'true' if @blurred clearInterval @globalRefreshInterval return diff --git a/SingularityUI/app/collections/Slaves.coffee b/SingularityUI/app/collections/Slaves.coffee index 52c2c683a6..41dcc48279 100644 --- a/SingularityUI/app/collections/Slaves.coffee +++ b/SingularityUI/app/collections/Slaves.coffee @@ -9,8 +9,4 @@ class Slaves extends Collection initialize: (models) => - parse: (slaves) -> - _.map slaves, (slave) => - slave - module.exports = Slaves diff --git a/SingularityUI/app/controllers/Slaves.coffee b/SingularityUI/app/controllers/Slaves.coffee index 00b567bda7..07fa021de4 100644 --- a/SingularityUI/app/controllers/Slaves.coffee +++ b/SingularityUI/app/controllers/Slaves.coffee @@ -18,6 +18,6 @@ class SlavesController extends Controller @refresh() refresh: -> - @collections.slaves.fetch() + @collections.slaves.fetch reset: true module.exports = SlavesController diff --git a/SingularityUI/app/controllers/Tail.coffee b/SingularityUI/app/controllers/Tail.coffee index 5a30dd2d2a..40f32e5476 100644 --- a/SingularityUI/app/controllers/Tail.coffee +++ b/SingularityUI/app/controllers/Tail.coffee @@ -1,6 +1,7 @@ Controller = require './Controller' LogLines = require '../collections/LogLines' +TaskHistory = require '../models/TaskHistory' TailView = require '../views/tail' @@ -8,12 +9,18 @@ class TailController extends Controller initialize: ({@taskId, @path}) -> @collections.logLines = new LogLines [], {@taskId, @path} - + @models.taskHistory = new TaskHistory {@taskId} + @setView new TailView _.extend {@taskId, @path}, collection: @collections.logLines + model: @models.taskHistory app.showView @view + @refresh() + refresh: -> @collections.logLines.fetchInitialData() + @models.taskHistory.fetch() + module.exports = TailController diff --git a/SingularityUI/app/controllers/TaskDetail.coffee b/SingularityUI/app/controllers/TaskDetail.coffee index 47f27ef780..ffa90cd973 100644 --- a/SingularityUI/app/controllers/TaskDetail.coffee +++ b/SingularityUI/app/controllers/TaskDetail.coffee @@ -82,17 +82,35 @@ class TaskDetailController extends Controller app.showView @view + + fetchResourceUsage: -> + @models.resourceUsage?.fetch() + .done => + # Store current resource usage to compare against future resource usage + @models.resourceUsage.setCpuUsage() if @models.resourceUsage.get('previousUsage') + @models.resourceUsage.set('previousUsage', @models.resourceUsage.toJSON()) + + if not @resourcesFetched + setTimeout (=> @fetchResourceUsage() ), 2000 + @resourcesFetched = true + + .error => + # If this 404s there's nothing to get so don't bother + app.caughtError() + delete @models.resourceUsage + refresh: -> - @models.task.fetch - error: => + @resourcesFetched = false + + @models.task.fetch() + .done => + @fetchResourceUsage() if @models.task.get('isStillRunning') + + .error => # If this 404s the task doesn't exist app.caughtError() app.router.notFound() - @models.resourceUsage?.fetch().error => - # If this 404s there's nothing to get so don't bother - app.caughtError() - delete @models.resourceUsage if @collections.s3Logs?.currentPage is 1 @collections.s3Logs.fetch().error => diff --git a/SingularityUI/app/handlebarsHelpers.coffee b/SingularityUI/app/handlebarsHelpers.coffee index 5a1d7ef7a0..e32deb1b8f 100644 --- a/SingularityUI/app/handlebarsHelpers.coffee +++ b/SingularityUI/app/handlebarsHelpers.coffee @@ -21,6 +21,12 @@ Handlebars.registerHelper "ifAll", (conditions..., options)-> Handlebars.registerHelper 'percentageOf', (v1, v2) -> (v1/v2) * 100 +# Override decimal rounding: {{fixedDecimal data.cpuUsage place="4"}} +Handlebars.registerHelper 'fixedDecimal', (value, options) -> + if options.hash.place then place = options.hash.place else place = 2 + +(value).toFixed(place) + + Handlebars.registerHelper 'ifInSubFilter', (needle, haystack, options) -> return options.fn @ if haystack is 'all' if haystack.indexOf(needle) isnt -1 @@ -76,21 +82,13 @@ Handlebars.registerHelper 'humanizeText', (text) -> text # 2121 => '2 KB' -Handlebars.registerHelper 'humanizeFileSize', (fileSize) -> - kilo = 1024 - mega = 1024 * 1024 - giga = 1024 * 1024 * 1024 - - shorten = (which) -> Math.round fileSize / which - - if fileSize > giga - return "#{ shorten giga } GB" - else if fileSize > mega - return "#{ shorten mega } MB" - else if fileSize > kilo - return "#{ shorten kilo } KB" - else - return "#{ fileSize } B" +Handlebars.registerHelper 'humanizeFileSize', (bytes) -> + k = 1024 + sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] + + return '0 B' if bytes is 0 + i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length-1) + return +(bytes / Math.pow(k, i)).toFixed(2) + ' ' + sizes[i] # 'sbacanu@hubspot.com' => 'sbacanu' # 'seb' => 'seb' diff --git a/SingularityUI/app/models/TaskResourceUsage.coffee b/SingularityUI/app/models/TaskResourceUsage.coffee index 6a202623ef..0ff4447556 100644 --- a/SingularityUI/app/models/TaskResourceUsage.coffee +++ b/SingularityUI/app/models/TaskResourceUsage.coffee @@ -6,4 +6,18 @@ class TaskResourceUsage extends Model initialize: ({ @taskId }) => -module.exports = TaskResourceUsage + # Calculate CPU Usage by comparing previous usage to current usage + setCpuUsage: -> + previous = @get('previousUsage') + currentTime = @get('cpusSystemTimeSecs') + @get('cpusUserTimeSecs') + previousTime = previous.cpusSystemTimeSecs + previous.cpusUserTimeSecs + timestampDiff = @get('timestamp') - previous.timestamp + cpus_used = (currentTime - previousTime) / timestampDiff + cpuUsageExceeding = (cpus_used / @get('cpusLimit')) > 1.10 + + if cpuUsageExceeding + @set 'cpuUsageExceeding', cpuUsageExceeding + @set 'cpuUsageClassStatus', 'danger' + @set 'cpuUsage', cpus_used + +module.exports = TaskResourceUsage \ No newline at end of file diff --git a/SingularityUI/app/styles/loader.styl b/SingularityUI/app/styles/loader.styl index 70a5f27cc3..a395fb19d7 100644 --- a/SingularityUI/app/styles/loader.styl +++ b/SingularityUI/app/styles/loader.styl @@ -31,6 +31,12 @@ &.cushy margin-top 3em margin-bottom 3em + + &.inline-left + display inline-block + margin-right 5px + + .page-loader-with-message text-align center diff --git a/SingularityUI/app/styles/table.styl b/SingularityUI/app/styles/table.styl index a7881bb86a..8f2142f153 100644 --- a/SingularityUI/app/styles/table.styl +++ b/SingularityUI/app/styles/table.styl @@ -39,9 +39,7 @@ a position relative overflow hidden - padding auto - padding-left 12px - padding-right 12px + padding 0 12px display inline-block background transparent text-decoration none diff --git a/SingularityUI/app/templates/dashboard.hbs b/SingularityUI/app/templates/dashboard.hbs index c585f941aa..38a659b01e 100644 --- a/SingularityUI/app/templates/dashboard.hbs +++ b/SingularityUI/app/templates/dashboard.hbs @@ -91,52 +91,11 @@ {{/if}}
-
+

Starred requests

- {{#if starredRequests.length}} - - - - - - - - - - - - {{#each starredRequests}} - - - - - - - - {{/each}} - -
RequestDeploy userInstances
- - - - - - {{ request.id }} - - - {{usernameFromEmail requestDeployState.activeDeploy.user}} - - {{ request.instances }} -
- {{else}} -

No starred requests

- {{/if}} + {{> requestsBody }}
{{else}} diff --git a/SingularityUI/app/templates/dashboardTable/dashboardStarred.hbs b/SingularityUI/app/templates/dashboardTable/dashboardStarred.hbs new file mode 100644 index 0000000000..4ba9bf7357 --- /dev/null +++ b/SingularityUI/app/templates/dashboardTable/dashboardStarred.hbs @@ -0,0 +1,48 @@ +{{#if haveStarredRequests}} + + + + + + + + + + + +{{/if}} + {{#each starredRequests}} + + + + + + + + {{/each}} + + {{#if haveStarredRequests}} + +
{{! Star column }}RequestDeploy userInstances
+ + + + + + {{ request.id }} + + + {{usernameFromEmail requestDeployState.activeDeploy.user}} + + {{ request.instances }} +
+{{else}} +

No starred requests

+{{/if}} + + + diff --git a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs index c110d1a534..38a2ce361a 100644 --- a/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs +++ b/SingularityUI/app/templates/taskDetail/taskResourceUsage.hbs @@ -11,17 +11,41 @@
- {{#ifAll data.memRssBytes data.memLimitBytes}} -
-
-

Memory (rss vs limit)

-
-
+
+ {{#ifAll data.memRssBytes data.memLimitBytes}} +
+
+

Memory (rss vs limit)

+
+
+
+
+
+ {{humanizeFileSize data.memRssBytes}} / {{humanizeFileSize data.memLimitBytes}} +
- {{humanizeFileSize data.memRssBytes}} / {{humanizeFileSize data.memLimitBytes}} -
+ {{/ifAll}} + {{#ifAll data.cpusUserTimeSecs data.cpusSystemTimeSecs }} +
+
+

CPU Usage

+
+
+ {{#ifAll data.cpuUsage}} + {{#if data.cpuUsageExceeding }} +

CPU usage > 110% allocated

+ {{/if}} +
+
+
+ {{fixedDecimal data.cpuUsage}} used / {{data.cpusLimit}} allocated CPUs + {{else}} + Calculating... + {{/ifAll}} +
+
+ {{/ifAll}}
- {{/ifAll}}
    {{#if data.cpusNrPeriods }} @@ -40,14 +64,7 @@
{{/if}} - {{#if data.cpusSystemTimeSecs }} -
  • -
    -

    System time (sec)

    -

    {{ data.cpusSystemTimeSecs }}

    -
    -
    • - {{/if}} + {{#if data.cpusThrottledTimeSecs }}
  • @@ -56,14 +73,6 @@
    • {{/if}} - {{#if data.cpusUserTimeSecs }} -
  • -
    -

    User time (sec)

    -

    {{ data.cpusUserTimeSecs }}

    -
    -
    • - {{/if}} {{#if data.memAnonBytes }}
  • diff --git a/SingularityUI/app/views/dashboard.coffee b/SingularityUI/app/views/dashboard.coffee index 0bd32f4be7..27626f31dd 100644 --- a/SingularityUI/app/views/dashboard.coffee +++ b/SingularityUI/app/views/dashboard.coffee @@ -2,12 +2,14 @@ View = require './view' class DashboardView extends View - template: require '../templates/dashboard' + templateBase: require '../templates/dashboard' + templateRequestsTable: require '../templates/dashboardTable/dashboardStarred' events: -> _.extend super, 'click [data-action="unstar"]': 'unstar' 'click [data-action="change-user"]': 'changeUser' + 'click th[data-sort-attribute]': 'sortTable' initialize: => @listenTo app.user, 'change', @render @@ -16,9 +18,9 @@ class DashboardView extends View render: => deployUser = app.user.get 'deployUser' - # Filter starred requests - starredRequests = @collection.getStarredOnly() - starredRequests = _.pluck starredRequests, 'attributes' + partials = + partials: + requestsBody: @templateRequestsTable # Count up the Requests for the clicky boxes userRequests = @collection.filter (model) -> @@ -27,12 +29,12 @@ class DashboardView extends View if not request.owners return false - + for owner in request.owners ownerTrimmed = owner.split("@")[0] if deployUserTrimmed == ownerTrimmed return true - + return false userRequestTotals = all: userRequests.length @@ -53,9 +55,68 @@ class DashboardView extends View deployUser: deployUser collectionSynced: @collection.synced userRequestTotals: userRequestTotals or { } - starredRequests: starredRequests or [] + haveStarredRequests: @collection.getStarredOnly().length + + @$el.html @templateBase context, partials + @renderTable() + + renderTable: => + @sortCollection() + requests = @currentRequests + + $contents = @templateRequestsTable + starredRequests: requests + requests: requests + + $table = @$ ".table-staged table" + $tableBody = $table.find "tbody" + $tableBody.html $contents + + sortCollection: => + requests = _.pluck @collection.getStarredOnly(), "attributes" + + # Sort the table if the user clicked on the table heading things + if @sortAttribute? + requests = _.sortBy requests, (request) => + + # Traverse through the properties to find what we're after + attributes = @sortAttribute.split '.' + value = request + for attribute in attributes + value = value[attribute] + value = '' if not value? + return value + + if not @sortAscending + requests = requests.reverse() + else + requests.reverse() + + @currentRequests = requests + + sortTable: (event) => + @isSorted = true + + $target = $ event.currentTarget + newSortAttribute = $target.attr "data-sort-attribute" + + $currentlySortedHeading = @$ "[data-sorted=true]" + $currentlySortedHeading.removeAttr "data-sorted" + $currentlySortedHeading.find('span').remove() + + + if newSortAttribute is @sortAttribute and @sortAscending? + @sortAscending = not @sortAscending + else + # timestamp should be DESC by default + @sortAscending = if newSortAttribute is "timestamp" then false else true + + @sortAttribute = newSortAttribute + + $target.attr "data-sorted", "true" + $target.append "" - @$el.html @template context + @renderTable() unstar: (e) => $target = $ e.currentTarget diff --git a/SingularityUI/app/views/formBaseView.coffee b/SingularityUI/app/views/formBaseView.coffee index 3271aa3c85..dde2473e97 100644 --- a/SingularityUI/app/views/formBaseView.coffee +++ b/SingularityUI/app/views/formBaseView.coffee @@ -96,6 +96,10 @@ class FormBaseView extends View val = $element.val() return val if val return if $element.parents('.required').length then "" else undefined + + base64Encode: (content) -> + return if not content? + btoa content multiMap: (selector) => $elements = @$ selector diff --git a/SingularityUI/app/views/newDeploy.coffee b/SingularityUI/app/views/newDeploy.coffee index 072f2f6ae6..9ebae1d6f8 100644 --- a/SingularityUI/app/views/newDeploy.coffee +++ b/SingularityUI/app/views/newDeploy.coffee @@ -139,7 +139,7 @@ class NewDeployView extends FormBaseView name: @valOrNothing '.name', $artifact filename: @valOrNothing '.filename', $artifact md5sum: @valOrNothing '.md5', $artifact - content: @valOrNothing '.content', $artifact + content: @base64Encode @valOrNothing '.content', $artifact else if type is 'external' deployObject.executorData.externalArtifacts = [] unless deployObject.executorData.externalArtifacts deployObject.executorData.externalArtifacts.push diff --git a/SingularityUI/app/views/tail.coffee b/SingularityUI/app/views/tail.coffee index a2e5f163c9..507f48fddb 100644 --- a/SingularityUI/app/views/tail.coffee +++ b/SingularityUI/app/views/tail.coffee @@ -28,6 +28,8 @@ class TailView extends View @$el.addClass 'fetching-data' @listenTo @collection, 'sync', => @$el.removeClass 'fetching-data' + + @listenTo @model, 'change:isStillRunning', => @stopTailing() unless @model.get 'isStillRunning' handleAjaxError: (response) => # ATM we get 404s if we request dirs and 500s if the file doesn't exist @@ -142,13 +144,15 @@ class TailView extends View @startTailing() startTailing: => - return if @isTailing is true - + return if @isTailing or not @model.get 'isStillRunning' + @isTailing = true @scrollToBottom() clearInterval @tailInterval @tailInterval = setInterval => + @stopTailing() if not @model.get 'isStillRunning' + @collection.fetchNext().done => # Only show the newly tail-ed lines if we are still tailing @scrollToBottom() if @isTailing diff --git a/pom.xml b/pom.xml index 962ecf0a05..8496f77a0c 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ 1.3.12 2.3.2 - 0.1.2 + 0.1.4 2.4.2 0.7.1 0.9.0