From 74230c59ea43ba64018b74de22626190cd5a39ec Mon Sep 17 00:00:00 2001 From: Yaliang <49084640+ylwu-amzn@users.noreply.github.com> Date: Tue, 19 Jan 2021 17:04:46 -0800 Subject: [PATCH] stop historical detector; support return AD task in get detector API (#359) * stop historical detector; support return AD task in get detector API * change to scaling executor * remove detector cache; handle node crash * add more comments/logs; tune method name --- .../ad/AnomalyDetectorPlugin.java | 37 +- .../ad/AnomalyDetectorProfileRunner.java | 15 +- .../ad/MemoryTracker.java | 2 +- .../exception/DuplicateTaskException.java | 24 + .../ad/constant/CommonErrorMessages.java | 3 + .../ad/constant/CommonName.java | 3 + .../ad/model/ADTask.java | 86 ++- .../ad/model/ADTaskAction.java | 21 + .../ad/model/ADTaskProfile.java | 300 +++++++++ .../ad/model/AnomalyDetector.java | 15 +- .../ad/model/DetectorProfile.java | 82 ++- .../ad/model/DetectorProfileName.java | 5 +- .../ad/rest/RestGetAnomalyDetectorAction.java | 2 + .../IndexAnomalyDetectorJobActionHandler.java | 8 +- .../ad/task/ADBatchTaskCache.java | 6 + .../ad/task/ADBatchTaskRunner.java | 35 +- .../ad/task/ADTaskCacheManager.java | 142 +++- .../ad/task/ADTaskCancellationState.java | 22 + .../ad/task/ADTaskManager.java | 625 ++++++++++++++++-- .../transport/ADBatchAnomalyResultAction.java | 4 +- .../ADBatchTaskRemoteExecutionAction.java | 4 +- ...tchTaskRemoteExecutionTransportAction.java | 4 +- .../ad/transport/ADCancelTaskAction.java | 33 + .../ad/transport/ADCancelTaskNodeRequest.java | 54 ++ .../transport/ADCancelTaskNodeResponse.java | 54 ++ .../ad/transport/ADCancelTaskRequest.java | 71 ++ .../ad/transport/ADCancelTaskResponse.java | 46 ++ .../ADCancelTaskTransportAction.java | 94 +++ .../ad/transport/ADTaskProfileAction.java | 33 + .../transport/ADTaskProfileNodeRequest.java | 46 ++ .../transport/ADTaskProfileNodeResponse.java | 63 ++ .../ad/transport/ADTaskProfileRequest.java | 63 ++ .../ad/transport/ADTaskProfileResponse.java | 47 ++ .../ADTaskProfileTransportAction.java | 85 +++ .../AnomalyDetectorJobTransportAction.java | 7 +- .../ad/transport/ForwardADTaskAction.java | 32 + .../ad/transport/ForwardADTaskRequest.java | 90 +++ .../ForwardADTaskTransportAction.java | 62 ++ .../transport/GetAnomalyDetectorRequest.java | 12 +- .../transport/GetAnomalyDetectorResponse.java | 31 + .../GetAnomalyDetectorTransportAction.java | 78 ++- .../ad/util/RestHandlerUtils.java | 1 + .../mappings/anomaly-detection-state.json | 6 + .../ad/ADIntegTestCase.java | 31 + .../ad/AbstractProfileRunnerTests.java | 6 +- .../ad/AnomalyDetectorProfileRunnerTests.java | 5 +- .../ad/AnomalyDetectorRestTestCase.java | 31 +- .../ad/HistoricalDetectorIntegTestCase.java | 10 + .../ad/HistoricalDetectorRestTestCase.java | 174 +++++ .../ad/MultiEntityProfileRunnerTests.java | 9 +- .../ad/TestHelpers.java | 8 + .../ad/mock/model/MockSimpleLog.java | 134 ++++ .../ad/mock/plugin/MockReindexPlugin.java | 162 +++++ .../MockAnomalyDetectorJobAction.java | 32 + ...alyDetectorJobTransportActionWithUser.java | 135 ++++ .../ad/rest/AnomalyDetectorRestApiIT.java | 7 +- .../ad/rest/HistoricalDetectorRestApiIT.java | 84 +++ .../ad/task/ADTaskCacheManagerTests.java | 26 +- .../ad/task/ADTaskManagerTests.java | 52 +- .../ad/transport/ADCancelTaskTests.java | 76 +++ .../ad/transport/ADTaskProfileTests.java | 149 +++++ .../ADTaskProfileTransportActionTests.java | 55 ++ ...nomalyDetectorJobTransportActionTests.java | 217 ++++-- .../ad/transport/ForwardADTaskTests.java | 84 +++ .../GetAnomalyDetectorActionTests.java | 5 +- .../ad/transport/GetAnomalyDetectorTests.java | 11 +- ...etAnomalyDetectorTransportActionTests.java | 27 +- 67 files changed, 3766 insertions(+), 217 deletions(-) create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/common/exception/DuplicateTaskException.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskAction.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskProfile.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCancellationState.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskAction.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeRequest.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeResponse.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskRequest.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskResponse.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTransportAction.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileAction.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeRequest.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeResponse.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileRequest.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileResponse.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportAction.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskAction.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskRequest.java create mode 100644 src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTransportAction.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorRestTestCase.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/model/MockSimpleLog.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/plugin/MockReindexPlugin.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobAction.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobTransportActionWithUser.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/HistoricalDetectorRestApiIT.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTests.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTests.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportActionTests.java create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTests.java diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java index 8184ec78..420f17c1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java @@ -44,6 +44,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -61,7 +62,7 @@ import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ExecutorBuilder; -import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; @@ -115,10 +116,14 @@ import com.amazon.opendistroforelasticsearch.ad.transport.ADBatchAnomalyResultTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADBatchTaskRemoteExecutionAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADBatchTaskRemoteExecutionTransportAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ADCancelTaskAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ADCancelTaskTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADResultBulkAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADResultBulkTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADStatsNodesAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADStatsNodesTransportAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ADTaskProfileAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ADTaskProfileTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobAction; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyResultAction; @@ -133,6 +138,8 @@ import com.amazon.opendistroforelasticsearch.ad.transport.EntityProfileTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.EntityResultAction; import com.amazon.opendistroforelasticsearch.ad.transport.EntityResultTransportAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ForwardADTaskAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ForwardADTaskTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.GetAnomalyDetectorAction; import com.amazon.opendistroforelasticsearch.ad.transport.GetAnomalyDetectorTransportAction; import com.amazon.opendistroforelasticsearch.ad.transport.IndexAnomalyDetectorAction; @@ -508,7 +515,16 @@ public Collection createComponents( ); adTaskCacheManager = new ADTaskCacheManager(settings, clusterService, memoryTracker); - adTaskManager = new ADTaskManager(settings, clusterService, client, xContentRegistry, anomalyDetectionIndices); + adTaskManager = new ADTaskManager( + settings, + clusterService, + client, + xContentRegistry, + anomalyDetectionIndices, + nodeFilter, + hashRing, + adTaskCacheManager + ); AnomalyResultBulkIndexHandler anomalyResultBulkIndexHandler = new AnomalyResultBulkIndexHandler( client, settings, @@ -579,18 +595,18 @@ protected Clock getClock() { public List> getExecutorBuilders(Settings settings) { return ImmutableList .of( - new FixedExecutorBuilder( - settings, + new ScalingExecutorBuilder( AD_THREAD_POOL_NAME, + 1, Math.max(1, EsExecutors.allocatedProcessors(settings) / 4), - AnomalyDetectorSettings.AD_THEAD_POOL_QUEUE_SIZE, + TimeValue.timeValueMinutes(10), AD_THREAD_POOL_PREFIX + AD_THREAD_POOL_NAME ), - new FixedExecutorBuilder( - settings, + new ScalingExecutorBuilder( AD_BATCH_TASK_THREAD_POOL_NAME, + 1, Math.max(1, EsExecutors.allocatedProcessors(settings) / 8), - AnomalyDetectorSettings.AD_THEAD_POOL_QUEUE_SIZE, + TimeValue.timeValueMinutes(10), AD_THREAD_POOL_PREFIX + AD_BATCH_TASK_THREAD_POOL_NAME ) ); @@ -671,7 +687,10 @@ public List getNamedXContent() { new ActionHandler<>(SearchAnomalyDetectorInfoAction.INSTANCE, SearchAnomalyDetectorInfoTransportAction.class), new ActionHandler<>(PreviewAnomalyDetectorAction.INSTANCE, PreviewAnomalyDetectorTransportAction.class), new ActionHandler<>(ADBatchAnomalyResultAction.INSTANCE, ADBatchAnomalyResultTransportAction.class), - new ActionHandler<>(ADBatchTaskRemoteExecutionAction.INSTANCE, ADBatchTaskRemoteExecutionTransportAction.class) + new ActionHandler<>(ADBatchTaskRemoteExecutionAction.INSTANCE, ADBatchTaskRemoteExecutionTransportAction.class), + new ActionHandler<>(ADTaskProfileAction.INSTANCE, ADTaskProfileTransportAction.class), + new ActionHandler<>(ADCancelTaskAction.INSTANCE, ADCancelTaskTransportAction.class), + new ActionHandler<>(ForwardADTaskAction.INSTANCE, ForwardADTaskTransportAction.class) ); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunner.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunner.java index b1c26ce4..1ae4ba97 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunner.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunner.java @@ -49,6 +49,7 @@ import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalCardinality; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.transport.TransportService; import com.amazon.opendistroforelasticsearch.ad.common.exception.ResourceNotFoundException; import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; @@ -62,6 +63,7 @@ import com.amazon.opendistroforelasticsearch.ad.model.DetectorState; import com.amazon.opendistroforelasticsearch.ad.model.InitProgressProfile; import com.amazon.opendistroforelasticsearch.ad.model.IntervalTimeConfiguration; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; import com.amazon.opendistroforelasticsearch.ad.transport.ProfileAction; import com.amazon.opendistroforelasticsearch.ad.transport.ProfileRequest; import com.amazon.opendistroforelasticsearch.ad.transport.ProfileResponse; @@ -77,12 +79,16 @@ public class AnomalyDetectorProfileRunner extends AbstractProfileRunner { private Client client; private NamedXContentRegistry xContentRegistry; private DiscoveryNodeFilterer nodeFilter; + private final TransportService transportService; + private final ADTaskManager adTaskManager; public AnomalyDetectorProfileRunner( Client client, NamedXContentRegistry xContentRegistry, DiscoveryNodeFilterer nodeFilter, - long requiredSamples + long requiredSamples, + TransportService transportService, + ADTaskManager adTaskManager ) { super(requiredSamples); this.client = client; @@ -91,6 +97,8 @@ public AnomalyDetectorProfileRunner( if (requiredSamples <= 0) { throw new IllegalArgumentException("required samples should be a positive number, but was " + requiredSamples); } + this.transportService = transportService; + this.adTaskManager = adTaskManager; } public void profile(String detectorId, ActionListener listener, Set profilesToCollect) { @@ -117,7 +125,10 @@ private void calculateTotalResponsesToWait( ) { ensureExpectedToken(XContentParser.Token.START_OBJECT, xContentParser.nextToken(), xContentParser); AnomalyDetector detector = AnomalyDetector.parse(xContentParser, detectorId); - + if (!detector.isRealTimeDetector() && profilesToCollect.contains(DetectorProfileName.AD_TASK)) { + adTaskManager.getLatestADTaskProfile(detectorId, transportService, listener); + return; + } prepareProfile(detector, listener, profilesToCollect); } catch (Exception e) { listener.onFailure(new RuntimeException(CommonErrorMessages.FAIL_TO_FIND_DETECTOR_MSG + detectorId, e)); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/MemoryTracker.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/MemoryTracker.java index 3c906117..0f438863 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/MemoryTracker.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/MemoryTracker.java @@ -194,7 +194,7 @@ public long estimateModelSize(AnomalyDetector detector, int numberOfTrees) { * @param numSamples The number of samples in RCF * @return estimated model size in bytes */ - private long estimateModelSize(int dimension, int numberOfTrees, int numSamples) { + public long estimateModelSize(int dimension, int numberOfTrees, int numSamples) { long totalSamples = (long) numberOfTrees * (long) numSamples; long rcfSize = totalSamples * (40 * dimension + 132); long samplerSize = totalSamples * 36; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/common/exception/DuplicateTaskException.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/common/exception/DuplicateTaskException.java new file mode 100644 index 00000000..cc853b3d --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/common/exception/DuplicateTaskException.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.common.exception; + +public class DuplicateTaskException extends AnomalyDetectionException { + + public DuplicateTaskException(String msg) { + super(msg); + this.countedInStats(false); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java index 253c6b30..69d90af8 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonErrorMessages.java @@ -35,4 +35,7 @@ public class CommonErrorMessages { public static String CATEGORICAL_FIELD_NUMBER_SURPASSED = "We don't support categorical fields more than "; public static String EMPTY_PROFILES_COLLECT = "profiles to collect are missing or invalid"; public static String FAIL_FETCH_ERR_MSG = "Fail to fetch profile for "; + public static String DETECTOR_IS_RUNNING = "Detector is already running"; + public static String DETECTOR_MISSING = "Detector is missing"; + public static String AD_TASK_ACTION_MISSING = "AD task action is missing"; } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonName.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonName.java index b57da50f..06c717c7 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonName.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/constant/CommonName.java @@ -67,6 +67,9 @@ public class CommonName { public static final String ACTIVE_ENTITIES = "active_entities"; public static final String ENTITY_INFO = "entity_info"; public static final String TOTAL_UPDATES = "total_updates"; + public static final String AD_TASK = "ad_task"; + public static final String AD_TASK_REMOTE = "ad_task_remote"; + public static final String CANCEL_TASK = "cancel_task"; // ====================================== // Index mapping diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTask.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTask.java index b14ec0fa..69440219 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTask.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTask.java @@ -51,6 +51,8 @@ public class ADTask implements ToXContentObject, Writeable { public static final String IS_LATEST_FIELD = "is_latest"; public static final String TASK_TYPE_FIELD = "task_type"; public static final String CHECKPOINT_ID_FIELD = "checkpoint_id"; + public static final String COORDINATING_NODE_FIELD = "coordinating_node"; + public static final String WORKER_NODE_FIELD = "worker_node"; public static final String DETECTOR_FIELD = "detector"; private String taskId = null; @@ -70,6 +72,9 @@ public class ADTask implements ToXContentObject, Writeable { private String checkpointId = null; private AnomalyDetector detector = null; + private String coordinatingNode = null; + private String workerNode = null; + private ADTask() {} public ADTask(StreamInput input) throws IOException { @@ -93,6 +98,8 @@ public ADTask(StreamInput input) throws IOException { this.lastUpdateTime = input.readOptionalInstant(); this.startedBy = input.readOptionalString(); this.stoppedBy = input.readOptionalString(); + this.coordinatingNode = input.readOptionalString(); + this.workerNode = input.readOptionalString(); } @Override @@ -118,6 +125,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalInstant(lastUpdateTime); out.writeOptionalString(startedBy); out.writeOptionalString(stoppedBy); + out.writeOptionalString(coordinatingNode); + out.writeOptionalString(workerNode); } public static Builder builder() { @@ -141,6 +150,8 @@ public static class Builder { private Instant lastUpdateTime = null; private String startedBy = null; private String stoppedBy = null; + private String coordinatingNode = null; + private String workerNode = null; public Builder() {} @@ -224,6 +235,16 @@ public Builder detector(AnomalyDetector detector) { return this; } + public Builder coordinatingNode(String coordinatingNode) { + this.coordinatingNode = coordinatingNode; + return this; + } + + public Builder workerNode(String workerNode) { + this.workerNode = workerNode; + return this; + } + public ADTask build() { ADTask adTask = new ADTask(); adTask.taskId = this.taskId; @@ -242,6 +263,8 @@ public ADTask build() { adTask.detector = this.detector; adTask.startedBy = this.startedBy; adTask.stoppedBy = this.stoppedBy; + adTask.coordinatingNode = this.coordinatingNode; + adTask.workerNode = this.workerNode; return adTask; } @@ -296,6 +319,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (checkpointId != null) { xContentBuilder.field(CHECKPOINT_ID_FIELD, checkpointId); } + if (coordinatingNode != null) { + xContentBuilder.field(COORDINATING_NODE_FIELD, coordinatingNode); + } + if (workerNode != null) { + xContentBuilder.field(WORKER_NODE_FIELD, workerNode); + } if (detector != null) { xContentBuilder.field(DETECTOR_FIELD, detector); } @@ -322,6 +351,9 @@ public static ADTask parse(XContentParser parser, String taskId) throws IOExcept String taskType = null; String checkpointId = null; AnomalyDetector detector = null; + String parsedTaskId = taskId; + String coordinatingNode = null; + String workerNode = null; ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { @@ -374,13 +406,44 @@ public static ADTask parse(XContentParser parser, String taskId) throws IOExcept case DETECTOR_FIELD: detector = AnomalyDetector.parse(parser); break; + case TASK_ID_FIELD: + parsedTaskId = parser.text(); + break; + case COORDINATING_NODE_FIELD: + coordinatingNode = parser.text(); + break; + case WORKER_NODE_FIELD: + workerNode = parser.text(); + break; default: parser.skipChildren(); break; } } + AnomalyDetector anomalyDetector = detector == null + ? null + : new AnomalyDetector( + detectorId, + detector.getVersion(), + detector.getName(), + detector.getDescription(), + detector.getTimeField(), + detector.getIndices(), + detector.getFeatureAttributes(), + detector.getFilterQuery(), + detector.getDetectionInterval(), + detector.getWindowDelay(), + detector.getShingleSize(), + detector.getUiMetadata(), + detector.getSchemaVersion(), + detector.getLastUpdateTime(), + detector.getCategoryField(), + detector.getUser(), + detector.getDetectorType(), + detector.getDetectionDateRange() + ); return new Builder() - .taskId(taskId) + .taskId(parsedTaskId) .lastUpdateTime(lastUpdateTime) .startedBy(startedBy) .stoppedBy(stoppedBy) @@ -395,7 +458,9 @@ public static ADTask parse(XContentParser parser, String taskId) throws IOExcept .isLatest(isLatest) .taskType(taskType) .checkpointId(checkpointId) - .detector(detector) + .coordinatingNode(coordinatingNode) + .workerNode(workerNode) + .detector(anomalyDetector) .build(); } @@ -422,6 +487,8 @@ public boolean equals(Object o) { && Objects.equal(getLatest(), that.getLatest()) && Objects.equal(getTaskType(), that.getTaskType()) && Objects.equal(getCheckpointId(), that.getCheckpointId()) + && Objects.equal(getCoordinatingNode(), that.getCoordinatingNode()) + && Objects.equal(getWorkerNode(), that.getWorkerNode()) && Objects.equal(getDetector(), that.getDetector()); } @@ -445,6 +512,8 @@ public int hashCode() { isLatest, taskType, checkpointId, + coordinatingNode, + workerNode, detector ); } @@ -477,6 +546,10 @@ public String getState() { return state; } + public void setState(String state) { + this.state = state; + } + public String getDetectorId() { return detectorId; } @@ -516,4 +589,13 @@ public String getCheckpointId() { public AnomalyDetector getDetector() { return detector; } + + public String getCoordinatingNode() { + return coordinatingNode; + } + + public String getWorkerNode() { + return workerNode; + } + } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskAction.java new file mode 100644 index 00000000..53ee6f1a --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskAction.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.model; + +public enum ADTaskAction { + START, + STOP +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskProfile.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskProfile.java new file mode 100644 index 00000000..baedc905 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/ADTaskProfile.java @@ -0,0 +1,300 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.model; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import com.amazon.opendistroforelasticsearch.ad.annotation.Generated; +import com.google.common.base.Objects; + +/** + * One anomaly detection task means one detector starts to run until stopped. + */ +public class ADTaskProfile implements ToXContentObject, Writeable { + + public static final String AD_TASK_FIELD = "ad_task"; + public static final String SHINGLE_SIZE_FIELD = "shingle_size"; + public static final String RCF_TOTAL_UPDATES_FIELD = "rcf_total_updates"; + public static final String THRESHOLD_MODEL_TRAINED_FIELD = "threshold_model_trained"; + public static final String THRESHOLD_MODEL_TRAINING_DATA_SIZE_FIELD = "threshold_model_training_data_size"; + public static final String MODEL_SIZE_IN_BYTES = "model_size_in_bytes"; + public static final String NODE_ID_FIELD = "node_id"; + + private ADTask adTask; + private Integer shingleSize; + private Long rcfTotalUpdates; + private Boolean thresholdModelTrained; + private Integer thresholdModelTrainingDataSize; + private Long modelSizeInBytes; + private String nodeId; + + public ADTaskProfile( + Integer shingleSize, + Long rcfTotalUpdates, + Boolean thresholdModelTrained, + Integer thresholdModelTrainingDataSize, + Long modelSizeInBytes, + String nodeId + ) { + this(null, shingleSize, rcfTotalUpdates, thresholdModelTrained, thresholdModelTrainingDataSize, modelSizeInBytes, nodeId); + } + + public ADTaskProfile( + ADTask adTask, + Integer shingleSize, + Long rcfTotalUpdates, + Boolean thresholdModelTrained, + Integer thresholdModelTrainingDataSize, + Long modelSizeInBytes, + String nodeId + ) { + this.adTask = adTask; + this.shingleSize = shingleSize; + this.rcfTotalUpdates = rcfTotalUpdates; + this.thresholdModelTrained = thresholdModelTrained; + this.thresholdModelTrainingDataSize = thresholdModelTrainingDataSize; + this.modelSizeInBytes = modelSizeInBytes; + this.nodeId = nodeId; + } + + public ADTaskProfile(StreamInput input) throws IOException { + if (input.readBoolean()) { + this.adTask = new ADTask(input); + } else { + this.adTask = null; + } + this.shingleSize = input.readOptionalInt(); + this.rcfTotalUpdates = input.readOptionalLong(); + this.thresholdModelTrained = input.readOptionalBoolean(); + this.thresholdModelTrainingDataSize = input.readOptionalInt(); + this.modelSizeInBytes = input.readOptionalLong(); + this.nodeId = input.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (adTask != null) { + out.writeBoolean(true); + adTask.writeTo(out); + } else { + out.writeBoolean(false); + } + + out.writeOptionalInt(shingleSize); + out.writeOptionalLong(rcfTotalUpdates); + out.writeOptionalBoolean(thresholdModelTrained); + out.writeOptionalInt(thresholdModelTrainingDataSize); + out.writeOptionalLong(modelSizeInBytes); + out.writeOptionalString(nodeId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + XContentBuilder xContentBuilder = builder.startObject(); + if (adTask != null) { + xContentBuilder.field(AD_TASK_FIELD, adTask); + } + if (shingleSize != null) { + xContentBuilder.field(SHINGLE_SIZE_FIELD, shingleSize); + } + if (rcfTotalUpdates != null) { + xContentBuilder.field(RCF_TOTAL_UPDATES_FIELD, rcfTotalUpdates); + } + if (thresholdModelTrained != null) { + xContentBuilder.field(THRESHOLD_MODEL_TRAINED_FIELD, thresholdModelTrained); + } + if (thresholdModelTrainingDataSize != null) { + xContentBuilder.field(THRESHOLD_MODEL_TRAINING_DATA_SIZE_FIELD, thresholdModelTrainingDataSize); + } + if (modelSizeInBytes != null) { + xContentBuilder.field(MODEL_SIZE_IN_BYTES, modelSizeInBytes); + } + if (nodeId != null) { + xContentBuilder.field(NODE_ID_FIELD, nodeId); + } + return xContentBuilder.endObject(); + } + + public static ADTaskProfile parse(XContentParser parser) throws IOException { + ADTask adTask = null; + Integer shingleSize = null; + Long rcfTotalUpdates = null; + Boolean thresholdModelTrained = null; + Integer thresholdNodelTrainingDataSize = null; + Long modelSizeInBytes = null; + String nodeId = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + + switch (fieldName) { + case AD_TASK_FIELD: + adTask = ADTask.parse(parser); + break; + case SHINGLE_SIZE_FIELD: + shingleSize = parser.intValue(); + break; + case RCF_TOTAL_UPDATES_FIELD: + rcfTotalUpdates = parser.longValue(); + break; + case THRESHOLD_MODEL_TRAINED_FIELD: + thresholdModelTrained = parser.booleanValue(); + break; + case THRESHOLD_MODEL_TRAINING_DATA_SIZE_FIELD: + thresholdNodelTrainingDataSize = parser.intValue(); + break; + case MODEL_SIZE_IN_BYTES: + modelSizeInBytes = parser.longValue(); + break; + case NODE_ID_FIELD: + nodeId = parser.text(); + break; + default: + parser.skipChildren(); + break; + } + } + return new ADTaskProfile( + adTask, + shingleSize, + rcfTotalUpdates, + thresholdModelTrained, + thresholdNodelTrainingDataSize, + modelSizeInBytes, + nodeId + ); + } + + @Generated + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ADTaskProfile that = (ADTaskProfile) o; + return Objects.equal(getAdTask(), that.getAdTask()) + && Objects.equal(getShingleSize(), that.getShingleSize()) + && Objects.equal(getRcfTotalUpdates(), that.getRcfTotalUpdates()) + && Objects.equal(getThresholdModelTrained(), that.getThresholdModelTrained()) + && Objects.equal(getModelSizeInBytes(), that.getModelSizeInBytes()) + && Objects.equal(getNodeId(), that.getNodeId()) + && Objects.equal(getThresholdModelTrainingDataSize(), that.getThresholdModelTrainingDataSize()); + } + + @Generated + @Override + public int hashCode() { + return Objects + .hashCode( + adTask, + shingleSize, + rcfTotalUpdates, + thresholdModelTrained, + thresholdModelTrainingDataSize, + modelSizeInBytes, + nodeId + ); + } + + public ADTask getAdTask() { + return adTask; + } + + public void setAdTask(ADTask adTask) { + this.adTask = adTask; + } + + public Integer getShingleSize() { + return shingleSize; + } + + public void setShingleSize(Integer shingleSize) { + this.shingleSize = shingleSize; + } + + public Long getRcfTotalUpdates() { + return rcfTotalUpdates; + } + + public void setRcfTotalUpdates(Long rcfTotalUpdates) { + this.rcfTotalUpdates = rcfTotalUpdates; + } + + public Boolean getThresholdModelTrained() { + return thresholdModelTrained; + } + + public void setThresholdModelTrained(Boolean thresholdModelTrained) { + this.thresholdModelTrained = thresholdModelTrained; + } + + public Integer getThresholdModelTrainingDataSize() { + return thresholdModelTrainingDataSize; + } + + public void setThresholdModelTrainingDataSize(Integer thresholdModelTrainingDataSize) { + this.thresholdModelTrainingDataSize = thresholdModelTrainingDataSize; + } + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public Long getModelSizeInBytes() { + return modelSizeInBytes; + } + + public void setModelSizeInBytes(Long modelSizeInBytes) { + this.modelSizeInBytes = modelSizeInBytes; + } + + @Override + public String toString() { + return "ADTaskProfile{" + + "adTask=" + + adTask + + ", shingleSize=" + + shingleSize + + ", rcfTotalUpdates=" + + rcfTotalUpdates + + ", thresholdModelTrained=" + + thresholdModelTrained + + ", thresholdNodelTrainingDataSize=" + + thresholdModelTrainingDataSize + + ", modelSizeInBytes=" + + modelSizeInBytes + + ", nodeId='" + + nodeId + + '\'' + + '}'; + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/AnomalyDetector.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/AnomalyDetector.java index e4cd2c70..4cc373a3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/AnomalyDetector.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/AnomalyDetector.java @@ -55,6 +55,9 @@ /** * An AnomalyDetector is used to represent anomaly detection model(RCF) related parameters. + * NOTE: If change detector config index mapping, you should change AD task index mapping as well. + * TODO: Will replace detector config mapping in AD task with detector config setting directly \ + * in code rather than config it in anomaly-detection-state.json file. */ public class AnomalyDetector implements Writeable, ToXContentObject { @@ -227,10 +230,10 @@ public AnomalyDetector( } public AnomalyDetector(StreamInput input) throws IOException { - detectorId = input.readString(); - version = input.readLong(); + detectorId = input.readOptionalString(); + version = input.readOptionalLong(); name = input.readString(); - description = input.readString(); + description = input.readOptionalString(); timeField = input.readString(); indices = input.readStringList(); featureAttributes = input.readList(Feature::new); @@ -265,10 +268,10 @@ public XContentBuilder toXContent(XContentBuilder builder) throws IOException { @Override public void writeTo(StreamOutput output) throws IOException { - output.writeString(detectorId); - output.writeLong(version); + output.writeOptionalString(detectorId); + output.writeOptionalLong(version); output.writeString(name); - output.writeString(description); + output.writeOptionalString(description); output.writeString(timeField); output.writeStringCollection(indices); output.writeList(featureAttributes); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfile.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfile.java index 86ba9126..e473577f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfile.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfile.java @@ -39,19 +39,28 @@ public class DetectorProfile implements Writeable, ToXContentObject, Mergeable { private InitProgressProfile initProgress; private Long totalEntities; private Long activeEntities; + private ADTaskProfile adTaskProfile; public XContentBuilder toXContent(XContentBuilder builder) throws IOException { return toXContent(builder, ToXContent.EMPTY_PARAMS); } public DetectorProfile(StreamInput in) throws IOException { - this.state = in.readEnum(DetectorState.class); - this.error = in.readString(); - this.modelProfile = in.readArray(ModelProfile::new, ModelProfile[]::new); - this.shingleSize = in.readInt(); - this.coordinatingNode = in.readString(); - this.totalSizeInBytes = in.readLong(); - this.initProgress = new InitProgressProfile(in); + if (in.readBoolean()) { + this.state = in.readEnum(DetectorState.class); + } + + this.error = in.readOptionalString(); + this.modelProfile = in.readOptionalArray(ModelProfile::new, ModelProfile[]::new); + this.shingleSize = in.readOptionalInt(); + this.coordinatingNode = in.readOptionalString(); + this.totalSizeInBytes = in.readOptionalLong(); + if (in.readBoolean()) { + this.initProgress = new InitProgressProfile(in); + } + if (in.readBoolean()) { + this.adTaskProfile = new ADTaskProfile(in); + } } private DetectorProfile() {} @@ -66,6 +75,7 @@ public static class Builder { private InitProgressProfile initProgress = null; private Long totalEntities; private Long activeEntities; + private ADTaskProfile adTaskProfile; public Builder() {} @@ -114,6 +124,11 @@ public Builder activeEntities(Long activeEntities) { return this; } + public Builder adTaskProfile(ADTaskProfile adTaskProfile) { + this.adTaskProfile = adTaskProfile; + return this; + } + public DetectorProfile build() { DetectorProfile profile = new DetectorProfile(); profile.state = this.state; @@ -125,6 +140,7 @@ public DetectorProfile build() { profile.initProgress = initProgress; profile.totalEntities = totalEntities; profile.activeEntities = activeEntities; + profile.adTaskProfile = adTaskProfile; return profile; } @@ -132,13 +148,30 @@ public DetectorProfile build() { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeEnum(state); - out.writeString(error); - out.writeArray(modelProfile); - out.writeInt(shingleSize); - out.writeString(coordinatingNode); - out.writeLong(totalSizeInBytes); - initProgress.writeTo(out); + if (state == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeEnum(state); + } + + out.writeOptionalString(error); + out.writeOptionalArray(modelProfile); + out.writeOptionalInt(shingleSize); + out.writeOptionalString(coordinatingNode); + out.writeOptionalLong(totalSizeInBytes); + if (initProgress == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + initProgress.writeTo(out); + } + if (adTaskProfile == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + adTaskProfile.writeTo(out); + } } @Override @@ -176,6 +209,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (activeEntities != null) { xContentBuilder.field(CommonName.ACTIVE_ENTITIES, activeEntities); } + if (adTaskProfile != null) { + xContentBuilder.field(CommonName.AD_TASK, adTaskProfile); + } return xContentBuilder.endObject(); } @@ -251,6 +287,14 @@ public void setActiveEntities(Long activeEntities) { this.activeEntities = activeEntities; } + public ADTaskProfile getAdTaskProfile() { + return adTaskProfile; + } + + public void setAdTaskProfile(ADTaskProfile adTaskProfile) { + this.adTaskProfile = adTaskProfile; + } + @Override public void merge(Mergeable other) { if (this == other || other == null || getClass() != other.getClass()) { @@ -284,6 +328,9 @@ public void merge(Mergeable other) { if (otherProfile.getActiveEntities() != null) { this.activeEntities = otherProfile.getActiveEntities(); } + if (otherProfile.getAdTaskProfile() != null) { + this.adTaskProfile = otherProfile.getAdTaskProfile(); + } } @Override @@ -325,6 +372,9 @@ public boolean equals(Object obj) { if (activeEntities != null) { equalsBuilder.append(activeEntities, other.activeEntities); } + if (adTaskProfile != null) { + equalsBuilder.append(adTaskProfile, other.adTaskProfile); + } return equalsBuilder.isEquals(); } return false; @@ -342,6 +392,7 @@ public int hashCode() { .append(initProgress) .append(totalEntities) .append(activeEntities) + .append(adTaskProfile) .toHashCode(); } @@ -376,6 +427,9 @@ public String toString() { if (activeEntities != null) { toStringBuilder.append(CommonName.ACTIVE_ENTITIES, activeEntities); } + if (adTaskProfile != null) { + toStringBuilder.append(CommonName.AD_TASK, adTaskProfile); + } return toStringBuilder.toString(); } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfileName.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfileName.java index 3ae6601b..5ca83db9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfileName.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/model/DetectorProfileName.java @@ -30,7 +30,8 @@ public enum DetectorProfileName implements Name { MODELS(CommonName.MODELS), INIT_PROGRESS(CommonName.INIT_PROGRESS), TOTAL_ENTITIES(CommonName.TOTAL_ENTITIES), - ACTIVE_ENTITIES(CommonName.ACTIVE_ENTITIES); + ACTIVE_ENTITIES(CommonName.ACTIVE_ENTITIES), + AD_TASK(CommonName.AD_TASK); private String name; @@ -68,6 +69,8 @@ public static DetectorProfileName getName(String name) { return TOTAL_ENTITIES; case CommonName.ACTIVE_ENTITIES: return ACTIVE_ENTITIES; + case CommonName.AD_TASK: + return AD_TASK; default: throw new IllegalArgumentException("Unsupported profile types"); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java index 1738f6aa..f051970c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/RestGetAnomalyDetectorAction.java @@ -64,11 +64,13 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli String entityValue = request.param(ENTITY); String rawPath = request.rawPath(); boolean returnJob = request.paramAsBoolean("job", false); + boolean returnTask = request.paramAsBoolean("task", false); boolean all = request.paramAsBoolean("_all", false); GetAnomalyDetectorRequest getAnomalyDetectorRequest = new GetAnomalyDetectorRequest( detectorId, RestActions.parseVersion(request), returnJob, + returnTask, typesStr, rawPath, all, diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/handler/IndexAnomalyDetectorJobActionHandler.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/handler/IndexAnomalyDetectorJobActionHandler.java index f1eb79bc..a9e32106 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/handler/IndexAnomalyDetectorJobActionHandler.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/rest/handler/IndexAnomalyDetectorJobActionHandler.java @@ -302,7 +302,13 @@ private ActionListener stopAdDetectorListener(String detec public void onResponse(StopDetectorResponse stopDetectorResponse) { if (stopDetectorResponse.success()) { logger.info("AD model deleted successfully for detector {}", detectorId); - AnomalyDetectorJobResponse anomalyDetectorJobResponse = new AnomalyDetectorJobResponse(null, 0, 0, 0, RestStatus.OK); + AnomalyDetectorJobResponse anomalyDetectorJobResponse = new AnomalyDetectorJobResponse( + detectorId, + 0, + 0, + 0, + RestStatus.OK + ); listener.onResponse(anomalyDetectorJobResponse); } else { logger.error("Failed to delete AD model for detector {}", detectorId); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskCache.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskCache.java index a301370a..a5fff9e6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskCache.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskCache.java @@ -41,6 +41,7 @@ */ public class ADBatchTaskCache { private final String detectorId; + private final String taskId; private RandomCutForest rcfModel; private ThresholdingModel thresholdModel; private boolean thresholdModelTrained; @@ -54,6 +55,7 @@ public class ADBatchTaskCache { protected ADBatchTaskCache(ADTask adTask) { this.detectorId = adTask.getDetectorId(); + this.taskId = adTask.getTaskId(); AnomalyDetector detector = adTask.getDetector(); rcfModel = RandomCutForest @@ -83,6 +85,10 @@ protected String getDetectorId() { return detectorId; } + protected String getTaskId() { + return taskId; + } + protected RandomCutForest getRcfModel() { return rcfModel; } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskRunner.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskRunner.java index c284c41f..29b3c5b2 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskRunner.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADBatchTaskRunner.java @@ -24,6 +24,7 @@ import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.INIT_PROGRESS_FIELD; import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.STATE_FIELD; import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.TASK_PROGRESS_FIELD; +import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.WORKER_NODE_FIELD; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_SIZE; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE; @@ -197,7 +198,7 @@ public void run(ADTask adTask, TransportService transportService, ActionListener node.getId(), adTask.getDetectorId() ); - startADBatchTask(adTask, false, delegatedListener); + startADBatchTask(adTask, false, transportService, delegatedListener); } else { // Execute batch task remotely logger @@ -273,19 +274,25 @@ private void dispatchTask(ADTask adTask, ActionListener listener) } /** - * Start AD task in dedicated batch task thread pool. + * Start AD task in dedicated batch task thread pool on worker node. * * @param adTask ad task * @param runTaskRemotely run task remotely or not + * @param transportService transport service * @param listener action listener */ - public void startADBatchTask(ADTask adTask, boolean runTaskRemotely, ActionListener listener) { + public void startADBatchTask( + ADTask adTask, + boolean runTaskRemotely, + TransportService transportService, + ActionListener listener + ) { try { // check if cluster is eligible to run AD currently, if not eligible like // circuit breaker open, will throw exception. checkClusterState(adTask); threadPool.executor(AD_BATCH_TASK_THREAD_POOL_NAME).execute(() -> { - ActionListener internalListener = internalBatchTaskListener(adTask); + ActionListener internalListener = internalBatchTaskListener(adTask, transportService); try { executeADBatchTask(adTask, internalListener); } catch (Exception e) { @@ -299,23 +306,29 @@ public void startADBatchTask(ADTask adTask, boolean runTaskRemotely, ActionListe } } - private ActionListener internalBatchTaskListener(ADTask adTask) { + private ActionListener internalBatchTaskListener(ADTask adTask, TransportService transportService) { String taskId = adTask.getTaskId(); ActionListener listener = ActionListener.wrap(response -> { // If batch task finished normally, remove task from cache and decrease executing task count by 1. adTaskCacheManager.remove(taskId); adStats.getStat(AD_EXECUTING_BATCH_TASK_COUNT.getName()).decrement(); + + adTaskManager + .cleanDetectorCache( + adTask, + transportService, + () -> adTaskManager.updateADTask(taskId, ImmutableMap.of(STATE_FIELD, ADTaskState.FINISHED.name())) + ); }, e -> { // If batch task failed, remove task from cache and decrease executing task count by 1. adTaskCacheManager.remove(taskId); adStats.getStat(AD_EXECUTING_BATCH_TASK_COUNT.getName()).decrement(); - handleException(adTask, e); + adTaskManager.cleanDetectorCache(adTask, transportService, () -> handleException(adTask, e)); }); return listener; } private void handleException(ADTask adTask, Exception e) { - logger.debug("Failed to run task " + adTask.getTaskId() + " for detector " + adTask.getDetectorId(), e); // Check if batch task was cancelled or not by exception type. // If it's cancelled, then increase cancelled task count by 1, otherwise increase failure count by 1. if (e instanceof ADTaskCancelledException) { @@ -333,7 +346,7 @@ private void executeADBatchTask(ADTask adTask, ActionListener internalLi adStats.getStat(StatNames.AD_TOTAL_BATCH_TASK_EXECUTION_COUNT.getName()).increment(); // put AD task into cache - adTaskCacheManager.put(adTask); + adTaskCacheManager.add(adTask); // start to run first piece Instant executeStartTime = Instant.now(); @@ -378,7 +391,9 @@ private void runFirstPiece(ADTask adTask, Instant executeStartTime, ActionListen TASK_PROGRESS_FIELD, 0.0f, INIT_PROGRESS_FIELD, - 0.0f + 0.0f, + WORKER_NODE_FIELD, + clusterService.localNode().getId() ), ActionListener.wrap(r -> { try { @@ -697,8 +712,6 @@ private void runNextPiece( taskId, ImmutableMap .of( - STATE_FIELD, - ADTaskState.FINISHED.name(), CURRENT_PIECE_FIELD, dataEndTime, TASK_PROGRESS_FIELD, diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManager.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManager.java index 764e6706..a9d4331b 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManager.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManager.java @@ -16,21 +16,30 @@ package com.amazon.opendistroforelasticsearch.ad.task; import static com.amazon.opendistroforelasticsearch.ad.MemoryTracker.Origin.HISTORICAL_SINGLE_ENTITY_DETECTOR; +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.NUM_TREES; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.THRESHOLD_MODEL_TRAINING_SIZE; import java.util.Deque; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import com.amazon.opendistroforelasticsearch.ad.MemoryTracker; +import com.amazon.opendistroforelasticsearch.ad.common.exception.DuplicateTaskException; import com.amazon.opendistroforelasticsearch.ad.common.exception.LimitExceededException; import com.amazon.opendistroforelasticsearch.ad.ml.ThresholdingModel; import com.amazon.opendistroforelasticsearch.ad.model.ADTask; @@ -38,12 +47,21 @@ import com.amazon.randomcutforest.RandomCutForest; public class ADTaskCacheManager { - + private final Logger logger = LogManager.getLogger(ADTaskCacheManager.class); private final Map taskCaches; private volatile Integer maxAdBatchTaskPerNode; private final MemoryTracker memoryTracker; private final int numberSize = 8; + // We use this field to record all detectors which running on the + // coordinating node to resolve race condition. We will check if + // detector id exists in cache or not first. If user starts + // multiple tasks for the same detector, we will put the first + // task in cache. For other tasks, we find the detector id exists, + // that means there is already one task running for this detector, + // so we will reject the task. + private Set detectors; + /** * Constructor to create AD task cache manager. * @@ -56,6 +74,7 @@ public ADTaskCacheManager(Settings settings, ClusterService clusterService, Memo clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_BATCH_TASK_PER_NODE, it -> maxAdBatchTaskPerNode = it); taskCaches = new ConcurrentHashMap<>(); this.memoryTracker = memoryTracker; + this.detectors = Sets.newConcurrentHashSet(); } /** @@ -66,13 +85,13 @@ public ADTaskCacheManager(Settings settings, ClusterService clusterService, Memo * * @param adTask AD task */ - public synchronized void put(ADTask adTask) { + public synchronized void add(ADTask adTask) { String taskId = adTask.getTaskId(); if (contains(taskId)) { - throw new IllegalArgumentException("AD task is already running"); + throw new DuplicateTaskException(DETECTOR_IS_RUNNING); } if (containsTaskOfDetector(adTask.getDetectorId())) { - throw new IllegalArgumentException("There is one task executing for detector"); + throw new DuplicateTaskException(DETECTOR_IS_RUNNING); } checkRunningTaskLimit(); long neededCacheSize = calculateADTaskCacheSize(adTask); @@ -85,6 +104,21 @@ public synchronized void put(ADTask adTask) { taskCaches.put(taskId, taskCache); } + /** + * Put detector id in running detector cache. + * + * @param detectorId detector id + * @throws LimitExceededException throw limit exceed exception when the detector id already in cache + */ + public synchronized void add(String detectorId) { + if (detectors.contains(detectorId)) { + logger.debug("detector is already in running detector cache, detectorId: " + detectorId); + throw new DuplicateTaskException(DETECTOR_IS_RUNNING); + } + logger.debug("add detector in running detector cache, detectorId: " + detectorId); + this.detectors.add(detectorId); + } + /** * check if current running batch task on current node exceeds * max running task limitation. @@ -131,10 +165,23 @@ public double[] getThresholdModelTrainingData(String taskId) { return getBatchTaskCache(taskId).getThresholdModelTrainingData(); } + /** + * Get threshhold model training data size in bytes. + * + * @param taskId task id + * @return training data size in bytes + */ public int getThresholdModelTrainingDataSize(String taskId) { return getBatchTaskCache(taskId).getThresholdModelTrainingDataSize().get(); } + /** + * Add threshold model training data. + * + * @param taskId task id + * @param data training data + * @return latest threshold model training data size after adding new data + */ public int addThresholdModelTrainingData(String taskId, double... data) { ADBatchTaskCache taskCache = getBatchTaskCache(taskId); double[] thresholdModelTrainingData = taskCache.getThresholdModelTrainingData(); @@ -203,6 +250,21 @@ public boolean containsTaskOfDetector(String detectorId) { return taskCaches.values().stream().filter(v -> Objects.equals(detectorId, v.getDetectorId())).findAny().isPresent(); } + /** + * Get task id list of detector. + * + * @param detectorId detector id + * @return list of task id + */ + public List getTasksOfDetector(String detectorId) { + return taskCaches + .values() + .stream() + .filter(v -> Objects.equals(detectorId, v.getDetectorId())) + .map(c -> c.getTaskId()) + .collect(Collectors.toList()); + } + /** * Get batch task cache. If task doesn't exist in cache, will throw * {@link java.lang.IllegalArgumentException} @@ -220,6 +282,10 @@ private ADBatchTaskCache getBatchTaskCache(String taskId) { return taskCaches.get(taskId); } + private List getBatchTaskCacheByDetectorId(String detectorId) { + return taskCaches.values().stream().filter(v -> Objects.equals(detectorId, v.getDetectorId())).collect(Collectors.toList()); + } + /** * Calculate AD task cache memory usage. * @@ -232,6 +298,19 @@ private long calculateADTaskCacheSize(ADTask adTask) { + shingleMemorySize(detector.getShingleSize(), detector.getEnabledFeatureIds().size()); } + /** + * Get RCF model size in bytes. + * + * @param taskId task id + * @return model size in bytes + */ + public long getModelSize(String taskId) { + ADBatchTaskCache batchTaskCache = getBatchTaskCache(taskId); + int dimensions = batchTaskCache.getRcfModel().getDimensions(); + int numberOfTrees = batchTaskCache.getRcfModel().getNumberOfTrees(); + return memoryTracker.estimateModelSize(dimensions, numberOfTrees, NUM_SAMPLES_PER_TREE); + } + /** * Remove task from cache. * @@ -239,8 +318,25 @@ private long calculateADTaskCacheSize(ADTask adTask) { */ public void remove(String taskId) { if (contains(taskId)) { - memoryTracker.releaseMemory(getBatchTaskCache(taskId).getCacheMemorySize().get(), true, HISTORICAL_SINGLE_ENTITY_DETECTOR); + ADBatchTaskCache taskCache = getBatchTaskCache(taskId); + memoryTracker.releaseMemory(taskCache.getCacheMemorySize().get(), true, HISTORICAL_SINGLE_ENTITY_DETECTOR); taskCaches.remove(taskId); + // can't remove detector id from cache here as it's possible that some task running on + // other worker nodes + } + } + + /** + * Remove detector id from running detector cache + * + * @param detectorId detector id + */ + public void removeDetector(String detectorId) { + if (detectors.contains(detectorId)) { + detectors.remove(detectorId); + logger.debug("Removed detector from AD task coordinating node cache, detectorId: " + detectorId); + } else { + logger.debug("Detector is not in AD task coordinating node cache"); } } @@ -250,9 +346,42 @@ public void remove(String taskId) { * @param taskId AD task id * @param reason why need to cancel task * @param userName user name + * @return AD task cancellation state */ - public void cancel(String taskId, String reason, String userName) { + public ADTaskCancellationState cancel(String taskId, String reason, String userName) { + if (!contains(taskId)) { + return ADTaskCancellationState.NOT_FOUND; + } + if (isCancelled(taskId)) { + return ADTaskCancellationState.ALREADY_CANCELLED; + } getBatchTaskCache(taskId).cancel(reason, userName); + return ADTaskCancellationState.CANCELLED; + } + + /** + * Cancel AD task by detector id. + * + * @param detectorId detector id + * @param reason why need to cancel task + * @param userName user name + * @return AD task cancellation state + */ + public ADTaskCancellationState cancelByDetectorId(String detectorId, String reason, String userName) { + List taskCaches = getBatchTaskCacheByDetectorId(detectorId); + + if (taskCaches.isEmpty()) { + return ADTaskCancellationState.NOT_FOUND; + } + + ADTaskCancellationState cancellationState = ADTaskCancellationState.ALREADY_CANCELLED; + for (ADBatchTaskCache cache : taskCaches) { + if (!cache.isCancelled()) { + cancellationState = ADTaskCancellationState.CANCELLED; + cache.cancel(reason, userName); + } + } + return cancellationState; } /** @@ -300,6 +429,7 @@ public int size() { */ public void clear() { taskCaches.clear(); + detectors.clear(); } /** diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCancellationState.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCancellationState.java new file mode 100644 index 00000000..893ed369 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCancellationState.java @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.task; + +public enum ADTaskCancellationState { + NOT_FOUND, + CANCELLED, + ALREADY_CANCELLED +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManager.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManager.java index 1391bdac..5aad2e21 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManager.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManager.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.ad.task; +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING; import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.DETECTOR_ID_FIELD; import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.ERROR_FIELD; import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD; @@ -22,19 +23,25 @@ import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.IS_LATEST_FIELD; import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD; import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.STATE_FIELD; +import static com.amazon.opendistroforelasticsearch.ad.model.ADTask.STOPPED_BY_FIELD; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT; import static com.amazon.opendistroforelasticsearch.ad.util.ExceptionUtil.getErrorMessage; import static com.amazon.opendistroforelasticsearch.ad.util.ExceptionUtil.getShardsFailure; import static org.elasticsearch.action.DocWriteResponse.Result.CREATED; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -42,7 +49,10 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; @@ -51,16 +61,18 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.reindex.DeleteByQueryAction; import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.index.reindex.UpdateByQueryAction; @@ -70,17 +82,37 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; +import com.amazon.opendistroforelasticsearch.ad.cluster.HashRing; +import com.amazon.opendistroforelasticsearch.ad.common.exception.ADTaskCancelledException; +import com.amazon.opendistroforelasticsearch.ad.common.exception.DuplicateTaskException; +import com.amazon.opendistroforelasticsearch.ad.common.exception.InternalFailure; +import com.amazon.opendistroforelasticsearch.ad.common.exception.LimitExceededException; +import com.amazon.opendistroforelasticsearch.ad.common.exception.ResourceNotFoundException; import com.amazon.opendistroforelasticsearch.ad.constant.CommonName; import com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices; import com.amazon.opendistroforelasticsearch.ad.model.ADTask; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskAction; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskProfile; import com.amazon.opendistroforelasticsearch.ad.model.ADTaskState; import com.amazon.opendistroforelasticsearch.ad.model.ADTaskType; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; +import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfile; +import com.amazon.opendistroforelasticsearch.ad.rest.handler.AnomalyDetectorFunction; import com.amazon.opendistroforelasticsearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler; import com.amazon.opendistroforelasticsearch.ad.transport.ADBatchAnomalyResultAction; import com.amazon.opendistroforelasticsearch.ad.transport.ADBatchAnomalyResultRequest; +import com.amazon.opendistroforelasticsearch.ad.transport.ADCancelTaskAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ADCancelTaskRequest; +import com.amazon.opendistroforelasticsearch.ad.transport.ADTaskProfileAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ADTaskProfileNodeResponse; +import com.amazon.opendistroforelasticsearch.ad.transport.ADTaskProfileRequest; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobResponse; +import com.amazon.opendistroforelasticsearch.ad.transport.ForwardADTaskAction; +import com.amazon.opendistroforelasticsearch.ad.transport.ForwardADTaskRequest; +import com.amazon.opendistroforelasticsearch.ad.util.DiscoveryNodeFilterer; import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; import com.amazon.opendistroforelasticsearch.commons.authuser.User; @@ -91,25 +123,45 @@ public class ADTaskManager { private final Logger logger = LogManager.getLogger(this.getClass()); private final Client client; + private final ClusterService clusterService; private final NamedXContentRegistry xContentRegistry; private final AnomalyDetectionIndices detectionIndices; + private final DiscoveryNodeFilterer nodeFilter; + private final ADTaskCacheManager adTaskCacheManager; + + private final HashRing hashRing; private volatile Integer maxAdTaskDocsPerDetector; + private volatile Integer pieceIntervalSeconds; + private volatile TimeValue requestTimeout; public ADTaskManager( Settings settings, ClusterService clusterService, Client client, NamedXContentRegistry xContentRegistry, - AnomalyDetectionIndices detectionIndices + AnomalyDetectionIndices detectionIndices, + DiscoveryNodeFilterer nodeFilter, + HashRing hashRing, + ADTaskCacheManager adTaskCacheManager ) { this.client = client; this.xContentRegistry = xContentRegistry; this.detectionIndices = detectionIndices; + this.nodeFilter = nodeFilter; + this.clusterService = clusterService; + this.adTaskCacheManager = adTaskCacheManager; + this.hashRing = hashRing; this.maxAdTaskDocsPerDetector = MAX_OLD_AD_TASK_DOCS_PER_DETECTOR.get(settings); clusterService .getClusterSettings() .addSettingsUpdateConsumer(MAX_OLD_AD_TASK_DOCS_PER_DETECTOR, it -> maxAdTaskDocsPerDetector = it); + + this.pieceIntervalSeconds = BATCH_TASK_PIECE_INTERVAL_SECONDS.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(BATCH_TASK_PIECE_INTERVAL_SECONDS, it -> pieceIntervalSeconds = it); + + this.requestTimeout = REQUEST_TIMEOUT.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(REQUEST_TIMEOUT, it -> requestTimeout = it); } /** @@ -119,18 +171,106 @@ public ADTaskManager( * @param detectorId detector id * @param handler anomaly detector job action handler * @param user user + * @param transportService transport service * @param listener action listener */ public void startDetector( String detectorId, IndexAnomalyDetectorJobActionHandler handler, User user, + TransportService transportService, ActionListener listener ) { getDetector( detectorId, (detector) -> handler.startAnomalyDetectorJob(detector), // run realtime detector - (detector) -> createADTaskIndex(detector, user, listener), // run historical detector + (detector) -> { + // run historical detector + Optional owningNode = hashRing.getOwningNode(detector.getDetectorId()); + if (!owningNode.isPresent()) { + logger.debug("Can't find eligible node to run as AD task's coordinating node"); + listener + .onFailure(new ElasticsearchStatusException("No eligible node to run detector", RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + forwardToCoordinatingNode(detector, user, ADTaskAction.START, transportService, owningNode.get(), listener); + }, + listener + ); + } + + /** + * We have three types of nodes in AD task process. + * + * 1.Forwarding node which receives external request. The request will \ + * be sent to coordinating node first. + * 2.Coordinating node which maintains running historical detector set.\ + * We use hash ring to find coordinating node with detector id. \ + * Coordinating node will find a worker node with least load and \ + * dispatch AD task to that worker node. + * 3.Worker node which will run AD task. + * + * This function is to forward the request to coordinating node. + * + * @param detector anomaly detector + * @param user user + * @param adTaskAction AD task action + * @param transportService transport service + * @param node ES node + * @param listener action listener + */ + protected void forwardToCoordinatingNode( + AnomalyDetector detector, + User user, + ADTaskAction adTaskAction, + TransportService transportService, + DiscoveryNode node, + ActionListener listener + ) { + TransportRequestOptions option = TransportRequestOptions + .builder() + .withType(TransportRequestOptions.Type.REG) + .withTimeout(requestTimeout) + .build(); + transportService + .sendRequest( + node, + ForwardADTaskAction.NAME, + new ForwardADTaskRequest(detector, user, adTaskAction), + option, + new ActionListenerResponseHandler<>(listener, AnomalyDetectorJobResponse::new) + ); + } + + /** + * Stop detector. + * For realtime detector, will set detector job as disabled. + * For historical detector, will set its AD task as cancelled. + * + * @param detectorId detector id + * @param handler AD job action handler + * @param user user + * @param transportService transport service + * @param listener action listener + */ + public void stopDetector( + String detectorId, + IndexAnomalyDetectorJobActionHandler handler, + User user, + TransportService transportService, + ActionListener listener + ) { + getDetector( + detectorId, + // stop realtime detector job + (detector) -> handler.stopAnomalyDetectorJob(detectorId), + // stop historical detector AD task + (detector) -> getLatestADTask( + detectorId, + (task) -> stopHistoricalDetector(detectorId, task, user, listener), + transportService, + listener + ), listener ); } @@ -139,15 +279,12 @@ private void getDetector( String detectorId, Consumer realTimeDetectorConsumer, Consumer historicalDetectorConsumer, - ActionListener listener + ActionListener listener ) { GetRequest getRequest = new GetRequest(AnomalyDetector.ANOMALY_DETECTORS_INDEX).id(detectorId); client.get(getRequest, ActionListener.wrap(response -> { if (!response.isExists()) { - listener - .onFailure( - new ElasticsearchStatusException("AnomalyDetector is not found with id: " + detectorId, RestStatus.NOT_FOUND) - ); + listener.onFailure(new ElasticsearchStatusException("AnomalyDetector is not found", RestStatus.NOT_FOUND)); return; } try ( @@ -170,69 +307,358 @@ private void getDetector( historicalDetectorConsumer.accept(detector); } } catch (Exception e) { - String message = "Failed to start anomaly detector"; + String message = "Failed to start anomaly detector " + detectorId; logger.error(message, e); listener.onFailure(new ElasticsearchStatusException(message, RestStatus.INTERNAL_SERVER_ERROR)); } }, exception -> listener.onFailure(exception))); } - private String validateDetector(AnomalyDetector detector) { - if (detector.getFeatureAttributes().size() == 0) { - return "Can't start detector job as no features configured"; + /** + * Get latest AD task and execute consumer function. + * + * @param detectorId detector id + * @param function consumer function + * @param transportService transport service + * @param listener action listener + * @param action listerner response + */ + public void getLatestADTask( + String detectorId, + Consumer> function, + TransportService transportService, + ActionListener listener + ) { + BoolQueryBuilder query = new BoolQueryBuilder(); + query.filter(new TermQueryBuilder(DETECTOR_ID_FIELD, detectorId)); + query.filter(new TermQueryBuilder(IS_LATEST_FIELD, true)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(query); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(sourceBuilder); + searchRequest.indices(CommonName.DETECTION_STATE_INDEX); + + client.search(searchRequest, ActionListener.wrap(r -> { + // https://github.com/opendistro-for-elasticsearch/anomaly-detection/pull/359#discussion_r558653132 + // getTotalHits will be null when we track_total_hits is false in the query request. + // Add more checking here to cover some unknown cases. + if (r == null || r.getHits().getTotalHits() == null || r.getHits().getTotalHits().value == 0) { + // don't throw exception here as consumer functions need to handle missing task + // in different way. + function.accept(Optional.empty()); + return; + } + SearchHit searchHit = r.getHits().getAt(0); + try (XContentParser parser = RestHandlerUtils.createXContentParserFromRegistry(xContentRegistry, searchHit.getSourceRef())) { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ADTask adTask = ADTask.parse(parser, searchHit.getId()); + + if (!isADTaskEnded(adTask) && lastUpdateTimeExpired(adTask)) { + // If AD task is still running, but its last updated time not refreshed + // for 2 pieces intervals, we will get task profile to check if it's + // really running and reset state as STOPPED if not running. + // For example, ES process crashes, then all tasks running on it will stay + // as running. We can reset the task state when next read happen. + getADTaskProfile(adTask, ActionListener.wrap(taskProfile -> { + if (taskProfile.getNodeId() == null) { + // If no node is running this task, reset it as STOPPED. + resetTaskStateAsStopped(adTask, transportService); + adTask.setState(ADTaskState.STOPPED.name()); + } + function.accept(Optional.of(adTask)); + }, e -> { + logger.error("Failed to get AD task profile for task " + adTask.getTaskId(), e); + listener.onFailure(e); + })); + } else { + function.accept(Optional.of(adTask)); + } + } catch (Exception e) { + String message = "Failed to parse AD task for detector " + detectorId; + logger.error(message, e); + listener.onFailure(new ElasticsearchStatusException(message, RestStatus.INTERNAL_SERVER_ERROR)); + } + }, e -> { + if (e instanceof IndexNotFoundException) { + function.accept(Optional.empty()); + } else { + logger.error("Failed to search AD task for detector " + detectorId, e); + listener.onFailure(e); + } + })); + } + + private void stopHistoricalDetector( + String detectorId, + Optional adTask, + User user, + ActionListener listener + ) { + if (!adTask.isPresent()) { + listener.onFailure(new ResourceNotFoundException(detectorId, "Detector not started")); + return; } - if (detector.getEnabledFeatureIds().size() == 0) { - return "Can't start detector job as no enabled features configured"; + + if (isADTaskEnded(adTask.get())) { + listener.onFailure(new ResourceNotFoundException(detectorId, "No running task found")); + return; + } + + String taskId = adTask.get().getTaskId(); + DiscoveryNode[] dataNodes = nodeFilter.getEligibleDataNodes(); + String userName = user == null ? null : user.getName(); + + ADCancelTaskRequest cancelTaskRequest = new ADCancelTaskRequest(detectorId, userName, dataNodes); + client + .execute( + ADCancelTaskAction.INSTANCE, + cancelTaskRequest, + ActionListener + .wrap(response -> { listener.onResponse(new AnomalyDetectorJobResponse(taskId, 0, 0, 0, RestStatus.OK)); }, e -> { + logger.error("Failed to cancel AD task " + taskId + ", detector id: " + detectorId, e); + listener.onFailure(e); + }) + ); + } + + private boolean lastUpdateTimeExpired(ADTask adTask) { + return adTask.getLastUpdateTime().plus(2 * pieceIntervalSeconds, ChronoUnit.SECONDS).isBefore(Instant.now()); + } + + private boolean isADTaskEnded(ADTask adTask) { + return ADTaskState.STOPPED.name().equals(adTask.getState()) + || ADTaskState.FINISHED.name().equals(adTask.getState()) + || ADTaskState.FAILED.name().equals(adTask.getState()); + } + + private void resetTaskStateAsStopped(ADTask adTask, TransportService transportService) { + if (!isADTaskEnded(adTask)) { + cleanDetectorCache(adTask, transportService, () -> { + Map updatedFields = new HashMap<>(); + updatedFields.put(STATE_FIELD, ADTaskState.STOPPED.name()); + updateADTask(adTask.getTaskId(), updatedFields); + logger.debug("reset task as stopped, task id " + adTask.getTaskId()); + }); } - return null; } - protected void createADTaskIndex(AnomalyDetector detector, User user, ActionListener listener) { - if (detectionIndices.doesDetectorStateIndexExist()) { - checkCurrentTaskState(detector, user, listener); + /** + * Clean detector cache on coordinating node. + * If task's coordinating node is still in cluster, will forward stop + * task request to coordinating node, then coordinating node will + * remove detector from cache. + * If task's coordinating node is not in cluster, we don't need to + * forward stop task request to coordinating node. + * + * @param adTask AD task + * @param transportService transport service + * @param function will execute it when detector cache cleaned successfully or coordinating node left cluster + */ + protected void cleanDetectorCache(ADTask adTask, TransportService transportService, AnomalyDetectorFunction function) { + String coordinatingNode = adTask.getCoordinatingNode(); + DiscoveryNode[] eligibleDataNodes = nodeFilter.getEligibleDataNodes(); + logger.debug("coordinatingNode is: " + coordinatingNode + " for task " + adTask.getTaskId()); + DiscoveryNode targetNode = null; + for (DiscoveryNode node : eligibleDataNodes) { + if (node.getId().equals(coordinatingNode)) { + targetNode = node; + break; + } + } + if (targetNode != null) { + logger.debug("coordinatingNode found, will clean detector cache on it, detectorId: " + adTask.getDetectorId()); + forwardToCoordinatingNode( + adTask.getDetector(), + null, + ADTaskAction.STOP, + transportService, + targetNode, + ActionListener + .wrap( + r -> { function.execute(); }, + e -> { logger.error("Failed to clear detector cache on coordinating node " + coordinatingNode, e); } + ) + ); } else { - detectionIndices.initDetectionStateIndex(ActionListener.wrap(r -> { - if (r.isAcknowledged()) { - logger.info("Created {} with mappings.", CommonName.DETECTION_STATE_INDEX); - executeHistoricalDetector(detector, user, listener); - } else { - String error = "Create index " + CommonName.DETECTION_STATE_INDEX + " with mappings not acknowledged"; - logger.warn(error); - listener.onFailure(new ElasticsearchStatusException(error, RestStatus.INTERNAL_SERVER_ERROR)); - } - }, e -> { - if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) { - executeHistoricalDetector(detector, user, listener); - } else { - logger.error("Failed to init anomaly detection state index", e); - listener.onFailure(e); - } - })); + logger + .warn( + "coordinating node" + + coordinatingNode + + " left cluster for detector " + + adTask.getDetectorId() + + ", task id " + + adTask.getTaskId() + ); + function.execute(); } } - private void checkCurrentTaskState(AnomalyDetector detector, User user, ActionListener listener) { - BoolQueryBuilder query = new BoolQueryBuilder(); - query.filter(new TermQueryBuilder(DETECTOR_ID_FIELD, detector.getDetectorId())); - query.filter(new TermsQueryBuilder(STATE_FIELD, ADTaskState.CREATED.name(), ADTaskState.INIT.name(), ADTaskState.RUNNING.name())); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(query); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.source(searchSourceBuilder); - searchRequest.indices(CommonName.DETECTION_STATE_INDEX); + /** + * Get AD task profile data. + * + * @param detectorId detector id + * @param transportService transport service + * @param listener action listener + */ + public void getLatestADTaskProfile(String detectorId, TransportService transportService, ActionListener listener) { + getLatestADTask(detectorId, adTask -> { + if (adTask.isPresent()) { + getADTaskProfile(adTask.get(), ActionListener.wrap(adTaskProfile -> { + DetectorProfile.Builder profileBuilder = new DetectorProfile.Builder(); + profileBuilder.adTaskProfile(adTaskProfile); + listener.onResponse(profileBuilder.build()); + }, e -> { + logger.error("Failed to get AD task profile for task " + adTask.get().getTaskId(), e); + listener.onFailure(e); + })); + } else { + listener.onFailure(new ResourceNotFoundException(detectorId, "Can't find task for detector")); + } + }, transportService, listener); + } - client.search(searchRequest, ActionListener.wrap(r -> { - if (r.getHits().getTotalHits().value > 0) { - listener.onFailure(new ElasticsearchStatusException("Detector is already running", RestStatus.BAD_REQUEST)); + private void getADTaskProfile(ADTask adTask, ActionListener listener) { + String detectorId = adTask.getDetectorId(); + + DiscoveryNode[] dataNodes = nodeFilter.getEligibleDataNodes(); + ADTaskProfileRequest adTaskProfileRequest = new ADTaskProfileRequest(detectorId, dataNodes); + client.execute(ADTaskProfileAction.INSTANCE, adTaskProfileRequest, ActionListener.wrap(response -> { + if (response.hasFailures()) { + listener.onFailure(response.failures().get(0)); + return; + } + + List nodeResponses = response + .getNodes() + .stream() + .filter(r -> r.getAdTaskProfile() != null) + .map(ADTaskProfileNodeResponse::getAdTaskProfile) + .collect(Collectors.toList()); + + if (nodeResponses.size() > 1) { + String error = nodeResponses.size() + + " tasks running for detector " + + adTask.getDetectorId() + + ". Please stop detector to kill all running tasks."; + logger.error(error); + listener.onFailure(new InternalFailure(adTask.getDetectorId(), error)); + return; + } + if (nodeResponses.size() == 0) { + ADTaskProfile adTaskProfile = new ADTaskProfile(adTask, null, null, null, null, null, null); + listener.onResponse(adTaskProfile); } else { - executeHistoricalDetector(detector, user, listener); + ADTaskProfile nodeResponse = nodeResponses.get(0); + ADTaskProfile adTaskProfile = new ADTaskProfile( + adTask, + nodeResponse.getShingleSize(), + nodeResponse.getRcfTotalUpdates(), + nodeResponse.getThresholdModelTrained(), + nodeResponse.getThresholdModelTrainingDataSize(), + nodeResponse.getModelSizeInBytes(), + nodeResponse.getNodeId() + ); + listener.onResponse(adTaskProfile); } }, e -> { - logger.error("Failed to search current running task for detector " + detector.getDetectorId(), e); + logger.error("Failed to get task profile for task " + adTask.getTaskId(), e); listener.onFailure(e); })); } + /** + * Get task profile for detector. + * + * @param detectorId detector id + * @return AD task profile + * @throws LimitExceededException if there are multiple tasks for the detector + */ + public ADTaskProfile getLocalADTaskProfileByDetectorId(String detectorId) { + ADTaskProfile adTaskProfile = null; + List tasksOfDetector = adTaskCacheManager.getTasksOfDetector(detectorId); + if (tasksOfDetector.size() > 1) { + String error = "Multiple tasks are running for detector: " + detectorId + ". You can stop detector to kill all running tasks."; + logger.warn(error); + throw new LimitExceededException(error); + } + if (tasksOfDetector.size() == 1) { + String taskId = tasksOfDetector.get(0); + adTaskProfile = new ADTaskProfile( + adTaskCacheManager.getShingle(taskId).size(), + adTaskCacheManager.getRcfModel(taskId).getTotalUpdates(), + adTaskCacheManager.isThresholdModelTrained(taskId), + adTaskCacheManager.getThresholdModelTrainingDataSize(taskId), + adTaskCacheManager.getModelSize(taskId), + clusterService.localNode().getId() + ); + } + return adTaskProfile; + } + + private String validateDetector(AnomalyDetector detector) { + if (detector.getFeatureAttributes().size() == 0) { + return "Can't start detector job as no features configured"; + } + if (detector.getEnabledFeatureIds().size() == 0) { + return "Can't start detector job as no enabled features configured"; + } + return null; + } + + /** + * Start historical detector on coordinating node. + * Will init task index if not exist and write new AD task to index. If task index + * exists, will check if there is task running. If no running task, reset old task + * as not latest and clean old tasks which exceeds limitation. Then find out node + * with least load and dispatch task to that node(worker node). + * + * @param detector anomaly detector + * @param user user + * @param transportService transport service + * @param listener action listener + */ + public void startHistoricalDetector( + AnomalyDetector detector, + User user, + TransportService transportService, + ActionListener listener + ) { + try { + if (detectionIndices.doesDetectorStateIndexExist()) { + // If detection index exist, check if latest AD task is running + getLatestADTask(detector.getDetectorId(), (adTask) -> { + if (!adTask.isPresent() || isADTaskEnded(adTask.get())) { + executeHistoricalDetector(detector, user, listener); + } else { + listener.onFailure(new ElasticsearchStatusException(DETECTOR_IS_RUNNING, RestStatus.BAD_REQUEST)); + } + }, transportService, listener); + } else { + // If detection index doesn't exist, create index and execute historical detector. + detectionIndices.initDetectionStateIndex(ActionListener.wrap(r -> { + if (r.isAcknowledged()) { + logger.info("Created {} with mappings.", CommonName.DETECTION_STATE_INDEX); + executeHistoricalDetector(detector, user, listener); + } else { + String error = "Create index " + CommonName.DETECTION_STATE_INDEX + " with mappings not acknowledged"; + logger.warn(error); + listener.onFailure(new ElasticsearchStatusException(error, RestStatus.INTERNAL_SERVER_ERROR)); + } + }, e -> { + if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) { + executeHistoricalDetector(detector, user, listener); + } else { + logger.error("Failed to init anomaly detection state index", e); + listener.onFailure(e); + } + })); + } + } catch (Exception e) { + logger.error("Failed to start historical detector " + detector.getDetectorId(), e); + listener.onFailure(e); + } + } + private void executeHistoricalDetector(AnomalyDetector detector, User user, ActionListener listener) { UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(); updateByQueryRequest.indices(CommonName.DETECTION_STATE_INDEX); @@ -271,6 +697,7 @@ private void createNewADTask(AnomalyDetector detector, User user, ActionListener .state(ADTaskState.CREATED.name()) .lastUpdateTime(now) .startedBy(userName) + .coordinatingNode(clusterService.localNode().getId()) .build(); IndexRequest request = new IndexRequest(CommonName.DETECTION_STATE_INDEX); @@ -314,9 +741,23 @@ private void onIndexADTaskResponse( } adTask.setTaskId(response.getId()); ActionListener delegatedListener = ActionListener.wrap(r -> { listener.onResponse(r); }, e -> { - listener.onFailure(e); handleADTaskException(adTask, e); + if (e instanceof DuplicateTaskException) { + listener.onFailure(new ElasticsearchStatusException(DETECTOR_IS_RUNNING, RestStatus.BAD_REQUEST)); + } else { + listener.onFailure(e); + adTaskCacheManager.removeDetector(adTask.getDetectorId()); + } }); + try { + // Put detector id in cache. If detector id already in cache, will throw + // DuplicateTaskException. This is to solve race condition when user send + // multiple start request for one historical detector. + adTaskCacheManager.add(adTask.getDetectorId()); + } catch (Exception e) { + delegatedListener.onFailure(e); + return; + } if (function != null) { function.accept(response, delegatedListener); } @@ -415,23 +856,46 @@ private void runBatchResultAction(IndexResponse response, ADTask adTask, ActionL */ public void handleADTaskException(ADTask adTask, Exception e) { // TODO: handle timeout exception - // TODO: handle TaskCancelledException + String state = ADTaskState.FAILED.name(); Map updatedFields = new HashMap<>(); - logger.error("Failed to execute AD batch task, task id: " + adTask.getTaskId() + ", detector id: " + adTask.getDetectorId(), e); - updatedFields.put(STATE_FIELD, ADTaskState.FAILED.name()); + if (e instanceof DuplicateTaskException) { + // If user send multiple start detector request, we will meet race condition. + // Cache manager will put first request in cache and throw DuplicateTaskException + // for the second request. We will delete the second task. + logger + .warn( + "There is already one running task for detector, detectorId:" + + adTask.getDetectorId() + + ". Will delete task " + + adTask.getTaskId() + ); + deleteADTask(adTask.getTaskId()); + return; + } + if (e instanceof ADTaskCancelledException) { + logger.warn("AD task cancelled: " + adTask.getTaskId()); + state = ADTaskState.STOPPED.name(); + String stoppedBy = ((ADTaskCancelledException) e).getCancelledBy(); + if (stoppedBy != null) { + updatedFields.put(STOPPED_BY_FIELD, stoppedBy); + } + } else { + logger.error("Failed to execute AD batch task, task id: " + adTask.getTaskId() + ", detector id: " + adTask.getDetectorId(), e); + } updatedFields.put(ERROR_FIELD, getErrorMessage(e)); + updatedFields.put(STATE_FIELD, state); updatedFields.put(EXECUTION_END_TIME_FIELD, Instant.now().toEpochMilli()); updateADTask(adTask.getTaskId(), updatedFields); } - private void updateADTask(String taskId, Map updatedFields) { + public void updateADTask(String taskId, Map updatedFields) { updateADTask(taskId, updatedFields, ActionListener.wrap(response -> { if (response.status() == RestStatus.OK) { - logger.info("Updated AD task successfully: {}", response.status()); + logger.debug("Updated AD task successfully: {}", response.status()); } else { logger.error("Failed to update AD task {}, status: {}", taskId, response.status()); } - }, e -> logger.error("Failed to update task: " + taskId, e))); + }, e -> { logger.error("Failed to update task: " + taskId, e); })); } /** @@ -448,11 +912,52 @@ public void updateADTask(String taskId, Map updatedFields, Actio updatedContent.put(LAST_UPDATE_TIME_FIELD, Instant.now().toEpochMilli()); updateRequest.doc(updatedContent); updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client - .update( - updateRequest, - ActionListener.wrap(response -> listener.onResponse(response), exception -> listener.onFailure(exception)) + client.update(updateRequest, listener); + } + + public void deleteADTask(String taskId) { + deleteADTask( + taskId, + ActionListener + .wrap( + r -> { logger.info("Deleted AD task {} with status: {}", taskId, r.status()); }, + e -> { logger.error("Failed to delete AD task " + taskId, e); } + ) + ); + } + + public void deleteADTask(String taskId, ActionListener listener) { + DeleteRequest deleteRequest = new DeleteRequest(CommonName.DETECTION_STATE_INDEX, taskId); + client.delete(deleteRequest, listener); + } + + /** + * Cancel running task by detector id. + * + * @param detectorId detector id + * @param reason reason to cancel AD task + * @param userName which user cancel the AD task + * @return AD task cancellation state + */ + public ADTaskCancellationState cancelLocalTaskByDetectorId(String detectorId, String reason, String userName) { + ADTaskCancellationState cancellationState = adTaskCacheManager.cancelByDetectorId(detectorId, reason, userName); + logger + .debug( + "Cancelled AD task for detector: {}, state: {}, cancelled by: {}, reason: {}", + detectorId, + cancellationState, + userName, + reason ); + return cancellationState; } + /** + * Remove detector from cache on coordinating node. + * + * @param detectorId detector id + */ + public void removeDetectorFromCache(String detectorId) { + adTaskCacheManager.removeDetector(detectorId); + } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchAnomalyResultAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchAnomalyResultAction.java index 43ce68a4..1f89404d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchAnomalyResultAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchAnomalyResultAction.java @@ -15,12 +15,14 @@ package com.amazon.opendistroforelasticsearch.ad.transport; +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonName.AD_TASK; + import org.elasticsearch.action.ActionType; import com.amazon.opendistroforelasticsearch.ad.constant.CommonValue; public class ADBatchAnomalyResultAction extends ActionType { - public static final String NAME = CommonValue.EXTERNAL_ACTION_PREFIX + "detector/ad_task"; + public static final String NAME = CommonValue.INTERNAL_ACTION_PREFIX + "detector/" + AD_TASK; public static final ADBatchAnomalyResultAction INSTANCE = new ADBatchAnomalyResultAction(); private ADBatchAnomalyResultAction() { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionAction.java index 66da10a3..bccd2fcb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionAction.java @@ -15,12 +15,14 @@ package com.amazon.opendistroforelasticsearch.ad.transport; +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonName.AD_TASK_REMOTE; + import org.elasticsearch.action.ActionType; import com.amazon.opendistroforelasticsearch.ad.constant.CommonValue; public class ADBatchTaskRemoteExecutionAction extends ActionType { - public static final String NAME = CommonValue.EXTERNAL_ACTION_PREFIX + "detector/ad_task_remote"; + public static final String NAME = CommonValue.INTERNAL_ACTION_PREFIX + "detector/" + AD_TASK_REMOTE; public static final ADBatchTaskRemoteExecutionAction INSTANCE = new ADBatchTaskRemoteExecutionAction(); private ADBatchTaskRemoteExecutionAction() { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionTransportAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionTransportAction.java index 95b76baf..56069fc1 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionTransportAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADBatchTaskRemoteExecutionTransportAction.java @@ -28,6 +28,7 @@ public class ADBatchTaskRemoteExecutionTransportAction extends HandledTransportAction { private final ADBatchTaskRunner adBatchTaskRunner; + private final TransportService transportService; @Inject public ADBatchTaskRemoteExecutionTransportAction( @@ -37,10 +38,11 @@ public ADBatchTaskRemoteExecutionTransportAction( ) { super(ADBatchTaskRemoteExecutionAction.NAME, transportService, actionFilters, ADBatchAnomalyResultRequest::new); this.adBatchTaskRunner = adBatchTaskRunner; + this.transportService = transportService; } @Override protected void doExecute(Task task, ADBatchAnomalyResultRequest request, ActionListener listener) { - adBatchTaskRunner.startADBatchTask(request.getAdTask(), true, listener); + adBatchTaskRunner.startADBatchTask(request.getAdTask(), true, transportService, listener); } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskAction.java new file mode 100644 index 00000000..d1c93d38 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskAction.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonName.CANCEL_TASK; + +import org.elasticsearch.action.ActionType; + +import com.amazon.opendistroforelasticsearch.ad.constant.CommonValue; + +public class ADCancelTaskAction extends ActionType { + + public static final String NAME = CommonValue.INTERNAL_ACTION_PREFIX + "detectors/" + CANCEL_TASK; + public static final ADCancelTaskAction INSTANCE = new ADCancelTaskAction(); + + private ADCancelTaskAction() { + super(NAME, ADCancelTaskResponse::new); + } + +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeRequest.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeRequest.java new file mode 100644 index 00000000..92fba5cf --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeRequest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; + +import org.elasticsearch.action.support.nodes.BaseNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +public class ADCancelTaskNodeRequest extends BaseNodeRequest { + private String detectorId; + private String userName; + + public ADCancelTaskNodeRequest(StreamInput in) throws IOException { + super(in); + this.detectorId = in.readOptionalString(); + this.userName = in.readOptionalString(); + } + + public ADCancelTaskNodeRequest(ADCancelTaskRequest request) { + this.detectorId = request.getDetectorId(); + this.userName = request.getUserName(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(detectorId); + out.writeOptionalString(userName); + } + + public String getDetectorId() { + return detectorId; + } + + public String getUserName() { + return userName; + } + +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeResponse.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeResponse.java new file mode 100644 index 00000000..7703ec9a --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskNodeResponse.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; + +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskCancellationState; + +public class ADCancelTaskNodeResponse extends BaseNodeResponse { + + private ADTaskCancellationState state; + + public ADCancelTaskNodeResponse(DiscoveryNode node, ADTaskCancellationState state) { + super(node); + this.state = state; + } + + public ADCancelTaskNodeResponse(StreamInput in) throws IOException { + super(in); + this.state = in.readEnum(ADTaskCancellationState.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeEnum(state); + } + + public static ADCancelTaskNodeResponse readNodeResponse(StreamInput in) throws IOException { + return new ADCancelTaskNodeResponse(in); + } + + public ADTaskCancellationState getState() { + return state; + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskRequest.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskRequest.java new file mode 100644 index 00000000..16a3f21d --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskRequest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +import java.io.IOException; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; + +public class ADCancelTaskRequest extends BaseNodesRequest { + + private String detectorId; + private String userName; + + public ADCancelTaskRequest(StreamInput in) throws IOException { + super(in); + this.detectorId = in.readOptionalString(); + this.userName = in.readOptionalString(); + } + + public ADCancelTaskRequest(String detectorId, String userName, DiscoveryNode... nodes) { + super(nodes); + this.detectorId = detectorId; + this.userName = userName; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isEmpty(detectorId)) { + validationException = addValidationError(CommonErrorMessages.AD_ID_MISSING_MSG, validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(detectorId); + out.writeOptionalString(userName); + } + + public String getDetectorId() { + return detectorId; + } + + public String getUserName() { + return userName; + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskResponse.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskResponse.java new file mode 100644 index 00000000..209cd218 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskResponse.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; +import java.util.List; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +public class ADCancelTaskResponse extends BaseNodesResponse { + + public ADCancelTaskResponse(StreamInput in) throws IOException { + super(new ClusterName(in), in.readList(ADCancelTaskNodeResponse::readNodeResponse), in.readList(FailedNodeException::new)); + } + + public ADCancelTaskResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + public void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public List readNodesFrom(StreamInput in) throws IOException { + return in.readList(ADCancelTaskNodeResponse::readNodeResponse); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTransportAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTransportAction.java new file mode 100644 index 00000000..d715c6ed --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTransportAction.java @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskCancellationState; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; + +public class ADCancelTaskTransportAction extends + TransportNodesAction { + private final Logger logger = LogManager.getLogger(ADCancelTaskTransportAction.class); + private Client client; + private ADTaskManager adTaskManager; + + @Inject + public ADCancelTaskTransportAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + ADTaskManager adTaskManager, + Client client + ) { + super( + ADCancelTaskAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + ADCancelTaskRequest::new, + ADCancelTaskNodeRequest::new, + ThreadPool.Names.MANAGEMENT, + ADCancelTaskNodeResponse.class + ); + this.adTaskManager = adTaskManager; + this.client = client; + } + + @Override + protected ADCancelTaskResponse newResponse( + ADCancelTaskRequest request, + List responses, + List failures + ) { + return new ADCancelTaskResponse(clusterService.getClusterName(), responses, failures); + } + + @Override + protected ADCancelTaskNodeRequest newNodeRequest(ADCancelTaskRequest request) { + return new ADCancelTaskNodeRequest(request); + } + + @Override + protected ADCancelTaskNodeResponse newNodeResponse(StreamInput in) throws IOException { + return new ADCancelTaskNodeResponse(in); + } + + @Override + protected ADCancelTaskNodeResponse nodeOperation(ADCancelTaskNodeRequest request) { + String reason = "Task cancelled by user"; + String userName = request.getUserName(); + String detectorId = request.getDetectorId(); + ADTaskCancellationState state = adTaskManager.cancelLocalTaskByDetectorId(detectorId, reason, userName); + logger.debug("Cancelled AD task for detector: {}", request.getDetectorId()); + return new ADCancelTaskNodeResponse(clusterService.localNode(), state); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileAction.java new file mode 100644 index 00000000..abe90826 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileAction.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonName.AD_TASK; + +import org.elasticsearch.action.ActionType; + +import com.amazon.opendistroforelasticsearch.ad.constant.CommonValue; + +public class ADTaskProfileAction extends ActionType { + + public static final String NAME = CommonValue.INTERNAL_ACTION_PREFIX + "detectors/profile/" + AD_TASK; + public static final ADTaskProfileAction INSTANCE = new ADTaskProfileAction(); + + private ADTaskProfileAction() { + super(NAME, ADTaskProfileResponse::new); + } + +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeRequest.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeRequest.java new file mode 100644 index 00000000..1a8c369d --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeRequest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; + +import org.elasticsearch.action.support.nodes.BaseNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +public class ADTaskProfileNodeRequest extends BaseNodeRequest { + private String detectorId; + + public ADTaskProfileNodeRequest(StreamInput in) throws IOException { + super(in); + this.detectorId = in.readString(); + } + + public ADTaskProfileNodeRequest(ADTaskProfileRequest request) { + this.detectorId = request.getDetectorId(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(detectorId); + } + + public String getDetectorId() { + return detectorId; + } + +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeResponse.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeResponse.java new file mode 100644 index 00000000..8ad6105d --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileNodeResponse.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; + +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskProfile; + +public class ADTaskProfileNodeResponse extends BaseNodeResponse { + + private ADTaskProfile adTaskProfile; + + public ADTaskProfileNodeResponse(DiscoveryNode node, ADTaskProfile adTaskProfile) { + super(node); + this.adTaskProfile = adTaskProfile; + } + + public ADTaskProfileNodeResponse(StreamInput in) throws IOException { + super(in); + if (in.readBoolean()) { + this.adTaskProfile = new ADTaskProfile(in); + } else { + this.adTaskProfile = null; + } + } + + public ADTaskProfile getAdTaskProfile() { + return adTaskProfile; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + if (adTaskProfile != null) { + out.writeBoolean(true); + adTaskProfile.writeTo(out); + } else { + out.writeBoolean(false); + } + } + + public static ADTaskProfileNodeResponse readNodeResponse(StreamInput in) throws IOException { + return new ADTaskProfileNodeResponse(in); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileRequest.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileRequest.java new file mode 100644 index 00000000..23398e20 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileRequest.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +import java.io.IOException; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; + +public class ADTaskProfileRequest extends BaseNodesRequest { + + private String detectorId; + + public ADTaskProfileRequest(StreamInput in) throws IOException { + super(in); + this.detectorId = in.readString(); + } + + public ADTaskProfileRequest(String detectorId, DiscoveryNode... nodes) { + super(nodes); + this.detectorId = detectorId; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isEmpty(detectorId)) { + validationException = addValidationError(CommonErrorMessages.AD_ID_MISSING_MSG, validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(detectorId); + } + + public String getDetectorId() { + return detectorId; + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileResponse.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileResponse.java new file mode 100644 index 00000000..edfa62a7 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileResponse.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; +import java.util.List; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +public class ADTaskProfileResponse extends BaseNodesResponse { + + public ADTaskProfileResponse(StreamInput in) throws IOException { + super(new ClusterName(in), in.readList(ADTaskProfileNodeResponse::readNodeResponse), in.readList(FailedNodeException::new)); + } + + public ADTaskProfileResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + public void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public List readNodesFrom(StreamInput in) throws IOException { + return in.readList(ADTaskProfileNodeResponse::readNodeResponse); + } + +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportAction.java new file mode 100644 index 00000000..0a276bcd --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportAction.java @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; +import java.util.List; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskProfile; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; + +public class ADTaskProfileTransportAction extends + TransportNodesAction { + + private ADTaskManager adTaskManager; + + @Inject + public ADTaskProfileTransportAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + ADTaskManager adTaskManager + ) { + super( + ADTaskProfileAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + ADTaskProfileRequest::new, + ADTaskProfileNodeRequest::new, + ThreadPool.Names.MANAGEMENT, + ADTaskProfileNodeResponse.class + ); + this.adTaskManager = adTaskManager; + } + + @Override + protected ADTaskProfileResponse newResponse( + ADTaskProfileRequest request, + List responses, + List failures + ) { + return new ADTaskProfileResponse(clusterService.getClusterName(), responses, failures); + } + + @Override + protected ADTaskProfileNodeRequest newNodeRequest(ADTaskProfileRequest request) { + return new ADTaskProfileNodeRequest(request); + } + + @Override + protected ADTaskProfileNodeResponse newNodeResponse(StreamInput in) throws IOException { + return new ADTaskProfileNodeResponse(in); + } + + @Override + protected ADTaskProfileNodeResponse nodeOperation(ADTaskProfileNodeRequest request) { + ADTaskProfile adTaskProfile = adTaskManager.getLocalADTaskProfileByDetectorId(request.getDetectorId()); + + return new ADTaskProfileNodeResponse(clusterService.localNode(), adTaskProfile); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportAction.java index e2224df9..beae687d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportAction.java @@ -51,6 +51,7 @@ public class AnomalyDetectorJobTransportAction extends HandledTransportAction { + // Internal Action which is not used for public facing RestAPIs. + public static final String NAME = CommonValue.INTERNAL_ACTION_PREFIX + "detector/" + AD_TASK + "/forward"; + public static final ForwardADTaskAction INSTANCE = new ForwardADTaskAction(); + + private ForwardADTaskAction() { + super(NAME, AnomalyDetectorJobResponse::new); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskRequest.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskRequest.java new file mode 100644 index 00000000..73b7409f --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskRequest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +import java.io.IOException; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskAction; +import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; +import com.amazon.opendistroforelasticsearch.commons.authuser.User; + +public class ForwardADTaskRequest extends ActionRequest { + private AnomalyDetector detector; + private User user; + private ADTaskAction adTaskAction; + + public ForwardADTaskRequest(AnomalyDetector detector, User user, ADTaskAction adTaskAction) { + this.detector = detector; + this.user = user; + this.adTaskAction = adTaskAction; + } + + public ForwardADTaskRequest(StreamInput in) throws IOException { + super(in); + this.detector = new AnomalyDetector(in); + if (in.readBoolean()) { + this.user = new User(in); + } + this.adTaskAction = in.readEnum(ADTaskAction.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + detector.writeTo(out); + if (user != null) { + out.writeBoolean(true); + user.writeTo(out); + } else { + out.writeBoolean(false); + } + out.writeEnum(adTaskAction); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (detector == null) { + validationException = addValidationError(CommonErrorMessages.DETECTOR_MISSING, validationException); + } else if (detector.getDetectorId() == null) { + validationException = addValidationError(CommonErrorMessages.AD_ID_MISSING_MSG, validationException); + } + if (adTaskAction == null) { + validationException = addValidationError(CommonErrorMessages.AD_TASK_ACTION_MISSING, validationException); + } + return validationException; + } + + public AnomalyDetector getDetector() { + return detector; + } + + public User getUser() { + return user; + } + + public ADTaskAction getAdTaskAction() { + return adTaskAction; + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTransportAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTransportAction.java new file mode 100644 index 00000000..902897e7 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTransportAction.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskAction; +import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; + +public class ForwardADTaskTransportAction extends HandledTransportAction { + + private final ADTaskManager adTaskManager; + private final TransportService transportService; + + @Inject + public ForwardADTaskTransportAction(ActionFilters actionFilters, TransportService transportService, ADTaskManager adTaskManager) { + super(ForwardADTaskAction.NAME, transportService, actionFilters, ForwardADTaskRequest::new); + this.adTaskManager = adTaskManager; + this.transportService = transportService; + } + + @Override + protected void doExecute(Task task, ForwardADTaskRequest request, ActionListener listener) { + ADTaskAction adTaskAction = request.getAdTaskAction(); + AnomalyDetector detector = request.getDetector(); + + switch (adTaskAction) { + case START: + adTaskManager.startHistoricalDetector(request.getDetector(), request.getUser(), transportService, listener); + break; + case STOP: + adTaskManager.removeDetectorFromCache(request.getDetector().getDetectorId()); + listener.onResponse(new AnomalyDetectorJobResponse(detector.getDetectorId(), 0, 0, 0, RestStatus.OK)); + break; + default: + listener.onFailure(new ElasticsearchStatusException("Unsupported AD task action " + adTaskAction, RestStatus.BAD_REQUEST)); + break; + } + + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorRequest.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorRequest.java index 8450b13d..b11f553f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorRequest.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorRequest.java @@ -27,6 +27,7 @@ public class GetAnomalyDetectorRequest extends ActionRequest { private String detectorID; private long version; private boolean returnJob; + private boolean returnTask; private String typeStr; private String rawPath; private boolean all; @@ -37,6 +38,7 @@ public GetAnomalyDetectorRequest(StreamInput in) throws IOException { detectorID = in.readString(); version = in.readLong(); returnJob = in.readBoolean(); + returnTask = in.readBoolean(); typeStr = in.readString(); rawPath = in.readString(); all = in.readBoolean(); @@ -49,16 +51,17 @@ public GetAnomalyDetectorRequest( String detectorID, long version, boolean returnJob, + boolean returnTask, String typeStr, String rawPath, boolean all, String entityValue - ) - throws IOException { + ) { super(); this.detectorID = detectorID; this.version = version; this.returnJob = returnJob; + this.returnTask = returnTask; this.typeStr = typeStr; this.rawPath = rawPath; this.all = all; @@ -77,6 +80,10 @@ public boolean isReturnJob() { return returnJob; } + public boolean isReturnTask() { + return returnTask; + } + public String getTypeStr() { return typeStr; } @@ -99,6 +106,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(detectorID); out.writeLong(version); out.writeBoolean(returnJob); + out.writeBoolean(returnTask); out.writeString(typeStr); out.writeString(rawPath); out.writeBoolean(all); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorResponse.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorResponse.java index 4c8720cf..843b2f88 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorResponse.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorResponse.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.RestStatus; +import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfile; @@ -39,11 +40,13 @@ public class GetAnomalyDetectorResponse extends ActionResponse implements ToXCon private long seqNo; private AnomalyDetector detector; private AnomalyDetectorJob adJob; + private ADTask adTask; private RestStatus restStatus; private DetectorProfile detectorProfile; private EntityProfile entityProfile; private boolean profileResponse; private boolean returnJob; + private boolean returnTask; public GetAnomalyDetectorResponse(StreamInput in) throws IOException { super(in); @@ -70,6 +73,12 @@ public GetAnomalyDetectorResponse(StreamInput in) throws IOException { } else { adJob = null; } + returnTask = in.readBoolean(); + if (returnTask) { + adTask = new ADTask(in); + } else { + adTask = null; + } } } @@ -81,6 +90,8 @@ public GetAnomalyDetectorResponse( AnomalyDetector detector, AnomalyDetectorJob adJob, boolean returnJob, + ADTask adTask, + boolean returnTask, RestStatus restStatus, DetectorProfile detectorProfile, EntityProfile entityProfile, @@ -93,11 +104,18 @@ public GetAnomalyDetectorResponse( this.detector = detector; this.restStatus = restStatus; this.returnJob = returnJob; + if (this.returnJob) { this.adJob = adJob; } else { this.adJob = null; } + this.returnTask = returnTask; + if (this.returnTask) { + this.adTask = adTask; + } else { + this.adTask = null; + } this.detectorProfile = detectorProfile; this.entityProfile = entityProfile; this.profileResponse = profileResponse; @@ -128,6 +146,12 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); // returnJob is false } + if (returnTask) { + out.writeBoolean(true); + adTask.writeTo(out); + } else { + out.writeBoolean(false); + } } } @@ -149,8 +173,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (returnJob) { builder.field(RestHandlerUtils.ANOMALY_DETECTOR_JOB, adJob); } + if (returnTask) { + builder.field(RestHandlerUtils.ANOMALY_DETECTION_TASK, adTask); + } builder.endObject(); } return builder; } + + public DetectorProfile getDetectorProfile() { + return detectorProfile; + } } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportAction.java index d3aa09fb..70a6c6d3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportAction.java @@ -27,6 +27,7 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -55,12 +56,14 @@ import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorProfileRunner; import com.amazon.opendistroforelasticsearch.ad.EntityProfileRunner; import com.amazon.opendistroforelasticsearch.ad.Name; +import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfile; import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfileName; import com.amazon.opendistroforelasticsearch.ad.model.EntityProfileName; import com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; import com.amazon.opendistroforelasticsearch.ad.util.DiscoveryNodeFilterer; import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; import com.amazon.opendistroforelasticsearch.commons.authuser.User; @@ -81,7 +84,9 @@ public class GetAnomalyDetectorTransportAction extends HandledTransportAction defaultEntityProfileTypes; private final NamedXContentRegistry xContentRegistry; private final DiscoveryNodeFilterer nodeFilter; + private final TransportService transportService; private volatile Boolean filterByEnabled; + private final ADTaskManager adTaskManager; @Inject public GetAnomalyDetectorTransportAction( @@ -91,7 +96,8 @@ public GetAnomalyDetectorTransportAction( ClusterService clusterService, Client client, Settings settings, - NamedXContentRegistry xContentRegistry + NamedXContentRegistry xContentRegistry, + ADTaskManager adTaskManager ) { super(GetAnomalyDetectorAction.NAME, transportService, actionFilters, GetAnomalyDetectorRequest::new); this.clusterService = clusterService; @@ -113,6 +119,8 @@ public GetAnomalyDetectorTransportAction( this.nodeFilter = nodeFilter; filterByEnabled = AnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES.get(settings); clusterService.getClusterSettings().addSettingsUpdateConsumer(FILTER_BY_BACKEND_ROLES, it -> filterByEnabled = it); + this.transportService = transportService; + this.adTaskManager = adTaskManager; } @Override @@ -138,12 +146,12 @@ protected void doExecute(Task task, GetAnomalyDetectorRequest request, ActionLis protected void getExecute(GetAnomalyDetectorRequest request, ActionListener listener) { String detectorID = request.getDetectorID(); - Long version = request.getVersion(); String typesStr = request.getTypeStr(); String rawPath = request.getRawPath(); String entityValue = request.getEntityValue(); boolean all = request.isAll(); boolean returnJob = request.isReturnJob(); + boolean returnTask = request.isReturnTask(); try { if (!Strings.isEmpty(typesStr) || rawPath.endsWith(PROFILE) || rawPath.endsWith(PROFILE + "/")) { @@ -164,7 +172,21 @@ protected void getExecute(GetAnomalyDetectorRequest request, ActionListener { listener .onResponse( - new GetAnomalyDetectorResponse(0, null, 0, 0, null, null, false, null, null, profile, true) + new GetAnomalyDetectorResponse( + 0, + null, + 0, + 0, + null, + null, + false, + null, + false, + null, + null, + profile, + true + ) ); }, e -> listener.onFailure(e) @@ -176,18 +198,24 @@ protected void getExecute(GetAnomalyDetectorRequest request, ActionListener getDetectorAndJob(detectorID, returnJob, returnTask, adTask, listener), + transportService, + listener + ); + } else { + getDetectorAndJob(detectorID, returnJob, returnTask, Optional.empty(), listener); } - client.multiGet(multiGetRequest, onMultiGetResponse(listener, returnJob, detectorID)); } } catch (Exception e) { LOG.error(e); @@ -195,9 +223,27 @@ protected void getExecute(GetAnomalyDetectorRequest request, ActionListener adTask, + ActionListener listener + ) { + MultiGetRequest.Item adItem = new MultiGetRequest.Item(ANOMALY_DETECTORS_INDEX, detectorID); + MultiGetRequest multiGetRequest = new MultiGetRequest().add(adItem); + if (returnJob) { + MultiGetRequest.Item adJobItem = new MultiGetRequest.Item(ANOMALY_DETECTOR_JOB_INDEX, detectorID); + multiGetRequest.add(adJobItem); + } + client.multiGet(multiGetRequest, onMultiGetResponse(listener, returnJob, returnTask, adTask, detectorID)); + } + private ActionListener onMultiGetResponse( ActionListener listener, boolean returnJob, + boolean returnTask, + Optional adTask, String detectorId ) { return new ActionListener() { @@ -267,6 +313,8 @@ public void onResponse(MultiGetResponse multiGetResponse) { detector, adJob, returnJob, + adTask.orElse(null), + returnTask, RestStatus.OK, null, null, @@ -282,14 +330,12 @@ public void onFailure(Exception e) { }; } - private ActionListener getProfileActionListener( - ActionListener listener, - String detectorId - ) { + private ActionListener getProfileActionListener(ActionListener listener) { return ActionListener.wrap(new CheckedConsumer() { @Override public void accept(DetectorProfile profile) throws Exception { - listener.onResponse(new GetAnomalyDetectorResponse(0, null, 0, 0, null, null, false, null, profile, null, true)); + listener + .onResponse(new GetAnomalyDetectorResponse(0, null, 0, 0, null, null, false, null, false, null, profile, null, true)); } }, exception -> { listener.onFailure(exception); }); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/ad/util/RestHandlerUtils.java b/src/main/java/com/amazon/opendistroforelasticsearch/ad/util/RestHandlerUtils.java index fdb085da..767feebe 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/ad/util/RestHandlerUtils.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/ad/util/RestHandlerUtils.java @@ -51,6 +51,7 @@ public final class RestHandlerUtils { public static final String DETECTOR_ID = "detectorID"; public static final String ANOMALY_DETECTOR = "anomaly_detector"; public static final String ANOMALY_DETECTOR_JOB = "anomaly_detector_job"; + public static final String ANOMALY_DETECTION_TASK = "anomaly_detection_task"; public static final String RUN = "_run"; public static final String PREVIEW = "_preview"; public static final String START_JOB = "_start"; diff --git a/src/main/resources/mappings/anomaly-detection-state.json b/src/main/resources/mappings/anomaly-detection-state.json index f618e1b4..e8767e01 100644 --- a/src/main/resources/mappings/anomaly-detection-state.json +++ b/src/main/resources/mappings/anomaly-detection-state.json @@ -53,6 +53,12 @@ "checkpoint_id": { "type": "keyword" }, + "coordinating_node": { + "type": "keyword" + }, + "worker_node": { + "type": "keyword" + }, "detector": { "properties": { "schema_version": { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/ADIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/ADIntegTestCase.java index a632737b..60daab9e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/ADIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/ADIntegTestCase.java @@ -20,8 +20,10 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -39,6 +41,8 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -57,6 +61,7 @@ import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; public abstract class ADIntegTestCase extends ESIntegTestCase { @@ -132,6 +137,13 @@ public AcknowledgedResponse deleteIndex(String indexName) { return admin().indices().delete(deleteIndexRequest).actionGet(timeout); } + public void deleteIndexIfExists(String indexName) { + if (indexExists(indexName)) { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); + admin().indices().delete(deleteIndexRequest).actionGet(timeout); + } + } + public String indexDoc(String indexName, XContentBuilder source) { IndexRequest indexRequest = new IndexRequest(indexName).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).source(source); IndexResponse indexResponse = client().index(indexRequest).actionGet(timeout); @@ -193,4 +205,23 @@ public ImmutableOpenMap getDataNodes() { return nodes.getDataNodes(); } + public Client getDataNodeClient() { + for (Client client : clients()) { + if (client instanceof NodeClient) { + return client; + } + } + return null; + } + + public DiscoveryNode[] getDataNodesArray() { + DiscoveryNodes nodes = clusterService().state().getNodes(); + Iterator> iterator = nodes.getDataNodes().iterator(); + List dataNodes = new ArrayList<>(); + while (iterator.hasNext()) { + dataNodes.add(iterator.next().value); + } + return dataNodes.toArray(new DiscoveryNode[0]); + } + } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AbstractProfileRunnerTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AbstractProfileRunnerTests.java index 5f3b4900..d00aaa63 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AbstractProfileRunnerTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AbstractProfileRunnerTests.java @@ -32,11 +32,13 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.TransportService; import org.junit.Before; import org.junit.BeforeClass; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfileName; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; import com.amazon.opendistroforelasticsearch.ad.util.DiscoveryNodeFilterer; public class AbstractProfileRunnerTests extends AbstractADTest { @@ -65,6 +67,8 @@ protected enum ErrorResultStatus { protected DiscoveryNodeFilterer nodeFilter; protected AnomalyDetector detector; protected ClusterService clusterService; + protected TransportService transportService; + protected ADTaskManager adTaskManager; protected static Set stateOnly; protected static Set stateNError; @@ -150,7 +154,7 @@ public void setUp() throws Exception { requiredSamples = 128; neededSamples = 5; - runner = new AnomalyDetectorProfileRunner(client, xContentRegistry(), nodeFilter, requiredSamples); + runner = new AnomalyDetectorProfileRunner(client, xContentRegistry(), nodeFilter, requiredSamples, transportService, adTaskManager); detectorIntervalMin = 3; detectorGetReponse = mock(GetResponse.class); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunnerTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunnerTests.java index 8bb18a16..c7314205 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunnerTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorProfileRunnerTests.java @@ -577,7 +577,10 @@ public void testInitNoIndex() throws IOException, InterruptedException { } public void testInvalidRequiredSamples() { - expectThrows(IllegalArgumentException.class, () -> new AnomalyDetectorProfileRunner(client, xContentRegistry(), nodeFilter, 0)); + expectThrows( + IllegalArgumentException.class, + () -> new AnomalyDetectorProfileRunner(client, xContentRegistry(), nodeFilter, 0, transportService, adTaskManager) + ); } public void testFailRCFPolling() throws IOException, InterruptedException { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java index 6c42d022..5eb24d31 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorRestTestCase.java @@ -44,6 +44,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; +import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorExecutionInput; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; @@ -118,7 +119,9 @@ protected AnomalyDetector createAnomalyDetector(AnomalyDetector detector, Boolea detector.getSchemaVersion(), detector.getLastUpdateTime(), detector.getCategoryField(), - detector.getUser() + detector.getUser(), + detector.getDetectorType(), + detector.getDetectionDateRange() ); } @@ -154,21 +157,26 @@ public AnomalyDetector getAnomalyDetector(String detectorId, RestClient client) } public AnomalyDetector getAnomalyDetector(String detectorId, BasicHeader header, RestClient client) throws IOException { - return (AnomalyDetector) getAnomalyDetector(detectorId, header, false, client)[0]; + return (AnomalyDetector) getAnomalyDetector(detectorId, header, false, false, client)[0]; } public ToXContentObject[] getAnomalyDetector(String detectorId, boolean returnJob, RestClient client) throws IOException { BasicHeader header = new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"); - return getAnomalyDetector(detectorId, header, returnJob, client); + return getAnomalyDetector(detectorId, header, returnJob, false, client); } - public ToXContentObject[] getAnomalyDetector(String detectorId, BasicHeader header, boolean returnJob, RestClient client) - throws IOException { + public ToXContentObject[] getAnomalyDetector( + String detectorId, + BasicHeader header, + boolean returnJob, + boolean returnTask, + RestClient client + ) throws IOException { Response response = TestHelpers .makeRequest( client, "GET", - TestHelpers.AD_BASE_DETECTORS_URI + "/" + detectorId + "?job=" + returnJob, + TestHelpers.AD_BASE_DETECTORS_URI + "/" + detectorId + "?job=" + returnJob + "&task=" + returnTask, null, "", ImmutableList.of(header) @@ -182,6 +190,7 @@ public ToXContentObject[] getAnomalyDetector(String detectorId, BasicHeader head Long version = null; AnomalyDetector detector = null; AnomalyDetectorJob detectorJob = null; + ADTask adTask = null; while (parser.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = parser.currentName(); parser.nextToken(); @@ -198,6 +207,9 @@ public ToXContentObject[] getAnomalyDetector(String detectorId, BasicHeader head case "anomaly_detector_job": detectorJob = AnomalyDetectorJob.parse(parser); break; + case "anomaly_detection_task": + adTask = ADTask.parse(parser); + break; } } @@ -218,9 +230,12 @@ public ToXContentObject[] getAnomalyDetector(String detectorId, BasicHeader head detector.getSchemaVersion(), detector.getLastUpdateTime(), null, - detector.getUser() + detector.getUser(), + detector.getDetectorType(), + detector.getDetectionDateRange() ), - detectorJob }; + detectorJob, + adTask }; } protected HttpEntity toHttpEntity(ToXContentObject object) throws IOException { diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorIntegTestCase.java index e3032dd0..12b9b136 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorIntegTestCase.java @@ -163,6 +163,16 @@ public List searchADTasks(String detectorId, Boolean isLatest, int size) return adTasks; } + public ADTask getADTask(String taskId) throws IOException { + ADTask adTask = toADTask(getDoc(CommonName.DETECTION_STATE_INDEX, taskId)); + adTask.setTaskId(taskId); + return adTask; + } + + public AnomalyDetectorJob getADJob(String detectorId) throws IOException { + return toADJob(getDoc(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, detectorId)); + } + public ADTask toADTask(GetResponse doc) throws IOException { return ADTask.parse(TestHelpers.parser(doc.getSourceAsString())); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorRestTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorRestTestCase.java new file mode 100644 index 00000000..8d6cd5f7 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/HistoricalDetectorRestTestCase.java @@ -0,0 +1,174 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad; + +import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.AD_BASE_DETECTORS_URI; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Map; +import java.util.function.ToDoubleFunction; +import java.util.function.ToIntFunction; + +import org.apache.http.HttpHeaders; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.junit.Before; + +import com.amazon.opendistroforelasticsearch.ad.mock.model.MockSimpleLog; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskProfile; +import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; +import com.amazon.opendistroforelasticsearch.ad.model.DetectionDateRange; +import com.amazon.opendistroforelasticsearch.ad.model.Feature; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public abstract class HistoricalDetectorRestTestCase extends AnomalyDetectorRestTestCase { + + protected String historicalDetectorTestIndex = "test_historical_detector_data"; + protected int detectionIntervalInMinutes = 1; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + updateClusterSettings(BATCH_TASK_PIECE_INTERVAL_SECONDS.getKey(), 1); + // ingest test data + ingestTestDataForHistoricalDetector(historicalDetectorTestIndex, detectionIntervalInMinutes); + } + + public ToXContentObject[] getHistoricalAnomalyDetector(String detectorId, boolean returnTask, RestClient client) throws IOException { + BasicHeader header = new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"); + return getAnomalyDetector(detectorId, header, false, returnTask, client); + } + + public ADTaskProfile getADTaskProfile(String detectorId) throws IOException { + Response profileResponse = TestHelpers + .makeRequest(client(), "GET", AD_BASE_DETECTORS_URI + "/" + detectorId + "/_profile/ad_task", ImmutableMap.of(), "", null); + return parseADTaskProfile(profileResponse); + } + + public Response ingestSimpleMockLog( + String indexName, + int startDays, + int totalDoc, + long intervalInMinutes, + ToDoubleFunction valueFunc, + ToIntFunction categoryFunc + ) throws IOException { + TestHelpers + .makeRequest( + client(), + "PUT", + indexName, + null, + toHttpEntity(MockSimpleLog.INDEX_MAPPING), + ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana")) + ); + + Response statsResponse = TestHelpers.makeRequest(client(), "GET", indexName, ImmutableMap.of(), "", null); + assertEquals(RestStatus.OK, restStatus(statsResponse)); + String result = EntityUtils.toString(statsResponse.getEntity()); + assertTrue(result.contains(indexName)); + + StringBuilder bulkRequestBuilder = new StringBuilder(); + Instant startTime = Instant.now().minus(startDays, ChronoUnit.DAYS); + for (int i = 0; i < totalDoc; i++) { + bulkRequestBuilder.append("{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_id\" : \"" + i + "\" } }\n"); + MockSimpleLog simpleLog = new MockSimpleLog( + startTime, + valueFunc.applyAsDouble(i), + "category" + categoryFunc.applyAsInt(i), + randomBoolean(), + randomAlphaOfLength(5) + ); + bulkRequestBuilder.append(TestHelpers.toJsonString(simpleLog)); + bulkRequestBuilder.append("\n"); + startTime = startTime.plus(intervalInMinutes, ChronoUnit.MINUTES); + } + Response bulkResponse = TestHelpers + .makeRequest( + client(), + "POST", + "_bulk?refresh=true", + null, + toHttpEntity(bulkRequestBuilder.toString()), + ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana")) + ); + return bulkResponse; + } + + public ADTaskProfile parseADTaskProfile(Response profileResponse) throws IOException { + String profileResult = EntityUtils.toString(profileResponse.getEntity()); + XContentParser parser = TestHelpers.parser(profileResult); + ADTaskProfile adTaskProfile = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + if ("ad_task".equals(fieldName)) { + adTaskProfile = ADTaskProfile.parse(parser); + } else { + parser.skipChildren(); + } + } + return adTaskProfile; + } + + protected void ingestTestDataForHistoricalDetector(String indexName, int detectionIntervalInMinutes) throws IOException { + ingestSimpleMockLog(indexName, 10, 3000, detectionIntervalInMinutes, (i) -> { + if (i % 500 == 0) { + return randomDoubleBetween(100, 1000, true); + } else { + return randomDoubleBetween(1, 10, true); + } + }, (i) -> 1); + } + + protected AnomalyDetector createHistoricalDetector() throws IOException { + AggregationBuilder aggregationBuilder = TestHelpers + .parseAggregation("{\"test\":{\"max\":{\"field\":\"" + MockSimpleLog.VALUE_FIELD + "\"}}}"); + Feature feature = new Feature(randomAlphaOfLength(5), randomAlphaOfLength(10), true, aggregationBuilder); + Instant endTime = Instant.now().truncatedTo(ChronoUnit.SECONDS); + Instant startTime = endTime.minus(10, ChronoUnit.DAYS).truncatedTo(ChronoUnit.SECONDS); + DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime); + AnomalyDetector detector = TestHelpers + .randomDetector( + dateRange, + ImmutableList.of(feature), + historicalDetectorTestIndex, + detectionIntervalInMinutes, + MockSimpleLog.TIME_FIELD + ); + return createAnomalyDetector(detector, true, client()); + } + + protected String startHistoricalDetector(String detectorId) throws IOException { + Response startDetectorResponse = startAnomalyDetector(detectorId, client()); + Map startDetectorResponseMap = responseAsMap(startDetectorResponse); + String taskId = (String) startDetectorResponseMap.get("_id"); + assertNotNull(taskId); + return taskId; + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/MultiEntityProfileRunnerTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/MultiEntityProfileRunnerTests.java index a9e87d28..4735abf6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/MultiEntityProfileRunnerTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/MultiEntityProfileRunnerTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.transport.TransportService; import org.junit.Before; import com.amazon.opendistroforelasticsearch.ad.constant.CommonName; @@ -55,6 +56,7 @@ import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfile; import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfileName; import com.amazon.opendistroforelasticsearch.ad.model.DetectorState; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; import com.amazon.opendistroforelasticsearch.ad.transport.ProfileAction; import com.amazon.opendistroforelasticsearch.ad.transport.ProfileNodeResponse; import com.amazon.opendistroforelasticsearch.ad.transport.ProfileResponse; @@ -83,6 +85,8 @@ public class MultiEntityProfileRunnerTests extends AbstractADTest { private int shingleSize; private AnomalyDetectorJob job; + private TransportService transportService; + private ADTaskManager adTaskManager; enum InittedEverResultStatus { INITTED, @@ -102,8 +106,9 @@ public void setUp() throws Exception { detector = TestHelpers.randomAnomalyDetectorUsingCategoryFields(detectorId, Arrays.asList("a")); result = new DetectorInternalState.Builder().lastUpdateTime(Instant.now()); job = TestHelpers.randomAnomalyDetectorJob(true); - - runner = new AnomalyDetectorProfileRunner(client, xContentRegistry(), nodeFilter, requiredSamples); + adTaskManager = mock(ADTaskManager.class); + transportService = mock(TransportService.class); + runner = new AnomalyDetectorProfileRunner(client, xContentRegistry(), nodeFilter, requiredSamples, transportService, adTaskManager); doAnswer(invocation -> { Object[] args = invocation.getArguments(); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/TestHelpers.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/TestHelpers.java index 472dfb98..2c7d57bd 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/TestHelpers.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/TestHelpers.java @@ -19,6 +19,7 @@ import static org.elasticsearch.cluster.node.DiscoveryNodeRole.BUILT_IN_ROLES; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; +import static org.elasticsearch.test.ESTestCase.buildNewFakeTransportAddress; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomDouble; @@ -135,6 +136,7 @@ import com.amazon.opendistroforelasticsearch.jobscheduler.spi.schedule.IntervalSchedule; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; public class TestHelpers { @@ -142,6 +144,8 @@ public class TestHelpers { public static final String AD_BASE_RESULT_URI = "/_opendistro/_anomaly_detection/detectors/results"; public static final String AD_BASE_PREVIEW_URI = "/_opendistro/_anomaly_detection/detectors/%s/_preview"; public static final String AD_BASE_STATS_URI = "/_opendistro/_anomaly_detection/stats"; + public static ImmutableSet historicalDetectorRunningStats = ImmutableSet + .of(ADTaskState.CREATED.name(), ADTaskState.INIT.name(), ADTaskState.RUNNING.name()); private static final Logger logger = LogManager.getLogger(TestHelpers.class); public static final Random random = new Random(42); @@ -939,4 +943,8 @@ public static SearchHits createSearchHits(int totalHits) { SearchHit[] hitArray = new SearchHit[hitList.size()]; return new SearchHits(hitList.toArray(hitArray), new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), 1.0F); } + + public static DiscoveryNode randomDiscoveryNode() { + return new DiscoveryNode(UUIDs.randomBase64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/model/MockSimpleLog.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/model/MockSimpleLog.java new file mode 100644 index 00000000..d0f62f73 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/model/MockSimpleLog.java @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.mock.model; + +import java.io.IOException; +import java.time.Instant; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +public class MockSimpleLog implements ToXContentObject, Writeable { + + public static final String TIME_FIELD = "timestamp"; + public static final String VALUE_FIELD = "value"; + public static final String CATEGORY_FIELD = "category"; + public static final String IS_ERROR_FIELD = "is_error"; + public static final String MESSAGE_FIELD = "message"; + + public static final String INDEX_MAPPING = "{\"mappings\":{\"properties\":{" + + "\"" + + TIME_FIELD + + "\":{\"type\":\"date\",\"format\":\"strict_date_time||epoch_millis\"}," + + "\"" + + VALUE_FIELD + + "\":{\"type\":\"double\"}," + + "\"" + + CATEGORY_FIELD + + "\":{\"type\":\"keyword\"}," + + "\"" + + IS_ERROR_FIELD + + "\":{\"type\":\"boolean\"}," + + "\"" + + MESSAGE_FIELD + + "\":{\"type\":\"text\"}}}}"; + + private Instant timestamp; + private Double value; + private String category; + private Boolean isError; + private String message; + + public MockSimpleLog(Instant timestamp, Double value, String category, Boolean isError, String message) { + this.timestamp = timestamp; + this.value = value; + this.category = category; + this.isError = isError; + this.message = message; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalInstant(timestamp); + out.writeOptionalDouble(value); + out.writeOptionalString(category); + out.writeOptionalBoolean(isError); + out.writeOptionalString(message); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + XContentBuilder xContentBuilder = builder.startObject(); + if (timestamp != null) { + xContentBuilder.field(TIME_FIELD, timestamp.toEpochMilli()); + } + if (value != null) { + xContentBuilder.field(VALUE_FIELD, value); + } + if (category != null) { + xContentBuilder.field(CATEGORY_FIELD, category); + } + if (isError != null) { + xContentBuilder.field(IS_ERROR_FIELD, isError); + } + if (message != null) { + xContentBuilder.field(MESSAGE_FIELD, message); + } + return xContentBuilder.endObject(); + } + + public Instant getTimestamp() { + return timestamp; + } + + public void setTimestamp(Instant timestamp) { + this.timestamp = timestamp; + } + + public Double getValue() { + return value; + } + + public void setValue(Double value) { + this.value = value; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public Boolean getError() { + return isError; + } + + public void setError(Boolean error) { + isError = error; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/plugin/MockReindexPlugin.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/plugin/MockReindexPlugin.java new file mode 100644 index 00000000..a463ffee --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/plugin/MockReindexPlugin.java @@ -0,0 +1,162 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.mock.plugin; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.UpdateByQueryAction; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonName; +import com.amazon.opendistroforelasticsearch.ad.mock.transport.MockAnomalyDetectorJobAction; +import com.amazon.opendistroforelasticsearch.ad.mock.transport.MockAnomalyDetectorJobTransportActionWithUser; +import com.google.common.collect.ImmutableList; + +public class MockReindexPlugin extends Plugin implements ActionPlugin { + + @Override + public List> getActions() { + return Arrays + .asList( + new ActionHandler<>(UpdateByQueryAction.INSTANCE, MockTransportUpdateByQueryAction.class), + new ActionHandler<>(DeleteByQueryAction.INSTANCE, MockTransportDeleteByQueryAction.class), + new ActionHandler<>(MockAnomalyDetectorJobAction.INSTANCE, MockAnomalyDetectorJobTransportActionWithUser.class) + ); + } + + public static class MockTransportUpdateByQueryAction extends HandledTransportAction { + + @Inject + public MockTransportUpdateByQueryAction(ActionFilters actionFilters, TransportService transportService) { + super(UpdateByQueryAction.NAME, transportService, actionFilters, UpdateByQueryRequest::new); + } + + @Override + protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener listener) { + BulkByScrollResponse response = null; + try { + XContentParser parser = TestHelpers + .parser( + "{\"slice_id\":1,\"total\":2,\"updated\":3,\"created\":0,\"deleted\":0,\"batches\":6," + + "\"version_conflicts\":0,\"noops\":0,\"retries\":{\"bulk\":0,\"search\":10}," + + "\"throttled_millis\":0,\"requests_per_second\":13.0,\"canceled\":\"reasonCancelled\"," + + "\"throttled_until_millis\":14}" + ); + parser.nextToken(); + response = new BulkByScrollResponse( + TimeValue.timeValueMillis(10), + BulkByScrollTask.Status.innerFromXContent(parser), + ImmutableList.of(), + ImmutableList.of(), + false + ); + } catch (IOException exception) { + exception.printStackTrace(); + } + listener.onResponse(response); + } + } + + public static class MockTransportDeleteByQueryAction extends HandledTransportAction { + + private Client client; + + @Inject + public MockTransportDeleteByQueryAction(ActionFilters actionFilters, TransportService transportService, Client client) { + super(DeleteByQueryAction.NAME, transportService, actionFilters, DeleteByQueryRequest::new); + this.client = client; + } + + @Override + protected void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { + try { + SearchRequest searchRequest = request.getSearchRequest(); + client.search(searchRequest, ActionListener.wrap(r -> { + long totalHits = r.getHits().getTotalHits().value; + Iterator iterator = r.getHits().iterator(); + BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); + while (iterator.hasNext()) { + String id = iterator.next().getId(); + DeleteRequest deleteRequest = new DeleteRequest(CommonName.DETECTION_STATE_INDEX, id); + bulkRequestBuilder.add(deleteRequest); + } + BulkRequest bulkRequest = bulkRequestBuilder.request().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + client + .execute( + BulkAction.INSTANCE, + bulkRequest, + ActionListener + .wrap( + res -> { listener.onResponse(mockBulkByScrollResponse(totalHits)); }, + ex -> { listener.onFailure(ex); } + ) + ); + + }, e -> { listener.onFailure(e); })); + } catch (Exception e) { + listener.onFailure(e); + } + } + + private BulkByScrollResponse mockBulkByScrollResponse(long totalHits) throws IOException { + XContentParser parser = TestHelpers + .parser( + "{\"slice_id\":1,\"total\":2,\"updated\":0,\"created\":0,\"deleted\":" + + totalHits + + ",\"batches\":6,\"version_conflicts\":0,\"noops\":0,\"retries\":{\"bulk\":0," + + "\"search\":10},\"throttled_millis\":0,\"requests_per_second\":13.0,\"canceled\":" + + "\"reasonCancelled\",\"throttled_until_millis\":14}" + ); + parser.nextToken(); + BulkByScrollResponse response = new BulkByScrollResponse( + TimeValue.timeValueMillis(10), + BulkByScrollTask.Status.innerFromXContent(parser), + ImmutableList.of(), + ImmutableList.of(), + false + ); + return response; + } + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobAction.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobAction.java new file mode 100644 index 00000000..a2bb4fda --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobAction.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.mock.transport; + +import org.elasticsearch.action.ActionType; + +import com.amazon.opendistroforelasticsearch.ad.constant.CommonValue; +import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobResponse; + +public class MockAnomalyDetectorJobAction extends ActionType { + // External Action which used for public facing RestAPIs. + public static final String NAME = CommonValue.EXTERNAL_ACTION_PREFIX + "detector/mockjobmanagement"; + public static final MockAnomalyDetectorJobAction INSTANCE = new MockAnomalyDetectorJobAction(); + + private MockAnomalyDetectorJobAction() { + super(NAME, AnomalyDetectorJobResponse::new); + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobTransportActionWithUser.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobTransportActionWithUser.java new file mode 100644 index 00000000..f09f3ab6 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/mock/transport/MockAnomalyDetectorJobTransportActionWithUser.java @@ -0,0 +1,135 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.mock.transport; + +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT; +import static com.amazon.opendistroforelasticsearch.ad.util.ParseUtils.resolveUserAndExecute; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +import com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices; +import com.amazon.opendistroforelasticsearch.ad.rest.handler.IndexAnomalyDetectorJobActionHandler; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; +import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobRequest; +import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobResponse; +import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobTransportAction; +import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; +import com.amazon.opendistroforelasticsearch.commons.authuser.User; + +public class MockAnomalyDetectorJobTransportActionWithUser extends + HandledTransportAction { + private final Logger logger = LogManager.getLogger(AnomalyDetectorJobTransportAction.class); + + private final Client client; + private final ClusterService clusterService; + private final Settings settings; + private final AnomalyDetectionIndices anomalyDetectionIndices; + private final NamedXContentRegistry xContentRegistry; + private volatile Boolean filterByEnabled; + private final ADTaskManager adTaskManager; + private final TransportService transportService; + + @Inject + public MockAnomalyDetectorJobTransportActionWithUser( + TransportService transportService, + ActionFilters actionFilters, + Client client, + ClusterService clusterService, + Settings settings, + AnomalyDetectionIndices anomalyDetectionIndices, + NamedXContentRegistry xContentRegistry, + ADTaskManager adTaskManager + ) { + super(MockAnomalyDetectorJobAction.NAME, transportService, actionFilters, AnomalyDetectorJobRequest::new); + this.transportService = transportService; + this.client = client; + this.clusterService = clusterService; + this.settings = settings; + this.anomalyDetectionIndices = anomalyDetectionIndices; + this.xContentRegistry = xContentRegistry; + this.adTaskManager = adTaskManager; + filterByEnabled = FILTER_BY_BACKEND_ROLES.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(FILTER_BY_BACKEND_ROLES, it -> filterByEnabled = it); + } + + @Override + protected void doExecute(Task task, AnomalyDetectorJobRequest request, ActionListener listener) { + String detectorId = request.getDetectorID(); + long seqNo = request.getSeqNo(); + long primaryTerm = request.getPrimaryTerm(); + String rawPath = request.getRawPath(); + TimeValue requestTimeout = REQUEST_TIMEOUT.get(settings); + String userStr = "user_name|backendrole1,backendrole2|roles1,role2"; + // By the time request reaches here, the user permissions are validated by Security plugin. + User user = User.parse(userStr); + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + resolveUserAndExecute( + user, + detectorId, + filterByEnabled, + listener, + () -> executeDetector(listener, detectorId, seqNo, primaryTerm, rawPath, requestTimeout, user), + client, + clusterService, + xContentRegistry + ); + } catch (Exception e) { + logger.error(e); + listener.onFailure(e); + } + } + + private void executeDetector( + ActionListener listener, + String detectorId, + long seqNo, + long primaryTerm, + String rawPath, + TimeValue requestTimeout, + User user + ) { + IndexAnomalyDetectorJobActionHandler handler = new IndexAnomalyDetectorJobActionHandler( + client, + listener, + anomalyDetectionIndices, + detectorId, + seqNo, + primaryTerm, + requestTimeout, + xContentRegistry + ); + if (rawPath.endsWith(RestHandlerUtils.START_JOB)) { + adTaskManager.startDetector(detectorId, handler, user, transportService, listener); + } else if (rawPath.endsWith(RestHandlerUtils.STOP_JOB)) { + // Stop detector + adTaskManager.stopDetector(detectorId, handler, user, transportService, listener); + } + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java index 1bebb3e1..2b2772fa 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/AnomalyDetectorRestApiIT.java @@ -823,7 +823,7 @@ public void testStartAdJobWithNonexistingDetector() throws Exception { TestHelpers .assertFailWith( ResponseException.class, - "AnomalyDetector is not found with id", + "AnomalyDetector is not found", () -> TestHelpers .makeRequest( client(), @@ -892,6 +892,7 @@ public void testStopAdJob() throws Exception { } public void testStopNonExistingAdJobIndex() throws Exception { + AnomalyDetector detector = createRandomAnomalyDetector(true, true, client()); TestHelpers .assertFailWith( ResponseException.class, @@ -900,7 +901,7 @@ public void testStopNonExistingAdJobIndex() throws Exception { .makeRequest( client(), "POST", - TestHelpers.AD_BASE_DETECTORS_URI + "/" + randomAlphaOfLength(10) + "/_stop", + TestHelpers.AD_BASE_DETECTORS_URI + "/" + detector.getDetectorId() + "/_stop", ImmutableMap.of(), "", null @@ -924,7 +925,7 @@ public void testStopNonExistingAdJob() throws Exception { TestHelpers .assertFailWith( ResponseException.class, - "Anomaly detector job not exist", + "AnomalyDetector is not found", () -> TestHelpers .makeRequest( client(), diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/HistoricalDetectorRestApiIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/HistoricalDetectorRestApiIT.java new file mode 100644 index 00000000..3ad07ce7 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/rest/HistoricalDetectorRestApiIT.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.rest; + +import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.AD_BASE_STATS_URI; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.rest.RestStatus; + +import com.amazon.opendistroforelasticsearch.ad.HistoricalDetectorRestTestCase; +import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.model.ADTask; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskProfile; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskState; +import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; +import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; +import com.google.common.collect.ImmutableMap; + +public class HistoricalDetectorRestApiIT extends HistoricalDetectorRestTestCase { + + public void testHistoricalDetectorWorkflow() throws Exception { + // create historical detector + AnomalyDetector detector = createHistoricalDetector(); + String detectorId = detector.getDetectorId(); + + // start historical detector + String taskId = startHistoricalDetector(detectorId); + + // get task stats + Response statsResponse = TestHelpers.makeRequest(client(), "GET", AD_BASE_STATS_URI, ImmutableMap.of(), "", null); + String statsResult = EntityUtils.toString(statsResponse.getEntity()); + assertTrue(statsResult.contains("\"ad_executing_batch_task_count\":1")); + + // get task profile + ADTaskProfile adTaskProfile = getADTaskProfile(detectorId); + ADTask adTask = adTaskProfile.getAdTask(); + assertEquals(taskId, adTask.getTaskId()); + assertTrue(TestHelpers.historicalDetectorRunningStats.contains(adTask.getState())); + + // get historical detector with AD task + ToXContentObject[] result = getHistoricalAnomalyDetector(detectorId, true, client()); + AnomalyDetector parsedDetector = (AnomalyDetector) result[0]; + AnomalyDetectorJob parsedJob = (AnomalyDetectorJob) result[1]; + ADTask parsedADTask = (ADTask) result[2]; + assertNull(parsedJob); + assertNotNull(parsedDetector); + assertNotNull(parsedADTask); + assertEquals(taskId, parsedADTask.getTaskId()); + + // stop historical detector + Response stopDetectorResponse = stopAnomalyDetector(detectorId, client()); + assertEquals(RestStatus.OK, restStatus(stopDetectorResponse)); + + // get task profile + ADTaskProfile stoppedAdTaskProfile = getADTaskProfile(detectorId); + int i = 0; + while (TestHelpers.historicalDetectorRunningStats.contains(stoppedAdTaskProfile.getAdTask().getState()) && i < 10) { + stoppedAdTaskProfile = getADTaskProfile(detectorId); + Thread.sleep(2000); + i++; + } + ADTask stoppedAdTask = stoppedAdTaskProfile.getAdTask(); + assertEquals(taskId, stoppedAdTask.getTaskId()); + assertEquals(ADTaskState.STOPPED.name(), stoppedAdTask.getState()); + updateClusterSettings(BATCH_TASK_PIECE_INTERVAL_SECONDS.getKey(), 1); + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManagerTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManagerTests.java index 1a7db42d..fdd4f52e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManagerTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskCacheManagerTests.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.ad.task; import static com.amazon.opendistroforelasticsearch.ad.MemoryTracker.Origin.HISTORICAL_SINGLE_ENTITY_DETECTOR; +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -40,6 +41,7 @@ import com.amazon.opendistroforelasticsearch.ad.MemoryTracker; import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.common.exception.DuplicateTaskException; import com.amazon.opendistroforelasticsearch.ad.common.exception.LimitExceededException; import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.ADTaskState; @@ -78,7 +80,7 @@ public void tearDown() throws Exception { public void testPutTask() throws IOException { when(memoryTracker.canAllocateReserved(anyString(), anyLong())).thenReturn(true); ADTask adTask = TestHelpers.randomAdTask(); - adTaskCacheManager.put(adTask); + adTaskCacheManager.add(adTask); assertEquals(1, adTaskCacheManager.size()); assertTrue(adTaskCacheManager.contains(adTask.getTaskId())); assertTrue(adTaskCacheManager.containsTaskOfDetector(adTask.getDetectorId())); @@ -94,10 +96,10 @@ public void testPutTask() throws IOException { public void testPutDuplicateTask() throws IOException { when(memoryTracker.canAllocateReserved(anyString(), anyLong())).thenReturn(true); ADTask adTask1 = TestHelpers.randomAdTask(); - adTaskCacheManager.put(adTask1); + adTaskCacheManager.add(adTask1); assertEquals(1, adTaskCacheManager.size()); - IllegalArgumentException e1 = expectThrows(IllegalArgumentException.class, () -> adTaskCacheManager.put(adTask1)); - assertEquals("AD task is already running", e1.getMessage()); + DuplicateTaskException e1 = expectThrows(DuplicateTaskException.class, () -> adTaskCacheManager.add(adTask1)); + assertEquals(DETECTOR_IS_RUNNING, e1.getMessage()); ADTask adTask2 = TestHelpers .randomAdTask( @@ -108,15 +110,15 @@ public void testPutDuplicateTask() throws IOException { adTask1.getDetectorId(), adTask1.getDetector() ); - IllegalArgumentException e2 = expectThrows(IllegalArgumentException.class, () -> adTaskCacheManager.put(adTask2)); - assertEquals("There is one task executing for detector", e2.getMessage()); + DuplicateTaskException e2 = expectThrows(DuplicateTaskException.class, () -> adTaskCacheManager.add(adTask2)); + assertEquals(DETECTOR_IS_RUNNING, e2.getMessage()); } public void testPutTaskWithMemoryExceedLimit() { when(memoryTracker.canAllocateReserved(anyString(), anyLong())).thenReturn(false); LimitExceededException exception = expectThrows( LimitExceededException.class, - () -> adTaskCacheManager.put(TestHelpers.randomAdTask()) + () -> adTaskCacheManager.add(TestHelpers.randomAdTask()) ); assertEquals("No enough memory to run detector", exception.getMessage()); } @@ -124,7 +126,7 @@ public void testPutTaskWithMemoryExceedLimit() { public void testThresholdModelTrained() throws IOException { when(memoryTracker.canAllocateReserved(anyString(), anyLong())).thenReturn(true); ADTask adTask = TestHelpers.randomAdTask(); - adTaskCacheManager.put(adTask); + adTaskCacheManager.add(adTask); assertEquals(1, adTaskCacheManager.size()); int size = adTaskCacheManager.addThresholdModelTrainingData(adTask.getTaskId(), randomDouble(), randomDouble()); long cacheSize = adTaskCacheManager.trainingDataMemorySize(size); @@ -137,7 +139,7 @@ public void testThresholdModelTrained() throws IOException { public void testCancel() throws IOException { when(memoryTracker.canAllocateReserved(anyString(), anyLong())).thenReturn(true); ADTask adTask = TestHelpers.randomAdTask(); - adTaskCacheManager.put(adTask); + adTaskCacheManager.add(adTask); assertEquals(1, adTaskCacheManager.size()); assertEquals(false, adTaskCacheManager.isCancelled(adTask.getTaskId())); String cancelReason = randomAlphaOfLength(10); @@ -163,10 +165,10 @@ public void testRemoveTaskWhichNotExist() { public void testExceedRunningTaskLimit() throws IOException { when(memoryTracker.canAllocateReserved(anyString(), anyLong())).thenReturn(true); - adTaskCacheManager.put(TestHelpers.randomAdTask()); - adTaskCacheManager.put(TestHelpers.randomAdTask()); + adTaskCacheManager.add(TestHelpers.randomAdTask()); + adTaskCacheManager.add(TestHelpers.randomAdTask()); assertEquals(2, adTaskCacheManager.size()); - LimitExceededException e = expectThrows(LimitExceededException.class, () -> adTaskCacheManager.put(TestHelpers.randomAdTask())); + LimitExceededException e = expectThrows(LimitExceededException.class, () -> adTaskCacheManager.add(TestHelpers.randomAdTask())); assertEquals("Can't run more than 2 historical detectors per data node", e.getMessage()); } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManagerTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManagerTests.java index 920cb050..853d9155 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManagerTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/task/ADTaskManagerTests.java @@ -19,7 +19,9 @@ import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.randomFeature; import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.randomUser; import static com.amazon.opendistroforelasticsearch.ad.constant.CommonName.ANOMALY_RESULT_INDEX_ALIAS; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.REQUEST_TIMEOUT; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -39,21 +41,33 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.transport.TransportService; import com.amazon.opendistroforelasticsearch.ad.ADUnitTestCase; +import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.cluster.HashRing; +import com.amazon.opendistroforelasticsearch.ad.common.exception.DuplicateTaskException; import com.amazon.opendistroforelasticsearch.ad.indices.AnomalyDetectionIndices; +import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.DetectionDateRange; import com.amazon.opendistroforelasticsearch.ad.transport.AnomalyDetectorJobResponse; +import com.amazon.opendistroforelasticsearch.ad.util.DiscoveryNodeFilterer; import com.google.common.collect.ImmutableList; public class ADTaskManagerTests extends ADUnitTestCase { private Settings settings; private Client client; + private ClusterService clusterService; private ClusterSettings clusterSettings; + private DiscoveryNodeFilterer nodeFilter; private AnomalyDetectionIndices anomalyDetectionIndices; + private ADTaskCacheManager adTaskCacheManager; + private HashRing hashRing; + private TransportService transportService; private ADTaskManager adTaskManager; private Instant startTime; @@ -67,15 +81,33 @@ public void setUp() throws Exception { startTime = now.minus(10, ChronoUnit.DAYS); endTime = now.minus(1, ChronoUnit.DAYS); - settings = Settings.builder().put(MAX_OLD_AD_TASK_DOCS_PER_DETECTOR.getKey(), 2).build(); + settings = Settings + .builder() + .put(MAX_OLD_AD_TASK_DOCS_PER_DETECTOR.getKey(), 2) + .put(BATCH_TASK_PIECE_INTERVAL_SECONDS.getKey(), 1) + .put(REQUEST_TIMEOUT.getKey(), TimeValue.timeValueSeconds(10)) + .build(); - clusterSettings = clusterSetting(settings, MAX_OLD_AD_TASK_DOCS_PER_DETECTOR); + clusterSettings = clusterSetting(settings, MAX_OLD_AD_TASK_DOCS_PER_DETECTOR, BATCH_TASK_PIECE_INTERVAL_SECONDS, REQUEST_TIMEOUT); - final ClusterService clusterService = new ClusterService(settings, clusterSettings, null); + clusterService = new ClusterService(settings, clusterSettings, null); client = mock(Client.class); + nodeFilter = mock(DiscoveryNodeFilterer.class); anomalyDetectionIndices = mock(AnomalyDetectionIndices.class); - adTaskManager = new ADTaskManager(settings, clusterService, client, NamedXContentRegistry.EMPTY, anomalyDetectionIndices); + adTaskCacheManager = mock(ADTaskCacheManager.class); + hashRing = mock(HashRing.class); + transportService = mock(TransportService.class); + adTaskManager = new ADTaskManager( + settings, + clusterService, + client, + NamedXContentRegistry.EMPTY, + anomalyDetectionIndices, + nodeFilter, + hashRing, + adTaskCacheManager + ); listener = spy(new ActionListener() { @Override @@ -100,7 +132,7 @@ public void testCreateTaskIndexNotAcknowledged() throws IOException { randomAlphaOfLength(5) ); - adTaskManager.createADTaskIndex(detector, randomUser(), listener); + adTaskManager.startHistoricalDetector(detector, randomUser(), transportService, listener); verify(listener, times(1)).onFailure(exceptionCaptor.capture()); assertEquals( "Create index .opendistro-anomaly-detection-state with mappings not acknowledged", @@ -122,7 +154,7 @@ public void testCreateTaskIndexWithResourceAlreadyExistsException() throws IOExc randomAlphaOfLength(5) ); - adTaskManager.createADTaskIndex(detector, randomUser(), listener); + adTaskManager.startHistoricalDetector(detector, randomUser(), transportService, listener); verify(listener, never()).onFailure(any()); } @@ -141,8 +173,14 @@ public void testCreateTaskIndexWithException() throws IOException { randomAlphaOfLength(5) ); - adTaskManager.createADTaskIndex(detector, randomUser(), listener); + adTaskManager.startHistoricalDetector(detector, randomUser(), transportService, listener); verify(listener, times(1)).onFailure(exceptionCaptor.capture()); assertEquals(error, exceptionCaptor.getValue().getMessage()); } + + public void testDeleteDuplicateTasks() throws IOException { + ADTask adTask = TestHelpers.randomAdTask(); + adTaskManager.handleADTaskException(adTask, new DuplicateTaskException("test")); + verify(client, times(1)).delete(any(), any()); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTests.java new file mode 100644 index 00000000..d913ef3b --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADCancelTaskTests.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.randomDiscoveryNode; + +import java.io.IOException; +import java.util.List; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.StreamInput; + +import com.amazon.opendistroforelasticsearch.ad.ADUnitTestCase; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskCancellationState; +import com.google.common.collect.ImmutableList; + +public class ADCancelTaskTests extends ADUnitTestCase { + + public void testADCancelTaskRequest() throws IOException { + ADCancelTaskRequest request = new ADCancelTaskRequest(randomAlphaOfLength(5), randomAlphaOfLength(5), randomDiscoveryNode()); + + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + NamedWriteableAwareStreamInput input = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry()); + ADCancelTaskRequest parsedRequest = new ADCancelTaskRequest(input); + assertEquals(request.getDetectorId(), parsedRequest.getDetectorId()); + assertEquals(request.getUserName(), parsedRequest.getUserName()); + } + + public void testInvalidADCancelTaskRequest() { + ADCancelTaskRequest request = new ADCancelTaskRequest(null, null, randomDiscoveryNode()); + ActionRequestValidationException validationException = request.validate(); + assertTrue(validationException.getMessage().contains(CommonErrorMessages.AD_ID_MISSING_MSG)); + } + + public void testSerializeResponse() throws IOException { + ADTaskCancellationState state = ADTaskCancellationState.CANCELLED; + ADCancelTaskNodeResponse nodeResponse = new ADCancelTaskNodeResponse(randomDiscoveryNode(), state); + + List nodes = ImmutableList.of(nodeResponse); + ADCancelTaskResponse response = new ADCancelTaskResponse(new ClusterName("test"), nodes, ImmutableList.of()); + + BytesStreamOutput output = new BytesStreamOutput(); + response.writeNodesTo(output, nodes); + StreamInput input = output.bytes().streamInput(); + + List adCancelTaskNodeResponses = response.readNodesFrom(input); + assertEquals(1, adCancelTaskNodeResponses.size()); + assertEquals(state, adCancelTaskNodeResponses.get(0).getState()); + + BytesStreamOutput output2 = new BytesStreamOutput(); + response.writeTo(output2); + StreamInput input2 = output2.bytes().streamInput(); + + ADCancelTaskResponse response2 = new ADCancelTaskResponse(input2); + assertEquals(1, response2.getNodes().size()); + assertEquals(state, response2.getNodes().get(0).getState()); + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTests.java new file mode 100644 index 00000000..a7f8ea54 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTests.java @@ -0,0 +1,149 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.randomDiscoveryNode; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; + +import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; +import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskProfile; +import com.google.common.collect.ImmutableList; + +public class ADTaskProfileTests extends ESSingleNodeTestCase { + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class, AnomalyDetectorPlugin.class); + } + + @Override + protected NamedWriteableRegistry writableRegistry() { + return getInstanceFromNode(NamedWriteableRegistry.class); + } + + public void testADTaskProfileRequest() throws IOException { + ADTaskProfileRequest request = new ADTaskProfileRequest(randomAlphaOfLength(5), randomDiscoveryNode()); + + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + NamedWriteableAwareStreamInput input = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry()); + ADTaskProfileRequest parsedRequest = new ADTaskProfileRequest(input); + assertEquals(request.getDetectorId(), parsedRequest.getDetectorId()); + } + + public void testInvalidADTaskProfileRequest() { + DiscoveryNode node = new DiscoveryNode(UUIDs.randomBase64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); + ADTaskProfileRequest request = new ADTaskProfileRequest(null, node); + ActionRequestValidationException validationException = request.validate(); + assertTrue(validationException.getMessage().contains(CommonErrorMessages.AD_ID_MISSING_MSG)); + } + + public void testADTaskProfileNodeResponse() throws IOException { + ADTaskProfile adTaskProfile = new ADTaskProfile( + randomInt(), + randomLong(), + randomBoolean(), + randomInt(), + randomLong(), + randomAlphaOfLength(5) + ); + ADTaskProfileNodeResponse response = new ADTaskProfileNodeResponse(randomDiscoveryNode(), adTaskProfile); + testADTaskProfileResponse(response); + } + + public void testADTaskProfileNodeResponseWithNullProfile() throws IOException { + ADTaskProfileNodeResponse response = new ADTaskProfileNodeResponse(randomDiscoveryNode(), null); + testADTaskProfileResponse(response); + } + + public void testADTaskProfileNodeResponseReadMethod() throws IOException { + ADTaskProfile adTaskProfile = new ADTaskProfile( + randomInt(), + randomLong(), + randomBoolean(), + randomInt(), + randomLong(), + randomAlphaOfLength(5) + ); + ADTaskProfileNodeResponse response = new ADTaskProfileNodeResponse(randomDiscoveryNode(), adTaskProfile); + testADTaskProfileResponse(response); + } + + public void testADTaskProfileNodeResponseReadMethodWithNullProfile() throws IOException { + ADTaskProfileNodeResponse response = new ADTaskProfileNodeResponse(randomDiscoveryNode(), null); + testADTaskProfileResponse(response); + } + + private void testADTaskProfileResponse(ADTaskProfileNodeResponse response) throws IOException { + BytesStreamOutput output = new BytesStreamOutput(); + response.writeTo(output); + NamedWriteableAwareStreamInput input = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry()); + ADTaskProfileNodeResponse parsedResponse = ADTaskProfileNodeResponse.readNodeResponse(input); + if (response.getAdTaskProfile() != null) { + assertTrue(response.getAdTaskProfile().equals(parsedResponse.getAdTaskProfile())); + } else { + assertNull(parsedResponse.getAdTaskProfile()); + } + } + + public void testSerializeResponse() throws IOException { + DiscoveryNode node = randomDiscoveryNode(); + ADTaskProfile profile = new ADTaskProfile( + TestHelpers.randomAdTask(), + randomInt(), + randomLong(), + randomBoolean(), + randomInt(), + randomLong(), + randomAlphaOfLength(5) + ); + ADTaskProfileNodeResponse nodeResponse = new ADTaskProfileNodeResponse(node, profile); + ImmutableList nodes = ImmutableList.of(nodeResponse); + ADTaskProfileResponse response = new ADTaskProfileResponse(new ClusterName("test"), nodes, ImmutableList.of()); + + BytesStreamOutput output = new BytesStreamOutput(); + response.writeNodesTo(output, nodes); + NamedWriteableAwareStreamInput input = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry()); + + List adTaskProfileNodeResponses = response.readNodesFrom(input); + assertEquals(1, adTaskProfileNodeResponses.size()); + assertEquals(profile, adTaskProfileNodeResponses.get(0).getAdTaskProfile()); + + BytesStreamOutput output2 = new BytesStreamOutput(); + response.writeTo(output2); + NamedWriteableAwareStreamInput input2 = new NamedWriteableAwareStreamInput(output2.bytes().streamInput(), writableRegistry()); + + ADTaskProfileResponse response2 = new ADTaskProfileResponse(input2); + assertEquals(1, response2.getNodes().size()); + assertEquals(profile, response2.getNodes().get(0).getAdTaskProfile()); + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportActionTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportActionTests.java new file mode 100644 index 00000000..de061e40 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ADTaskProfileTransportActionTests.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; +import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import org.elasticsearch.common.settings.Settings; +import org.junit.Before; + +import com.amazon.opendistroforelasticsearch.ad.HistoricalDetectorIntegTestCase; + +public class ADTaskProfileTransportActionTests extends HistoricalDetectorIntegTestCase { + + private Instant startTime; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + startTime = Instant.now().minus(10, ChronoUnit.DAYS); + ingestTestData(testIndex, startTime, detectionIntervalInMinutes, "error", 2000); + createDetectorIndex(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings + .builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(BATCH_TASK_PIECE_INTERVAL_SECONDS.getKey(), 1) + .put(MAX_BATCH_TASK_PER_NODE.getKey(), 1) + .build(); + } + + public void testProfile() { + + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportActionTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportActionTests.java index 572b1ff3..304a0f0d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportActionTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyDetectorJobTransportActionTests.java @@ -15,10 +15,13 @@ package com.amazon.opendistroforelasticsearch.ad.transport; +import static com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages.DETECTOR_IS_RUNNING; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.BATCH_TASK_PIECE_INTERVAL_SECONDS; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.MAX_BATCH_TASK_PER_NODE; import static com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings.MAX_OLD_AD_TASK_DOCS_PER_DETECTOR; +import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.PROFILE; import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.START_JOB; +import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.STOP_JOB; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -29,29 +32,38 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.junit.After; import org.junit.Before; import com.amazon.opendistroforelasticsearch.ad.HistoricalDetectorIntegTestCase; import com.amazon.opendistroforelasticsearch.ad.TestHelpers; import com.amazon.opendistroforelasticsearch.ad.constant.CommonName; +import com.amazon.opendistroforelasticsearch.ad.mock.plugin.MockReindexPlugin; +import com.amazon.opendistroforelasticsearch.ad.mock.transport.MockAnomalyDetectorJobAction; import com.amazon.opendistroforelasticsearch.ad.model.ADTask; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskProfile; import com.amazon.opendistroforelasticsearch.ad.model.ADTaskState; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.model.DetectionDateRange; -import com.amazon.opendistroforelasticsearch.ad.plugin.MockReindexPlugin; +import com.amazon.opendistroforelasticsearch.ad.stats.StatNames; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 2) public class AnomalyDetectorJobTransportActionTests extends HistoricalDetectorIntegTestCase { @@ -86,18 +98,14 @@ protected Collection> getMockPlugins() { final ArrayList> plugins = new ArrayList<>(); plugins.add(MockReindexPlugin.class); plugins.addAll(super.getMockPlugins()); + plugins.remove(MockTransportService.TestPlugin.class); return Collections.unmodifiableList(plugins); } public void testDetectorIndexNotFound() { deleteDetectorIndex(); String detectorId = randomAlphaOfLength(5); - AnomalyDetectorJobRequest request = new AnomalyDetectorJobRequest( - detectorId, - UNASSIGNED_SEQ_NO, - UNASSIGNED_PRIMARY_TERM, - START_JOB - ); + AnomalyDetectorJobRequest request = startDetectorJobRequest(detectorId); IndexNotFoundException exception = expectThrows( IndexNotFoundException.class, () -> client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(3000) @@ -107,20 +115,22 @@ public void testDetectorIndexNotFound() { public void testDetectorNotFound() { String detectorId = randomAlphaOfLength(5); - AnomalyDetectorJobRequest request = new AnomalyDetectorJobRequest( - detectorId, - UNASSIGNED_SEQ_NO, - UNASSIGNED_PRIMARY_TERM, - START_JOB - ); + AnomalyDetectorJobRequest request = startDetectorJobRequest(detectorId); ElasticsearchStatusException exception = expectThrows( ElasticsearchStatusException.class, - () -> client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(3000) + () -> client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000) ); assertTrue(exception.getMessage().contains("AnomalyDetector is not found")); } public void testValidHistoricalDetector() throws IOException, InterruptedException { + ADTask adTask = startHistoricalDetector(); + Thread.sleep(10000); + ADTask finishedTask = getADTask(adTask.getTaskId()); + assertEquals(ADTaskState.FINISHED.name(), finishedTask.getState()); + } + + private ADTask startHistoricalDetector() throws IOException { DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime); AnomalyDetector detector = TestHelpers .randomDetector(dateRange, ImmutableList.of(maxValueFeature()), testIndex, detectionIntervalInMinutes, timeField); @@ -132,12 +142,10 @@ public void testValidHistoricalDetector() throws IOException, InterruptedExcepti START_JOB ); AnomalyDetectorJobResponse response = client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000); - Thread.sleep(10000); - GetResponse doc = getDoc(CommonName.DETECTION_STATE_INDEX, response.getId()); - assertEquals(ADTaskState.FINISHED.name(), doc.getSourceAsMap().get(ADTask.STATE_FIELD)); + return getADTask(response.getId()); } - public void testRunMultipleTasksForHistoricalDetector() throws IOException, InterruptedException { + public void testStartHistoricalDetectorWithUser() throws IOException, InterruptedException { DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime); AnomalyDetector detector = TestHelpers .randomDetector(dateRange, ImmutableList.of(maxValueFeature()), testIndex, detectionIntervalInMinutes, timeField); @@ -148,17 +156,57 @@ public void testRunMultipleTasksForHistoricalDetector() throws IOException, Inte UNASSIGNED_PRIMARY_TERM, START_JOB ); + Client nodeClient = getDataNodeClient(); + if (nodeClient != null) { + AnomalyDetectorJobResponse response = nodeClient.execute(MockAnomalyDetectorJobAction.INSTANCE, request).actionGet(100000); + ADTask adTask = getADTask(response.getId()); + assertNotNull(adTask.getStartedBy()); + } + } + + public void testRunMultipleTasksForHistoricalDetector() throws IOException, InterruptedException { + DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime); + AnomalyDetector detector = TestHelpers + .randomDetector(dateRange, ImmutableList.of(maxValueFeature()), testIndex, detectionIntervalInMinutes, timeField); + String detectorId = createDetector(detector); + AnomalyDetectorJobRequest request = startDetectorJobRequest(detectorId); AnomalyDetectorJobResponse response = client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000); assertNotNull(response.getId()); ElasticsearchStatusException exception = expectThrows( ElasticsearchStatusException.class, () -> client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000) ); - assertTrue(exception.getMessage().contains("Detector is already running")); + assertTrue(exception.getMessage().contains(DETECTOR_IS_RUNNING)); + assertEquals(DETECTOR_IS_RUNNING, exception.getMessage()); + Thread.sleep(10000); + List adTasks = searchADTasks(detectorId, null, 100); + assertEquals(1, adTasks.size()); + assertEquals(ADTaskState.FINISHED.name(), adTasks.get(0).getState()); } - public void testCleanOldTaskDocs() throws IOException, InterruptedException { - updateTransientSettings(ImmutableMap.of(BATCH_TASK_PIECE_INTERVAL_SECONDS.getKey(), 1)); + public void testRaceConditionByStartingMultipleTasks() throws IOException, InterruptedException { + DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime); + AnomalyDetector detector = TestHelpers + .randomDetector(dateRange, ImmutableList.of(maxValueFeature()), testIndex, detectionIntervalInMinutes, timeField); + String detectorId = createDetector(detector); + AnomalyDetectorJobRequest request = new AnomalyDetectorJobRequest( + detectorId, + UNASSIGNED_SEQ_NO, + UNASSIGNED_PRIMARY_TERM, + START_JOB + ); + client().execute(AnomalyDetectorJobAction.INSTANCE, request); + client().execute(AnomalyDetectorJobAction.INSTANCE, request); + + Thread.sleep(5000); + List adTasks = searchADTasks(detectorId, null, 100); + + assertEquals(1, adTasks.size()); + assertTrue(adTasks.get(0).getLatest()); + assertNotEquals(ADTaskState.FAILED.name(), adTasks.get(0).getState()); + } + + public void testCleanOldTaskDocs() throws InterruptedException, IOException { DetectionDateRange dateRange = new DetectionDateRange(startTime, endTime); AnomalyDetector detector = TestHelpers .randomDetector(dateRange, ImmutableList.of(maxValueFeature()), testIndex, detectionIntervalInMinutes, timeField); @@ -174,6 +222,7 @@ public void testCleanOldTaskDocs() throws IOException, InterruptedException { assertEquals(states.size(), count); AnomalyDetectorJobRequest request = new AnomalyDetectorJobRequest(detectorId, randomLong(), randomLong(), START_JOB); + AtomicReference response = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); client().execute(AnomalyDetectorJobAction.INSTANCE, request, ActionListener.wrap(r -> { @@ -187,22 +236,31 @@ public void testCleanOldTaskDocs() throws IOException, InterruptedException { assertEquals(maxOldAdTaskDocsPerDetector + 1, count); } + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + // delete index will clear search context, this can avoid in-flight contexts error + deleteIndexIfExists(AnomalyDetector.ANOMALY_DETECTORS_INDEX); + deleteIndexIfExists(CommonName.DETECTION_STATE_INDEX); + } + public void testStartRealtimeDetector() throws IOException { + String detectorId = startRealtimeDetector(); + GetResponse doc = getDoc(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, detectorId); + AnomalyDetectorJob job = toADJob(doc); + assertTrue(job.isEnabled()); + assertEquals(detectorId, job.getName()); + } + + private String startRealtimeDetector() throws IOException { AnomalyDetector detector = TestHelpers .randomDetector(null, ImmutableList.of(maxValueFeature()), testIndex, detectionIntervalInMinutes, timeField); String detectorId = createDetector(detector); - AnomalyDetectorJobRequest request = new AnomalyDetectorJobRequest( - detectorId, - UNASSIGNED_SEQ_NO, - UNASSIGNED_PRIMARY_TERM, - START_JOB - ); + AnomalyDetectorJobRequest request = startDetectorJobRequest(detectorId); AnomalyDetectorJobResponse response = client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000); assertEquals(detectorId, response.getId()); - GetResponse doc = getDoc(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, detectorId); - AnomalyDetectorJob job = toADJob(doc); - assertTrue(job.isEnabled()); - assertEquals(detectorId, job.getName()); + return response.getId(); } public void testRealtimeDetectorWithoutFeature() throws IOException { @@ -242,16 +300,99 @@ public void testHistoricalDetectorWithoutEnabledFeature() throws IOException { private void testInvalidDetector(AnomalyDetector detector, String error) throws IOException { String detectorId = createDetector(detector); - AnomalyDetectorJobRequest request = new AnomalyDetectorJobRequest( - detectorId, - UNASSIGNED_SEQ_NO, - UNASSIGNED_PRIMARY_TERM, - START_JOB - ); + AnomalyDetectorJobRequest request = startDetectorJobRequest(detectorId); ElasticsearchStatusException exception = expectThrows( ElasticsearchStatusException.class, () -> client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000) ); assertEquals(error, exception.getMessage()); } + + private AnomalyDetectorJobRequest startDetectorJobRequest(String detectorId) { + return new AnomalyDetectorJobRequest(detectorId, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, START_JOB); + } + + private AnomalyDetectorJobRequest stopDetectorJobRequest(String detectorId) { + return new AnomalyDetectorJobRequest(detectorId, UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM, STOP_JOB); + } + + public void testStopRealtimeDetector() throws IOException { + String detectorId = startRealtimeDetector(); + AnomalyDetectorJobRequest request = stopDetectorJobRequest(detectorId); + client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000); + GetResponse doc = getDoc(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, detectorId); + AnomalyDetectorJob job = toADJob(doc); + assertFalse(job.isEnabled()); + assertEquals(detectorId, job.getName()); + } + + public void testStopHistoricalDetector() throws IOException, InterruptedException { + ADTask adTask = startHistoricalDetector(); + assertEquals(ADTaskState.INIT.name(), adTask.getState()); + AnomalyDetectorJobRequest request = stopDetectorJobRequest(adTask.getDetectorId()); + client().execute(AnomalyDetectorJobAction.INSTANCE, request).actionGet(10000); + Thread.sleep(10000); + ADTask stoppedTask = getADTask(adTask.getTaskId()); + assertEquals(ADTaskState.STOPPED.name(), stoppedTask.getState()); + assertEquals(0, getExecutingADTask()); + } + + public void testProfileHistoricalDetector() throws IOException, InterruptedException { + ADTask adTask = startHistoricalDetector(); + GetAnomalyDetectorRequest request = taskProfileRequest(adTask.getDetectorId()); + GetAnomalyDetectorResponse response = client().execute(GetAnomalyDetectorAction.INSTANCE, request).actionGet(10000); + assertNotNull(response.getDetectorProfile().getAdTaskProfile()); + + ADTask finishedTask = getADTask(adTask.getTaskId()); + int i = 0; + while (TestHelpers.historicalDetectorRunningStats.contains(finishedTask.getState()) && i < 10) { + finishedTask = getADTask(adTask.getTaskId()); + Thread.sleep(2000); + i++; + } + assertEquals(ADTaskState.FINISHED.name(), finishedTask.getState()); + + response = client().execute(GetAnomalyDetectorAction.INSTANCE, request).actionGet(10000); + assertNull(response.getDetectorProfile().getAdTaskProfile().getNodeId()); + ADTask profileAdTask = response.getDetectorProfile().getAdTaskProfile().getAdTask(); + assertEquals(finishedTask.getTaskId(), profileAdTask.getTaskId()); + assertEquals(finishedTask.getDetectorId(), profileAdTask.getDetectorId()); + assertEquals(finishedTask.getDetector(), profileAdTask.getDetector()); + assertEquals(finishedTask.getState(), profileAdTask.getState()); + } + + public void testProfileWithMultipleRunningTask() throws IOException { + ADTask adTask1 = startHistoricalDetector(); + ADTask adTask2 = startHistoricalDetector(); + + GetAnomalyDetectorRequest request1 = taskProfileRequest(adTask1.getDetectorId()); + GetAnomalyDetectorRequest request2 = taskProfileRequest(adTask2.getDetectorId()); + GetAnomalyDetectorResponse response1 = client().execute(GetAnomalyDetectorAction.INSTANCE, request1).actionGet(10000); + GetAnomalyDetectorResponse response2 = client().execute(GetAnomalyDetectorAction.INSTANCE, request2).actionGet(10000); + ADTaskProfile taskProfile1 = response1.getDetectorProfile().getAdTaskProfile(); + ADTaskProfile taskProfile2 = response2.getDetectorProfile().getAdTaskProfile(); + assertNotNull(taskProfile1.getNodeId()); + assertNotNull(taskProfile2.getNodeId()); + assertNotEquals(taskProfile1.getNodeId(), taskProfile2.getNodeId()); + } + + private GetAnomalyDetectorRequest taskProfileRequest(String detectorId) throws IOException { + return new GetAnomalyDetectorRequest(detectorId, Versions.MATCH_ANY, false, false, "", PROFILE, true, null); + } + + private long getExecutingADTask() { + ADStatsRequest adStatsRequest = new ADStatsRequest(getDataNodesArray()); + Set validStats = ImmutableSet.of(StatNames.AD_EXECUTING_BATCH_TASK_COUNT.getName()); + adStatsRequest.addAll(validStats); + StatsAnomalyDetectorResponse statsResponse = client().execute(StatsAnomalyDetectorAction.INSTANCE, adStatsRequest).actionGet(5000); + AtomicLong totalExecutingTask = new AtomicLong(0); + statsResponse + .getAdStatsResponse() + .getADStatsNodesResponse() + .getNodes() + .forEach( + node -> { totalExecutingTask.getAndAdd((Long) node.getStatsMap().get(StatNames.AD_EXECUTING_BATCH_TASK_COUNT.getName())); } + ); + return totalExecutingTask.get(); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTests.java new file mode 100644 index 00000000..0ddab8e7 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/ForwardADTaskTests.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.ad.transport; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; + +import com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorPlugin; +import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; +import com.amazon.opendistroforelasticsearch.ad.model.ADTaskAction; +import com.google.common.collect.ImmutableMap; + +public class ForwardADTaskTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class, AnomalyDetectorPlugin.class); + } + + @Override + protected NamedWriteableRegistry writableRegistry() { + return getInstanceFromNode(NamedWriteableRegistry.class); + } + + public void testForwardADTaskRequest() throws IOException { + ForwardADTaskRequest request = new ForwardADTaskRequest( + TestHelpers.randomAnomalyDetector(ImmutableMap.of(), Instant.now()), + TestHelpers.randomUser(), + ADTaskAction.START + ); + testForwardADTaskRequest(request); + } + + public void testForwardADTaskRequestWithoutUser() throws IOException { + ForwardADTaskRequest request = new ForwardADTaskRequest( + TestHelpers.randomAnomalyDetector(ImmutableMap.of(), Instant.now()), + null, + ADTaskAction.START + ); + testForwardADTaskRequest(request); + } + + public void testInvalidForwardADTaskRequest() { + ForwardADTaskRequest request = new ForwardADTaskRequest(null, TestHelpers.randomUser(), ADTaskAction.START); + + ActionRequestValidationException exception = request.validate(); + assertTrue(exception.getMessage().contains(CommonErrorMessages.DETECTOR_MISSING)); + } + + private void testForwardADTaskRequest(ForwardADTaskRequest request) throws IOException { + BytesStreamOutput output = new BytesStreamOutput(); + request.writeTo(output); + NamedWriteableAwareStreamInput input = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry()); + ForwardADTaskRequest parsedRequest = new ForwardADTaskRequest(input); + if (request.getUser() != null) { + assertTrue(request.getUser().equals(parsedRequest.getUser())); + } else { + assertNull(parsedRequest.getUser()); + } + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorActionTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorActionTests.java index 5c834972..7e6cefc1 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorActionTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorActionTests.java @@ -29,6 +29,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.model.DetectorProfile; @@ -44,7 +45,7 @@ public void setUp() throws Exception { @Test public void testGetRequest() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); - GetAnomalyDetectorRequest request = new GetAnomalyDetectorRequest("1234", 4321, false, "nonempty", "", false, null); + GetAnomalyDetectorRequest request = new GetAnomalyDetectorRequest("1234", 4321, false, false, "nonempty", "", false, null); request.writeTo(out); StreamInput input = out.bytes().streamInput(); GetAnomalyDetectorRequest newRequest = new GetAnomalyDetectorRequest(input); @@ -66,6 +67,8 @@ public void testGetResponse() throws Exception { detector, detectorJob, false, + Mockito.mock(ADTask.class), + false, RestStatus.OK, Mockito.mock(DetectorProfile.class), null, diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTests.java index 9a96e17d..b88ffd6e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTests.java @@ -44,6 +44,7 @@ import com.amazon.opendistroforelasticsearch.ad.AbstractADTest; import com.amazon.opendistroforelasticsearch.ad.constant.CommonErrorMessages; import com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; import com.amazon.opendistroforelasticsearch.ad.util.DiscoveryNodeFilterer; public class GetAnomalyDetectorTests extends AbstractADTest { @@ -58,6 +59,7 @@ public class GetAnomalyDetectorTests extends AbstractADTest { private String typeStr; private String rawPath; private PlainActionFuture future; + private ADTaskManager adTaskManager; @BeforeClass public static void setUpBeforeClass() { @@ -96,6 +98,8 @@ public void setUp() throws Exception { client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); + adTaskManager = mock(ADTaskManager.class); + action = new GetAnomalyDetectorTransportAction( transportService, nodeFilter, @@ -103,7 +107,8 @@ public void setUp() throws Exception { clusterService, client, Settings.EMPTY, - xContentRegistry() + xContentRegistry(), + adTaskManager ); } @@ -112,7 +117,7 @@ public void testInvalidRequest() throws IOException { rawPath = "_opendistro/_anomaly_detection/detectors/T4c3dXUBj-2IZN7itix_/_profile"; - request = new GetAnomalyDetectorRequest(detectorId, 0L, false, typeStr, rawPath, false, entityValue); + request = new GetAnomalyDetectorRequest(detectorId, 0L, false, false, typeStr, rawPath, false, entityValue); future = new PlainActionFuture<>(); action.doExecute(null, request, future); @@ -137,7 +142,7 @@ public void testValidRequest() throws IOException { rawPath = "_opendistro/_anomaly_detection/detectors/T4c3dXUBj-2IZN7itix_/_profile"; - request = new GetAnomalyDetectorRequest(detectorId, 0L, false, typeStr, rawPath, false, entityValue); + request = new GetAnomalyDetectorRequest(detectorId, 0L, false, false, typeStr, rawPath, false, entityValue); future = new PlainActionFuture<>(); action.doExecute(null, request, future); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportActionTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportActionTests.java index 7608e2f9..2fa4f0a3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportActionTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/GetAnomalyDetectorTransportActionTests.java @@ -46,10 +46,12 @@ import org.mockito.Mockito; import com.amazon.opendistroforelasticsearch.ad.TestHelpers; +import com.amazon.opendistroforelasticsearch.ad.model.ADTask; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetector; import com.amazon.opendistroforelasticsearch.ad.model.AnomalyDetectorJob; import com.amazon.opendistroforelasticsearch.ad.model.EntityProfile; import com.amazon.opendistroforelasticsearch.ad.settings.AnomalyDetectorSettings; +import com.amazon.opendistroforelasticsearch.ad.task.ADTaskManager; import com.amazon.opendistroforelasticsearch.ad.util.DiscoveryNodeFilterer; import com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils; import com.google.common.collect.ImmutableMap; @@ -58,6 +60,7 @@ public class GetAnomalyDetectorTransportActionTests extends ESSingleNodeTestCase private GetAnomalyDetectorTransportAction action; private Task task; private ActionListener response; + private ADTaskManager adTaskManager; @Override @Before @@ -69,6 +72,7 @@ public void setUp() throws Exception { Collections.unmodifiableSet(new HashSet<>(Arrays.asList(AnomalyDetectorSettings.FILTER_BY_BACKEND_ROLES))) ); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + adTaskManager = mock(ADTaskManager.class); action = new GetAnomalyDetectorTransportAction( Mockito.mock(TransportService.class), Mockito.mock(DiscoveryNodeFilterer.class), @@ -76,7 +80,8 @@ public void setUp() throws Exception { clusterService, client(), Settings.EMPTY, - xContentRegistry() + xContentRegistry(), + adTaskManager ); task = Mockito.mock(Task.class); response = new ActionListener() { @@ -102,6 +107,7 @@ public void testGetTransportAction() throws IOException { "1234", 4321, false, + false, "nonempty", "", false, @@ -112,7 +118,16 @@ public void testGetTransportAction() throws IOException { @Test public void testGetTransportActionWithReturnJob() throws IOException { - GetAnomalyDetectorRequest getAnomalyDetectorRequest = new GetAnomalyDetectorRequest("1234", 4321, true, "", "abcd", false, null); + GetAnomalyDetectorRequest getAnomalyDetectorRequest = new GetAnomalyDetectorRequest( + "1234", + 4321, + true, + false, + "", + "abcd", + false, + null + ); action.doExecute(task, getAnomalyDetectorRequest, response); } @@ -124,7 +139,7 @@ public void testGetAction() { @Test public void testGetAnomalyDetectorRequest() throws IOException { - GetAnomalyDetectorRequest request = new GetAnomalyDetectorRequest("1234", 4321, true, "", "abcd", false, "value"); + GetAnomalyDetectorRequest request = new GetAnomalyDetectorRequest("1234", 4321, true, false, "", "abcd", false, "value"); BytesStreamOutput out = new BytesStreamOutput(); request.writeTo(out); StreamInput input = out.bytes().streamInput(); @@ -136,7 +151,7 @@ public void testGetAnomalyDetectorRequest() throws IOException { @Test public void testGetAnomalyDetectorRequestNoEntityValue() throws IOException { - GetAnomalyDetectorRequest request = new GetAnomalyDetectorRequest("1234", 4321, true, "", "abcd", false, null); + GetAnomalyDetectorRequest request = new GetAnomalyDetectorRequest("1234", 4321, true, false, "", "abcd", false, null); BytesStreamOutput out = new BytesStreamOutput(); request.writeTo(out); StreamInput input = out.bytes().streamInput(); @@ -158,6 +173,8 @@ public void testGetAnomalyDetectorResponse() throws IOException { detector, adJob, false, + mock(ADTask.class), + false, RestStatus.OK, null, null, @@ -189,6 +206,8 @@ public void testGetAnomalyDetectorProfileResponse() throws IOException { detector, adJob, false, + mock(ADTask.class), + false, RestStatus.OK, null, entityProfile,