Upgrade to ES 6.6 (#8)

* Upgrade to ES 6.6.2 and Gradle 5.1.1 (upstream Gradle version used by ES 6.6)

Known issues: Invoking 'clean' along with other gradle tasks like './gradlew clean build' is broken because of the way Gradle extracts the jacoco agent jar (in a sudirectory of build/tmp that gets deleted when clean task runs and doesn't get recreated). Workaround is to run ./gradlew clean separately instead of along with other commands.

* Remove ElasticAPI and ElasticTestAPI.

We now maintain separate branches for each ES version so there's no need
for an API translation layer that handles multiple elastic versions.

alerting/build.gradle

@@ -62,9 +62,6 @@ dependencies {
 
     compile project(":alerting-core")
     compile project(":alerting-notification")
-    compile project(':alerting-elastic-api')
-
-    testCompile project(path: ':alerting-elastic-api', configuration: 'testOutput')
 }
 
 javadoc.enabled = false // turn off javadoc as it barfs on Kotlin code
@@ -99,7 +96,7 @@ task startMultiNode(dependsOn: startMultiNode1)
 def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
 es_tmp_dir.mkdirs()
 
-test {
+unitTest {
     systemProperty 'tests.security.manager', 'false'
 }
 
@@ -123,8 +120,4 @@ run {
     distribution = "oss-zip"
 }
 
-afterEvaluate {
-    repositories.remove(repositories.findByName("MavenRepo"))
-}
-
 apply from: '../build-tools/rpmbuild.gradle'

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/MonitorRunner.kt

@@ -48,10 +48,10 @@ import com.amazon.opendistroforelasticsearch.alerting.settings.AlertingSettings.
 import com.amazon.opendistroforelasticsearch.alerting.core.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX
 import com.amazon.opendistroforelasticsearch.alerting.core.model.ScheduledJob.Companion.SCHEDULED_JOB_TYPE
 import com.amazon.opendistroforelasticsearch.alerting.core.model.SearchInput
-import com.amazon.opendistroforelasticsearch.alerting.elasticapi.ElasticAPI
 import com.amazon.opendistroforelasticsearch.alerting.elasticapi.convertToMap
 import com.amazon.opendistroforelasticsearch.alerting.elasticapi.firstFailureOrNull
 import com.amazon.opendistroforelasticsearch.alerting.elasticapi.retry
+import org.apache.logging.log4j.LogManager
 import org.elasticsearch.ExceptionsHelper
 import org.elasticsearch.action.DocWriteRequest
 import org.elasticsearch.action.bulk.BackoffPolicy
@@ -69,11 +69,14 @@ import org.elasticsearch.common.settings.Settings
 import org.elasticsearch.common.unit.TimeValue
 import org.elasticsearch.common.util.concurrent.EsExecutors
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler
 import org.elasticsearch.common.xcontent.NamedXContentRegistry
 import org.elasticsearch.common.xcontent.ToXContent
 import org.elasticsearch.common.xcontent.XContentFactory
+import org.elasticsearch.common.xcontent.XContentHelper
 import org.elasticsearch.common.xcontent.XContentParser
 import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken
+import org.elasticsearch.common.xcontent.XContentType
 import org.elasticsearch.index.query.QueryBuilders
 import org.elasticsearch.rest.RestStatus
 import org.elasticsearch.script.Script
@@ -95,7 +98,7 @@ class MonitorRunner(
     clusterService: ClusterService
 ) : JobRunner {
 
-    private val logger = ElasticAPI.INSTANCE.getLogger(MonitorRunner::class.java, settings)
+    private val logger = LogManager.getLogger(MonitorRunner::class.java)
 
     @Volatile private var searchTimeout = INPUT_TIMEOUT.get(settings)
     @Volatile private var bulkTimeout = BULK_TIMEOUT.get(settings)
@@ -250,7 +253,7 @@ class MonitorRunner(
                                 .execute()
 
                         val searchRequest = SearchRequest().indices(*input.indices.toTypedArray())
-                        ElasticAPI.INSTANCE.jsonParser(xContentRegistry, searchSource).use {
+                        XContentType.JSON.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, searchSource).use {
                             searchRequest.source(SearchSourceBuilder.fromXContent(it))
                         }
                         results += client.search(searchRequest).actionGet(searchTimeout).convertToMap()
@@ -303,7 +306,8 @@ class MonitorRunner(
     }
 
     private fun contentParser(bytesReference: BytesReference): XContentParser {
-        val xcp = ElasticAPI.INSTANCE.jsonParser(xContentRegistry, bytesReference)
+        val xcp = XContentHelper.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE,
+                bytesReference, XContentType.JSON)
         ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation)
         return xcp
     }
@@ -409,7 +413,7 @@ class MonitorRunner(
         }
 
         val jobSource = getResponse.sourceAsBytesRef
-        val xcp = ElasticAPI.INSTANCE.jsonParser(xContentRegistry, jobSource)
+        val xcp = XContentHelper.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, jobSource, XContentType.JSON)
         ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation)
         ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp::getTokenLocation)
         ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation)

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/alerts/AlertIndices.kt

@@ -22,7 +22,7 @@ import com.amazon.opendistroforelasticsearch.alerting.settings.AlertingSettings.
 import com.amazon.opendistroforelasticsearch.alerting.settings.AlertingSettings.Companion.ALERT_HISTORY_MAX_DOCS
 import com.amazon.opendistroforelasticsearch.alerting.settings.AlertingSettings.Companion.ALERT_HISTORY_ROLLOVER_PERIOD
 import com.amazon.opendistroforelasticsearch.alerting.settings.AlertingSettings.Companion.REQUEST_TIMEOUT
-import com.amazon.opendistroforelasticsearch.alerting.elasticapi.ElasticAPI
+import org.apache.logging.log4j.LogManager
 import org.elasticsearch.ResourceAlreadyExistsException
 import org.elasticsearch.action.admin.indices.alias.Alias
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest
@@ -88,9 +88,9 @@ class AlertIndices(
         @JvmStatic
         fun alertMapping() =
                 AlertIndices::class.java.getResource("alert_mapping.json").readText()
-    }
 
-    private val logger = ElasticAPI.INSTANCE.getLogger(AlertIndices::class.java, settings)
+        private val logger = LogManager.getLogger(AlertIndices::class.java)
+    }
 
     @Volatile private var historyMaxDocs = AlertingSettings.ALERT_HISTORY_MAX_DOCS.get(settings)
 
@@ -185,7 +185,7 @@ class AlertIndices(
 
         // We have to pass null for newIndexName in order to get Elastic to increment the index count.
         val request = RolloverRequest(HISTORY_WRITE_INDEX, null)
-        ElasticAPI.INSTANCE.getCreateIndexRequest(request).index(HISTORY_INDEX_PATTERN)
+        request.createIndexRequest.index(HISTORY_INDEX_PATTERN)
                 .mapping(MAPPING_TYPE, alertMapping(), XContentType.JSON)
         request.addMaxIndexDocsCondition(historyMaxDocs)
         request.addMaxIndexAgeCondition(historyMaxAge)

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/alerts/AlertMover.kt

@@ -20,7 +20,6 @@ import com.amazon.opendistroforelasticsearch.alerting.alerts.AlertIndices.Compan
 import com.amazon.opendistroforelasticsearch.alerting.alerts.AlertIndices.Companion.HISTORY_WRITE_INDEX
 import com.amazon.opendistroforelasticsearch.alerting.model.Alert
 import com.amazon.opendistroforelasticsearch.alerting.model.Monitor
-import com.amazon.opendistroforelasticsearch.alerting.elasticapi.ElasticAPI
 import org.apache.logging.log4j.Logger
 import org.elasticsearch.action.ActionListener
 import org.elasticsearch.action.bulk.BulkRequest
@@ -32,11 +31,14 @@ import org.elasticsearch.action.search.SearchResponse
 import org.elasticsearch.client.Client
 import org.elasticsearch.common.bytes.BytesReference
 import org.elasticsearch.common.unit.TimeValue
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler
 import org.elasticsearch.common.xcontent.NamedXContentRegistry
 import org.elasticsearch.common.xcontent.ToXContent
 import org.elasticsearch.common.xcontent.XContentFactory
+import org.elasticsearch.common.xcontent.XContentHelper
 import org.elasticsearch.common.xcontent.XContentParser
 import org.elasticsearch.common.xcontent.XContentParserUtils
+import org.elasticsearch.common.xcontent.XContentType
 import org.elasticsearch.index.VersionType
 import org.elasticsearch.index.query.QueryBuilders
 import org.elasticsearch.search.builder.SearchSourceBuilder
@@ -155,7 +157,8 @@ class AlertMover(
     }
 
     private fun alertContentParser(bytesReference: BytesReference): XContentParser {
-        val xcp = ElasticAPI.INSTANCE.jsonParser(NamedXContentRegistry.EMPTY, bytesReference)
+        val xcp = XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE,
+                bytesReference, XContentType.JSON)
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation)
         return xcp
     }

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/model/MonitorRunResult.kt

@@ -17,8 +17,8 @@ package com.amazon.opendistroforelasticsearch.alerting.model
 
 import com.amazon.opendistroforelasticsearch.alerting.alerts.AlertError
 import com.amazon.opendistroforelasticsearch.alerting.elasticapi.optionalTimeField
+import org.apache.logging.log4j.LogManager
 import org.elasticsearch.ElasticsearchException
-import org.elasticsearch.common.logging.Loggers
 import org.elasticsearch.common.xcontent.ToXContent
 import org.elasticsearch.common.xcontent.XContentBuilder
 import org.elasticsearch.script.ScriptException
@@ -116,7 +116,7 @@ data class ActionRunResult(
     }
 }
 
-private val logger = Loggers.getLogger("UserError")
+private val logger = LogManager.getLogger(MonitorRunResult::class.java)
 
 /** Constructs an error message from an exception suitable for human consumption. */
 private fun Throwable.userErrorMessage(): String {

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/model/destination/Destination.kt

@@ -16,7 +16,6 @@
 package com.amazon.opendistroforelasticsearch.alerting.model.destination
 
 import com.amazon.opendistroforelasticsearch.alerting.destination.Notification
-import com.amazon.opendistroforelasticsearch.alerting.destination.client.DestinationHttpClient
 import com.amazon.opendistroforelasticsearch.alerting.destination.message.BaseMessage
 import com.amazon.opendistroforelasticsearch.alerting.destination.message.ChimeMessage
 import com.amazon.opendistroforelasticsearch.alerting.destination.message.CustomWebhookMessage
@@ -26,7 +25,7 @@ import com.amazon.opendistroforelasticsearch.alerting.elasticapi.convertToMap
 import com.amazon.opendistroforelasticsearch.alerting.elasticapi.instant
 import com.amazon.opendistroforelasticsearch.alerting.elasticapi.optionalTimeField
 import com.amazon.opendistroforelasticsearch.alerting.util.DestinationType
-import org.elasticsearch.common.logging.Loggers
+import org.apache.logging.log4j.LogManager
 import org.elasticsearch.common.xcontent.ToXContent
 import org.elasticsearch.common.xcontent.XContentBuilder
 import org.elasticsearch.common.xcontent.XContentParser
@@ -49,8 +48,6 @@ data class Destination(
     val customWebhook: CustomWebhook?
 ) : ToXContent {
 
-    private val logger = Loggers.getLogger(DestinationHttpClient::class.java)
-
     override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
         builder.startObject()
         if (params.paramAsBoolean("with_type", false)) builder.startObject(DESTINATION)
@@ -79,6 +76,8 @@ data class Destination(
         // This constant is used for test actions created part of integ tests
         const val TEST_ACTION = "test"
 
+        private val logger = LogManager.getLogger(Destination::class.java)
+
         @JvmStatic
         @JvmOverloads
         @Throws(IOException::class)

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/resthandler/RestAcknowledgeAlertAction.kt

@@ -22,7 +22,6 @@ import com.amazon.opendistroforelasticsearch.alerting.model.Alert.State.ACTIVE
 import com.amazon.opendistroforelasticsearch.alerting.model.Alert.State.COMPLETED
 import com.amazon.opendistroforelasticsearch.alerting.model.Alert.State.ERROR
 import com.amazon.opendistroforelasticsearch.alerting.util.REFRESH
-import com.amazon.opendistroforelasticsearch.alerting.elasticapi.ElasticAPI
 import com.amazon.opendistroforelasticsearch.alerting.AlertingPlugin
 import com.amazon.opendistroforelasticsearch.alerting.elasticapi.optionalTimeField
 import org.elasticsearch.action.ActionListener
@@ -35,10 +34,13 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy
 import org.elasticsearch.action.update.UpdateRequest
 import org.elasticsearch.client.node.NodeClient
 import org.elasticsearch.common.settings.Settings
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler
 import org.elasticsearch.common.xcontent.XContentBuilder
 import org.elasticsearch.common.xcontent.XContentFactory
+import org.elasticsearch.common.xcontent.XContentHelper
 import org.elasticsearch.common.xcontent.XContentParser
 import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken
+import org.elasticsearch.common.xcontent.XContentType
 import org.elasticsearch.index.query.QueryBuilders
 import org.elasticsearch.rest.BaseRestHandler
 import org.elasticsearch.rest.BaseRestHandler.RestChannelConsumer
@@ -107,7 +109,8 @@ class RestAcknowledgeAlertAction(settings: Settings, controller: RestController)
 
         private fun onSearchResponse(response: SearchResponse) {
             val updateRequests = response.hits.flatMap { hit ->
-                val xcp = ElasticAPI.INSTANCE.jsonParser(channel.request().xContentRegistry, hit.sourceRef)
+                val xcp = XContentHelper.createParser(channel.request().xContentRegistry, LoggingDeprecationHandler.INSTANCE,
+                        hit.sourceRef, XContentType.JSON)
                 ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp::getTokenLocation)
                 val alert = Alert.parse(xcp, hit.id, hit.version)
                 alerts[alert.id] = alert

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/resthandler/RestExecuteMonitorAction.kt

@@ -18,13 +18,13 @@ package com.amazon.opendistroforelasticsearch.alerting.resthandler
 import com.amazon.opendistroforelasticsearch.alerting.core.model.ScheduledJob
 import com.amazon.opendistroforelasticsearch.alerting.MonitorRunner
 import com.amazon.opendistroforelasticsearch.alerting.model.Monitor
-import com.amazon.opendistroforelasticsearch.alerting.elasticapi.ElasticAPI
 import com.amazon.opendistroforelasticsearch.alerting.AlertingPlugin
 import org.elasticsearch.action.get.GetRequest
 import org.elasticsearch.action.get.GetResponse
 import org.elasticsearch.client.node.NodeClient
 import org.elasticsearch.common.settings.Settings
 import org.elasticsearch.common.unit.TimeValue
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler
 import org.elasticsearch.common.xcontent.XContentParser.Token.START_OBJECT
 import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken
 import org.elasticsearch.common.xcontent.XContentType
@@ -97,8 +97,9 @@ class RestExecuteMonitorAction(
                     this.channel.sendResponse(BytesRestResponse(RestStatus.NOT_FOUND, ret))
                 }
 
-                val xcp = ElasticAPI.INSTANCE.createParser(this.channel.request().xContentRegistry,
-                        response.sourceAsBytesRef, this.channel.request().xContentType ?: XContentType.JSON)
+                val xcp = (this.channel.request().xContentType ?: XContentType.JSON).xContent()
+                        .createParser(this.channel.request().xContentRegistry, LoggingDeprecationHandler.INSTANCE,
+                                response.sourceAsBytesRef.streamInput())
                 val monitor = xcp.use {
                     ScheduledJob.parse(xcp, response.id, response.version) as Monitor
                 }

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/resthandler/RestGetMonitorAction.kt

@@ -20,12 +20,14 @@ import com.amazon.opendistroforelasticsearch.alerting.core.model.ScheduledJob.Co
 import com.amazon.opendistroforelasticsearch.alerting.util._ID
 import com.amazon.opendistroforelasticsearch.alerting.util._VERSION
 import com.amazon.opendistroforelasticsearch.alerting.util.context
-import com.amazon.opendistroforelasticsearch.alerting.elasticapi.ElasticAPI
 import com.amazon.opendistroforelasticsearch.alerting.AlertingPlugin
 import org.elasticsearch.action.get.GetRequest
 import org.elasticsearch.action.get.GetResponse
 import org.elasticsearch.client.node.NodeClient
 import org.elasticsearch.common.settings.Settings
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler
+import org.elasticsearch.common.xcontent.XContentHelper
+import org.elasticsearch.common.xcontent.XContentType
 import org.elasticsearch.rest.BaseRestHandler
 import org.elasticsearch.rest.BaseRestHandler.RestChannelConsumer
 import org.elasticsearch.rest.BytesRestResponse
@@ -82,8 +84,8 @@ class RestGetMonitorAction(settings: Settings, controller: RestController) : Bas
                         .field(_ID, response.id)
                         .field(_VERSION, response.version)
                 if (!response.isSourceEmpty) {
-                    ElasticAPI.INSTANCE
-                            .jsonParser(channel.request().xContentRegistry, response.sourceAsBytesRef).use { xcp ->
+                    XContentHelper.createParser(channel.request().xContentRegistry, LoggingDeprecationHandler.INSTANCE,
+                            response.sourceAsBytesRef, XContentType.JSON).use { xcp ->
                                 val monitor = ScheduledJob.parse(xcp, response.id, response.version)
                                 builder.field("monitor", monitor)
                             }

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/resthandler/RestIndexMonitorAction.kt

@@ -25,7 +25,6 @@ import com.amazon.opendistroforelasticsearch.alerting.settings.AlertingSettings.
 import com.amazon.opendistroforelasticsearch.alerting.util.REFRESH
 import com.amazon.opendistroforelasticsearch.alerting.util._ID
 import com.amazon.opendistroforelasticsearch.alerting.util._VERSION
-import com.amazon.opendistroforelasticsearch.alerting.elasticapi.ElasticAPI
 import com.amazon.opendistroforelasticsearch.alerting.AlertingPlugin
 import org.elasticsearch.action.ActionListener
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse
@@ -39,9 +38,12 @@ import org.elasticsearch.action.support.WriteRequest
 import org.elasticsearch.client.node.NodeClient
 import org.elasticsearch.cluster.service.ClusterService
 import org.elasticsearch.common.settings.Settings
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler
 import org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS
+import org.elasticsearch.common.xcontent.XContentHelper
 import org.elasticsearch.common.xcontent.XContentParser.Token
 import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken
+import org.elasticsearch.common.xcontent.XContentType
 import org.elasticsearch.index.query.QueryBuilders
 import org.elasticsearch.rest.BaseRestHandler
 import org.elasticsearch.rest.BaseRestHandler.RestChannelConsumer
@@ -187,7 +189,8 @@ class RestIndexMonitorAction(
                         .endObject()
                 return channel.sendResponse(BytesRestResponse(RestStatus.NOT_FOUND, response.toXContent(builder, EMPTY_PARAMS)))
             }
-            val xcp = ElasticAPI.INSTANCE.jsonParser(channel.request().xContentRegistry, response.sourceAsBytesRef)
+            val xcp = XContentHelper.createParser(channel.request().xContentRegistry, LoggingDeprecationHandler.INSTANCE,
+                    response.sourceAsBytesRef, XContentType.JSON)
             val currentMonitor = ScheduledJob.parse(xcp, monitorId) as Monitor
             // If both are enabled, use the current existing monitor enabled time, otherwise the next execution will be
             // incorrect.