[RORDEV-1252] ES 8.15.0 support (#1037)

azure-pipelines.yml

@@ -108,6 +108,8 @@ stages:
         strategy:
           maxParallel: 99
           matrix:
+            IT_es815x:
+              ROR_TASK: integration_es815x
             IT_es814x:
               ROR_TASK: integration_es814x
             IT_es813x:
@@ -164,8 +166,10 @@ stages:
         strategy:
           maxParallel: 99
           matrix:
-            IT_es814x:
-              ROR_TASK: integration_es814x
+            IT_es815x:
+              ROR_TASK: integration_es815x
+            IT_es810x:
+              ROR_TASK: integration_es810x
             IT_es80x:
               ROR_TASK: integration_es80x
       - job:

ci/run-pipeline.sh

@@ -50,6 +50,10 @@ run_integration_tests() {
   ./gradlew integration-tests:test "-PesModule=$ES_MODULE" || (find . | grep hs_err | xargs cat && exit 1)
 }
 
+if [[ -z $TRAVIS ]] || [[ $ROR_TASK == "integration_es815x" ]]; then
+  run_integration_tests "es815x"
+fi
+
 if [[ -z $TRAVIS ]] || [[ $ROR_TASK == "integration_es814x" ]]; then
   run_integration_tests "es814x"
 fi

ci/supported-es-versions/es8x.txt

@@ -1,3 +1,4 @@
+8.15.0
 8.14.3
 8.14.2
 8.14.1

core/build.gradle

@@ -122,7 +122,7 @@ dependencies {
     testImplementation  group: 'org.scalamock',              name: 'scalamock_3',                version: '6.0.0'
     testImplementation  group: 'org.scalatestplus',          name: 'scalacheck-1-16_3',          version: '3.2.14.0'
     testImplementation  group: 'org.scalatest',              name: 'scalatest_3',                version: '3.2.16'
-    testImplementation  group: 'com.dimafeng',               name: 'testcontainers-scala_3',     version: '0.40.17'
+    testImplementation  group: 'com.dimafeng',               name: 'testcontainers-scala_3',     version: '0.41.4'
 
     constraints {
         api group: 'io.netty',              name: 'netty-codec-http2',                  version: '4.1.108.Final'

core/src/main/scala/tech/beshu/ror/configuration/loader/distributed/domain.scala

@@ -25,5 +25,5 @@ final case class NodeConfig(loadedConfig: Either[LoadedRorConfig.Error, LoadedRo
 final case class Timeout(nanos: Long) extends AnyVal
 final case class NodeConfigRequest(timeout: Timeout)
 object NodeConfigRequest {
-  val defaultTimeout: Timeout = Timeout(nanos = (1 minute).toNanos)
+  val defaultTimeout: Timeout = Timeout(nanos = (10 seconds).toNanos)
 }

es815x/Dockerfile

@@ -0,0 +1,27 @@
+ARG ES_VERSION
+ARG ROR_VERSION
+
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
+
+ARG ES_VERSION
+ARG ROR_VERSION
+
+ENV KIBANA_USER_PASS=kibana
+ENV ADMIN_USER_PASS=admin
+
+USER root
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends gosu && \
+    rm -rf /var/lib/apt/lists/* && \
+    gosu nobody true
+
+USER 1000
+
+COPY readonlyrest-${ROR_VERSION}_es${ES_VERSION}.zip /tmp/readonlyrest.zip
+COPY init-readonlyrest.yml /usr/share/elasticsearch/config/readonlyrest.yml
+COPY ror-entrypoint.sh /usr/local/bin/ror-entrypoint.sh
+
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:///tmp/readonlyrest.zip
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/ror-entrypoint.sh"]

es815x/build.gradle

@@ -0,0 +1,165 @@
+/*
+ *    This file is part of ReadonlyREST.
+ *
+ *    ReadonlyREST is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation, either version 3 of the License, or
+ *    (at your option) any later version.
+ *
+ *    ReadonlyREST is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with ReadonlyREST.  If not, see http://www.gnu.org/licenses/
+ */
+import com.bmuschko.gradle.docker.tasks.image.*
+import org.gradle.crypto.checksum.Checksum
+
+plugins {
+    id 'com.bmuschko.docker-remote-api'
+    id 'org.gradle.crypto.checksum'
+    id "readonlyrest.plugin-common-conventions"
+}
+
+def pluginFullName = pluginName + '-' + version
+def projectJavaLanguageVersion = JavaLanguageVersion.of(17)
+def moduleEsVersion = project.property('esVersion') != null ? project.property('esVersion') : project.property('latestSupportedEsVersion')
+def dockerImageLatest = 'beshultd/elasticsearch-readonlyrest:' + moduleEsVersion + '-ror-latest'
+def dockerImageVersion = 'beshultd/elasticsearch-readonlyrest:' + moduleEsVersion + '-ror-' + pluginVersion
+
+java {
+    toolchain {
+        languageVersion = projectJavaLanguageVersion
+    }
+}
+
+dependencies {
+    implementation  project(path: ':core')
+    implementation  project(path: ':ror-tools', configuration: 'shadow')
+    implementation  project(path: ':ror-tools-core')
+    implementation  group: 'org.elasticsearch',                 name: 'elasticsearch' ,                         version: moduleEsVersion
+    implementation  group: 'org.elasticsearch.client',          name: 'elasticsearch-rest-client',              version: moduleEsVersion
+    implementation  group: 'org.apache.httpcomponents',         name: 'httpclient',                             version: '4.5.13'
+    implementation  group: 'org.apache.logging.log4j',          name: 'log4j-core',                             version: '2.11.1'
+    compileOnly     group: 'org.locationtech.spatial4j',        name: 'spatial4j',                              version: '0.7'
+    // if you don't have this dependency in local maven, please run publishToMavenLocal task first
+    compileOnly     group: 'org.elasticsearch.plugin',          name: 'transport-netty4',                       version: moduleEsVersion
+}
+
+configurations {
+    wagon
+    distJars {
+        exclude group: 'org.elasticsearch'
+        exclude group: 'lucene-core'
+        exclude module: 'log4j-api'
+        exclude module: 'log4j-core'
+        exclude group: 'lucene-analyzers-common'
+        exclude group: 'org.apache.commons'
+        exclude group: 'org.yaml'
+        exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core'
+    }
+}
+
+tasks.register('cleanOldData') {
+    doLast {
+        delete 'build/tmp/' + pluginFullName
+    }
+}
+
+tasks.register('jarHellCheck', JavaExec) {
+    outputs.upToDateWhen { false }
+    mainClass.set("org.elasticsearch.jdk.JarHell")
+    classpath = project.sourceSets.main.compileClasspath.filter { it.exists() }
+    javaLauncher = javaToolchains.launcherFor {
+        languageVersion = projectJavaLanguageVersion
+    }
+}
+
+tasks.register('resolvePluginDescriptorTemplate', Copy) {
+    outputs.upToDateWhen { false }
+    from './plugin-metadata'
+    into 'build/tmp/' + pluginFullName
+    expand([
+            'descriptor': [
+                    'name'         : pluginName,
+                    'pluginVersion': pluginVersion,
+                    'esVersion'    : moduleEsVersion
+            ]
+    ])
+}
+
+tasks.register('buildRorPluginZip') {
+    dependsOn(packageRorPlugin, createRorPluginChecksums)
+}
+
+tasks.register('packageRorPlugin', Zip) {
+    dependsOn(cleanOldData, jarHellCheck, toJar, resolvePluginDescriptorTemplate)
+    outputs.upToDateWhen { false }
+    archivesBaseName = pluginName
+    into('.') {
+        from configurations.distJars.filter { x -> !x.name.contains('spatial4j') && !x.name.contains('jts') }
+        from 'build/libs/' + pluginFullName + '.jar'
+        from 'build/tmp/' + pluginFullName
+    }
+}
+
+tasks.register('createRorPluginChecksums', Checksum) {
+    dependsOn(packageRorPlugin)
+    def distributionsDir = layout.buildDirectory.dir("distributions")
+
+    outputs.upToDateWhen { false }
+    inputFiles.setFrom(packageRorPlugin.archiveFile)
+    outputDirectory.set(distributionsDir)
+    checksumAlgorithm.set(Checksum.Algorithm.SHA512)
+}
+
+tasks.register('uploadRorPluginToS3', Exec) {
+    dependsOn(packageRorPlugin, createRorPluginChecksums)
+
+    def distributionsDir = layout.buildDirectory.get().asFile.path + "/distributions"
+    def pluginFileZip = distributionsDir + "/" + pluginFullName + ".zip"
+    def pluginSha1 = distributionsDir + "/" + pluginFullName + ".zip.sha1"
+    def pluginSha512 = distributionsDir + "/" + pluginFullName + ".zip.sha512"
+    def targetS3Dir = "build/" + pluginVersion + "/"
+
+    commandLine '../ci/upload-files-to-s3.sh', pluginFileZip, pluginSha512, pluginSha1, targetS3Dir
+}
+
+tasks.register('prepareDockerImageFiles', Copy) {
+    dependsOn packageRorPlugin
+    outputs.upToDateWhen { false }
+
+    from layout.projectDirectory.file("Dockerfile")
+    from layout.buildDirectory.file("distributions/" + pluginFullName + ".zip")
+    from rootProject.files("docker-image")
+
+    into layout.buildDirectory.dir("docker-image")
+}
+
+tasks.register('buildRorDockerImage', DockerBuildImage) {
+    dependsOn('packageRorPlugin', 'prepareDockerImageFiles')
+
+    inputDir = layout.buildDirectory.dir("docker-image")
+    buildArgs = [
+            ES_VERSION: project.properties['esVersion'],
+            ROR_VERSION: pluginVersion
+    ]
+    images.add(dockerImageLatest)
+    images.add(dockerImageVersion)
+}
+
+tasks.register('removeRorDockerImage', DockerRemoveImage) {
+    dependsOn buildRorDockerImage
+    force = true
+    targetImageId buildRorDockerImage.getImageId()
+}
+
+tasks.register('pushRorDockerImage', DockerPushImage) {
+    dependsOn buildRorDockerImage
+
+    images.add(dockerImageLatest)
+    images.add(dockerImageVersion)
+}
+pushRorDockerImage.configure { finalizedBy removeRorDockerImage }

es815x/gradle.properties

@@ -0,0 +1 @@
+latestSupportedEsVersion=8.15.0

es815x/plugin-metadata/plugin-descriptor.properties

@@ -0,0 +1,27 @@
+#
+# This file is part of ReadonlyREST.
+#
+#     ReadonlyREST is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     ReadonlyREST is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with ReadonlyREST.  If not, see http://www.gnu.org/licenses/
+#
+#
+# Elasticsearch plugin descriptor file
+# This file must exist as 'plugin-descriptor.properties' in a folder named `elasticsearch`
+# inside all plugins.
+#
+name=${descriptor.name}
+version=${descriptor.pluginVersion}
+description=Safely expose Elasticsearch REST API
+classname=tech.beshu.ror.es.ReadonlyRestPlugin
+java.version=1.8
+elasticsearch.version=${descriptor.esVersion}

es815x/plugin-metadata/plugin-security.policy

@@ -0,0 +1,14 @@
+grant {
+  permission java.io.FilePermission "/usr/share/elasticsearch", "read";
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.misc.*";
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.lang.RuntimePermission "setContextClassLoader";
+  permission java.net.SocketPermission "*", "accept, resolve, connect";
+  permission java.security.SecurityPermission "insertProvider";
+  permission java.security.SecurityPermission "putProviderProperty.BCFIPS";
+  permission java.security.SecurityPermission "putProviderProperty.BCJSSE";
+  permission java.util.PropertyPermission "*", "read,write";
+};