build.gradle

/*
 * Copyright OpenSearch Contributors
 * SPDX-License-Identifier: Apache-2.0
 */
import java.util.concurrent.Callable
import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.opensearch.gradle.test.RestIntegTestTask
buildscript {
    ext {
        distribution = 'oss-zip'
        opensearch_group = "org.opensearch"
        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
        opensearch_plugin_version = System.getProperty("bwc.version", "2.13.0.0")
        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
        // 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT
        version_tokens = opensearch_version.tokenize('-')
        opensearch_build = version_tokens[0] + '.0'
        if (buildVersionQualifier) {
            opensearch_build += "-${buildVersionQualifier}"
        }
        if (isSnapshot) {
            opensearch_build += "-SNAPSHOT"
        }
        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
    }

    repositories {
        mavenLocal()
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    }

    dependencies {
        classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}"
        classpath "org.jacoco:org.jacoco.agent:0.8.11"
    }
}
//****************************************************************************/
// Build configurations
//****************************************************************************/

plugins {
    id 'com.netflix.nebula.ospackage' version "11.6.0"
    id 'checkstyle'
}

repositories {
    mavenLocal()
    mavenCentral()
    maven { url "https://plugins.gradle.org/m2/" }
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
}
apply plugin: 'java'
apply plugin: 'jacoco'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.testclusters'
apply plugin: 'opensearch.rest-test'
apply plugin: 'opensearch.pluginzip'


checkstyle {
    toolVersion = '10.12.1'
    configFile file("checkstyle/checkstyle.xml")
}

def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
def usingMultiNode = project.properties.containsKey('numNodes')
// Only apply jacoco test coverage if we are running a local single node cluster
if (!usingRemoteCluster) {
    if (!usingMultiNode) {
        apply from: 'build-tools/plugin-coverage.gradle'
    }
}

ext {
    projectSubstitutions = [:]
    licenseFile = rootProject.file('LICENSE.txt')
    noticeFile = rootProject.file('NOTICE.txt')
}

java {
    sourceCompatibility = JavaVersion.VERSION_11
    targetCompatibility = JavaVersion.VERSION_11
}

allprojects {
    group 'org.opensearch'
    version = opensearch_build
}

publishing {
    repositories {
        maven {
            name = "Snapshots"
            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
            credentials {
                username "$System.env.SONATYPE_USERNAME"
                password "$System.env.SONATYPE_PASSWORD"
            }
        }
    }
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = "opensearch-asynchronous-search"
                description = "OpenSearch Asynchronous Search plugin"
                groupId = "org.opensearch.plugin"
                licenses {
                    license {
                        name = "The Apache License, Version 2.0"
                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                    }
                }
                developers {
                    developer {
                        name = "OpenSearch"
                        url = "https://github.com/opensearch-project/asynchronous-search"
                    }
                }
            }
        }
    }
}

dependencies {
    testImplementation "org.antlr:antlr4-runtime:${versions.antlr4}"
    testImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
    testImplementation "org.opensearch.plugin:lang-painless:${opensearch_version}"
    testImplementation "org.opensearch.test:framework:${opensearch_version}"
    compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}"
    compileOnly "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
    compileOnly "org.opensearch:opensearch:${opensearch_version}"
    implementation "org.opensearch:common-utils:${common_utils_version}"
    configurations.all {
        resolutionStrategy {
            force "com.google.guava:guava:32.1.1-jre"
            force "com.puppycrawl.tools:checkstyle:${project.checkstyle.toolVersion}"
        }
    }
}
compileTestJava {
    classpath = classpath.filter{ File file ->
        !file.name.equals( "hamcrest-core-1.3.jar" )
    }
}

check.dependsOn jacocoTestReport

opensearchplugin {
    name 'opensearch-asynchronous-search'
    description 'Provides support for asynchronous search'
    classname 'org.opensearch.search.asynchronous.plugin.AsynchronousSearchPlugin'
}

tasks.named("integTest").configure {
    it.dependsOn(project.tasks.named("bundlePlugin"))
}

licenseHeaders.enabled = true
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
validateNebulaPom.enabled = false
loggerUsageCheck.enabled = false

def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

def securityEnabled = System.getProperty("security", "false") == "true"

test {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'es.set.netty.runtime.available.processors', 'false'

}

File repo = file("$buildDir/testclusters/repo")
def _numNodes = findProperty('numNodes') as Integer ?: 1

Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
testClusters.integTest {
    testDistribution = "ARCHIVE"
    plugin(project.tasks.bundlePlugin.archiveFile)
    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug opensearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("opensearch.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
            debugPort += 1
        }
    }
}

integTest {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    systemProperty 'buildDir', buildDir.path
    systemProperty "https", System.getProperty("https", securityEnabled.toString())
    systemProperty "user", System.getProperty("user", "admin")
    systemProperty "password", System.getProperty("password", "myStrongPassword123!") // this change is needed for >= 2.12 and should not be backported to < 2.12
    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
    doFirst {
        systemProperty 'cluster.debug', getDebug()
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }
    // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
    }
    if (System.getProperty("tests.rest.bwcsuite") == null) {
        filter {
            excludeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT"
        }
    }
}


task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    if (System.getProperty("tests.rest.bwcsuite") == null) {
        filter {
            excludeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT"
        }
    }

    // Only rest case can run with remote cluster
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.search.asynchronous.rest*"
        }
    }
}

ext.getPluginResource = { download_to_folder, download_from_src ->
    project.mkdir download_to_folder
    ant.get(src: download_from_src,
            dest: download_to_folder,
            httpusecaches: false)
    return fileTree(download_to_folder).getSingleFile()
}

String baseName = "asynSearchCluster"
String bwcVersionShort = "2.13.0"
String bwcVersion = bwcVersionShort + ".0"
String bwcFilePath = "src/test/resources/org/opensearch/search/asynchronous/bwc/"
String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/"+ bwcVersionShort + "/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-asynchronous-search-"+ bwcVersion +".zip"
//getPluginResource(bwcFilePath + bwcVersion, bwcRemoteFile)
// Creates two test clusters of previous version and loads opensearch plugin of bwcVersion
2.times { i ->
    testClusters {
        "${baseName}$i" {
            testDistribution = "ARCHIVE"
            versions = [bwcVersionShort,opensearch_version]
            numberOfNodes = 3
            plugin(provider(new Callable<RegularFile>() {
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            return fileTree(bwcFilePath + opensearch_plugin_version).getSingleFile()
                        }
                    }
                }
            }))
            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
            setting 'http.content_type.required', 'true'
            systemProperty "java.library.path", "$rootDir/src/test/resources/org/opensearch/search/asynchronous/lib:$rootDir/jni/release"
        }
    }
}

// upgradeNodeAndPluginToNextVersion(plugins) upgrades plugin on the upgraded node with project.version binary file in bwcFilePath
// upgradeAllNodesAndPluginsToNextVersion(plugins) upgrades plugins on all the 3 nodes after upgrading the nodes
List<Provider<RegularFile>> plugins = [
        provider(new Callable<RegularFile>(){
            @Override
            RegularFile call() throws Exception {
                return new RegularFile() {
                    @Override
                    File getAsFile() {
                        return fileTree(bwcFilePath + project.version).getSingleFile()
                    }
                }
            }
        })
]

// Ensure the artifact for the current project version is available to be used for the bwc tests
task prepareBwcTests {
    dependsOn bundle
    doLast {
        plugins = [
                project.getObjects().fileProperty().value(bundle.getArchiveFile())
        ]
    }
}

// Creates 2 test clusters with 3 nodes of the old version.
2.times { i ->
    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
        dependsOn 'prepareBwcTests'
        useCluster testClusters."${baseName}$i"
        filter {
            includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT"
        }
        systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
        systemProperty 'tests.rest.bwcsuite_round', 'old'
        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
    }
}

// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
    useCluster testClusters."${baseName}0"
    dependsOn "${baseName}#oldVersionClusterTask0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'first'
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}
// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#mixedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'second'
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT"
    }
    mustRunAfter "${baseName}#mixedClusterTask"
    systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'third'
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#oldVersionClusterTask1"
    useCluster testClusters."${baseName}1"
    doFirst {
        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'fullrestart'
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
}


// A bwc test suite which runs all the bwc tasks combined.
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
    exclude '**/*Test*'
    exclude '**/*IT*'
    dependsOn tasks.named("${baseName}#mixedClusterTask")
    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
}


run {
    useCluster project.testClusters.integTest
    doFirst {
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }
}

apply from: 'build-tools/pkgbuild.gradle'

// updateVersion: Task to auto increment to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
        // String tokenization to support -SNAPSHOT
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
    }
}