build.gradle

/*
 * This file was generated by the Gradle 'init' task.
 *
 * This is a general purpose Gradle build.
 * Learn more about Gradle by exploring our samples at https://docs.gradle.org/7.5.1/samples
 * This project uses @Incubating APIs which are subject to change.
 */
import org.opensearch.gradle.test.RestIntegTestTask
import java.util.concurrent.Callable

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.pluginzip'
apply plugin: 'jacoco'
apply plugin: "com.diffplug.spotless"
apply plugin: 'io.freefair.lombok'

def pluginName = 'neural-search'
def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem'
def projectPath = 'org.opensearch'
def pathToPlugin = 'neuralsearch.plugin'
def pluginClassName = 'NeuralSearch'

tasks.register("preparePluginPathDirs") {
    mustRunAfter clean
    doLast {
        def newPath = pathToPlugin.replace(".", "/")
        mkdir "src/main/java/org/opensearch/$newPath"
        mkdir "src/test/java/org/opensearch/$newPath"
    }
}

publishing {
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = pluginName
                description = pluginDescription
                licenses {
                    license {
                        name = "The Apache License, Version 2.0"
                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                    }
                }
                developers {
                    developer {
                        name = "OpenSearch"
                        url = "https://github.com/opensearch-project/neural-search"
                    }
                }
            }
        }
    }
}
opensearchplugin {
    name pluginName
    description pluginDescription
    classname "${projectPath}.${pathToPlugin}.${pluginClassName}"
    licenseFile rootProject.file('LICENSE')
    noticeFile rootProject.file('NOTICE')
}

dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
loggerUsageCheck.enabled = false
// No need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false

buildscript {
    ext {
        // as we don't have 3.0.0, 2.4.0 version for K-NN on darwin we need to keep OpenSearch version as 2.3 for now.
        // Github issue: https://github.com/opensearch-project/opensearch-build/issues/2662
        opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT")
        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
        version_tokens = opensearch_version.tokenize('-')
        opensearch_build = version_tokens[0] + '.0'
        plugin_no_snapshot = opensearch_build
        if (buildVersionQualifier) {
            opensearch_build += "-${buildVersionQualifier}"
            plugin_no_snapshot += "-${buildVersionQualifier}"
        }
        if (isSnapshot) {
            opensearch_build += "-SNAPSHOT"
        }
        opensearch_group = "org.opensearch"
        opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
        k_NN_resource_folder = "build/resources/k-NN"
        ml_common_resource_folder = "build/resources/ml-commons"
        //TODO: we need a better way to construct this URL as, this URL is valid for released version of K-NN, ML-Plugin.
        // Github issue: https://github.com/opensearch-project/opensearch-build/issues/2662
        k_NN_build_download_url = "https://aws.oss.sonatype.org/content/repositories/releases/org/opensearch/plugin/opensearch-knn/" + opensearch_no_snapshot + "/opensearch-knn-" + opensearch_no_snapshot +".zip"
        ml_common_build_download_url = "https://aws.oss.sonatype.org/content/repositories/releases/org/opensearch/plugin/opensearch-ml-plugin/" + opensearch_no_snapshot + "/opensearch-ml-plugin-" + opensearch_no_snapshot +".zip"
    }

    repositories {
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
    }

    dependencies {
        classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}"
        classpath "com.diffplug.spotless:spotless-plugin-gradle:5.6.1"
        classpath "io.freefair.gradle:lombok-plugin:6.4.3"
    }
}

ext {
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

allprojects {
    group = opensearch_group
    version = "${opensearch_build}"
    targetCompatibility = JavaVersion.VERSION_11
    sourceCompatibility = JavaVersion.VERSION_11
}

repositories {
    mavenLocal()
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    mavenCentral()
    maven { url "https://plugins.gradle.org/m2/" }
}

configurations {
    zipArchive
}

def knnJarDirectory = "$buildDir/dependencies/opensearch-knn"

dependencies {
    api "org.opensearch:opensearch:${opensearch_version}"
    zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
    api fileTree(dir: knnJarDirectory, include: '*.jar')
    api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
}

// From maven, we can get the k-NN plugin as a zip. In order to add the jar to the classpath, we need to unzip the
// k-NN zip and then copy it into a directory that we specify as a dependency.
task unzip(type: Copy) {
    configurations.zipArchive.asFileTree.each {
        from(zipTree(it))
    }
    into knnJarDirectory
}

compileJava {
    dependsOn unzip
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}
compileTestJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}

def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()
def _numNodes = findProperty('numNodes') as Integer ?: 1

test {
    include '**/*Tests.class'
    systemProperty 'tests.security.manager', 'false'
}

// Setting up Integration Tests
task integTest(type: RestIntegTestTask) {
    description = "Run tests against a cluster"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    // allows integration test classes to access test resource from project root path
    systemProperty('project.root', project.rootDir.absolutePath)

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
    }
}

testClusters.integTest {
    testDistribution = "ARCHIVE"
    // Install ML-Plugin on the integTest cluster nodes
    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    if (new File("$project.rootDir/$ml_common_resource_folder").exists()) {
                        project.delete(files("$project.rootDir/$ml_common_resource_folder"))
                    }
                    project.mkdir ml_common_resource_folder
                    ant.get(src: ml_common_build_download_url,
                            dest: ml_common_resource_folder,
                            httpusecaches: false)
                    return fileTree(ml_common_resource_folder).getSingleFile()
                }
            }
        }
    }))

    // Install K-NN plugin on the integTest cluster nodes
    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    if (new File("$project.rootDir/$k_NN_resource_folder").exists()) {
                        project.delete(files("$project.rootDir/$k_NN_resource_folder"))
                    }
                    project.mkdir k_NN_resource_folder
                    ant.get(src: k_NN_build_download_url,
                            dest: k_NN_resource_folder,
                            httpusecaches: false)
                    return fileTree(k_NN_resource_folder).getSingleFile()
                }
            }
        }
    }))

    // This installs our neural-search plugin into the testClusters
    plugin(project.tasks.bundlePlugin.archiveFile)
    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug opensearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("cluster.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=${debugPort}")
            debugPort += 1
        }
    }
}

// Remote Integration Tests
task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    systemProperty 'cluster.number_of_nodes', "${_numNodes}"

    systemProperty 'tests.security.manager', 'false'

    // Run tests with remote cluster only if rest case is defined
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.neuralsearch.*IT"
        }
    }
}


run {
    useCluster testClusters.integTest
}

spotless {
    java {
        removeUnusedImports()
        importOrder 'java', 'javax', 'org', 'com'
        eclipse().configFile rootProject.file('formatterConfig.xml')
        trimTrailingWhitespace()
        endWithNewline()
    }
}

jacocoTestReport {
    dependsOn integTest, test
    reports {
        xml.enabled = true
        html.enabled = true
    }
}

check.dependsOn jacocoTestCoverageVerification
jacocoTestCoverageVerification.dependsOn jacocoTestReport


// updateVersion: Task to auto update version to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
        // String tokenization to support -SNAPSHOT
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
    }
}