diff --git a/data-prepper-main/build.gradle b/data-prepper-main/build.gradle index 4cccbe14d5..63b6c57b99 100644 --- a/data-prepper-main/build.gradle +++ b/data-prepper-main/build.gradle @@ -27,3 +27,21 @@ jar { 'Main-Class': 'org.opensearch.dataprepper.DataPrepperExecute') } } + +configurations { + allDependencyJars { + canBeConsumed = true + canBeResolved = true + extendsFrom runtimeClasspath + } +} + +artifacts { + /** + * Shares a configuration which has all the necessary dependencies and can be used + * as part of the release to load all dependencies. + * https://docs.gradle.org/current/userguide/cross_project_publications.html + */ + allDependencyJars(jar) +} + diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle index 95831fc8ed..d20d91f168 100644 --- a/e2e-test/build.gradle +++ b/e2e-test/build.gradle @@ -10,10 +10,17 @@ subprojects { gradlePluginPortal() } dependencies { - classpath 'com.bmuschko:gradle-docker-plugin:7.0.0' + classpath 'com.bmuschko:gradle-docker-plugin:9.3.2' } } + ext { + dataPrepperJarImageFilepath = 'bin/data-prepper/' + targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11' + targetOpenTelemetryVersion = project.hasProperty('openTelemetryVersion') ? project.getProperty('openTelemetryVersion') : "${libs.versions.opentelemetry.get()}" + dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre" + } + sourceSets { integrationTest { java { @@ -40,17 +47,10 @@ subprojects { duplicatesStrategy = DuplicatesStrategy.EXCLUDE from project(':data-prepper-main').jar.archivePath from project(':data-prepper-main').configurations.runtimeClasspath - into("${project.buildDir}/bin/data-prepper") + into("${project.buildDir}/docker/${dataPrepperJarImageFilepath}") } dependencies { testImplementation testLibs.junit.vintage } - - ext { - dataPrepperJarFilepath = "${project.buildDir.name}/bin/data-prepper/" - targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11' - targetOpenTelemetryVersion = project.hasProperty('openTelemetryVersion') ? project.getProperty('openTelemetryVersion') : "${libs.versions.opentelemetry.get()}" - dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre" - } } diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index f3516d406e..382dcc5100 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -41,16 +41,15 @@ task createDataPrepperDockerFile(type: Dockerfile) { dependsOn copyDataPrepperJar destFile = project.file('build/docker/Dockerfile') from(dataPrepperBaseImage) - workingDir("/app/data-prepper") - copyFile("${dataPrepperJarFilepath}", "/app/data-prepper/lib") + workingDir('/app/data-prepper') + copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib') defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute') } task buildDataPrepperDockerImage(type: DockerBuildImage) { dependsOn createDataPrepperDockerFile - inputDir = file(".") - dockerFile = file("build/docker/Dockerfile") - images.add("e2e-test-log-pipeline-image") + dockerFile = file('build/docker/Dockerfile') + images.add('e2e-test-log-pipeline-image') } def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int sourcePort, diff --git a/e2e-test/peerforwarder/build.gradle b/e2e-test/peerforwarder/build.gradle index a93dc88100..4a0a0aad0f 100644 --- a/e2e-test/peerforwarder/build.gradle +++ b/e2e-test/peerforwarder/build.gradle @@ -36,23 +36,29 @@ def DATA_PREPPER_CONFIG_STATIC = "data_prepper_static.yml" /** * DataPrepper Docker tasks */ + +task copyTestResources(type: Copy) { + from 'src/integrationTest/resources/' + into "${project.buildDir}/docker/test-resources" +} + task createDataPrepperDockerFile(type: Dockerfile) { dependsOn copyDataPrepperJar + dependsOn copyTestResources destFile = project.file('build/docker/Dockerfile') from(dataPrepperBaseImage) exposePort(2021) - workingDir("/app/data-prepper") - copyFile("${dataPrepperJarFilepath}", "/app/data-prepper/lib") - copyFile("src/integrationTest/resources/default_certificate.pem", "/app/data-prepper/config/default_certificate.pem") - copyFile("src/integrationTest/resources/default_private_key.pem", "/app/data-prepper/config/default_private_key.pem") + workingDir('/app/data-prepper') + copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib') + copyFile('test-resources/default_certificate.pem', '/app/data-prepper/config/default_certificate.pem') + copyFile('test-resources/default_private_key.pem', '/app/data-prepper/config/default_private_key.pem') defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute') } task buildDataPrepperDockerImage(type: DockerBuildImage) { dependsOn createDataPrepperDockerFile - inputDir = file(".") - dockerFile = file("build/docker/Dockerfile") - images.add("integ-test-pipeline-image") + dockerFile = file('build/docker/Dockerfile') + images.add('integ-test-pipeline-image') } def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int sourcePort, diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle index bc97baf9ce..0f811a26a7 100644 --- a/e2e-test/trace/build.gradle +++ b/e2e-test/trace/build.gradle @@ -45,16 +45,15 @@ task createDataPrepperDockerFile(type: Dockerfile) { destFile = project.file('build/docker/Dockerfile') from(dataPrepperBaseImage) exposePort(21890) - workingDir("/app/data-prepper") - copyFile("${dataPrepperJarFilepath}", "/app/data-prepper/lib") + workingDir('/app/data-prepper') + copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib') defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute') } task buildDataPrepperDockerImage(type: DockerBuildImage) { dependsOn createDataPrepperDockerFile - inputDir = file(".") - dockerFile = file("build/docker/Dockerfile") - images.add("integ-test-pipeline-image") + dockerFile = file('build/docker/Dockerfile') + images.add('integ-test-pipeline-image') } def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int grpcPort, diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 249e5832f0..7f93135c49 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ae04661ee7..ac72c34e8a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index a69d9cb6c2..0adc8e1a53 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +80,11 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +131,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,6 +198,10 @@ if "$cygwin" || "$msys" ; then done fi + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + # Collect all arguments for the java command; # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # shell script including quotes and variable substitutions, so put them in diff --git a/gradlew.bat b/gradlew.bat index f127cfd49d..93e3f59f13 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% diff --git a/performance-test/build.gradle b/performance-test/build.gradle index 6e9a8a5902..0a9c1ce2c2 100644 --- a/performance-test/build.gradle +++ b/performance-test/build.gradle @@ -5,7 +5,7 @@ plugins { id 'java' - id 'io.gatling.gradle' version '3.9.0' + id 'io.gatling.gradle' version '3.9.5.5' } configurations.all { diff --git a/release/archives/build.gradle b/release/archives/build.gradle index dcb19752f8..327a84ec16 100644 --- a/release/archives/build.gradle +++ b/release/archives/build.gradle @@ -13,11 +13,24 @@ subprojects { apply plugin: 'de.undercouch.download' apply plugin: 'distribution' + configurations { + allDependencyJarsFromMain { + canBeConsumed = false + canBeResolved = true + } + } + dependencies { /* required to resolve below issue with aws sdk JAXB is unavailable. Will fallback to SDK implementation which may be less performant */ implementation 'javax.xml.bind:jaxb-api:2.3.1' + + /** + * Creates a configuration on the data-prepper-main runtimeClasspath. + * https://docs.gradle.org/current/userguide/cross_project_publications.html + */ + allDependencyJarsFromMain(project(path: ':data-prepper-main', configuration: 'allDependencyJars')) } ext { @@ -111,14 +124,14 @@ subprojects { dependsOn tarTask.name file = tarTask.archiveFile.get().asFile.absolutePath bucket = awsS3Bucket - key = "${destinationKeyPath}/${tarTask.archiveName}" + key = "${destinationKeyPath}/${tarTask.archiveFileName.get()}" } tasks.create(name: "upload${platformWithArchitecture}TarWithJDKToS3", type: S3Upload) { dependsOn tarWithJDKTask.name file = tarWithJDKTask.archiveFile.get().asFile.absolutePath bucket = awsS3Bucket - key = "${destinationKeyPath}/${tarWithJDKTask.archiveName}" + key = "${destinationKeyPath}/${tarWithJDKTask.archiveFileName.get()}" } } @@ -135,7 +148,9 @@ CopySpec archiveToTar() { duplicatesStrategy = DuplicatesStrategy.EXCLUDE into('lib') { from project(':data-prepper-main').jar.archivePath - from project(':data-prepper-main').configurations.runtimeClasspath + from configurations.runtimeClasspath + //from allDependencyJarsFromMain(project(path: ':data-prepper-main', configuration: 'allDependencyJars')).runtimeClasspath + //from project(':data-prepper-main').configurations.allDependencyJars fileMode 0755 } into('examples') { diff --git a/release/docker/build.gradle b/release/docker/build.gradle index faf10b0b44..be96705aaf 100644 --- a/release/docker/build.gradle +++ b/release/docker/build.gradle @@ -12,8 +12,8 @@ docker { tag "${project.rootProject.name}", "${project.version}" files project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archivePath files "${project.projectDir}/config/default-data-prepper-config.yaml", "${project.projectDir}/config/default-keystore.p12" - buildArgs(['ARCHIVE_FILE' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveName, - 'ARCHIVE_FILE_UNPACKED' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveName.replace('.tar.gz', ''), + buildArgs(['ARCHIVE_FILE' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get(), + 'ARCHIVE_FILE_UNPACKED' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get().replace('.tar.gz', ''), 'CONFIG_FILEPATH' : '/usr/share/data-prepper/config/data-prepper-config.yaml', 'PIPELINE_FILEPATH' : '/usr/share/data-prepper/pipelines/pipelines.yaml']) dockerfile file('Dockerfile')