Updates to Gradle 8.3; fixes deprecated Gradle behavior

data-prepper-main/build.gradle

@@ -27,3 +27,21 @@ jar {
                 'Main-Class': 'org.opensearch.dataprepper.DataPrepperExecute')
     }
 }
+
+configurations {
+  allDependencyJars {
+      canBeConsumed = true
+      canBeResolved = true
+      extendsFrom runtimeClasspath
+  }
+}
+
+artifacts {
+    /**
+     * Shares a configuration which has all the necessary dependencies and can be used
+     * as part of the release to load all dependencies.
+     * https://docs.gradle.org/current/userguide/cross_project_publications.html
+     */
+    allDependencyJars(jar)
+}
+

e2e-test/build.gradle

@@ -10,10 +10,17 @@ subprojects {
             gradlePluginPortal()
         }
         dependencies {
-            classpath 'com.bmuschko:gradle-docker-plugin:7.0.0'
+            classpath 'com.bmuschko:gradle-docker-plugin:9.3.2'
         }
     }
 
+    ext {
+        dataPrepperJarImageFilepath = 'bin/data-prepper/'
+        targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11'
+        targetOpenTelemetryVersion = project.hasProperty('openTelemetryVersion') ? project.getProperty('openTelemetryVersion') : "${libs.versions.opentelemetry.get()}"
+        dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre"
+    }
+
     sourceSets {
         integrationTest {
             java {
@@ -40,17 +47,10 @@ subprojects {
         duplicatesStrategy = DuplicatesStrategy.EXCLUDE
         from project(':data-prepper-main').jar.archivePath
         from project(':data-prepper-main').configurations.runtimeClasspath
-        into("${project.buildDir}/bin/data-prepper")
+        into("${project.buildDir}/docker/${dataPrepperJarImageFilepath}")
     }
 
     dependencies {
         testImplementation testLibs.junit.vintage
     }
-
-    ext {
-        dataPrepperJarFilepath = "${project.buildDir.name}/bin/data-prepper/"
-        targetJavaVersion = project.hasProperty('endToEndJavaVersion') ? project.getProperty('endToEndJavaVersion') : '11'
-        targetOpenTelemetryVersion = project.hasProperty('openTelemetryVersion') ? project.getProperty('openTelemetryVersion') : "${libs.versions.opentelemetry.get()}"
-        dataPrepperBaseImage = "eclipse-temurin:${targetJavaVersion}-jre"
-    }
 }

e2e-test/log/build.gradle

@@ -41,16 +41,15 @@ task createDataPrepperDockerFile(type: Dockerfile) {
     dependsOn copyDataPrepperJar
     destFile = project.file('build/docker/Dockerfile')
     from(dataPrepperBaseImage)
-    workingDir("/app/data-prepper")
-    copyFile("${dataPrepperJarFilepath}", "/app/data-prepper/lib")
+    workingDir('/app/data-prepper')
+    copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib')
     defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute')
 }
 
 task buildDataPrepperDockerImage(type: DockerBuildImage) {
     dependsOn createDataPrepperDockerFile
-    inputDir  = file(".")
-    dockerFile  = file("build/docker/Dockerfile")
-    images.add("e2e-test-log-pipeline-image")
+    dockerFile  = file('build/docker/Dockerfile')
+    images.add('e2e-test-log-pipeline-image')
 }
 
 def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int sourcePort,

e2e-test/peerforwarder/build.gradle

@@ -36,23 +36,29 @@ def DATA_PREPPER_CONFIG_STATIC = "data_prepper_static.yml"
 /**
  * DataPrepper Docker tasks
  */
+
+task copyTestResources(type: Copy) {
+    from 'src/integrationTest/resources/'
+    into "${project.buildDir}/docker/test-resources"
+}
+
 task createDataPrepperDockerFile(type: Dockerfile) {
     dependsOn copyDataPrepperJar
+    dependsOn copyTestResources
     destFile = project.file('build/docker/Dockerfile')
     from(dataPrepperBaseImage)
     exposePort(2021)
-    workingDir("/app/data-prepper")
-    copyFile("${dataPrepperJarFilepath}", "/app/data-prepper/lib")
-    copyFile("src/integrationTest/resources/default_certificate.pem", "/app/data-prepper/config/default_certificate.pem")
-    copyFile("src/integrationTest/resources/default_private_key.pem", "/app/data-prepper/config/default_private_key.pem")
+    workingDir('/app/data-prepper')
+    copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib')
+    copyFile('test-resources/default_certificate.pem', '/app/data-prepper/config/default_certificate.pem')
+    copyFile('test-resources/default_private_key.pem', '/app/data-prepper/config/default_private_key.pem')
     defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute')
 }
 
 task buildDataPrepperDockerImage(type: DockerBuildImage) {
     dependsOn createDataPrepperDockerFile
-    inputDir  = file(".")
-    dockerFile  = file("build/docker/Dockerfile")
-    images.add("integ-test-pipeline-image")
+    dockerFile  = file('build/docker/Dockerfile')
+    images.add('integ-test-pipeline-image')
 }
 
 def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int sourcePort,

e2e-test/trace/build.gradle

@@ -45,16 +45,15 @@ task createDataPrepperDockerFile(type: Dockerfile) {
     destFile = project.file('build/docker/Dockerfile')
     from(dataPrepperBaseImage)
     exposePort(21890)
-    workingDir("/app/data-prepper")
-    copyFile("${dataPrepperJarFilepath}", "/app/data-prepper/lib")
+    workingDir('/app/data-prepper')
+    copyFile("${dataPrepperJarImageFilepath}", '/app/data-prepper/lib')
     defaultCommand('java', '-Ddata-prepper.dir=/app/data-prepper', '-cp', '/app/data-prepper/lib/*', 'org.opensearch.dataprepper.DataPrepperExecute')
 }
 
 task buildDataPrepperDockerImage(type: DockerBuildImage) {
     dependsOn createDataPrepperDockerFile
-    inputDir  = file(".")
-    dockerFile  = file("build/docker/Dockerfile")
-    images.add("integ-test-pipeline-image")
+    dockerFile  = file('build/docker/Dockerfile')
+    images.add('integ-test-pipeline-image')
 }
 
 def createDataPrepperDockerContainer(final String taskBaseName, final String dataPrepperName, final int grpcPort,

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -55,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -80,13 +80,11 @@ do
     esac
 done
 
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-APP_NAME="Gradle"
+# This is normally unused
+# shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,22 +131,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -193,6 +198,10 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
 # Collect all arguments for the java command;
 #   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
 #     shell script including quotes and variable substitutions, so put them in

gradlew.bat

@@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
 if "%DIRNAME%"=="" set DIRNAME=.
+@rem This is normally unused
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 

performance-test/build.gradle

@@ -5,7 +5,7 @@
 
 plugins {
     id 'java'
-    id 'io.gatling.gradle' version '3.9.0'
+    id 'io.gatling.gradle' version '3.9.5.5'
 }
 
 configurations.all {

release/archives/build.gradle

@@ -13,11 +13,24 @@ subprojects {
     apply plugin: 'de.undercouch.download'
     apply plugin: 'distribution'
 
+    configurations {
+        allDependencyJarsFromMain {
+            canBeConsumed = false
+            canBeResolved = true
+        }
+    }
+
     dependencies {
         /* required to resolve below issue with aws sdk
           JAXB is unavailable. Will fallback to SDK implementation which may be less performant
          */
         implementation 'javax.xml.bind:jaxb-api:2.3.1'
+
+        /**
+         * Creates a configuration on the data-prepper-main runtimeClasspath.
+         * https://docs.gradle.org/current/userguide/cross_project_publications.html
+         */
+        allDependencyJarsFromMain(project(path: ':data-prepper-main', configuration: 'allDependencyJars'))
     }
 
     ext {
@@ -111,14 +124,14 @@ subprojects {
                 dependsOn tarTask.name
                 file = tarTask.archiveFile.get().asFile.absolutePath
                 bucket = awsS3Bucket
-                key = "${destinationKeyPath}/${tarTask.archiveName}"
+                key = "${destinationKeyPath}/${tarTask.archiveFileName.get()}"
             }
 
             tasks.create(name: "upload${platformWithArchitecture}TarWithJDKToS3", type: S3Upload) {
                 dependsOn tarWithJDKTask.name
                 file = tarWithJDKTask.archiveFile.get().asFile.absolutePath
                 bucket = awsS3Bucket
-                key = "${destinationKeyPath}/${tarWithJDKTask.archiveName}"
+                key = "${destinationKeyPath}/${tarWithJDKTask.archiveFileName.get()}"
             }
         }
 
@@ -135,7 +148,9 @@ CopySpec archiveToTar() {
         duplicatesStrategy = DuplicatesStrategy.EXCLUDE
         into('lib') {
             from project(':data-prepper-main').jar.archivePath
-            from project(':data-prepper-main').configurations.runtimeClasspath
+            from configurations.runtimeClasspath
+            //from allDependencyJarsFromMain(project(path: ':data-prepper-main', configuration: 'allDependencyJars')).runtimeClasspath
+            //from project(':data-prepper-main').configurations.allDependencyJars
             fileMode 0755
         }
         into('examples') {

release/docker/build.gradle

@@ -12,8 +12,8 @@ docker {
     tag "${project.rootProject.name}", "${project.version}"
     files project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archivePath
     files "${project.projectDir}/config/default-data-prepper-config.yaml", "${project.projectDir}/config/default-keystore.p12"
-    buildArgs(['ARCHIVE_FILE' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveName,
-               'ARCHIVE_FILE_UNPACKED' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveName.replace('.tar.gz', ''),
+    buildArgs(['ARCHIVE_FILE' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get(),
+               'ARCHIVE_FILE_UNPACKED' : project(':release:archives:linux').tasks.getByName('linuxx64DistTar').archiveFileName.get().replace('.tar.gz', ''),
                'CONFIG_FILEPATH' : '/usr/share/data-prepper/config/data-prepper-config.yaml',
                'PIPELINE_FILEPATH' : '/usr/share/data-prepper/pipelines/pipelines.yaml'])
     dockerfile file('Dockerfile')