KAFKA-9366 Upgrade log4j to log4j2 (apache#17373)

This pull request replaces Log4j with Log4j2 across the entire project, including dependencies, configurations, and code. The notable changes are listed below: 1. Introduce Log4j2 Instead of Log4j 2. Change Configuration File Format from Properties to YAML 3. Adds warnings to notify users if they are still using Log4j properties, encouraging them to transition to Log4j2 configurations Co-authored-by: Lee Dongjin <[email protected]> Reviewers: Luke Chen <[email protected]>, Mickael Maison <[email protected]>, Chia-Ping Tsai <[email protected]>
confluentinc · Dec 13, 2024 · b37b89c · b37b89c
1 parent b94defa
commit b37b89c
Show file tree

Hide file tree

Showing 94 changed files with 2,043 additions and 941 deletions.
diff --git a/LICENSE-binary b/LICENSE-binary
@@ -220,6 +220,7 @@ jackson-annotations-2.16.2
 jackson-core-2.16.2
 jackson-databind-2.16.2
 jackson-dataformat-csv-2.16.2
+jackson-dataformat-yaml-2.16.2
 jackson-datatype-jdk8-2.16.2
 jackson-jaxrs-base-2.16.2
 jackson-jaxrs-json-provider-2.16.2
@@ -239,6 +240,11 @@ jetty-servlets-9.4.56.v20240826
 jetty-util-9.4.56.v20240826
 jetty-util-ajax-9.4.56.v20240826
 jose4j-0.9.4
+log4j-api-2.24.1
+log4j-core-2.24.1
+log4j-core-test-2.24.1
+log4j-slf4j-impl-2.24.1
+log4j-1.2-api-2.24.1
 lz4-java-1.8.0
 maven-artifact-3.9.6
 metrics-core-4.1.12.1
@@ -254,7 +260,6 @@ netty-transport-native-epoll-4.1.115.Final
 netty-transport-native-unix-common-4.1.115.Final
 opentelemetry-proto-1.0.0-alpha
 plexus-utils-3.5.1
-reload4j-1.2.25
 rocksdbjni-7.9.2
 scala-library-2.13.15
 scala-logging_2.13-3.9.5
@@ -312,7 +317,6 @@ argparse4j-0.7.0, see: licenses/argparse-MIT
 classgraph-4.8.173, see: licenses/classgraph-MIT
 jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
 slf4j-api-1.7.36, see: licenses/slf4j-MIT
-slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT
 pcollections-4.0.1, see: licenses/pcollections-MIT
 
 ---------------------------------------

diff --git a/README.md b/README.md
@@ -50,10 +50,10 @@ Follow instructions in https://kafka.apache.org/quickstart
     ./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate
 
 ### Running a particular unit/integration test with log4j output ###
-By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j.properties` file in the module's `src/test/resources` directory.
+By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j2.yml` file in the module's `src/test/resources` directory.
 
-For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j.properties#L21) in `clients/src/test/resources/log4j.properties` 
-to `log4j.logger.org.apache.kafka=INFO` and then run:
+For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j2.yml#L35) in `clients/src/test/resources/log4j2.yml` 
+to `level: INFO` and then run:
 
     ./gradlew cleanTest clients:test --tests NetworkClientTest   
 

diff --git a/bin/connect-distributed.sh b/bin/connect-distributed.sh
@@ -22,8 +22,12 @@ fi
 
 base_dir=$(dirname $0)
 
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
-    export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+    export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
+elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then
+    echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
+    echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
+    echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
 fi
 
 if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then

diff --git a/bin/connect-mirror-maker.sh b/bin/connect-mirror-maker.sh
@@ -22,8 +22,12 @@ fi
 
 base_dir=$(dirname $0)
 
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
-    export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+    export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
+elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then
+    echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
+    echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
+    echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
 fi
 
 if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then

diff --git a/bin/connect-standalone.sh b/bin/connect-standalone.sh
@@ -22,8 +22,12 @@ fi
 
 base_dir=$(dirname $0)
 
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
-    export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+    export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
+elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then
+    echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
+    echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
+    echo You can also use the \$KAFKA_HOME/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
 fi
 
 if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then

diff --git a/bin/kafka-server-start.sh b/bin/kafka-server-start.sh
@@ -21,8 +21,12 @@ then
 fi
 base_dir=$(dirname $0)
 
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
-    export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+    export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/log4j2.yaml"
+elif echo "$KAFKA_LOG4J_OPTS" | grep -qE "log4j\.[^[:space:]]+$"; then
+    echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
+    echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
+    echo You can also use the \$KAFKA_HOME/config/log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
 fi
 
 if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then

diff --git a/bin/windows/connect-distributed.bat b/bin/windows/connect-distributed.bat
@@ -27,7 +27,14 @@ popd
 
 rem Log4j settings
 IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
-	set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/connect-log4j.properties
+        set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%BASE_DIR%/config/connect-log4j2.yaml
+) ELSE (
+        echo %KAFKA_LOG4J_OPTS% | findstr /r /c:"log4j\.[^ ]*$" >nul
+        IF %ERRORLEVEL% == 0 (
+            echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended.
+            echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
+            echo You can also use the %BASE_DIR%/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
+        )
 )
 
 "%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectDistributed %*

diff --git a/bin/windows/connect-standalone.bat b/bin/windows/connect-standalone.bat
@@ -27,7 +27,14 @@ popd
 
 rem Log4j settings
 IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
-	set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/connect-log4j.properties
+        set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%BASE_DIR%/config/connect-log4j2.yaml
+) ELSE (
+        echo %KAFKA_LOG4J_OPTS% | findstr /r /c:"log4j\.[^ ]*$" >nul
+        IF %ERRORLEVEL% == 0 (
+            echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended.
+            echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration.
+            echo You can also use the %BASE_DIR%/config/connect-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration.
+        )
 )
 
 "%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectStandalone %*

diff --git a/bin/windows/kafka-server-start.bat b/bin/windows/kafka-server-start.bat
@@ -21,7 +21,14 @@ IF [%1] EQU [] (
 
 SetLocal
 IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
-    set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%~dp0../../config/log4j.properties
+    set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%~dp0../../config/log4j2.yaml
+) ELSE (
+    echo %KAFKA_LOG4J_OPTS% | findstr /r /c:"log4j\.[^ ]*$" >nul
+    IF %ERRORLEVEL% == 0 (
+        echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended.
+        echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration.
+        echo You can also use the %~dp0../../config/log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration.
+    )
 )
 IF ["%KAFKA_HEAP_OPTS%"] EQU [""] (
     rem detect OS architecture