[Feature-#1787]Support hadoop 3 compilation and packaging

[Feature-#1787]Support hadoop 3 compilation and packaging [Feature-#1787]Support hadoop 3 compilation and packaging
DTStack · Aug 15, 2023 · a6f8970 · a6f8970
1 parent 74c1591
commit a6f8970
Show file tree

Hide file tree

Showing 10 changed files with 34 additions and 11 deletions.
diff --git a/chunjun-connectors/chunjun-connector-binlog/pom.xml b/chunjun-connectors/chunjun-connector-binlog/pom.xml
@@ -47,10 +47,11 @@
 			<artifactId>canal.parse</artifactId>
 			<version>${canal.version}</version>
 			<exclusions>
-				<exclusion>
-					<artifactId>commons-lang</artifactId>
-					<groupId>commons-lang</groupId>
-				</exclusion>
+			<!-- 为了适配hadoop 3, 解决 java.lang.ClassNotFoundException: org.apache.commons.lang.exception.NestableRuntimeException -->
+<!--				<exclusion>-->
+<!--					<artifactId>commons-lang</artifactId>-->
+<!--					<groupId>commons-lang</groupId>-->
+<!--				</exclusion>-->
 				<exclusion>
 					<artifactId>logback-classic</artifactId>
 					<groupId>ch.qos.logback</groupId>

diff --git a/...plum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java b/...plum/src/main/java/com/dtstack/chunjun/connector/greenplum/sink/GreenplumSinkFactory.java
@@ -24,7 +24,7 @@
 import com.dtstack.chunjun.connector.jdbc.sink.JdbcSinkFactory;
 import com.dtstack.chunjun.connector.postgresql.dialect.PostgresqlDialect;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import static com.dtstack.chunjun.connector.greenplum.sink.GreenplumOutputFormat.INSERT_SQL_MODE_TYPE;
 

diff --git a/chunjun-connectors/chunjun-connector-hdfs/pom.xml b/chunjun-connectors/chunjun-connector-hdfs/pom.xml
@@ -133,6 +133,14 @@
 					<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
 					<groupId>org.apache.hadoop</groupId>
 				</exclusion>
+				<exclusion>
+					<artifactId>hadoop-auth</artifactId>
+					<groupId>org.apache.hadoop</groupId>
+				</exclusion>
+				<exclusion>
+					<artifactId>hadoop-common</artifactId>
+					<groupId>org.apache.hadoop</groupId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 
@@ -205,6 +213,10 @@
 					<artifactId>parquet-hadoop-bundle</artifactId>
 					<groupId>com.twitter</groupId>
 				</exclusion>
+				<exclusion>
+					<artifactId>hadoop-common</artifactId>
+					<groupId>org.apache.hadoop</groupId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 
@@ -250,6 +262,12 @@
 									</excludes>
 								</filter>
 							</filters>
+							<relocations>
+								<relocation>
+									<pattern>com.google.common</pattern>
+									<shadedPattern>com.dtstack.chunjun.connector.hdfs.com.google.common</shadedPattern>
+								</relocation>
+							</relocations>
 						</configuration>
 					</execution>
 				</executions>

diff --git a/chunjun-connectors/chunjun-connector-hive/pom.xml b/chunjun-connectors/chunjun-connector-hive/pom.xml
@@ -344,6 +344,10 @@
 									<pattern>com.google.protobuf</pattern>
 									<shadedPattern>com.dtstack.chunjun.connector.hive.com.google.protobuf</shadedPattern>
 								</relocation>
+								<relocation>
+									<pattern>com.google.common</pattern>
+									<shadedPattern>com.dtstack.chunjun.connector.hive.com.google.common</shadedPattern>
+								</relocation>
 							</relocations>
 							<transformers>
 								<transformer

diff --git a/...n-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java b/...n-connector-jdbc-base/src/main/java/com/dtstack/chunjun/connector/jdbc/util/JdbcUtil.java
@@ -233,7 +233,7 @@ public static List<String> getTablePrimaryKey(TableIdentify tableIdentify, Conne
         List<String> indexList = new LinkedList<>();
         while (rs.next()) {
             String index = rs.getString(4);
-            if (org.apache.commons.lang.StringUtils.isNotBlank(index)) indexList.add(index);
+            if (StringUtils.isNotBlank(index)) indexList.add(index);
         }
         return indexList;
     }

diff --git a/.../chunjun-connector-s3/src/main/java/com/dtstack/chunjun/connector/s3/config/S3Config.java b/.../chunjun-connector-s3/src/main/java/com/dtstack/chunjun/connector/s3/config/S3Config.java
@@ -21,9 +21,9 @@
 import com.dtstack.chunjun.config.CommonConfig;
 
 import com.amazonaws.regions.Regions;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import lombok.Getter;
 import lombok.Setter;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 
 import java.io.Serializable;
 import java.util.List;

diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/config/FieldConfig.java b/chunjun-core/src/main/java/com/dtstack/chunjun/config/FieldConfig.java
@@ -22,7 +22,7 @@
 
 import lombok.Data;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.Serializable;
 import java.util.ArrayList;

diff --git a/chunjun-core/src/main/java/com/dtstack/chunjun/element/column/StringColumn.java b/chunjun-core/src/main/java/com/dtstack/chunjun/element/column/StringColumn.java
@@ -25,7 +25,7 @@
 
 import cn.hutool.core.date.LocalDateTimeUtil;
 import com.alibaba.fastjson.JSON;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.math.NumberUtils;
 
 import java.math.BigDecimal;

diff --git a/...re/src/main/java/com/dtstack/chunjun/typeutil/serializer/base/StringColumnSerializer.java b/...re/src/main/java/com/dtstack/chunjun/typeutil/serializer/base/StringColumnSerializer.java
@@ -29,7 +29,7 @@
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 

diff --git a/pom.xml b/pom.xml
@@ -121,7 +121,7 @@
 		<maven-source-plugin.version>3.0.1</maven-source-plugin.version>
 		<maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
 		<maven-deploy-plugin.version>2.8.2</maven-deploy-plugin.version>
-		<maven-scala-plugin.version>4.8.1</maven-scala-plugin.version>
+		<maven-scala-plugin.version>3.2.2</maven-scala-plugin.version>
 		<maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version>
 		<maven-pmd-plugin.version>3.8</maven-pmd-plugin.version>
 		<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>