[feat_spark320][taier-all] support spark 3.2.0 version

DTStack · Nov 1, 2023 · 10c03cb · 10c03cb
1 parent c96e3bd
commit 10c03cb
Show file tree

Hide file tree

Showing 82 changed files with 5,730 additions and 4,781 deletions.
diff --git a/pom.xml b/pom.xml
@@ -206,6 +206,11 @@
             <id>oss-sonatype</id>
             <url>https://oss.sonatype.org/content/repositories/releases/</url>
         </repository>
+
+        <repository>
+            <id>huawei</id>
+            <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+        </repository>
     </repositories>
 
     <build>
@@ -283,7 +288,7 @@
                 <configuration>
                     <aggregate>true</aggregate>
                     <reportOutputDirectory>javadocs</reportOutputDirectory>
-                    <destDir>engine-java-docs</destDir>
+                    <destDir>taier-java-docs</destDir>
                 </configuration>
             </plugin>
         </plugins>

diff --git a/sql/1.5/1.5_increment.sql b/sql/1.5/1.5_increment.sql
diff --git a/sql/init.sql b/sql/init.sql
diff --git a/taier-common/pom.xml b/taier-common/pom.xml
@@ -222,7 +222,7 @@
                 <version>3.0.1</version>
                 <configuration>
                     <reportOutputDirectory>../javadocs</reportOutputDirectory>
-                    <destDir>engine-api-client</destDir>
+                    <destDir>taier-api-client</destDir>
                     <additionalJOptions>
                         <additionalJOption>-Xdoclint:none</additionalJOption>
                     </additionalJOptions>

diff --git a/taier-data-develop/pom.xml b/taier-data-develop/pom.xml
@@ -131,7 +131,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>2.4.3</version>
+                <version>3.2.4</version>
                 <dependencies>
                     <dependency>
                         <groupId>org.springframework.boot</groupId>

diff --git a/...src/main/java/com/dtstack/taier/develop/datasource/convert/utils/AbstractAssertUtils.java b/...src/main/java/com/dtstack/taier/develop/datasource/convert/utils/AbstractAssertUtils.java
diff --git a/...src/main/java/com/dtstack/taier/develop/service/develop/impl/DevelopSelectSqlService.java b/...src/main/java/com/dtstack/taier/develop/service/develop/impl/DevelopSelectSqlService.java
@@ -150,6 +150,7 @@ public String sendSqlTask(String sql, String taskParams, String jobId, Task task
         paramActionExt.setTenantId(task.getTenantId());
         paramActionExt.setQueueName(task.getQueueName());
         paramActionExt.setDatasourceId(task.getDatasourceId());
+        paramActionExt.setComponentVersion(task.getComponentVersion());
         actionService.start(paramActionExt);
         return jobId;
     }

diff --git a/...elop/src/main/java/com/dtstack/taier/develop/service/develop/saver/AbstractTaskSaver.java b/...elop/src/main/java/com/dtstack/taier/develop/service/develop/saver/AbstractTaskSaver.java
@@ -21,6 +21,7 @@
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.dtstack.taier.common.enums.EComponentType;
+import com.dtstack.taier.common.enums.EComputeType;
 import com.dtstack.taier.common.enums.EScheduleJobType;
 import com.dtstack.taier.common.enums.EScheduleStatus;
 import com.dtstack.taier.common.enums.ESubmitStatus;
@@ -183,6 +184,9 @@ public TaskVO updateTaskInfo(TaskResourceParam taskResourceParam) {
         Task task = developTaskService.getOne(Wrappers.lambdaQuery(Task.class)
                 .eq(Task::getName, taskVO.getName())
                 .eq(Task::getTenantId, taskVO.getTenantId()));
+        if(EComputeType.BATCH.getType() == taskVO.getComputeType()){
+            taskVO.setJobId(null);
+        }
 
         if (taskVO.getId() != null && taskVO.getId() > 0) {
             //update

diff --git a/...velop/src/main/java/com/dtstack/taier/develop/service/develop/saver/DefaultTaskSaver.java b/...velop/src/main/java/com/dtstack/taier/develop/service/develop/saver/DefaultTaskSaver.java
@@ -26,6 +26,7 @@
 import com.dtstack.taier.develop.dto.devlop.TaskVO;
 import com.dtstack.taier.develop.service.develop.impl.DevelopTaskTaskService;
 import com.dtstack.taier.develop.service.user.UserService;
+import com.dtstack.taier.pluginapi.enums.ComputeType;
 import com.dtstack.taier.pluginapi.enums.EJobType;
 import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
@@ -50,13 +51,13 @@ public class DefaultTaskSaver extends AbstractTaskSaver {
     @Autowired
     private UserService userService;
 
-    @Autowired
-    private DevelopTaskTaskService developTaskTaskService;
 
     @Override
     public TaskResourceParam beforeProcessing(TaskResourceParam taskResourceParam) {
         // sql 任务必须选择数据源
         EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskResourceParam.getTaskType());
+        taskResourceParam.setTaskParams(taskResourceParam.getTaskParams() == null ? taskTemplateService.getTaskTemplate(taskResourceParam.getTaskType(), taskResourceParam.getComponentVersion()).getParams() : taskResourceParam.getTaskParams());
+        taskResourceParam.setComputeType(ComputeType.BATCH.getType());
         if (EComputeType.BATCH.getType() == scheduleJobType.getComputeType().getType() && EJobType.SQL.getType() == scheduleJobType.getEngineJobType()) {
             if (null == taskResourceParam.getDatasourceId()) {
                 throw new TaierDefineException(ErrorCode.DATA_SOURCE_NOT_SET);

diff --git a/taier-datasource/pom.xml b/taier-datasource/pom.xml
@@ -89,6 +89,24 @@
         </dependencies>
     </dependencyManagement>
 
+
+    <repositories>
+        <repository>
+            <id>central</id>
+            <url>https://repo1.maven.org/maven2/</url>
+        </repository>
+
+        <repository>
+            <id>oss-sonatype</id>
+            <url>https://oss.sonatype.org/content/repositories/releases/</url>
+        </repository>
+
+        <repository>
+            <id>huawei</id>
+            <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+        </repository>
+    </repositories>
+
     <build>
         <plugins>
             <plugin>

diff --git a/...taier-datasource-api/src/main/java/com/dtstack/taier/datasource/api/base/ClientCache.java b/...taier-datasource-api/src/main/java/com/dtstack/taier/datasource/api/base/ClientCache.java
@@ -25,11 +25,15 @@
 import com.dtstack.taier.datasource.api.client.IRestful;
 import com.dtstack.taier.datasource.api.client.ITable;
 import com.dtstack.taier.datasource.api.client.IYarn;
+import com.dtstack.taier.datasource.api.config.Configuration;
+import com.dtstack.taier.datasource.api.context.ClientEnvironment;
 import com.dtstack.taier.datasource.api.exception.InitializeException;
+import com.dtstack.taier.datasource.api.manager.ManagerFactory;
 import com.dtstack.taier.datasource.api.manager.list.ClientManager;
 import com.dtstack.taier.datasource.api.source.DataSourceType;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.HashMap;
 import java.util.Objects;
 
 /**
@@ -78,6 +82,18 @@ public static IClient getClient(Integer dataSourceType) {
         return getClientByType(IClient.class, dataSourceType);
     }
 
+    public static void main(String[] args) {
+        Configuration configuration = new Configuration(new HashMap<>());
+        ClientEnvironment clientEnvironment = new ClientEnvironment(configuration);
+        clientEnvironment.start();
+        ClientCache.setEnv(clientEnvironment.getManagerFactory().getManager(ClientManager.class));
+        ClientManager clientManager = new ClientManager();
+        clientManager.setManagerFactory(new ManagerFactory());
+        setEnv(clientManager);
+        IClient client = getClient(DataSourceType.KAFKA.getVal());
+        System.out.println(client);
+    }
+
     /**
      * 获取 HDFS 文件客户端
      *

diff --git a/taier-datasource/taier-datasource-plugin/pom.xml b/taier-datasource/taier-datasource-plugin/pom.xml
@@ -209,6 +209,12 @@
                 <version>1.7.26</version>
                 <scope>test</scope>
             </dependency>
+
+            <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+                <version>4.4</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-aws_s3/pom.xml b/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-aws_s3/pom.xml
@@ -51,7 +51,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-csp_s3/pom.xml b/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-csp_s3/pom.xml
@@ -50,7 +50,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-hive1/pom.xml b/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-hive1/pom.xml
@@ -22,10 +22,10 @@
         <lzo-hadoop.version>1.0.5</lzo-hadoop.version>
         <parquet-hadoop.version>1.8.3</parquet-hadoop.version>
         <snappy-java.version>1.1.2.6</snappy-java.version>
-        <hive-exec.version>1.1.1</hive-exec.version>
+        <hive-exec.version>1.2.1</hive-exec.version>
         <hadoop-client.version>2.7.3</hadoop-client.version>
         <orc-core.version>1.6.3</orc-core.version>
-        <hive-jdbc.version>1.1.1</hive-jdbc.version>
+        <hive-jdbc.version>1.2.1</hive-jdbc.version>
     </properties>
 
     <dependencies>
@@ -424,6 +424,10 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
     </dependencies>
 
     <build>