[feat_spark320][taier-all] support spark 3.2.0 version

DTStack · Nov 1, 2023 · 4fe3458 · 4fe3458
1 parent c96e3bd
commit 4fe3458
Show file tree

Hide file tree

Showing 79 changed files with 5,531 additions and 4,719 deletions.
diff --git a/pom.xml b/pom.xml
@@ -206,6 +206,11 @@
             <id>oss-sonatype</id>
             <url>https://oss.sonatype.org/content/repositories/releases/</url>
         </repository>
+
+        <repository>
+            <id>huawei</id>
+            <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+        </repository>
     </repositories>
 
     <build>
@@ -283,7 +288,7 @@
                 <configuration>
                     <aggregate>true</aggregate>
                     <reportOutputDirectory>javadocs</reportOutputDirectory>
-                    <destDir>engine-java-docs</destDir>
+                    <destDir>taier-java-docs</destDir>
                 </configuration>
             </plugin>
         </plugins>

diff --git a/sql/1.5/1.5_increment.sql b/sql/1.5/1.5_increment.sql
diff --git a/sql/init.sql b/sql/init.sql
diff --git a/taier-common/pom.xml b/taier-common/pom.xml
@@ -222,7 +222,7 @@
                 <version>3.0.1</version>
                 <configuration>
                     <reportOutputDirectory>../javadocs</reportOutputDirectory>
-                    <destDir>engine-api-client</destDir>
+                    <destDir>taier-api-client</destDir>
                     <additionalJOptions>
                         <additionalJOption>-Xdoclint:none</additionalJOption>
                     </additionalJOptions>

diff --git a/taier-data-develop/pom.xml b/taier-data-develop/pom.xml
@@ -131,7 +131,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>2.4.3</version>
+                <version>3.2.4</version>
                 <dependencies>
                     <dependency>
                         <groupId>org.springframework.boot</groupId>

diff --git a/...src/main/java/com/dtstack/taier/develop/datasource/convert/utils/AbstractAssertUtils.java b/...src/main/java/com/dtstack/taier/develop/datasource/convert/utils/AbstractAssertUtils.java
diff --git a/...src/main/java/com/dtstack/taier/develop/service/develop/impl/DevelopSelectSqlService.java b/...src/main/java/com/dtstack/taier/develop/service/develop/impl/DevelopSelectSqlService.java
@@ -150,6 +150,7 @@ public String sendSqlTask(String sql, String taskParams, String jobId, Task task
         paramActionExt.setTenantId(task.getTenantId());
         paramActionExt.setQueueName(task.getQueueName());
         paramActionExt.setDatasourceId(task.getDatasourceId());
+        paramActionExt.setComponentVersion(task.getComponentVersion());
         actionService.start(paramActionExt);
         return jobId;
     }

diff --git a/...elop/src/main/java/com/dtstack/taier/develop/service/develop/saver/AbstractTaskSaver.java b/...elop/src/main/java/com/dtstack/taier/develop/service/develop/saver/AbstractTaskSaver.java
@@ -21,6 +21,7 @@
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.dtstack.taier.common.enums.EComponentType;
+import com.dtstack.taier.common.enums.EComputeType;
 import com.dtstack.taier.common.enums.EScheduleJobType;
 import com.dtstack.taier.common.enums.EScheduleStatus;
 import com.dtstack.taier.common.enums.ESubmitStatus;
@@ -183,6 +184,9 @@ public TaskVO updateTaskInfo(TaskResourceParam taskResourceParam) {
         Task task = developTaskService.getOne(Wrappers.lambdaQuery(Task.class)
                 .eq(Task::getName, taskVO.getName())
                 .eq(Task::getTenantId, taskVO.getTenantId()));
+        if(EComputeType.BATCH.getType() == taskVO.getComputeType()){
+            taskVO.setJobId(null);
+        }
 
         if (taskVO.getId() != null && taskVO.getId() > 0) {
             //update

diff --git a/...velop/src/main/java/com/dtstack/taier/develop/service/develop/saver/DefaultTaskSaver.java b/...velop/src/main/java/com/dtstack/taier/develop/service/develop/saver/DefaultTaskSaver.java
@@ -26,6 +26,7 @@
 import com.dtstack.taier.develop.dto.devlop.TaskVO;
 import com.dtstack.taier.develop.service.develop.impl.DevelopTaskTaskService;
 import com.dtstack.taier.develop.service.user.UserService;
+import com.dtstack.taier.pluginapi.enums.ComputeType;
 import com.dtstack.taier.pluginapi.enums.EJobType;
 import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
@@ -50,13 +51,13 @@ public class DefaultTaskSaver extends AbstractTaskSaver {
     @Autowired
     private UserService userService;
 
-    @Autowired
-    private DevelopTaskTaskService developTaskTaskService;
 
     @Override
     public TaskResourceParam beforeProcessing(TaskResourceParam taskResourceParam) {
         // sql 任务必须选择数据源
         EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskResourceParam.getTaskType());
+        taskResourceParam.setTaskParams(taskResourceParam.getTaskParams() == null ? taskTemplateService.getTaskTemplate(taskResourceParam.getTaskType(), taskResourceParam.getComponentVersion()).getParams() : taskResourceParam.getTaskParams());
+        taskResourceParam.setComputeType(ComputeType.BATCH.getType());
         if (EComputeType.BATCH.getType() == scheduleJobType.getComputeType().getType() && EJobType.SQL.getType() == scheduleJobType.getEngineJobType()) {
             if (null == taskResourceParam.getDatasourceId()) {
                 throw new TaierDefineException(ErrorCode.DATA_SOURCE_NOT_SET);

diff --git a/taier-datasource/pom.xml b/taier-datasource/pom.xml
@@ -89,6 +89,24 @@
         </dependencies>
     </dependencyManagement>
 
+
+    <repositories>
+        <repository>
+            <id>central</id>
+            <url>https://repo1.maven.org/maven2/</url>
+        </repository>
+
+        <repository>
+            <id>oss-sonatype</id>
+            <url>https://oss.sonatype.org/content/repositories/releases/</url>
+        </repository>
+
+        <repository>
+            <id>huawei</id>
+            <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+        </repository>
+    </repositories>
+
     <build>
         <plugins>
             <plugin>

diff --git a/...taier-datasource-api/src/main/java/com/dtstack/taier/datasource/api/base/ClientCache.java b/...taier-datasource-api/src/main/java/com/dtstack/taier/datasource/api/base/ClientCache.java
@@ -25,11 +25,15 @@
 import com.dtstack.taier.datasource.api.client.IRestful;
 import com.dtstack.taier.datasource.api.client.ITable;
 import com.dtstack.taier.datasource.api.client.IYarn;
+import com.dtstack.taier.datasource.api.config.Configuration;
+import com.dtstack.taier.datasource.api.context.ClientEnvironment;
 import com.dtstack.taier.datasource.api.exception.InitializeException;
+import com.dtstack.taier.datasource.api.manager.ManagerFactory;
 import com.dtstack.taier.datasource.api.manager.list.ClientManager;
 import com.dtstack.taier.datasource.api.source.DataSourceType;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.HashMap;
 import java.util.Objects;
 
 /**
@@ -78,6 +82,18 @@ public static IClient getClient(Integer dataSourceType) {
         return getClientByType(IClient.class, dataSourceType);
     }
 
+    public static void main(String[] args) {
+        Configuration configuration = new Configuration(new HashMap<>());
+        ClientEnvironment clientEnvironment = new ClientEnvironment(configuration);
+        clientEnvironment.start();
+        ClientCache.setEnv(clientEnvironment.getManagerFactory().getManager(ClientManager.class));
+        ClientManager clientManager = new ClientManager();
+        clientManager.setManagerFactory(new ManagerFactory());
+        setEnv(clientManager);
+        IClient client = getClient(DataSourceType.KAFKA.getVal());
+        System.out.println(client);
+    }
+
     /**
      * 获取 HDFS 文件客户端
      *

diff --git a/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-aws_s3/pom.xml b/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-aws_s3/pom.xml
@@ -51,7 +51,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-csp_s3/pom.xml b/taier-datasource/taier-datasource-plugin/taier-datasource-plugin-csp_s3/pom.xml
@@ -50,7 +50,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/taier-ui/src/components/scaffolds/create.tsx b/taier-ui/src/components/scaffolds/create.tsx
@@ -70,7 +70,7 @@ const ComponentVersion = ({ onChange }: ICreateFormProps) => {
     const [versions, setVersions] = useState<{ label: string; value: string }[]>([]);
 
     useEffect(() => {
-        if (taskType) {
+        if (taskType !== undefined) {
             api.getComponentVersionByTaskType<{ componentVersion: string; default: boolean; componentName: string }[]>({
                 taskType,
             }).then((res) => {

diff --git a/.../taier-worker-api/src/main/java/com/dtstack/taier/pluginapi/constrant/ConfigConstant.java b/.../taier-worker-api/src/main/java/com/dtstack/taier/pluginapi/constrant/ConfigConstant.java
@@ -113,4 +113,11 @@ public class ConfigConstant {
 
     public static final String DATAX_PYTHON_BIN = "DataX.python.path";
 
+
+    public static final String SPARK_KERBEROS_REMOTE_KRB5 = "spark.kerberos.remotekrb5";
+
+    public static final String SPARK_KERBEROS_REMOTE_KEYTAB = "spark.kerberos.remotekeytab";
+
+    public static final String SPARK_HADOOP_CONF_REMOTE_DIR = "spark.hadoopconf.remotedir";
+
 }
diff --git a/...ker/taier-worker-plugin/base/src/main/java/com/dtstack/taier/base/util/KerberosUtils.java b/...ker/taier-worker-plugin/base/src/main/java/com/dtstack/taier/base/util/KerberosUtils.java
@@ -432,7 +432,7 @@ public static synchronized String[] getKerberosFile(BaseConfig config, String lo
     public static String getKeytabPath(BaseConfig config) {
         String fileName = config.getPrincipalFile();
         String remoteDir = config.getRemoteDir();
-        String localDir = USER_DIR + remoteDir;
+        String localDir = ConfigConstant.LOCAL_KEYTAB_DIR_PARENT + remoteDir;
 
         File path = new File(localDir);
         if (!path.exists()) {

diff --git a/taier-worker/taier-worker-plugin/dummy/pom.xml b/taier-worker/taier-worker-plugin/dummy/pom.xml
@@ -41,7 +41,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/taier-worker/taier-worker-plugin/flink/flink112-standalone/pom.xml b/taier-worker/taier-worker-plugin/flink/flink112-standalone/pom.xml
@@ -258,7 +258,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>

diff --git a/taier-worker/taier-worker-plugin/flink/pom.xml b/taier-worker/taier-worker-plugin/flink/pom.xml
@@ -18,8 +18,8 @@
     <modules>
         <module>common</module>
         <module>yarn-hdfs-flink112-core</module>
-        <module>yarn3-hdfs3-flink112</module>
         <module>yarn2-hdfs2-flink112</module>
+        <module>yarn3-hdfs3-flink112</module>
         <module>flink112-standalone</module>
         <module>flink-base</module>
     </modules>

diff --git a/taier-worker/taier-worker-plugin/flink/yarn2-hdfs2-flink112/pom.xml b/taier-worker/taier-worker-plugin/flink/yarn2-hdfs2-flink112/pom.xml
@@ -247,6 +247,25 @@
 
     </dependencies>
 
+
+    <repositories>
+        <repository>
+            <id>central</id>
+            <url>https://repo1.maven.org/maven2/</url>
+        </repository>
+
+        <repository>
+            <id>oss-sonatype</id>
+            <url>https://oss.sonatype.org/content/repositories/releases/</url>
+        </repository>
+
+        <repository>
+            <id>huawei</id>
+            <url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
+        </repository>
+    </repositories>
+
+
     <build>
         <plugins>
             <plugin>
@@ -259,7 +278,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>3.2.4</version>
                 <executions>
                     <execution>
                         <phase>package</phase>