[feat_spark320][taier-all] support spark 3.2.0 version

DTStack · Oct 31, 2023 · adaae24 · adaae24
1 parent c96e3bd
commit adaae24
Show file tree

Hide file tree

Showing 67 changed files with 5,481 additions and 4,706 deletions.
diff --git a/pom.xml b/pom.xml
@@ -206,6 +206,19 @@
             <id>oss-sonatype</id>
             <url>https://oss.sonatype.org/content/repositories/releases/</url>
         </repository>
+
+        <repository>
+            <id>pentaho</id>
+            <name>public.nexus.pentaho.org</name>
+            <url>https://public.nexus.pentaho.org/repository/proxy-public-3rd-party-release</url>
+            <layout>default</layout>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
     </repositories>
 
     <build>
@@ -283,7 +296,7 @@
                 <configuration>
                     <aggregate>true</aggregate>
                     <reportOutputDirectory>javadocs</reportOutputDirectory>
-                    <destDir>engine-java-docs</destDir>
+                    <destDir>taier-java-docs</destDir>
                 </configuration>
             </plugin>
         </plugins>

diff --git a/sql/1.5/1.5_increment.sql b/sql/1.5/1.5_increment.sql
diff --git a/sql/init.sql b/sql/init.sql
diff --git a/taier-common/pom.xml b/taier-common/pom.xml
@@ -222,7 +222,7 @@
                 <version>3.0.1</version>
                 <configuration>
                     <reportOutputDirectory>../javadocs</reportOutputDirectory>
-                    <destDir>engine-api-client</destDir>
+                    <destDir>taier-api-client</destDir>
                     <additionalJOptions>
                         <additionalJOption>-Xdoclint:none</additionalJOption>
                     </additionalJOptions>

diff --git a/...src/main/java/com/dtstack/taier/develop/datasource/convert/utils/AbstractAssertUtils.java b/...src/main/java/com/dtstack/taier/develop/datasource/convert/utils/AbstractAssertUtils.java
diff --git a/...src/main/java/com/dtstack/taier/develop/service/develop/impl/DevelopSelectSqlService.java b/...src/main/java/com/dtstack/taier/develop/service/develop/impl/DevelopSelectSqlService.java
@@ -150,6 +150,7 @@ public String sendSqlTask(String sql, String taskParams, String jobId, Task task
         paramActionExt.setTenantId(task.getTenantId());
         paramActionExt.setQueueName(task.getQueueName());
         paramActionExt.setDatasourceId(task.getDatasourceId());
+        paramActionExt.setComponentVersion(task.getComponentVersion());
         actionService.start(paramActionExt);
         return jobId;
     }

diff --git a/...elop/src/main/java/com/dtstack/taier/develop/service/develop/saver/AbstractTaskSaver.java b/...elop/src/main/java/com/dtstack/taier/develop/service/develop/saver/AbstractTaskSaver.java
@@ -21,6 +21,7 @@
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.dtstack.taier.common.enums.EComponentType;
+import com.dtstack.taier.common.enums.EComputeType;
 import com.dtstack.taier.common.enums.EScheduleJobType;
 import com.dtstack.taier.common.enums.EScheduleStatus;
 import com.dtstack.taier.common.enums.ESubmitStatus;
@@ -183,6 +184,9 @@ public TaskVO updateTaskInfo(TaskResourceParam taskResourceParam) {
         Task task = developTaskService.getOne(Wrappers.lambdaQuery(Task.class)
                 .eq(Task::getName, taskVO.getName())
                 .eq(Task::getTenantId, taskVO.getTenantId()));
+        if(EComputeType.BATCH.getType() == taskVO.getComputeType()){
+            taskVO.setJobId(null);
+        }
 
         if (taskVO.getId() != null && taskVO.getId() > 0) {
             //update

diff --git a/...velop/src/main/java/com/dtstack/taier/develop/service/develop/saver/DefaultTaskSaver.java b/...velop/src/main/java/com/dtstack/taier/develop/service/develop/saver/DefaultTaskSaver.java
@@ -26,6 +26,7 @@
 import com.dtstack.taier.develop.dto.devlop.TaskVO;
 import com.dtstack.taier.develop.service.develop.impl.DevelopTaskTaskService;
 import com.dtstack.taier.develop.service.user.UserService;
+import com.dtstack.taier.pluginapi.enums.ComputeType;
 import com.dtstack.taier.pluginapi.enums.EJobType;
 import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang.StringUtils;
@@ -50,13 +51,13 @@ public class DefaultTaskSaver extends AbstractTaskSaver {
     @Autowired
     private UserService userService;
 
-    @Autowired
-    private DevelopTaskTaskService developTaskTaskService;
 
     @Override
     public TaskResourceParam beforeProcessing(TaskResourceParam taskResourceParam) {
         // sql 任务必须选择数据源
         EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskResourceParam.getTaskType());
+        taskResourceParam.setTaskParams(taskResourceParam.getTaskParams() == null ? taskTemplateService.getTaskTemplate(taskResourceParam.getTaskType(), taskResourceParam.getComponentVersion()).getParams() : taskResourceParam.getTaskParams());
+        taskResourceParam.setComputeType(ComputeType.BATCH.getType());
         if (EComputeType.BATCH.getType() == scheduleJobType.getComputeType().getType() && EJobType.SQL.getType() == scheduleJobType.getEngineJobType()) {
             if (null == taskResourceParam.getDatasourceId()) {
                 throw new TaierDefineException(ErrorCode.DATA_SOURCE_NOT_SET);

diff --git a/taier-datasource/pom.xml b/taier-datasource/pom.xml
@@ -30,7 +30,7 @@
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-api</artifactId>
                 <version>1.7.21</version>
-                <scope>provided</scope>
+<!--                <scope>provided</scope>-->
             </dependency>
 
             <dependency>

diff --git a/...taier-datasource-api/src/main/java/com/dtstack/taier/datasource/api/base/ClientCache.java b/...taier-datasource-api/src/main/java/com/dtstack/taier/datasource/api/base/ClientCache.java
@@ -25,11 +25,15 @@
 import com.dtstack.taier.datasource.api.client.IRestful;
 import com.dtstack.taier.datasource.api.client.ITable;
 import com.dtstack.taier.datasource.api.client.IYarn;
+import com.dtstack.taier.datasource.api.config.Configuration;
+import com.dtstack.taier.datasource.api.context.ClientEnvironment;
 import com.dtstack.taier.datasource.api.exception.InitializeException;
+import com.dtstack.taier.datasource.api.manager.ManagerFactory;
 import com.dtstack.taier.datasource.api.manager.list.ClientManager;
 import com.dtstack.taier.datasource.api.source.DataSourceType;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.HashMap;
 import java.util.Objects;
 
 /**
@@ -78,6 +82,18 @@ public static IClient getClient(Integer dataSourceType) {
         return getClientByType(IClient.class, dataSourceType);
     }
 
+    public static void main(String[] args) {
+        Configuration configuration = new Configuration(new HashMap<>());
+        ClientEnvironment clientEnvironment = new ClientEnvironment(configuration);
+        clientEnvironment.start();
+        ClientCache.setEnv(clientEnvironment.getManagerFactory().getManager(ClientManager.class));
+        ClientManager clientManager = new ClientManager();
+        clientManager.setManagerFactory(new ManagerFactory());
+        setEnv(clientManager);
+        IClient client = getClient(DataSourceType.KAFKA.getVal());
+        System.out.println(client);
+    }
+
     /**
      * 获取 HDFS 文件客户端
      *

diff --git a/taier-ui/src/components/scaffolds/create.tsx b/taier-ui/src/components/scaffolds/create.tsx
@@ -70,7 +70,7 @@ const ComponentVersion = ({ onChange }: ICreateFormProps) => {
     const [versions, setVersions] = useState<{ label: string; value: string }[]>([]);
 
     useEffect(() => {
-        if (taskType) {
+        if (taskType !== undefined) {
             api.getComponentVersionByTaskType<{ componentVersion: string; default: boolean; componentName: string }[]>({
                 taskType,
             }).then((res) => {

diff --git a/.../taier-worker-api/src/main/java/com/dtstack/taier/pluginapi/constrant/ConfigConstant.java b/.../taier-worker-api/src/main/java/com/dtstack/taier/pluginapi/constrant/ConfigConstant.java
@@ -113,4 +113,11 @@ public class ConfigConstant {
 
     public static final String DATAX_PYTHON_BIN = "DataX.python.path";
 
+
+    public static final String SPARK_KERBEROS_REMOTE_KRB5 = "spark.kerberos.remotekrb5";
+
+    public static final String SPARK_KERBEROS_REMOTE_KEYTAB = "spark.kerberos.remotekeytab";
+
+    public static final String SPARK_HADOOP_CONF_REMOTE_DIR = "spark.hadoopconf.remotedir";
+
 }
diff --git a/...ker/taier-worker-plugin/base/src/main/java/com/dtstack/taier/base/util/KerberosUtils.java b/...ker/taier-worker-plugin/base/src/main/java/com/dtstack/taier/base/util/KerberosUtils.java
@@ -432,7 +432,7 @@ public static synchronized String[] getKerberosFile(BaseConfig config, String lo
     public static String getKeytabPath(BaseConfig config) {
         String fileName = config.getPrincipalFile();
         String remoteDir = config.getRemoteDir();
-        String localDir = USER_DIR + remoteDir;
+        String localDir = ConfigConstant.LOCAL_KEYTAB_DIR_PARENT + remoteDir;
 
         File path = new File(localDir);
         if (!path.exists()) {

diff --git a/taier-worker/taier-worker-plugin/flink/yarn3-hdfs3-flink112/pom.xml b/taier-worker/taier-worker-plugin/flink/yarn3-hdfs3-flink112/pom.xml
@@ -252,6 +252,21 @@
 
     </dependencies>
 
+    <repositories>
+        <repository>
+            <id>pentaho</id>
+            <name>public.nexus.pentaho.org</name>
+            <url>https://public.nexus.pentaho.org/repository/proxy-public-3rd-party-release</url>
+            <layout>default</layout>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
     <build>
         <plugins>
             <plugin>

diff --git a/taier-worker/taier-worker-plugin/pom.xml b/taier-worker/taier-worker-plugin/pom.xml
@@ -164,7 +164,7 @@
                 <configuration>
                     <aggregate>true</aggregate>
                     <reportOutputDirectory>javadocs</reportOutputDirectory>
-                    <destDir>engine-java-docs</destDir>
+                    <destDir>taier-java-docs</destDir>
                 </configuration>
             </plugin>
         </plugins>

diff --git a/taier-worker/taier-worker-plugin/script/yarn2-hdfs2-script/script-client/pom.xml b/taier-worker/taier-worker-plugin/script/yarn2-hdfs2-script/script-client/pom.xml
@@ -31,12 +31,6 @@
             <version>1.0.0</version>
         </dependency>
 
-        <!--<dependency>-->
-        <!--    <groupId>com.dtstack.engine</groupId>-->
-        <!--    <artifactId>native-hadoop2</artifactId>-->
-        <!--    <version>1.0.0</version>-->
-        <!--</dependency>-->
-
         <!-- hadoop2 -->
         <dependency>
             <groupId>org.apache.hadoop</groupId>

diff --git a/taier-worker/taier-worker-plugin/script/yarn3-hdfs3-script/script-client/pom.xml b/taier-worker/taier-worker-plugin/script/yarn3-hdfs3-script/script-client/pom.xml
@@ -31,11 +31,6 @@
             <version>1.0.0</version>
         </dependency>
 
-        <!--<dependency>-->
-        <!--    <groupId>com.dtstack.engine</groupId>-->
-        <!--    <artifactId>native-hadoop3</artifactId>-->
-        <!--    <version>1.0.0</version>-->
-        <!--</dependency>-->
 
         <dependency>
             <groupId>org.apache.hadoop</groupId>