Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feat_spark320][taier-all] support spark 3.2.0 version #1126

Merged
merged 1 commit into from
Nov 8, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
[feat_spark320][taier-all] support spark 3.2.0 version
vainhope committed Nov 1, 2023

Verified

This commit was signed with the committer’s verified signature.
snyk-bot Snyk bot
commit 0668cce78cacda1447e91059b84a7eb294a661dc
7 changes: 6 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
@@ -206,6 +206,11 @@
<id>oss-sonatype</id>
<url>https://oss.sonatype.org/content/repositories/releases/</url>
</repository>

<repository>
<id>huawei</id>
<url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
</repository>
</repositories>

<build>
@@ -283,7 +288,7 @@
<configuration>
<aggregate>true</aggregate>
<reportOutputDirectory>javadocs</reportOutputDirectory>
<destDir>engine-java-docs</destDir>
<destDir>taier-java-docs</destDir>
</configuration>
</plugin>
</plugins>
95 changes: 95 additions & 0 deletions sql/1.5/1.5_increment.sql

Large diffs are not rendered by default.

99 changes: 84 additions & 15 deletions sql/init.sql

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion taier-common/pom.xml
Original file line number Diff line number Diff line change
@@ -222,7 +222,7 @@
<version>3.0.1</version>
<configuration>
<reportOutputDirectory>../javadocs</reportOutputDirectory>
<destDir>engine-api-client</destDir>
<destDir>taier-api-client</destDir>
<additionalJOptions>
<additionalJOption>-Xdoclint:none</additionalJOption>
</additionalJOptions>
2 changes: 1 addition & 1 deletion taier-data-develop/pom.xml
Original file line number Diff line number Diff line change
@@ -131,7 +131,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<version>3.2.4</version>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>

This file was deleted.

Original file line number Diff line number Diff line change
@@ -150,6 +150,7 @@ public String sendSqlTask(String sql, String taskParams, String jobId, Task task
paramActionExt.setTenantId(task.getTenantId());
paramActionExt.setQueueName(task.getQueueName());
paramActionExt.setDatasourceId(task.getDatasourceId());
paramActionExt.setComponentVersion(task.getComponentVersion());
actionService.start(paramActionExt);
return jobId;
}
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.dtstack.taier.common.enums.EComponentType;
import com.dtstack.taier.common.enums.EComputeType;
import com.dtstack.taier.common.enums.EScheduleJobType;
import com.dtstack.taier.common.enums.EScheduleStatus;
import com.dtstack.taier.common.enums.ESubmitStatus;
@@ -183,6 +184,9 @@ public TaskVO updateTaskInfo(TaskResourceParam taskResourceParam) {
Task task = developTaskService.getOne(Wrappers.lambdaQuery(Task.class)
.eq(Task::getName, taskVO.getName())
.eq(Task::getTenantId, taskVO.getTenantId()));
if(EComputeType.BATCH.getType() == taskVO.getComputeType()){
taskVO.setJobId(null);
}

if (taskVO.getId() != null && taskVO.getId() > 0) {
//update
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@
import com.dtstack.taier.develop.dto.devlop.TaskVO;
import com.dtstack.taier.develop.service.develop.impl.DevelopTaskTaskService;
import com.dtstack.taier.develop.service.user.UserService;
import com.dtstack.taier.pluginapi.enums.ComputeType;
import com.dtstack.taier.pluginapi.enums.EJobType;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
@@ -50,13 +51,13 @@ public class DefaultTaskSaver extends AbstractTaskSaver {
@Autowired
private UserService userService;

@Autowired
private DevelopTaskTaskService developTaskTaskService;

@Override
public TaskResourceParam beforeProcessing(TaskResourceParam taskResourceParam) {
// sql 任务必须选择数据源
EScheduleJobType scheduleJobType = EScheduleJobType.getByTaskType(taskResourceParam.getTaskType());
taskResourceParam.setTaskParams(taskResourceParam.getTaskParams() == null ? taskTemplateService.getTaskTemplate(taskResourceParam.getTaskType(), taskResourceParam.getComponentVersion()).getParams() : taskResourceParam.getTaskParams());
taskResourceParam.setComputeType(ComputeType.BATCH.getType());
if (EComputeType.BATCH.getType() == scheduleJobType.getComputeType().getType() && EJobType.SQL.getType() == scheduleJobType.getEngineJobType()) {
if (null == taskResourceParam.getDatasourceId()) {
throw new TaierDefineException(ErrorCode.DATA_SOURCE_NOT_SET);
18 changes: 18 additions & 0 deletions taier-datasource/pom.xml
Original file line number Diff line number Diff line change
@@ -89,6 +89,24 @@
</dependencies>
</dependencyManagement>


<repositories>
<repository>
<id>central</id>
<url>https://repo1.maven.org/maven2/</url>
</repository>

<repository>
<id>oss-sonatype</id>
<url>https://oss.sonatype.org/content/repositories/releases/</url>
</repository>

<repository>
<id>huawei</id>
<url>https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
</repository>
</repositories>

<build>
<plugins>
<plugin>
Original file line number Diff line number Diff line change
@@ -25,11 +25,15 @@
import com.dtstack.taier.datasource.api.client.IRestful;
import com.dtstack.taier.datasource.api.client.ITable;
import com.dtstack.taier.datasource.api.client.IYarn;
import com.dtstack.taier.datasource.api.config.Configuration;
import com.dtstack.taier.datasource.api.context.ClientEnvironment;
import com.dtstack.taier.datasource.api.exception.InitializeException;
import com.dtstack.taier.datasource.api.manager.ManagerFactory;
import com.dtstack.taier.datasource.api.manager.list.ClientManager;
import com.dtstack.taier.datasource.api.source.DataSourceType;
import lombok.extern.slf4j.Slf4j;

import java.util.HashMap;
import java.util.Objects;

/**
@@ -78,6 +82,18 @@ public static IClient getClient(Integer dataSourceType) {
return getClientByType(IClient.class, dataSourceType);
}

public static void main(String[] args) {
Configuration configuration = new Configuration(new HashMap<>());
ClientEnvironment clientEnvironment = new ClientEnvironment(configuration);
clientEnvironment.start();
ClientCache.setEnv(clientEnvironment.getManagerFactory().getManager(ClientManager.class));
ClientManager clientManager = new ClientManager();
clientManager.setManagerFactory(new ManagerFactory());
setEnv(clientManager);
IClient client = getClient(DataSourceType.KAFKA.getVal());
System.out.println(client);
}

/**
* 获取 HDFS 文件客户端
*
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
Original file line number Diff line number Diff line change
@@ -50,7 +50,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
Original file line number Diff line number Diff line change
@@ -22,10 +22,10 @@
<lzo-hadoop.version>1.0.5</lzo-hadoop.version>
<parquet-hadoop.version>1.8.3</parquet-hadoop.version>
<snappy-java.version>1.1.2.6</snappy-java.version>
<hive-exec.version>1.1.1</hive-exec.version>
<hive-exec.version>1.2.1</hive-exec.version>
<hadoop-client.version>2.7.3</hadoop-client.version>
<orc-core.version>1.6.3</orc-core.version>
<hive-jdbc.version>1.1.1</hive-jdbc.version>
<hive-jdbc.version>1.2.1</hive-jdbc.version>
</properties>

<dependencies>
@@ -424,6 +424,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
</dependencies>

<build>
Loading