Merge remote-tracking branch 'origin/master' into mergeUpstream

# Conflicts: # tdenginereader/pom.xml # tdenginewriter/pom.xml # tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java
alibaba · Oct 26, 2022 · 8fadb0c · 8fadb0c
2 parents bef6244 + c91dc5f
commit 8fadb0c
Show file tree

Hide file tree

Showing 176 changed files with 8,633 additions and 2,514 deletions.
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ DataX本身作为数据同步框架，将不同数据源的同步抽象为从源
 
 # Quick Start
 
-##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/20220530/datax.tar.gz)
+##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202209/datax.tar.gz)
 
 
 ##### 请点击：[Quick Start](https://github.com/alibaba/DataX/blob/master/userGuid.md)
@@ -44,6 +44,8 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
 |              | SQLServer  |     √     |     √     |[读](https://github.com/alibaba/DataX/blob/master/sqlserverreader/doc/sqlserverreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/sqlserverwriter/doc/sqlserverwriter.md)|
 |              | PostgreSQL |     √     |     √     |[读](https://github.com/alibaba/DataX/blob/master/postgresqlreader/doc/postgresqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/postgresqlwriter/doc/postgresqlwriter.md)|
 |              | DRDS |     √     |     √     |[读](https://github.com/alibaba/DataX/blob/master/drdsreader/doc/drdsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/drdswriter/doc/drdswriter.md)|
+|              | Apache Doris |          |     √     |[写](https://github.com/alibaba/DataX/blob/master/doriswriter/doc/doriswriter.md)|
+|              | StarRocks |          |     √     |[写](https://github.com/alibaba/DataX/blob/master/starrockswriter/doc/starrockswriter.md)|
 |              | 通用RDBMS(支持所有关系型数据库)         |     √     |     √     |[读](https://github.com/alibaba/DataX/blob/master/rdbmsreader/doc/rdbmsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/rdbmswriter/doc/rdbmswriter.md)|
 | 阿里云数仓数据存储    | ODPS       |     √     |     √     |[读](https://github.com/alibaba/DataX/blob/master/odpsreader/doc/odpsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/odpswriter/doc/odpswriter.md)|
 |              | ADS        |           |     √     |[写](https://github.com/alibaba/DataX/blob/master/adswriter/doc/adswriter.md)|
@@ -95,6 +97,9 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
 
 DataX 后续计划月度迭代更新，也欢迎感兴趣的同学提交 Pull requests，月度更新内容会介绍介绍如下。
 
+- [datax_v202209]（https://github.com/alibaba/DataX/releases/tag/datax_v202209)
+    - 涉及通道能力更新（MaxCompute、Datahub、SLS等）、安全漏洞更新、通用打包更新等
+
 - [datax_v202205]（https://github.com/alibaba/DataX/releases/tag/datax_v202205)
     - 涉及通道能力更新（MaxCompute、Hologres、OSS、Tdengine等）、安全漏洞更新、通用打包更新等
 

diff --git a/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java b/adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/DataType.java
@@ -70,7 +70,7 @@ public static byte convertToDataType(String type) throws IllegalArgumentExceptio
         } else if ("datetime".equals(type)) {
             return DATETIME;
         } else {
-            throw new IllegalArgumentException("unkown type: " + type);
+            throw new IllegalArgumentException("unknown type: " + type);
         }
     }
 

diff --git a/...iter/src/main/java/com/alibaba/datax/plugin/writer/clickhousewriter/ClickhouseWriter.java b/...iter/src/main/java/com/alibaba/datax/plugin/writer/clickhousewriter/ClickhouseWriter.java
@@ -68,7 +68,7 @@ public void init() {
 
 			this.commonRdbmsWriterSlave = new CommonRdbmsWriter.Task(DATABASE_TYPE) {
 				@Override
-				protected PreparedStatement fillPreparedStatementColumnType(PreparedStatement preparedStatement, int columnIndex, int columnSqltype, Column column) throws SQLException {
+				protected PreparedStatement fillPreparedStatementColumnType(PreparedStatement preparedStatement, int columnIndex, int columnSqltype, String typeName, Column column) throws SQLException {
 					try {
 						if (column.getRawData() == null) {
 							preparedStatement.setNull(columnIndex + 1, columnSqltype);

diff --git a/clickhousewriter/src/main/resources/plugin.json b/clickhousewriter/src/main/resources/plugin.json
@@ -2,5 +2,5 @@
     "name": "clickhousewriter",
     "class": "com.alibaba.datax.plugin.writer.clickhousewriter.ClickhouseWriter",
     "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql.",
-    "developer": "jiye.tjy"
+    "developer": "alibaba"
 }
diff --git a/common/src/main/java/com/alibaba/datax/common/util/Configuration.java b/common/src/main/java/com/alibaba/datax/common/util/Configuration.java
@@ -411,6 +411,15 @@ public List<Object> getList(final String path) {
 		return list;
 	}
 
+	public <T> List<T> getListWithJson(final String path, Class<T> t) {
+		Object object = this.get(path, List.class);
+		if (null == object) {
+			return null;
+		}
+
+		return JSON.parseArray(JSON.toJSONString(object),t);
+	}
+
 	/**
 	 * 根据用户提供的json path，寻址List对象，如果对象不存在，返回null
 	 */

diff --git a/common/src/main/java/com/alibaba/datax/common/util/StrUtil.java b/common/src/main/java/com/alibaba/datax/common/util/StrUtil.java
@@ -3,6 +3,8 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.text.DecimalFormat;
 import java.util.HashMap;
 import java.util.Map;
@@ -82,4 +84,20 @@ public static String compressMiddle(String s, int headLength, int tailLength) {
         return s.substring(0, headLength) + "..." + s.substring(s.length() - tailLength);
     }
 
+    public static String getMd5(String plainText) {
+        try {
+            StringBuilder builder = new StringBuilder();
+            for (byte b : MessageDigest.getInstance("MD5").digest(plainText.getBytes())) {
+                int i = b & 0xff;
+                if (i < 0x10) {
+                    builder.append('0');
+                }
+                builder.append(Integer.toHexString(i));
+            }
+            return builder.toString();
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
 }
diff --git a/core/pom.xml b/core/pom.xml
@@ -41,7 +41,7 @@
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
-            <version>4.5</version>
+            <version>4.5.13</version>
         </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

diff --git a/core/src/main/java/com/alibaba/datax/core/transport/transformer/DigestTransformer.java b/core/src/main/java/com/alibaba/datax/core/transport/transformer/DigestTransformer.java
@@ -0,0 +1,87 @@
+package com.alibaba.datax.core.transport.transformer;
+
+import com.alibaba.datax.common.element.Column;
+import com.alibaba.datax.common.element.Record;
+import com.alibaba.datax.common.element.StringColumn;
+import com.alibaba.datax.common.exception.DataXException;
+import com.alibaba.datax.transformer.Transformer;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Arrays;
+
+/**
+ * no comments.
+ *
+ * @author XuDaojie
+ * @since 2021-08-16
+ */
+public class DigestTransformer extends Transformer {
+
+    private static final String MD5 = "md5";
+    private static final String SHA1 = "sha1";
+    private static final String TO_UPPER_CASE = "toUpperCase";
+    private static final String TO_LOWER_CASE = "toLowerCase";
+
+    public DigestTransformer() {
+        setTransformerName("dx_digest");
+    }
+
+    @Override
+    public Record evaluate(Record record, Object... paras) {
+
+        int columnIndex;
+        String type;
+        String charType;
+
+        try {
+            if (paras.length != 3) {
+                throw new RuntimeException("dx_digest paras length must be 3");
+            }
+
+            columnIndex = (Integer) paras[0];
+            type = (String) paras[1];
+            charType = (String) paras[2];
+
+            if (!StringUtils.equalsIgnoreCase(MD5, type) && !StringUtils.equalsIgnoreCase(SHA1, type)) {
+                throw new RuntimeException("dx_digest paras index 1 must be md5 or sha1");
+            }
+            if (!StringUtils.equalsIgnoreCase(TO_UPPER_CASE, charType) && !StringUtils.equalsIgnoreCase(TO_LOWER_CASE, charType)) {
+                throw new RuntimeException("dx_digest paras index 2 must be toUpperCase or toLowerCase");
+            }
+        } catch (Exception e) {
+            throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, "paras:" + Arrays.asList(paras) + " => " + e.getMessage());
+        }
+
+        Column column = record.getColumn(columnIndex);
+
+        try {
+            String oriValue = column.asString();
+
+            // 如果字段为空，作为空字符串处理
+            if (oriValue == null) {
+                oriValue = "";
+            }
+            String newValue;
+            if (MD5.equals(type)) {
+                newValue = DigestUtils.md5Hex(oriValue);
+            } else {
+                newValue = DigestUtils.sha1Hex(oriValue);
+            }
+
+            if (TO_UPPER_CASE.equals(charType)) {
+                newValue = newValue.toUpperCase();
+            } else {
+                newValue = newValue.toLowerCase();
+            }
+
+            record.setColumn(columnIndex, new StringColumn(newValue));
+
+        } catch (Exception e) {
+            throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e);
+        }
+        return record;
+    }
+
+}
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java b/core/src/main/java/com/alibaba/datax/core/transport/transformer/FilterTransformer.java
@@ -61,7 +61,7 @@ public Record evaluate(Record record, Object... paras) {
             } else if (code.equalsIgnoreCase("<=")) {
                 return doLess(record, value, column, true);
             } else {
-                throw new RuntimeException("dx_filter can't suport code:" + code);
+                throw new RuntimeException("dx_filter can't support code:" + code);
             }
         } catch (Exception e) {
             throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e);

diff --git a/...c/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java b/...c/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java
@@ -1,10 +1,18 @@
 package com.alibaba.datax.core.transport.transformer;
 
+import org.apache.commons.codec.digest.DigestUtils;
+
 /**
  * GroovyTransformer的帮助类，供groovy代码使用，必须全是static的方法
  * Created by liqiang on 16/3/4.
  */
 public class GroovyTransformerStaticUtil  {
 
+    public static String md5(final String data) {
+        return DigestUtils.md5Hex(data);
+    }
 
+    public static String sha1(final String data) {
+        return DigestUtils.sha1Hex(data);
+    }
 }
diff --git a/core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerRegistry.java b/core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerRegistry.java
@@ -36,6 +36,7 @@ public class TransformerRegistry {
         registTransformer(new ReplaceTransformer());
         registTransformer(new FilterTransformer());
         registTransformer(new GroovyTransformer());
+        registTransformer(new DigestTransformer());
     }
 
     public static void loadTransformerFromLocalStorage() {

diff --git a/core/src/main/job/job.json b/core/src/main/job/job.json
@@ -2,7 +2,7 @@
     "job": {
         "setting": {
             "speed": {
-                "byte":10485760
+                "channel":1
             },
             "errorLimit": {
                 "record": 0,

diff --git a/datahubreader/pom.xml b/datahubreader/pom.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>datax-all</artifactId>
+        <groupId>com.alibaba.datax</groupId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>datahubreader</artifactId>
+
+    <version>0.0.1-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.datax</groupId>
+            <artifactId>datax-common</artifactId>
+            <version>${datax-project-version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>slf4j-log4j12</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun.datahub</groupId>
+            <artifactId>aliyun-sdk-datahub</artifactId>
+            <version>2.21.6-public</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- compiler plugin -->
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>${jdk-version}</source>
+                    <target>${jdk-version}</target>
+                    <encoding>${project-sourceEncoding}</encoding>
+                </configuration>
+            </plugin>
+            <!-- assembly plugin -->
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <descriptor>src/main/assembly/package.xml</descriptor>
+                    </descriptors>
+                    <finalName>datax</finalName>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>dwzip</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/datahubreader/src/main/assembly/package.xml b/datahubreader/src/main/assembly/package.xml
@@ -0,0 +1,34 @@
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <id></id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <includes>
+                <include>plugin.json</include>
+            </includes>
+            <outputDirectory>plugin/reader/datahubreader</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>target/</directory>
+            <includes>
+                <include>datahubreader-0.0.1-SNAPSHOT.jar</include>
+            </includes>
+            <outputDirectory>plugin/reader/datahubreader</outputDirectory>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>plugin/reader/datahubreader/libs</outputDirectory>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java b/datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java
@@ -0,0 +1,8 @@
+package com.alibaba.datax.plugin.reader.datahubreader;
+
+public class Constant {
+
+    public static String DATETIME_FORMAT = "yyyyMMddHHmmss";
+    public static String DATE_FORMAT = "yyyyMMdd";
+
+}