Merge branch 'master' into 0.7.x

apache · Jul 16, 2024 · d8e2784 · d8e2784
2 parents 3a89437 + bac3fb6
commit d8e2784
Show file tree

Hide file tree

Showing 10 changed files with 36 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -115,7 +115,7 @@ Amoro is built using Maven with JDK 8 and JDK 17(only for `amoro-mixed-format/am
 * Build all modules without `amoro-mixed-format-trino`: `mvn clean package`
 * Build and skip tests: `mvn clean package -DskipTests`
 * Build and skip dashboard: `mvn clean package -Pskip-dashboard-build`
-* Build and support disk storage, RocksDB will be introduced to avoid memory overflow: `mvn clean package -DskipTests -Psupport-disk-storage`
+* Build and disable disk storage, RocksDB will NOT be introduced to avoid memory overflow: `mvn clean package -DskipTests -Pno-extented-disk-storage`
 * Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package -DskipTests -Phadoop2`
 * Specify Flink version for Flink optimizer(the default is 1.18.1): `mvn clean package -DskipTests -Dflink-optimizer.flink-version=1.15.4`
   * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6`

diff --git a/...oro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java b/...oro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java
@@ -43,12 +43,6 @@ public class OptimizerConfig implements Serializable {
       required = true)
   private int executionParallel;
 
-  /** @deprecated This parameter is deprecated and will be removed in version 0.7.0. */
-  @Deprecated
-  @Option(
-      name = "-m",
-      aliases = "--" + OptimizerProperties.OPTIMIZER_MEMORY_SIZE,
-      usage = "Optimizer memory size(MB)")
   private int memorySize;
 
   @Option(

diff --git a/amoro-ams/amoro-ams-server/pom.xml b/amoro-ams/amoro-ams-server/pom.xml
@@ -387,6 +387,8 @@
                     </execution>
                 </executions>
                 <configuration>
+                    <failOnNoGitDirectory>false</failOnNoGitDirectory>
+                    <failOnUnableToExtractRepoInfo>false</failOnUnableToExtractRepoInfo>
                     <dotGitDirectory>${project.basedir}/.git</dotGitDirectory>
                     <generateGitPropertiesFile>true</generateGitPropertiesFile>
                     <generateGitPropertiesFilename>

diff --git a/...ro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java b/...ro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java
@@ -46,7 +46,7 @@ public static Map<Integer, RewriteFilesInput> loadTaskInputs(long processId) {
     List<byte[]> bytes =
         persistence.getAs(
             OptimizingMapper.class, mapper -> mapper.selectProcessInputFiles(processId));
-    if (bytes == null) {
+    if (bytes == null || bytes.isEmpty()) {
       return Collections.emptyMap();
     } else {
       return SerializationUtil.simpleDeserialize(CompressUtil.unGzip(bytes.get(0)));

diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql
@@ -82,7 +82,7 @@ CREATE TABLE `table_identifier`
     `format`          VARCHAR(32)  NOT NULL COMMENT 'Table Format',
     PRIMARY KEY (`table_id`),
     UNIQUE KEY `table_name_index` (`catalog_name`,`db_name`,`table_name`)
-);
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Table identifier for AMS' ROW_FORMAT=DYNAMIC;
 
 CREATE TABLE `table_metadata`
 (
@@ -131,7 +131,7 @@ CREATE TABLE `table_runtime`
     `pending_input`                 mediumtext,
     PRIMARY KEY (`table_id`),
     UNIQUE KEY `table_index` (`catalog_name`,`db_name`,`table_name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table';
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table' ROW_FORMAT=DYNAMIC;
 
 CREATE TABLE `table_optimizing_process`
 (
@@ -221,4 +221,4 @@ CREATE TABLE `table_blocker` (
   `properties` mediumtext COMMENT 'Blocker properties',
   PRIMARY KEY (`blocker_id`),
   KEY `table_index` (`catalog_name`,`db_name`,`table_name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers';
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers' ROW_FORMAT=DYNAMIC;
diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql
@@ -24,7 +24,7 @@ CREATE TABLE `table_blocker` (
   `properties` mediumtext COMMENT 'Blocker properties',
   PRIMARY KEY (`blocker_id`),
   KEY `table_index` (`catalog_name`,`db_name`,`table_name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers';
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers' ROW_FORMAT=DYNAMIC;
 
 ALTER TABLE `optimize_group` ADD COLUMN `scheduling_policy`   varchar(20) COMMENT 'Optimize group scheduling policy' after `name`;
 ALTER TABLE `optimize_task` CHANGE `max_change_transaction_id` `to_sequence` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'to sequence';

diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql
@@ -59,7 +59,7 @@ CREATE TABLE `table_identifier`
     `table_name`      varchar(128) NOT NULL COMMENT 'Table name',
     PRIMARY KEY (`table_id`),
     UNIQUE KEY `table_name_index` (`catalog_name`,`db_name`,`table_name`)
-);
+) ROW_FORMAT=DYNAMIC;
 INSERT INTO `table_identifier` (`catalog_name`, `db_name`, `table_name`) SELECT `catalog_name`, `db_name`, `table_name` FROM `table_metadata`;
 
 -- table_metadata
@@ -126,7 +126,7 @@ CREATE TABLE `table_runtime`
     `optimizing_config`             mediumtext,
     `pending_input`                 mediumtext,
     PRIMARY KEY (`table_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table';
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table' ROW_FORMAT=DYNAMIC;
 
 
 

diff --git a/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml b/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml
@@ -92,27 +92,27 @@ ams:
     connection-pool-max-idle: 16
     connection-pool-max-wait-millis: 1000
 
-#    MySQL database configuration.
-#    database:
-#      type: mysql
-#      jdbc-driver-class: com.mysql.cj.jdbc.Driver
-#      url: jdbc:mysql://127.0.0.1:3306/db?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false
-#      username: root
-#      password: root
-#      connection-pool-max-total: 20
-#      connection-pool-max-idle: 16
-#      connection-pool-max-wait-millis: 1000
-
-  #  Postgres database configuration.
-  #  database:
-  #    type: postgres
-  #    jdbc-driver-class: org.postgresql.Driver
-  #    url: jdbc:postgresql://127.0.0.1:5432/db
-  #    username: user
-  #    password: passwd
-  #    connection-pool-max-total: 20
-  #    connection-pool-max-idle: 16
-  #    connection-pool-max-wait-millis: 1000
+#  MySQL database configuration.
+#  database:
+#    type: mysql
+#    jdbc-driver-class: com.mysql.cj.jdbc.Driver
+#    url: jdbc:mysql://127.0.0.1:3306/db?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false
+#    username: root
+#    password: root
+#    connection-pool-max-total: 20
+#    connection-pool-max-idle: 16
+#    connection-pool-max-wait-millis: 1000
+
+#  Postgres database configuration.
+#  database:
+#    type: postgres
+#    jdbc-driver-class: org.postgresql.Driver
+#    url: jdbc:postgresql://127.0.0.1:5432/db
+#    username: user
+#    password: passwd
+#    connection-pool-max-total: 20
+#    connection-pool-max-idle: 16
+#    connection-pool-max-wait-millis: 1000
 
   terminal:
     backend: local

diff --git a/charts/amoro/templates/amoro-deployment.yaml b/charts/amoro/templates/amoro-deployment.yaml
@@ -83,7 +83,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}}
           {{- if .Values.livenessProbe.enabled }}
           livenessProbe:
             httpGet:
-              path: /health/status
+              path: /ams/v1/health/status
               port: rest
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
@@ -94,7 +94,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}}
           {{- if .Values.readinessProbe.enabled }}
           readinessProbe:
             httpGet:
-              path: /versionInfo
+              path: ams/v1/versionInfo
               port: rest
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}

diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh
@@ -68,10 +68,10 @@ rsync -a \
   --exclude "*/dependency-reduced-pom.xml" \
   . amoro-$RELEASE_VERSION
 
-tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tgz amoro-$RELEASE_VERSION
-gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tgz
+tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz amoro-$RELEASE_VERSION
+gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz
 cd ${RELEASE_DIR}
-$SHASUM apache-amoro-$RELEASE_VERSION-src.tgz > apache-amoro-$RELEASE_VERSION-src.tgz.sha512
+$SHASUM apache-amoro-$RELEASE_VERSION-src.tar.gz > apache-amoro-$RELEASE_VERSION-src.tar.gz.sha512
 
 cd ${CURR_DIR}
 rm -rf ${CLONE_DIR}