diff --git a/README.md b/README.md index 0748a8832f..b06093c2ba 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Amoro is built using Maven with JDK 8 and JDK 17(only for `amoro-mixed-format/am * Build all modules without `amoro-mixed-format-trino`: `mvn clean package` * Build and skip tests: `mvn clean package -DskipTests` * Build and skip dashboard: `mvn clean package -Pskip-dashboard-build` -* Build and support disk storage, RocksDB will be introduced to avoid memory overflow: `mvn clean package -DskipTests -Psupport-disk-storage` +* Build and disable disk storage, RocksDB will NOT be introduced to avoid memory overflow: `mvn clean package -DskipTests -Pno-extented-disk-storage` * Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package -DskipTests -Phadoop2` * Specify Flink version for Flink optimizer(the default is 1.18.1): `mvn clean package -DskipTests -Dflink-optimizer.flink-version=1.15.4` * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6` diff --git a/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java index 152eb2b162..d800216ca0 100644 --- a/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java +++ b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java @@ -43,12 +43,6 @@ public class OptimizerConfig implements Serializable { required = true) private int executionParallel; - /** @deprecated This parameter is deprecated and will be removed in version 0.7.0. */ - @Deprecated - @Option( - name = "-m", - aliases = "--" + OptimizerProperties.OPTIMIZER_MEMORY_SIZE, - usage = "Optimizer memory size(MB)") private int memorySize; @Option( diff --git a/amoro-ams/amoro-ams-server/pom.xml b/amoro-ams/amoro-ams-server/pom.xml index f75523582e..1b4280d18a 100644 --- a/amoro-ams/amoro-ams-server/pom.xml +++ b/amoro-ams/amoro-ams-server/pom.xml @@ -387,6 +387,8 @@ + false + false ${project.basedir}/.git true diff --git a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java index 9e8f040aa6..5bbba3abdd 100644 --- a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java +++ b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/persistence/TaskFilesPersistence.java @@ -46,7 +46,7 @@ public static Map loadTaskInputs(long processId) { List bytes = persistence.getAs( OptimizingMapper.class, mapper -> mapper.selectProcessInputFiles(processId)); - if (bytes == null) { + if (bytes == null || bytes.isEmpty()) { return Collections.emptyMap(); } else { return SerializationUtil.simpleDeserialize(CompressUtil.unGzip(bytes.get(0))); diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql index ce2c88ad34..82fe7072d3 100644 --- a/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql +++ b/amoro-ams/amoro-ams-server/src/main/resources/mysql/ams-mysql-init.sql @@ -82,7 +82,7 @@ CREATE TABLE `table_identifier` `format` VARCHAR(32) NOT NULL COMMENT 'Table Format', PRIMARY KEY (`table_id`), UNIQUE KEY `table_name_index` (`catalog_name`,`db_name`,`table_name`) -); +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Table identifier for AMS' ROW_FORMAT=DYNAMIC; CREATE TABLE `table_metadata` ( @@ -131,7 +131,7 @@ CREATE TABLE `table_runtime` `pending_input` mediumtext, PRIMARY KEY (`table_id`), UNIQUE KEY `table_index` (`catalog_name`,`db_name`,`table_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table' ROW_FORMAT=DYNAMIC; CREATE TABLE `table_optimizing_process` ( @@ -221,4 +221,4 @@ CREATE TABLE `table_blocker` ( `properties` mediumtext COMMENT 'Blocker properties', PRIMARY KEY (`blocker_id`), KEY `table_index` (`catalog_name`,`db_name`,`table_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers' ROW_FORMAT=DYNAMIC; diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql index 609499e114..c4d705d860 100644 --- a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql +++ b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.0-to-0.4.1.sql @@ -24,7 +24,7 @@ CREATE TABLE `table_blocker` ( `properties` mediumtext COMMENT 'Blocker properties', PRIMARY KEY (`blocker_id`), KEY `table_index` (`catalog_name`,`db_name`,`table_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Table blockers' ROW_FORMAT=DYNAMIC; ALTER TABLE `optimize_group` ADD COLUMN `scheduling_policy` varchar(20) COMMENT 'Optimize group scheduling policy' after `name`; ALTER TABLE `optimize_task` CHANGE `max_change_transaction_id` `to_sequence` bigint(20) NOT NULL DEFAULT '-1' COMMENT 'to sequence'; diff --git a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql index 89292b95bb..f293d19587 100644 --- a/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql +++ b/amoro-ams/amoro-ams-server/src/main/resources/mysql/upgrade-0.4.1-to-0.5.0.sql @@ -59,7 +59,7 @@ CREATE TABLE `table_identifier` `table_name` varchar(128) NOT NULL COMMENT 'Table name', PRIMARY KEY (`table_id`), UNIQUE KEY `table_name_index` (`catalog_name`,`db_name`,`table_name`) -); +) ROW_FORMAT=DYNAMIC; INSERT INTO `table_identifier` (`catalog_name`, `db_name`, `table_name`) SELECT `catalog_name`, `db_name`, `table_name` FROM `table_metadata`; -- table_metadata @@ -126,7 +126,7 @@ CREATE TABLE `table_runtime` `optimizing_config` mediumtext, `pending_input` mediumtext, PRIMARY KEY (`table_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Optimize running information of each table' ROW_FORMAT=DYNAMIC; diff --git a/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml b/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml index 9402ab2fb1..cdea700842 100644 --- a/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml +++ b/amoro-ams/dist/src/main/amoro-bin/conf/config.yaml @@ -92,27 +92,27 @@ ams: connection-pool-max-idle: 16 connection-pool-max-wait-millis: 1000 -# MySQL database configuration. -# database: -# type: mysql -# jdbc-driver-class: com.mysql.cj.jdbc.Driver -# url: jdbc:mysql://127.0.0.1:3306/db?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false -# username: root -# password: root -# connection-pool-max-total: 20 -# connection-pool-max-idle: 16 -# connection-pool-max-wait-millis: 1000 - - # Postgres database configuration. - # database: - # type: postgres - # jdbc-driver-class: org.postgresql.Driver - # url: jdbc:postgresql://127.0.0.1:5432/db - # username: user - # password: passwd - # connection-pool-max-total: 20 - # connection-pool-max-idle: 16 - # connection-pool-max-wait-millis: 1000 +# MySQL database configuration. +# database: +# type: mysql +# jdbc-driver-class: com.mysql.cj.jdbc.Driver +# url: jdbc:mysql://127.0.0.1:3306/db?useUnicode=true&characterEncoding=UTF8&autoReconnect=true&useAffectedRows=true&allowPublicKeyRetrieval=true&useSSL=false +# username: root +# password: root +# connection-pool-max-total: 20 +# connection-pool-max-idle: 16 +# connection-pool-max-wait-millis: 1000 + +# Postgres database configuration. +# database: +# type: postgres +# jdbc-driver-class: org.postgresql.Driver +# url: jdbc:postgresql://127.0.0.1:5432/db +# username: user +# password: passwd +# connection-pool-max-total: 20 +# connection-pool-max-idle: 16 +# connection-pool-max-wait-millis: 1000 terminal: backend: local diff --git a/charts/amoro/templates/amoro-deployment.yaml b/charts/amoro/templates/amoro-deployment.yaml index da9397b719..bce9dc1bb0 100644 --- a/charts/amoro/templates/amoro-deployment.yaml +++ b/charts/amoro/templates/amoro-deployment.yaml @@ -83,7 +83,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}} {{- if .Values.livenessProbe.enabled }} livenessProbe: httpGet: - path: /health/status + path: /ams/v1/health/status port: rest initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} @@ -94,7 +94,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}} {{- if .Values.readinessProbe.enabled }} readinessProbe: httpGet: - path: /versionInfo + path: ams/v1/versionInfo port: rest initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh index fd4a2cc3e8..ffb86a0fe5 100644 --- a/tools/releasing/create_source_release.sh +++ b/tools/releasing/create_source_release.sh @@ -68,10 +68,10 @@ rsync -a \ --exclude "*/dependency-reduced-pom.xml" \ . amoro-$RELEASE_VERSION -tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tgz amoro-$RELEASE_VERSION -gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tgz +tar czf ${RELEASE_DIR}/apache-amoro-${RELEASE_VERSION}-src.tar.gz amoro-$RELEASE_VERSION +gpg --armor --detach-sig ${RELEASE_DIR}/apache-amoro-$RELEASE_VERSION-src.tar.gz cd ${RELEASE_DIR} -$SHASUM apache-amoro-$RELEASE_VERSION-src.tgz > apache-amoro-$RELEASE_VERSION-src.tgz.sha512 +$SHASUM apache-amoro-$RELEASE_VERSION-src.tar.gz > apache-amoro-$RELEASE_VERSION-src.tar.gz.sha512 cd ${CURR_DIR} rm -rf ${CLONE_DIR} \ No newline at end of file