Skip to content

Commit

Permalink
[AMORO-3118][Improvement]: Move iceberg codes from amoro-core to a ne…
Browse files Browse the repository at this point in the history
…w module amoro-iceberg (#3108)

* move mixed-format codes from core to a new module amoro-iceberg

* module refactor

* fix reviews
  • Loading branch information
baiyangtx authored Aug 23, 2024
1 parent 8c351a0 commit 7f70f01
Show file tree
Hide file tree
Showing 295 changed files with 1,815 additions and 214 deletions.
13 changes: 13 additions & 0 deletions amoro-ams/amoro-ams-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,19 @@
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-iceberg-format</artifactId>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.apache.amoro.table.TableProperties;
import org.apache.amoro.utils.CompatiblePropertyUtil;
import org.apache.iceberg.util.PropertyUtil;
import org.apache.amoro.utils.PropertyUtil;

import java.util.Map;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,7 @@ public int hashCode() {
}

public static TableConfiguration parseConfig(Map<String, String> properties) {
boolean gcEnabled =
CompatiblePropertyUtil.propertyAsBoolean(
properties, org.apache.iceberg.TableProperties.GC_ENABLED, true);
boolean gcEnabled = CompatiblePropertyUtil.propertyAsBoolean(properties, "gc.enabled", true);
return new TableConfiguration()
.setExpireSnapshotEnabled(
gcEnabled
Expand Down
13 changes: 13 additions & 0 deletions amoro-ams/amoro-ams-server/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@
<artifactId>amoro-ams-api</artifactId>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-iceberg-format</artifactId>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-mixed-format-hive</artifactId>
Expand Down Expand Up @@ -351,6 +356,14 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-iceberg-format</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.amoro</groupId>
<artifactId>amoro-ams-api</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.amoro.utils.TablePropertyUtil;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableOperations;
Expand Down Expand Up @@ -435,7 +435,7 @@ private InternalCatalog getCatalog(String catalog) {
ServerCatalog internalCatalog = tableService.getServerCatalog(catalog);
Preconditions.checkArgument(
internalCatalog instanceof InternalCatalog, "The catalog is not an iceberg rest catalog");
Set<TableFormat> tableFormats = MixedCatalogUtil.tableFormats(internalCatalog.getMetadata());
Set<TableFormat> tableFormats = CatalogUtil.tableFormats(internalCatalog.getMetadata());
Preconditions.checkArgument(
tableFormats.size() == 1
&& (tableFormats.contains(TableFormat.ICEBERG)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableMap;
import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;

import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -68,7 +68,7 @@ private static String getAmsURI(Configurations serviceConfig) {
public static ServerCatalog buildServerCatalog(
CatalogMeta catalogMeta, Configurations serverConfiguration) {
String type = catalogMeta.getCatalogType();
Set<TableFormat> tableFormats = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> tableFormats = CatalogUtil.tableFormats(catalogMeta);

Preconditions.checkState(
formatSupportedMatrix.containsKey(type), "unsupported catalog type: %s", type);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import org.apache.amoro.server.persistence.mapper.TableMetaMapper;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -46,7 +46,7 @@ public class ExternalCatalog extends ServerCatalog {

protected ExternalCatalog(CatalogMeta metadata) {
super(metadata);
this.tableMetaStore = MixedCatalogUtil.buildMetaStore(metadata);
this.tableMetaStore = CatalogUtil.buildMetaStore(metadata);
this.unifiedCatalog =
this.tableMetaStore.doAs(
() -> new CommonUnifiedCatalog(this::getMetadata, Maps.newHashMap()));
Expand Down Expand Up @@ -76,7 +76,7 @@ public void disposeTable(String database, String tableName) {
@Override
public void updateMetadata(CatalogMeta metadata) {
super.updateMetadata(metadata);
this.tableMetaStore = MixedCatalogUtil.buildMetaStore(metadata);
this.tableMetaStore = CatalogUtil.buildMetaStore(metadata);
this.unifiedCatalog.refresh();
updateDatabaseFilter(metadata);
updateTableFilter(metadata);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import org.apache.amoro.server.table.internal.InternalTableHandler;
import org.apache.amoro.server.utils.InternalTableUtil;
import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.iceberg.BaseTable;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.TableOperations;
Expand Down Expand Up @@ -100,7 +100,7 @@ public AmoroTable<?> loadTable(String database, String tableName) {
return IcebergTable.newIcebergTable(
tableIdentifier,
table,
MixedCatalogUtil.buildMetaStore(getMetadata()),
CatalogUtil.buildMetaStore(getMetadata()),
getMetadata().getCatalogProperties());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import org.apache.amoro.server.table.TableMetadata;
import org.apache.amoro.server.table.internal.InternalTableCreator;
import org.apache.amoro.server.table.internal.InternalTableHandler;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.iceberg.rest.requests.CreateTableRequest;
import org.apache.thrift.TException;
Expand All @@ -44,7 +44,7 @@ protected MixedHiveCatalogImpl(CatalogMeta catalogMeta) {
super(catalogMeta);
this.tables =
new MixedHiveTables(
catalogMeta.getCatalogProperties(), MixedCatalogUtil.buildMetaStore(catalogMeta));
catalogMeta.getCatalogProperties(), CatalogUtil.buildMetaStore(catalogMeta));
hiveClientPool = ((MixedHiveTables) tables()).getHiveClientPool();
}

Expand All @@ -53,8 +53,7 @@ public void updateMetadata(CatalogMeta metadata) {
super.updateMetadata(metadata);
hiveClientPool = ((MixedHiveTables) tables()).getHiveClientPool();
this.tables =
new MixedHiveTables(
metadata.getCatalogProperties(), MixedCatalogUtil.buildMetaStore(metadata));
new MixedHiveTables(metadata.getCatalogProperties(), CatalogUtil.buildMetaStore(metadata));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.amoro.table.TableProperties;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.AwsClientProperties;
Expand Down Expand Up @@ -249,12 +249,12 @@ private void fillAuthConfigs2CatalogMeta(
metaAuthConfig.put(
AUTH_CONFIGS_KEY_SECRET_KEY, serverAuthConfig.get(AUTH_CONFIGS_KEY_SECRET_KEY));

MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
serverAuthConfig,
catalogMeta.getCatalogProperties(),
AUTH_CONFIGS_KEY_ACCESS_KEY,
S3FileIOProperties.ACCESS_KEY_ID);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
serverAuthConfig,
catalogMeta.getCatalogProperties(),
AUTH_CONFIGS_KEY_SECRET_KEY,
Expand Down Expand Up @@ -295,12 +295,12 @@ private Map<String, Object> extractAuthConfigsFromCatalogMeta(
catalogName, CONFIG_TYPE_AUTH, AUTH_CONFIGS_KEY_KRB5.replace("\\.", "-"))));
break;
case AUTH_CONFIGS_VALUE_TYPE_AK_SK:
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
serverAuthConfig,
S3FileIOProperties.ACCESS_KEY_ID,
AUTH_CONFIGS_KEY_ACCESS_KEY);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
serverAuthConfig,
S3FileIOProperties.SECRET_ACCESS_KEY,
Expand All @@ -315,7 +315,7 @@ private Map<String, Object> extractStorageConfigsFromCatalogMeta(
String catalogName, CatalogMeta catalogMeta) {
Map<String, Object> storageConfig = new HashMap<>();
Map<String, String> config = catalogMeta.getStorageConfigs();
String storageType = MixedCatalogUtil.getCompatibleStorageType(config);
String storageType = CatalogUtil.getCompatibleStorageType(config);
storageConfig.put(STORAGE_CONFIGS_KEY_TYPE, storageType);
if (STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equals(storageType)) {
storageConfig.put(
Expand Down Expand Up @@ -345,12 +345,12 @@ private Map<String, Object> extractStorageConfigsFromCatalogMeta(
CONFIG_TYPE_STORAGE,
STORAGE_CONFIGS_KEY_HIVE_SITE.replace("\\.", "-"))));
} else if (STORAGE_CONFIGS_VALUE_TYPE_S3.equals(storageType)) {
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
storageConfig,
AwsClientProperties.CLIENT_REGION,
STORAGE_CONFIGS_KEY_REGION);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
catalogMeta.getCatalogProperties(),
storageConfig,
S3FileIOProperties.ENDPOINT,
Expand Down Expand Up @@ -421,12 +421,12 @@ private CatalogMeta constructCatalogMeta(CatalogRegisterInfo info, CatalogMeta o
}
}
} else if (storageType.equals(STORAGE_CONFIGS_VALUE_TYPE_S3)) {
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
info.getStorageConfig(),
catalogMeta.getCatalogProperties(),
STORAGE_CONFIGS_KEY_REGION,
AwsClientProperties.CLIENT_REGION);
MixedCatalogUtil.copyProperty(
CatalogUtil.copyProperty(
info.getStorageConfig(),
catalogMeta.getCatalogProperties(),
STORAGE_CONFIGS_KEY_ENDPOINT,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.table.TableProperties;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
Expand Down Expand Up @@ -171,7 +171,7 @@ public void getHiveTableDetail(Context ctx) {
"catalog.database.tableName can not be empty in any element");
ServerCatalog serverCatalog = tableService.getServerCatalog(catalog);
CatalogMeta catalogMeta = serverCatalog.getMetadata();
TableMetaStore tableMetaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore tableMetaStore = CatalogUtil.buildMetaStore(catalogMeta);
HMSClientPool hmsClientPool =
new CachedHiveClientPool(tableMetaStore, catalogMeta.getCatalogProperties());

Expand Down Expand Up @@ -212,9 +212,9 @@ public void upgradeHiveTable(Context ctx) {
CatalogMeta catalogMeta = serverCatalog.getMetadata();
String amsUri = AmsUtil.getAMSThriftAddress(serviceConfig, Constants.THRIFT_TABLE_SERVICE_NAME);
catalogMeta.putToCatalogProperties(CatalogMetaProperties.AMS_URI, amsUri);
TableMetaStore tableMetaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore tableMetaStore = CatalogUtil.buildMetaStore(catalogMeta);
// check whether catalog support MIXED_HIVE format.
Set<TableFormat> tableFormats = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> tableFormats = CatalogUtil.tableFormats(catalogMeta);
Preconditions.checkState(
tableFormats.contains(TableFormat.MIXED_HIVE),
"Catalog %s does not support MIXED_HIVE format",
Expand Down Expand Up @@ -527,7 +527,7 @@ public void getTableList(Context ctx) {
String catalogType = serverCatalog.getMetadata().getCatalogType();
if (catalogType.equals(CATALOG_TYPE_HIVE)) {
CatalogMeta catalogMeta = serverCatalog.getMetadata();
TableMetaStore tableMetaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore tableMetaStore = CatalogUtil.buildMetaStore(catalogMeta);
HMSClientPool hmsClientPool =
new CachedHiveClientPool(tableMetaStore, catalogMeta.getCatalogProperties());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.PrimaryKeySpec;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.amoro.utils.TablePropertyUtil;
import org.apache.hadoop.fs.Path;
import org.apache.iceberg.TableOperations;
Expand Down Expand Up @@ -65,7 +65,7 @@ private TableOperations newTableOperations(boolean changeStore) {
if (InternalTableUtil.isLegacyMixedIceberg(tableMetadata())) {
String tableLocation =
changeStore ? tableMetadata().getChangeLocation() : tableMetadata().getBaseLocation();
TableMetaStore metaStore = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore metaStore = CatalogUtil.buildMetaStore(catalogMeta);

MixedHadoopTableOperations ops =
new MixedHadoopTableOperations(new Path(tableLocation), io, metaStore.getConfiguration());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.iceberg.CatalogProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -251,7 +251,7 @@ public void dispose() {

private String catalogConnectorType(CatalogMeta catalogMeta) {
String catalogType = catalogMeta.getCatalogType();
Set<TableFormat> tableFormatSet = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> tableFormatSet = CatalogUtil.tableFormats(catalogMeta);

if (catalogType.equalsIgnoreCase(CatalogType.AMS.name())) {
if (tableFormatSet.contains(TableFormat.MIXED_ICEBERG)) {
Expand Down Expand Up @@ -294,7 +294,7 @@ private TableMetaStore getCatalogTableMetaStore(CatalogMeta catalogMeta) {
if (catalogMeta.getStorageConfigs() != null) {
Map<String, String> storageConfigs = catalogMeta.getStorageConfigs();
if (CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HADOOP.equalsIgnoreCase(
MixedCatalogUtil.getCompatibleStorageType(storageConfigs))) {
CatalogUtil.getCompatibleStorageType(storageConfigs))) {
builder
.withBase64MetaStoreSite(
catalogMeta
Expand Down Expand Up @@ -379,7 +379,7 @@ private TerminalSessionFactory loadTerminalSessionFactory(Configurations conf) {
}

private void applyClientProperties(CatalogMeta catalogMeta) {
Set<TableFormat> formats = MixedCatalogUtil.tableFormats(catalogMeta);
Set<TableFormat> formats = CatalogUtil.tableFormats(catalogMeta);
String catalogType = catalogMeta.getCatalogType();
if (formats.contains(TableFormat.ICEBERG)) {
if (CatalogMetaProperties.CATALOG_TYPE_AMS.equalsIgnoreCase(catalogType)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import org.apache.amoro.io.AuthenticatedFileIOs;
import org.apache.amoro.properties.CatalogMetaProperties;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
Expand Down Expand Up @@ -82,7 +82,7 @@ public static boolean isKeyedMixedTable(
*/
public static AuthenticatedFileIO newIcebergFileIo(CatalogMeta meta) {
Map<String, String> catalogProperties = meta.getCatalogProperties();
TableMetaStore store = MixedCatalogUtil.buildMetaStore(meta);
TableMetaStore store = CatalogUtil.buildMetaStore(meta);
Configuration conf = store.getConfiguration();
String warehouse = meta.getCatalogProperties().get(CatalogMetaProperties.KEY_WAREHOUSE);
String defaultImpl = HADOOP_FILE_IO_IMPL;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@
import org.apache.amoro.table.PrimaryKeySpec;
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import org.apache.iceberg.CatalogUtil;
import org.apache.amoro.utils.CatalogUtil;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.rest.RESTCatalog;
Expand Down Expand Up @@ -99,10 +98,10 @@ protected RESTCatalog loadIcebergCatalog(Map<String, String> clientProperties) {
clientProperties.putIfAbsent("warehouse", catalogName());

CatalogMeta catalogMeta = serverCatalog.getMetadata();
TableMetaStore store = MixedCatalogUtil.buildMetaStore(catalogMeta);
TableMetaStore store = CatalogUtil.buildMetaStore(catalogMeta);

return (RESTCatalog)
CatalogUtil.loadCatalog(
org.apache.iceberg.CatalogUtil.loadCatalog(
"org.apache.iceberg.rest.RESTCatalog",
"test",
clientProperties,
Expand Down
Loading

0 comments on commit 7f70f01

Please sign in to comment.