[Enhance](fe) Iceberg table in HMS catalog supports broker scan

apache · Jan 2, 2024 · 57432eb · 57432eb
1 parent 236753c
commit 57432eb
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 2 deletions.
diff --git a/docs/en/docs/lakehouse/multi-catalog/hive.md b/docs/en/docs/lakehouse/multi-catalog/hive.md
@@ -391,6 +391,13 @@ Add following setting when creating an HMS catalog, file splitting and scanning
 "broker.name" = "test_broker"
 ```
 
+
+Doris has implemented Broker query support for HMS Catalog Iceberg based on the Iceberg `FileIO` interface. If needed, the following configuration can be added when creating the HMS Catalog.
+
+```sql
+"io-impl" = "org.apache.doris.datasource.iceberg.broker.IcebergBrokerIO"
+```
+
 ## Integrate with Apache Ranger
 
 Apache Ranger is a security framework for monitoring, enabling services, and comprehensive data security access management on the Hadoop platform.

diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
@@ -373,6 +373,12 @@ CREATE CATALOG hive PROPERTIES (
 "broker.name" = "test_broker"
 ```
 
+Doris 基于 Iceberg `FileIO` 接口实现了 Broker 查询 HMS Catalog Iceberg 的支持。如有需求，可以在创建 HMS Catalog 时增加如下配置。
+
+```sql
+"io-impl" = "org.apache.doris.datasource.iceberg.broker.IcebergBrokerIO"
+```
+
 ## 使用 Ranger 进行权限校验
 
 Apache Ranger是一个用来在Hadoop平台上进行监控，启用服务，以及全方位数据安全访问管理的安全框架。

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -39,6 +39,7 @@
 import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
 import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
 import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.external.iceberg.IcebergSplit;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
@@ -202,8 +202,6 @@ private Table createIcebergTable(String uri, Map<String, String> hdfsConf, Strin
         hiveCatalog.setConf(conf);
 
         if (props.containsKey(HMSExternalCatalog.BIND_BROKER_NAME)) {
-            // Set Iceberg FileIO implementation as `IcebergBrokerIO` when Catalog binding broker is specified.
-            props.put("io-impl", "org.apache.doris.datasource.iceberg.broker.IcebergBrokerIO");
             props.put(HMSProperties.HIVE_METASTORE_URIS, uri);
             props.put("uri", uri);
             hiveCatalog.initialize("hive", props);