[2.1][improvement](jdbc catalog) Optimize JdbcCatalog case mapping st…

…ability (#43256) cherry-pick #41510
apache · Dec 26, 2024 · 1396f5a · 1396f5a
1 parent 879bb79
commit 1396f5a
Show file tree

Hide file tree

Showing 66 changed files with 3,924 additions and 566 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -220,4 +220,14 @@ default Optional<TableValuedFunction> getMetaTableFunction(String dbName, String
     default Optional<TableValuedFunctionRef> getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) {
         return Optional.empty();
     }
+
+    // Convert from remote database name to local database name, overridden by subclass if necessary
+    default String fromRemoteDatabaseName(String remoteDatabaseName) {
+        return remoteDatabaseName;
+    }
+
+    // Convert from remote table name to local table name, overridden by subclass if necessary
+    default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) {
+        return remoteTableName;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -48,6 +48,7 @@
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalDatabase;
 import org.apache.doris.datasource.hive.HMSExternalTable;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.persist.OperationType;
@@ -655,8 +656,8 @@ public boolean externalTableExistInLocal(String dbName, String tableName, String
     }
 
     public void registerExternalTableFromEvent(String dbName, String tableName,
-                                               String catalogName, long updateTime,
-                                               boolean ignoreIfExists) throws DdlException {
+            String catalogName, long updateTime,
+            boolean ignoreIfExists) throws DdlException {
         CatalogIf catalog = nameToCatalog.get(catalogName);
         if (catalog == null) {
             throw new DdlException("No catalog found with name: " + catalogName);
@@ -686,7 +687,8 @@ public void registerExternalTableFromEvent(String dbName, String tableName,
 
         db.writeLock();
         try {
-            HMSExternalTable namedTable = new HMSExternalTable(tblId, tableName, dbName, (HMSExternalCatalog) catalog);
+            HMSExternalTable namedTable = ((HMSExternalDatabase) db)
+                    .buildTableForInit(tableName, tableName, tblId, hmsCatalog, (HMSExternalDatabase) db, false);
             namedTable.setUpdateTime(updateTime);
             db.registerTable(namedTable);
         } finally {
@@ -732,7 +734,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName)
     }
 
     public void addExternalPartitions(String catalogName, String dbName, String tableName,
-                                      List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
+            List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
             throws DdlException {
         CatalogIf catalog = nameToCatalog.get(catalogName);
         if (catalog == null) {
@@ -767,7 +769,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl
     }
 
     public void dropExternalPartitions(String catalogName, String dbName, String tableName,
-                                       List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
+            List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
             throws DdlException {
         CatalogIf catalog = nameToCatalog.get(catalogName);
         if (catalog == null) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.cluster.ClusterNamespace;
 import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
 import org.apache.doris.common.ThreadPoolManager;
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
 import org.apache.doris.datasource.hive.HMSExternalTable;
@@ -302,7 +303,7 @@ public void invalidatePartitionsCache(long catalogId, String dbName, String tabl
 
     public <T> MetaCache<T> buildMetaCache(String name,
             OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize,
-            CacheLoader<String, List<String>> namesCacheLoader,
+            CacheLoader<String, List<Pair<String, String>>> namesCacheLoader,
             CacheLoader<String, Optional<T>> metaObjCacheLoader,
             RemovalListener<String, Optional<T>> removalListener) {
         MetaCache<T> metaCache = new MetaCache<>(name, commonRefreshExecutor, expireAfterWriteSec, refreshAfterWriteSec,

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -71,10 +71,13 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
     protected long id;
     @SerializedName(value = "name")
     protected String name;
+    @SerializedName(value = "remoteName")
+    protected String remoteName;
     @SerializedName(value = "type")
     protected TableType type = null;
     @SerializedName(value = "timestamp")
     protected long timestamp;
+    // dbName is temporarily retained and will be deleted later. To use dbName, please use db.getFullName()
     @SerializedName(value = "dbName")
     protected String dbName;
     @SerializedName(value = "ta")
@@ -86,6 +89,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
     protected long dbId;
     protected boolean objectCreated;
     protected ExternalCatalog catalog;
+    protected ExternalDatabase db;
 
     /**
      * No args constructor for persist.
@@ -99,15 +103,19 @@ public ExternalTable() {
      *
      * @param id Table id.
      * @param name Table name.
+     * @param remoteName Remote table name.
      * @param catalog ExternalCatalog this table belongs to.
-     * @param dbName Name of the db the this table belongs to.
+     * @param db ExternalDatabase this table belongs to.
      * @param type Table type.
      */
-    public ExternalTable(long id, String name, ExternalCatalog catalog, String dbName, TableType type) {
+    public ExternalTable(long id, String name, String remoteName, ExternalCatalog catalog, ExternalDatabase db,
+            TableType type) {
         this.id = id;
         this.name = name;
+        this.remoteName = remoteName;
         this.catalog = catalog;
-        this.dbName = dbName;
+        this.db = db;
+        this.dbName = db.getFullName();
         this.type = type;
         this.objectCreated = false;
     }
@@ -116,6 +124,14 @@ public void setCatalog(ExternalCatalog catalog) {
         this.catalog = catalog;
     }
 
+    public void setDb(ExternalDatabase db) {
+        this.db = db;
+    }
+
+    public void setRemoteName(String remoteName) {
+        this.remoteName = remoteName;
+    }
+
     public boolean isView() {
         return false;
     }
@@ -141,6 +157,10 @@ public String getName() {
         return name;
     }
 
+    public String getRemoteName() {
+        return remoteName;
+    }
+
     @Override
     public TableType getType() {
         return type;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
@@ -62,6 +62,9 @@ public enum Type {
     @SerializedName(value = "createDbNames")
     private List<String> createDbNames;
 
+    @SerializedName(value = "remoteDbNames")
+    private List<String> remoteDbNames;
+
     @SerializedName(value = "type")
     private Type type;
 
@@ -75,6 +78,7 @@ public InitCatalogLog() {
         refreshDbIds = Lists.newArrayList();
         createDbIds = Lists.newArrayList();
         createDbNames = Lists.newArrayList();
+        remoteDbNames = Lists.newArrayList();
         type = Type.UNKNOWN;
     }
 
@@ -83,10 +87,11 @@ public void addRefreshDb(long id) {
         refreshDbIds.add(id);
     }
 
-    public void addCreateDb(long id, String name) {
+    public void addCreateDb(long id, String name, String remoteName) {
         createCount += 1;
         createDbIds.add(id);
         createDbNames.add(name);
+        remoteDbNames.add(remoteName);
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
@@ -66,6 +66,9 @@ public enum Type {
     @SerializedName(value = "createTableNames")
     private List<String> createTableNames;
 
+    @SerializedName(value = "remoteTableNames")
+    private List<String> remoteTableNames;
+
     @SerializedName(value = "type")
     private Type type;
 
@@ -80,6 +83,7 @@ public InitDatabaseLog() {
         refreshTableIds = Lists.newArrayList();
         createTableIds = Lists.newArrayList();
         createTableNames = Lists.newArrayList();
+        remoteTableNames = Lists.newArrayList();
         type = Type.UNKNOWN;
     }
 
@@ -88,10 +92,11 @@ public void addRefreshTable(long id) {
         refreshTableIds.add(id);
     }
 
-    public void addCreateTable(long id, String name) {
+    public void addCreateTable(long id, String name, String remoteName) {
         createCount += 1;
         createTableIds.add(id);
         createTableNames.add(name);
+        remoteTableNames.add(remoteName);
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java
@@ -32,14 +32,18 @@ public class EsExternalDatabase extends ExternalDatabase<EsExternalTable> {
      * @param extCatalog External data source this database belongs to.
      * @param id database id.
      * @param name database name.
+     * @param remoteName remote database name.
      */
-    public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
-        super(extCatalog, id, name, InitDatabaseLog.Type.ES);
+    public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) {
+        super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ES);
     }
 
     @Override
-    protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog);
+    public EsExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new EsExternalTable(tblId, localTableName, remoteTableName, (EsExternalCatalog) extCatalog,
+                (EsExternalDatabase) db);
     }
 
     public void addTableForTest(EsExternalTable tbl) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java
@@ -42,11 +42,12 @@ public class EsExternalTable extends ExternalTable {
      *
      * @param id Table id.
      * @param name Table name.
-     * @param dbName Database name.
-     * @param catalog HMSExternalDataSource.
+     * @param remoteName Remote table name.
+     * @param catalog EsExternalDataSource.
+     * @param db Database.
      */
-    public EsExternalTable(long id, String name, String dbName, EsExternalCatalog catalog) {
-        super(id, name, catalog, dbName, TableType.ES_EXTERNAL_TABLE);
+    public EsExternalTable(long id, String name, String remoteName, EsExternalCatalog catalog, EsExternalDatabase db) {
+        super(id, name, remoteName, catalog, db, TableType.ES_EXTERNAL_TABLE);
     }
 
     protected synchronized void makeSureInitialized() {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -263,7 +263,7 @@ public void registerDatabase(long dbId, String dbName) {
             LOG.debug("create database [{}]", dbName);
         }
 
-        ExternalDatabase<? extends ExternalTable> db = buildDbForInit(dbName, dbId, logType, false);
+        ExternalDatabase<? extends ExternalTable> db = buildDbForInit(dbName, null,  dbId, logType, false);
         if (useMetaCache.get()) {
             if (isInitialized()) {
                 metaCache.updateCache(dbName, db, Util.genIdByName(getQualifiedName(dbName)));

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
@@ -32,14 +32,18 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
      * @param extCatalog External catalog this database belongs to.
      * @param id database id.
      * @param name database name.
+     * @param remoteName remote database name.
      */
-    public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
-        super(extCatalog, id, name, InitDatabaseLog.Type.HMS);
+    public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) {
+        super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.HMS);
     }
 
     @Override
-    protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog);
+    public HMSExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new HMSExternalTable(tblId, localTableName, remoteTableName, (HMSExternalCatalog) extCatalog,
+                (HMSExternalDatabase) db);
     }
 
     public void addTableForTest(HMSExternalTable tbl) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -169,11 +169,13 @@ public enum DLAType {
      *
      * @param id Table id.
      * @param name Table name.
-     * @param dbName Database name.
-     * @param catalog HMSExternalCatalog.
+     * @param remoteName Remote table name.
+     * @param catalog HMSExternalDataSource.
+     * @param db Database.
      */
-    public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog) {
-        super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE);
+    public HMSExternalTable(long id, String name, String remoteName, HMSExternalCatalog catalog,
+            HMSExternalDatabase db) {
+        super(id, name, remoteName, catalog, db, TableType.HMS_EXTERNAL_TABLE);
     }
 
     // Will throw NotSupportedException if not supported hms table.

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java
@@ -28,13 +28,16 @@
 
 public class IcebergExternalDatabase extends ExternalDatabase<IcebergExternalTable> {
 
-    public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name) {
-        super(extCatalog, id, name, InitDatabaseLog.Type.ICEBERG);
+    public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) {
+        super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ICEBERG);
     }
 
     @Override
-    protected IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new IcebergExternalTable(tblId, tableName, name, (IcebergExternalCatalog) extCatalog);
+    public IcebergExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new IcebergExternalTable(tblId, localTableName, remoteTableName, (IcebergExternalCatalog) extCatalog,
+                (IcebergExternalDatabase) db);
     }
 
     public String getLocation() {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
@@ -36,8 +36,9 @@
 
 public class IcebergExternalTable extends ExternalTable {
 
-    public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) {
-        super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);
+    public IcebergExternalTable(long id, String name, String remoteName, IcebergExternalCatalog catalog,
+            IcebergExternalDatabase db) {
+        super(id, name, remoteName, catalog, db, TableType.ICEBERG_EXTERNAL_TABLE);
     }
 
     public String getIcebergCatalogType() {

diff --git a/...core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java b/...core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java
@@ -36,16 +36,18 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase {
      * @param dbId The id of this database.
      */
     public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) {
-        super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB);
+        super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB);
     }
 
     public static List<String> listTableNames() {
         return Lists.newArrayList(SchemaTable.TABLE_MAP.keySet());
     }
 
     @Override
-    protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new ExternalInfoSchemaTable(tblId, tableName, catalog);
+    public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new ExternalInfoSchemaTable(tblId, localTableName, catalog, db);
     }
 
     @Override

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java
@@ -18,9 +18,9 @@
 package org.apache.doris.datasource.infoschema;
 
 import org.apache.doris.analysis.SchemaTableType;
-import org.apache.doris.catalog.InfoSchemaDb;
 import org.apache.doris.catalog.SchemaTable;
 import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
 import org.apache.doris.thrift.TSchemaTable;
@@ -31,8 +31,8 @@
 
 public class ExternalInfoSchemaTable extends ExternalTable {
 
-    public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) {
-        super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA);
+    public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) {
+        super(id, name, name, catalog, db, TableType.SCHEMA);
     }
 
     @Override