[HUDI-3654] Add new module hudi-metaserver (apache#5064)

Co-authored-by: gengxiaoyu <[email protected]>
Co-authored-by: Raymond Xu <[email protected]>

.github/workflows/bot.yml

@@ -137,4 +137,3 @@ jobs:
         run: |
           HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
           ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION
-

.gitignore

@@ -3,7 +3,7 @@
 target/
 metastore_db/
 .mvn/
- 
+
 # OS Files #
 .DS_Store
 

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java

@@ -481,7 +481,7 @@ private Stream<HoodieInstant> getCommitInstantsToArchive() throws IOException {
 
   private Stream<HoodieInstant> getInstantsToArchive() throws IOException {
     Stream<HoodieInstant> instants = Stream.concat(getCleanInstantsToArchive(), getCommitInstantsToArchive());
-    if (config.isMetastoreEnabled()) {
+    if (config.isMetaserverEnabled()) {
       return Stream.empty();
     }
 

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java

@@ -28,7 +28,7 @@
 import org.apache.hudi.common.config.HoodieCommonConfig;
 import org.apache.hudi.common.config.HoodieConfig;
 import org.apache.hudi.common.config.HoodieMetadataConfig;
-import org.apache.hudi.common.config.HoodieMetastoreConfig;
+import org.apache.hudi.common.config.HoodieMetaserverConfig;
 import org.apache.hudi.common.config.HoodieStorageConfig;
 import org.apache.hudi.common.config.TypedProperties;
 import org.apache.hudi.common.engine.EngineType;
@@ -558,7 +558,7 @@ public class HoodieWriteConfig extends HoodieConfig {
   private FileSystemViewStorageConfig viewStorageConfig;
   private HoodiePayloadConfig hoodiePayloadConfig;
   private HoodieMetadataConfig metadataConfig;
-  private HoodieMetastoreConfig metastoreConfig;
+  private HoodieMetaserverConfig metaserverConfig;
   private HoodieCommonConfig commonConfig;
   private HoodieStorageConfig storageConfig;
   private EngineType engineType;
@@ -951,7 +951,7 @@ protected HoodieWriteConfig(EngineType engineType, Properties props) {
     this.viewStorageConfig = clientSpecifiedViewStorageConfig;
     this.hoodiePayloadConfig = HoodiePayloadConfig.newBuilder().fromProperties(newProps).build();
     this.metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(props).build();
-    this.metastoreConfig = HoodieMetastoreConfig.newBuilder().fromProperties(props).build();
+    this.metaserverConfig = HoodieMetaserverConfig.newBuilder().fromProperties(props).build();
     this.commonConfig = HoodieCommonConfig.newBuilder().fromProperties(props).build();
     this.storageConfig = HoodieStorageConfig.newBuilder().fromProperties(props).build();
   }
@@ -2274,8 +2274,8 @@ public HoodieStorageLayout.LayoutType getLayoutType() {
   /**
    * Metastore configs.
    */
-  public boolean isMetastoreEnabled() {
-    return metastoreConfig.enableMetastore();
+  public boolean isMetaserverEnabled() {
+    return metaserverConfig.isMetaserverEnabled();
   }
 
   /**

hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetastoreConfig.java → hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetaserverConfig.java

@@ -29,68 +29,68 @@
 @ConfigClassProperty(name = "Metastore Configs",
     groupName = ConfigGroups.Names.WRITE_CLIENT,
     description = "Configurations used by the Hudi Metastore.")
-public class HoodieMetastoreConfig extends HoodieConfig {
+public class HoodieMetaserverConfig extends HoodieConfig {
 
-  public static final String METASTORE_PREFIX = "hoodie.metastore";
+  public static final String METASERVER_PREFIX = "hoodie.metaserver";
 
-  public static final ConfigProperty<Boolean> METASTORE_ENABLE = ConfigProperty
-      .key(METASTORE_PREFIX + ".enable")
+  public static final ConfigProperty<Boolean> METASERVER_ENABLE = ConfigProperty
+      .key(METASERVER_PREFIX + ".enabled")
       .defaultValue(false)
-      .withDocumentation("Use metastore server to store hoodie table metadata");
+      .withDocumentation("Enable Hudi metaserver for storing Hudi tables' metadata.");
 
-  public static final ConfigProperty<String> METASTORE_URLS = ConfigProperty
-      .key(METASTORE_PREFIX + ".uris")
+  public static final ConfigProperty<String> METASERVER_URLS = ConfigProperty
+      .key(METASERVER_PREFIX + ".uris")
       .defaultValue("thrift://localhost:9090")
       .withDocumentation("Metastore server uris");
 
-  public static final ConfigProperty<Integer> METASTORE_CONNECTION_RETRIES = ConfigProperty
-      .key(METASTORE_PREFIX + ".connect.retries")
+  public static final ConfigProperty<Integer> METASERVER_CONNECTION_RETRIES = ConfigProperty
+      .key(METASERVER_PREFIX + ".connect.retries")
       .defaultValue(3)
       .withDocumentation("Number of retries while opening a connection to metastore");
 
-  public static final ConfigProperty<Integer> METASTORE_CONNECTION_RETRY_DELAY = ConfigProperty
-      .key(METASTORE_PREFIX + ".connect.retry.delay")
+  public static final ConfigProperty<Integer> METASERVER_CONNECTION_RETRY_DELAY = ConfigProperty
+      .key(METASERVER_PREFIX + ".connect.retry.delay")
       .defaultValue(1)
       .withDocumentation("Number of seconds for the client to wait between consecutive connection attempts");
 
-  public static HoodieMetastoreConfig.Builder newBuilder() {
-    return new HoodieMetastoreConfig.Builder();
+  public static HoodieMetaserverConfig.Builder newBuilder() {
+    return new HoodieMetaserverConfig.Builder();
   }
 
-  public boolean enableMetastore() {
-    return getBoolean(METASTORE_ENABLE);
+  public boolean isMetaserverEnabled() {
+    return getBoolean(METASERVER_ENABLE);
   }
 
-  public String getMetastoreUris() {
-    return getStringOrDefault(METASTORE_URLS);
+  public String getMetaserverUris() {
+    return getStringOrDefault(METASERVER_URLS);
   }
 
   public int getConnectionRetryLimit() {
-    return getIntOrDefault(METASTORE_CONNECTION_RETRIES);
+    return getIntOrDefault(METASERVER_CONNECTION_RETRIES);
   }
 
   public int getConnectionRetryDelay() {
-    return getIntOrDefault(METASTORE_CONNECTION_RETRY_DELAY);
+    return getIntOrDefault(METASERVER_CONNECTION_RETRY_DELAY);
   }
 
   /**
-   * Builder for {@link HoodieMetastoreConfig}.
+   * Builder for {@link HoodieMetaserverConfig}.
    */
   public static class Builder {
-    private final HoodieMetastoreConfig config = new HoodieMetastoreConfig();
+    private final HoodieMetaserverConfig config = new HoodieMetaserverConfig();
 
     public Builder fromProperties(Properties props) {
       this.config.getProps().putAll(props);
       return this;
     }
 
     public Builder setUris(String uris) {
-      config.setValue(METASTORE_URLS, uris);
+      config.setValue(METASERVER_URLS, uris);
       return this;
     }
 
-    public HoodieMetastoreConfig build() {
-      config.setDefaults(HoodieMetastoreConfig.class.getName());
+    public HoodieMetaserverConfig build() {
+      config.setDefaults(HoodieMetaserverConfig.class.getName());
       return config;
     }
   }

hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java

@@ -19,7 +19,7 @@
 package org.apache.hudi.common.table;
 
 import org.apache.hudi.common.config.HoodieConfig;
-import org.apache.hudi.common.config.HoodieMetastoreConfig;
+import org.apache.hudi.common.config.HoodieMetaserverConfig;
 import org.apache.hudi.common.config.SerializableConfiguration;
 import org.apache.hudi.common.fs.ConsistencyGuardConfig;
 import org.apache.hudi.common.fs.FSUtils;
@@ -116,7 +116,7 @@ public class HoodieTableMetaClient implements Serializable {
   protected HoodieActiveTimeline activeTimeline;
   private ConsistencyGuardConfig consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().build();
   private FileSystemRetryConfig fileSystemRetryConfig = FileSystemRetryConfig.newBuilder().build();
-  protected HoodieMetastoreConfig metastoreConfig;
+  protected HoodieMetaserverConfig metaserverConfig;
 
   /**
    *
@@ -378,11 +378,11 @@ public synchronized HoodieArchivedTimeline getArchivedTimeline() {
     return getArchivedTimeline(StringUtils.EMPTY_STRING);
   }
 
-  public HoodieMetastoreConfig getMetastoreConfig() {
-    if (metastoreConfig == null) {
-      metastoreConfig = new HoodieMetastoreConfig();
+  public HoodieMetaserverConfig getMetaserverConfig() {
+    if (metaserverConfig == null) {
+      metaserverConfig = new HoodieMetaserverConfig();
     }
-    return metastoreConfig;
+    return metaserverConfig;
   }
 
   /**
@@ -681,14 +681,14 @@ public void initializeBootstrapDirsIfNotExists() throws IOException {
   private static HoodieTableMetaClient newMetaClient(Configuration conf, String basePath, boolean loadActiveTimelineOnLoad,
       ConsistencyGuardConfig consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion,
       String payloadClassName, String mergerStrategy, FileSystemRetryConfig fileSystemRetryConfig, Properties props) {
-    HoodieMetastoreConfig metastoreConfig = null == props
-        ? new HoodieMetastoreConfig.Builder().build()
-        : new HoodieMetastoreConfig.Builder().fromProperties(props).build();
-    return metastoreConfig.enableMetastore()
-        ? (HoodieTableMetaClient) ReflectionUtils.loadClass("org.apache.hudi.common.table.HoodieTableMetastoreClient",
-            new Class<?>[]{Configuration.class, ConsistencyGuardConfig.class, FileSystemRetryConfig.class, String.class, String.class, HoodieMetastoreConfig.class},
-            conf, consistencyGuardConfig, fileSystemRetryConfig,
-            props.getProperty(HoodieTableConfig.DATABASE_NAME.key()), props.getProperty(HoodieTableConfig.NAME.key()), metastoreConfig)
+    HoodieMetaserverConfig metaserverConfig = null == props
+        ? new HoodieMetaserverConfig.Builder().build()
+        : new HoodieMetaserverConfig.Builder().fromProperties(props).build();
+    return metaserverConfig.isMetaserverEnabled()
+        ? (HoodieTableMetaClient) ReflectionUtils.loadClass("org.apache.hudi.common.table.HoodieTableMetaserverClient",
+        new Class<?>[] {Configuration.class, ConsistencyGuardConfig.class, String.class, FileSystemRetryConfig.class, String.class, String.class, HoodieMetaserverConfig.class},
+        conf, consistencyGuardConfig, mergerStrategy, fileSystemRetryConfig,
+        props.getProperty(HoodieTableConfig.DATABASE_NAME.key()), props.getProperty(HoodieTableConfig.NAME.key()), metaserverConfig)
         : new HoodieTableMetaClient(conf, basePath,
         loadActiveTimelineOnLoad, consistencyGuardConfig, layoutVersion, payloadClassName, mergerStrategy, fileSystemRetryConfig);
   }

hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java

@@ -20,7 +20,7 @@
 
 import org.apache.hudi.common.config.HoodieCommonConfig;
 import org.apache.hudi.common.config.HoodieMetadataConfig;
-import org.apache.hudi.common.config.HoodieMetastoreConfig;
+import org.apache.hudi.common.config.HoodieMetaserverConfig;
 import org.apache.hudi.common.config.SerializableConfiguration;
 import org.apache.hudi.common.engine.HoodieEngineContext;
 import org.apache.hudi.common.function.SerializableSupplier;
@@ -61,7 +61,7 @@
 public class FileSystemViewManager {
   private static final Logger LOG = LogManager.getLogger(FileSystemViewManager.class);
 
-  private static final String HOODIE_METASTORE_FILE_SYSTEM_VIEW_CLASS = "org.apache.hudi.common.table.view.HoodieMetastoreFileSystemView";
+  private static final String HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS = "org.apache.hudi.common.table.view.HoodieMetaserverFileSystemView";
 
   private final SerializableConfiguration conf;
   // The View Storage config used to store file-system views
@@ -169,10 +169,10 @@ private static HoodieTableFileSystemView createInMemoryFileSystemView(HoodieMeta
       return new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(),
           metadataSupplier.get());
     }
-    if (metaClient.getMetastoreConfig().enableMetastore()) {
-      return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASTORE_FILE_SYSTEM_VIEW_CLASS,
-          new Class<?>[] {HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetastoreConfig.class},
-          metaClient, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(), metaClient.getMetastoreConfig());
+    if (metaClient.getMetaserverConfig().isMetaserverEnabled()) {
+      return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS,
+          new Class<?>[] {HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetaserverConfig.class},
+          metaClient, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(), metaClient.getMetaserverConfig());
     }
     return new HoodieTableFileSystemView(metaClient, timeline, viewConf.isIncrementalTimelineSyncEnabled());
   }
@@ -193,10 +193,10 @@ public static HoodieTableFileSystemView createInMemoryFileSystemViewWithTimeline
     if (metadataConfig.enabled()) {
       return new HoodieMetadataFileSystemView(engineContext, metaClient, timeline, metadataConfig);
     }
-    if (metaClient.getMetastoreConfig().enableMetastore()) {
-      return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASTORE_FILE_SYSTEM_VIEW_CLASS,
+    if (metaClient.getMetaserverConfig().isMetaserverEnabled()) {
+      return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS,
           new Class<?>[] {HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetadataConfig.class},
-          metaClient, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(), metaClient.getMetastoreConfig());
+          metaClient, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(), metaClient.getMetaserverConfig());
     }
     return new HoodieTableFileSystemView(metaClient, timeline);
   }

hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java

@@ -26,6 +26,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -167,4 +168,25 @@ private static List<String> findClasses(File directory, String packageName) {
   public static boolean isSameClass(Comparable<?> v, Comparable<?> o) {
     return v.getClass() == o.getClass();
   }
+
+  /**
+   * Invoke a static method of a class.
+   * @param clazz
+   * @param methodName
+   * @param args
+   * @param parametersType
+   * @return the return value of the method
+   */
+  public static Object invokeStaticMethod(String clazz, String methodName, Object[] args, Class<?>... parametersType) {
+    try {
+      Method method = Class.forName(clazz).getMethod(methodName, parametersType);
+      return method.invoke(null, args);
+    } catch (ClassNotFoundException e) {
+      throw new HoodieException("Unable to find the class " + clazz, e);
+    } catch (NoSuchMethodException e) {
+      throw new HoodieException(String.format("Unable to find the method %s of the class %s ",  methodName, clazz), e);
+    } catch (InvocationTargetException | IllegalAccessException e) {
+      throw new HoodieException(String.format("Unable to invoke the methond %s of the class %s ",  methodName, clazz), e);
+    }
+  }
 }

hudi-common/src/main/java/org/apache/hudi/common/util/RetryHelper.java

@@ -74,7 +74,7 @@ public RetryHelper<T, R> tryWith(CheckedFunction<T, R> func) {
     return this;
   }
 
-  public <R extends Exception> T start(CheckedFunction<T, R> func) throws R {
+  public T start(CheckedFunction<T, R> func) throws R {
     int retries = 0;
     T functionResult = null;
 

hudi-platform-service/hudi-metaserver/README.md

@@ -0,0 +1,91 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+## How to compile
+
+There are two ways, if there is a thrift binary in the local, please see the first one. 
+If docker is more convenient for you, please see the second way.
+
+### Compile with local Thrift binary
+
+Firstly, make sure `/usr/local/bin/thrift` exists, and its version is the same with the one declared in the pom.xml.
+
+Then compile the module with maven options `-Pthrift-gen-source`. For example,
+`mvn install -pl :hudi-metaserver -DskipTests=true -Phudi-platform-service`
+
+### Compile with Docker
+
+Firstly, make sure there is a docker in the local.
+
+Then just compile the module as a normal one. For example,
+`mvn install -pl :hudi-metaserver -DskipTests=true -Phudi-platform-service`
+
+Attention: Apple m1 cannot install thrift by docker successfully. The script will use homebrew to do it. So make sure homebrew exits.
+
+
+### Source code generated by Thrift
+
+After packaging, the generated source code are placed in `target/generated-sources/thrift/gen-java`.
+It looks like,
+
+```shell
+├── gen-java
+│       └── org
+│           └── apache
+│               └── hudi
+│                   └── metaserver
+│                       └── thrift
+│                           ├── AlreadyExistException.java
+│                           ├── FieldSchema.java
+│                           ├── ...
+```
+
+## How to run a job with Hudi Metaserver
+
+### Start Hudi Metaserver
+
+1. modify the `hikariPool.properties` and config the mysql address. For example,
+```text
+jdbcUrl=jdbc:mysql://localhost:3306
+dataSource.user=root
+dataSource.password=password
+```
+2. start the server
+
+   make sure `hudi-metaserver-${project.version}.jar` is under the directory,
+```shell
+sh start_hudi_metaserver.sh
+```
+
+### Write client configurations
+
+```shell
+hoodie.database.name=default
+hoodie.table.name=test
+hoodie.base.path=${path}
+hoodie.metaserver.enabled=true
+hoodie.metadata.enabled=false
+hoodie.metaserver.uris=thrift://${serverIP}:9090
+```
+
+## How to test
+
+### Run a unit test with Hudi Metaserver
+
+Add the configurations mentioned in the `Write client configurations` part, and then set `hoodie.metaserver.uris=""`.
+
+The metaserver runs as an embedded one with h2 database that performs like mysql running locally.