Add hive hook configuration parameters for each hms (#322)

* Add hive hook configuration parameters for each hms --------- Co-authored-by: yangyx <[email protected]>
ExpediaGroup · Jul 30, 2024 · bb3861f · bb3861f
1 parent a58f325
commit bb3861f
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 2 deletions.
diff --git a/...e/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java b/...e/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java
@@ -108,6 +108,12 @@ private MetaStoreFilterHook loadMetastoreFilterHook(AbstractMetaStore metaStore)
         conf.set(property.getKey(), property.getValue());
       }
     }
+    Map<String, String> metaConfigurationProperties = metaStore.getConfigurationProperties();
+    if (metaConfigurationProperties != null) {
+      for (Map.Entry<String, String> property : metaConfigurationProperties.entrySet()) {
+        conf.set(property.getKey(), property.getValue());
+      }
+    }
     conf.set(HiveConf.ConfVars.METASTORE_FILTER_HOOK.varname, metaStoreFilterHook);
     try {
       Class<? extends MetaStoreFilterHook> filterHookClass = conf

diff --git a/...c/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/...c/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java
@@ -18,6 +18,7 @@
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.verify;
@@ -26,8 +27,13 @@
 import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance;
 
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
+import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.thrift.TException;
 import org.junit.Before;
 import org.junit.Rule;
@@ -149,4 +155,26 @@ public void loadDefaultMetastoreFilterHook() {
     assertThat(mapping, is(notNullValue()));
     assertThat(mapping.getMetastoreFilter(), instanceOf(DefaultMetaStoreFilterHookImpl.class));
   }
+
+  @Test
+  public void loadMetastoreFilterHookWithCustomConfig() throws Exception{
+    AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri());
+    federatedMetaStore.setHiveMetastoreFilterHook(PrefixingMetastoreFilter.class.getName());
+    Map<String,String> metaStoreConfigurationProperties = new HashMap<>();
+    metaStoreConfigurationProperties.put(PrefixingMetastoreFilter.PREFIX_KEY,"prefix-test-");
+    federatedMetaStore.setConfigurationProperties(metaStoreConfigurationProperties);
+
+    MetaStoreMapping mapping = factory.newInstance(federatedMetaStore);
+    assertThat(mapping, is(notNullValue()));
+    MetaStoreFilterHook filterHook = mapping.getMetastoreFilter();
+    assertThat(filterHook, instanceOf(PrefixingMetastoreFilter.class));
+
+    Table table = new Table();
+    StorageDescriptor sd = new StorageDescriptor();
+    sd.setLocation("file:///tmp/local_database/local_table");
+    table.setSd(sd);
+
+    String oldLocation=sd.getLocation();
+    assertThat(filterHook.filterTable(table).getSd().getLocation(), equalTo("prefix-test-" + oldLocation ) );
+  }
 }
diff --git a/...core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java b/...core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java
@@ -33,9 +33,12 @@
  * */
 public class PrefixingMetastoreFilter implements MetaStoreFilterHook {
 
-  public static final String PREFIX = "prefix-";
+  public static final String PREFIX_KEY = "waggledance.hook.prefix";
+  public static final String PREFIX_DEFAULT = "prefix-";
+  private final String prefix;
 
   public PrefixingMetastoreFilter(HiveConf conf) {
+    prefix = conf.get(PREFIX_KEY,PREFIX_DEFAULT);
   }
 
   @Override
@@ -140,7 +143,7 @@ private void setLocationPrefix(Partition partition) {
 
   private void setLocationPrefix(StorageDescriptor sd) {
     String location = sd.getLocation();
-    sd.setLocation(PREFIX + location);
+    sd.setLocation(prefix + location);
   }
 
 }