From bb3861f8e261ed48b813ea4e1254c73cdb284ba8 Mon Sep 17 00:00:00 2001 From: yangyuxia <13853186257@139.com> Date: Tue, 30 Jul 2024 15:35:34 +0800 Subject: [PATCH] Add hive hook configuration parameters for each hms (#322) * Add hive hook configuration parameters for each hms --------- Co-authored-by: yangyx <360508847@qq.com> --- .../model/MetaStoreMappingFactoryImpl.java | 6 ++++ .../MetaStoreMappingFactoryImplTest.java | 28 +++++++++++++++++++ .../model/PrefixingMetastoreFilter.java | 7 +++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java index 4e74cc535..ecba6d2a9 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java @@ -108,6 +108,12 @@ private MetaStoreFilterHook loadMetastoreFilterHook(AbstractMetaStore metaStore) conf.set(property.getKey(), property.getValue()); } } + Map metaConfigurationProperties = metaStore.getConfigurationProperties(); + if (metaConfigurationProperties != null) { + for (Map.Entry property : metaConfigurationProperties.entrySet()) { + conf.set(property.getKey(), property.getValue()); + } + } conf.set(HiveConf.ConfVars.METASTORE_FILTER_HOOK.varname, metaStoreFilterHook); try { Class filterHookClass = conf diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index acf5557a3..f5c24c308 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -18,6 +18,7 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; @@ -26,8 +27,13 @@ import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl; +import org.apache.hadoop.hive.metastore.MetaStoreFilterHook; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.thrift.TException; import org.junit.Before; import org.junit.Rule; @@ -149,4 +155,26 @@ public void loadDefaultMetastoreFilterHook() { assertThat(mapping, is(notNullValue())); assertThat(mapping.getMetastoreFilter(), instanceOf(DefaultMetaStoreFilterHookImpl.class)); } + + @Test + public void loadMetastoreFilterHookWithCustomConfig() throws Exception{ + AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); + federatedMetaStore.setHiveMetastoreFilterHook(PrefixingMetastoreFilter.class.getName()); + Map metaStoreConfigurationProperties = new HashMap<>(); + metaStoreConfigurationProperties.put(PrefixingMetastoreFilter.PREFIX_KEY,"prefix-test-"); + federatedMetaStore.setConfigurationProperties(metaStoreConfigurationProperties); + + MetaStoreMapping mapping = factory.newInstance(federatedMetaStore); + assertThat(mapping, is(notNullValue())); + MetaStoreFilterHook filterHook = mapping.getMetastoreFilter(); + assertThat(filterHook, instanceOf(PrefixingMetastoreFilter.class)); + + Table table = new Table(); + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("file:///tmp/local_database/local_table"); + table.setSd(sd); + + String oldLocation=sd.getLocation(); + assertThat(filterHook.filterTable(table).getSd().getLocation(), equalTo("prefix-test-" + oldLocation ) ); + } } diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java index b7c12010c..c2afe03de 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java @@ -33,9 +33,12 @@ * */ public class PrefixingMetastoreFilter implements MetaStoreFilterHook { - public static final String PREFIX = "prefix-"; + public static final String PREFIX_KEY = "waggledance.hook.prefix"; + public static final String PREFIX_DEFAULT = "prefix-"; + private final String prefix; public PrefixingMetastoreFilter(HiveConf conf) { + prefix = conf.get(PREFIX_KEY,PREFIX_DEFAULT); } @Override @@ -140,7 +143,7 @@ private void setLocationPrefix(Partition partition) { private void setLocationPrefix(StorageDescriptor sd) { String location = sd.getLocation(); - sd.setLocation(PREFIX + location); + sd.setLocation(prefix + location); } }