FLUME-3448 fixed unit test errors related to bump up

Signed-off-by: nikita15p <[email protected]>
apache · Mar 27, 2023 · 4923efe · 4923efe
1 parent 3c27b3e
commit 4923efe
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 23 deletions.
diff --git a/...ender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java b/...ender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java
@@ -71,8 +71,7 @@ private static int getFreePort() throws Exception {
   @Before
   public void setUp() throws Exception {
     URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc");
-    Files.copy(Resources.newInputStreamSupplier(schemaUrl),
-        new File("/tmp/myrecord.avsc"));
+    Resources.asByteSource(schemaUrl).copyTo(Files.asByteSink(new File("/tmp/myrecord.avsc")));
 
     port = getFreePort();
     source = new AvroSource();

diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java
@@ -23,7 +23,7 @@
 package org.apache.flume.source;
 
 import com.google.common.base.Preconditions;
-import com.google.common.cache.Cache;
+import com.google.common.cache.LoadingCache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.collect.Maps;
@@ -59,7 +59,7 @@ public class SyslogParser {
   private static final int RFC5424_PREFIX_LEN = 19;
   private final DateTimeFormatter timeParser;
 
-  private Cache<String, Long> timestampCache;
+  private LoadingCache<String, Long> timestampCache;
 
   public SyslogParser() {
     timeParser = DateTimeFormat.forPattern(timePat).withZoneUTC();

diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.session.SessionState;
@@ -105,8 +104,8 @@ public TestHiveSink() throws Exception {
     TestUtil.setConfValues(conf);
 
     // 1) prepare hive
-    TxnDbUtil.cleanDb();
-    TxnDbUtil.prepDb();
+    TxnDbUtil.cleanDb(conf);
+    TxnDbUtil.prepDb(conf);
 
     // 2) Setup Hive client
     SessionState.start(new CliSessionState(conf));
@@ -283,7 +282,7 @@ public void testSingleWriterUseHeaders()
 
   @Test
   public void testHeartBeat()
-          throws EventDeliveryException, IOException, CommandNeedRetryException {
+          throws EventDeliveryException, IOException {
     int batchSize = 2;
     int batchCount = 3;
     int totalRecords = batchCount * batchSize;
@@ -407,7 +406,7 @@ private static Channel startSink(HiveSink sink, Context context, Channel pChanne
   }
 
   private void checkRecordCountInTable(int expectedCount, String db, String tbl)
-          throws CommandNeedRetryException, IOException {
+          throws IOException {
     int count = TestUtil.listRecordsInTable(driver, db, tbl).size();
     Assert.assertEquals(expectedCount, count);
   }

diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java
@@ -26,8 +26,8 @@
 import org.apache.flume.instrumentation.SinkCounter;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hive.hcatalog.streaming.HiveEndPoint;
@@ -99,8 +99,8 @@ public TestHiveWriter() throws Exception {
   @Before
   public void setUp() throws Exception {
     // 1) prepare hive
-    TxnDbUtil.cleanDb();
-    TxnDbUtil.prepDb();
+    TxnDbUtil.cleanDb(conf);
+    TxnDbUtil.prepDb(conf);
 
     // 1) Setup tables
     TestUtil.dropDB(conf, dbName);
@@ -207,7 +207,7 @@ public void testTxnBatchConsumption() throws Exception {
   }
 
   private void checkRecordCountInTable(int expectedCount)
-          throws CommandNeedRetryException, IOException {
+          throws IOException {
     int count = TestUtil.listRecordsInTable(driver, dbName, tblName).size();
     Assert.assertEquals(expectedCount, count);
   }

diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestUtil.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestUtil.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.shims.ShimLoader;
@@ -58,6 +57,17 @@ public static void setConfValues(HiveConf conf) {
     conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, txnMgr);
     conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
     conf.set("fs.raw.impl", RawFileSystem.class.getName());
+    try{
+      conf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false );
+      conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:;databaseName=metastore_db;create=true");
+      conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "org.apache.derby.jdbc.EmbeddedDriver");
+      conf.setBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, true);
+      conf.setIntVar(HiveConf.ConfVars.METASTORE_SERVER_PORT, 0);
+      conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("java.io.tmpdir"));
+    }catch (Throwable t){
+      t.printStackTrace();
+    }
+
   }
 
   public static void createDbAndTable(Driver driver, String databaseName,
@@ -148,7 +158,7 @@ private static String getTablePartsStr2(String[] partNames, List<String> partVal
   }
 
   public static ArrayList<String> listRecordsInTable(Driver driver, String dbName, String tblName)
-      throws CommandNeedRetryException, IOException {
+      throws  IOException {
     driver.run("select * from " + dbName + "." + tblName);
     ArrayList<String> res = new ArrayList<String>();
     driver.getResults(res);
@@ -158,7 +168,7 @@ public static ArrayList<String> listRecordsInTable(Driver driver, String dbName,
   public static ArrayList<String> listRecordsInPartition(Driver driver, String dbName,
                                                          String tblName, String continent,
                                                          String country)
-      throws CommandNeedRetryException, IOException {
+      throws IOException {
     driver.run("select * from " + dbName + "." + tblName + " where continent='"
             + continent + "' and country='" + country + "'");
     ArrayList<String> res = new ArrayList<String>();
@@ -217,15 +227,8 @@ public FileStatus getFileStatus(Path path) throws IOException {
   private static boolean runDDL(Driver driver, String sql) throws QueryFailedException {
     int retryCount = 1; // # of times to retry if first attempt fails
     for (int attempt = 0; attempt <= retryCount; ++attempt) {
-      try {
         driver.run(sql);
-        return true;
-      } catch (CommandNeedRetryException e) {
-        if (attempt == retryCount) {
-          throw new QueryFailedException(sql, e);
-        }
         continue;
-      }
     } // for
     return false;
   }