diff --git a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java index 9ccebc8e60..23c2d7e8d8 100644 --- a/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java +++ b/flume-ng-clients/flume-ng-log4jappender/src/test/java/org/apache/flume/clients/log4jappender/TestLog4jAppenderWithAvro.java @@ -71,8 +71,7 @@ private static int getFreePort() throws Exception { @Before public void setUp() throws Exception { URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc"); - Files.copy(Resources.newInputStreamSupplier(schemaUrl), - new File("/tmp/myrecord.avsc")); + Resources.asByteSource(schemaUrl).copyTo(Files.asByteSink(new File("/tmp/myrecord.avsc"))); port = getFreePort(); source = new AvroSource(); diff --git a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java index 9ad306b464..f1db1553da 100644 --- a/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java +++ b/flume-ng-core/src/main/java/org/apache/flume/source/SyslogParser.java @@ -23,7 +23,7 @@ package org.apache.flume.source; import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; +import com.google.common.cache.LoadingCache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.collect.Maps; @@ -59,7 +59,7 @@ public class SyslogParser { private static final int RFC5424_PREFIX_LEN = 19; private final DateTimeFormatter timeParser; - private Cache timestampCache; + private LoadingCache timestampCache; public SyslogParser() { timeParser = DateTimeFormat.forPattern(timePat).withZoneUTC(); diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java index fbb2de2c87..d07ef4becd 100644 --- a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java +++ b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveSink.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; @@ -105,8 +104,8 @@ public TestHiveSink() throws Exception { TestUtil.setConfValues(conf); // 1) prepare hive - TxnDbUtil.cleanDb(); - TxnDbUtil.prepDb(); + TxnDbUtil.cleanDb(conf); + TxnDbUtil.prepDb(conf); // 2) Setup Hive client SessionState.start(new CliSessionState(conf)); @@ -283,7 +282,7 @@ public void testSingleWriterUseHeaders() @Test public void testHeartBeat() - throws EventDeliveryException, IOException, CommandNeedRetryException { + throws EventDeliveryException, IOException { int batchSize = 2; int batchCount = 3; int totalRecords = batchCount * batchSize; @@ -407,7 +406,7 @@ private static Channel startSink(HiveSink sink, Context context, Channel pChanne } private void checkRecordCountInTable(int expectedCount, String db, String tbl) - throws CommandNeedRetryException, IOException { + throws IOException { int count = TestUtil.listRecordsInTable(driver, db, tbl).size(); Assert.assertEquals(expectedCount, count); } diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java index 4d7c9bb8ca..42b1fb3612 100644 --- a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java +++ b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestHiveWriter.java @@ -26,8 +26,8 @@ import org.apache.flume.instrumentation.SinkCounter; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.streaming.HiveEndPoint; @@ -99,8 +99,8 @@ public TestHiveWriter() throws Exception { @Before public void setUp() throws Exception { // 1) prepare hive - TxnDbUtil.cleanDb(); - TxnDbUtil.prepDb(); + TxnDbUtil.cleanDb(conf); + TxnDbUtil.prepDb(conf); // 1) Setup tables TestUtil.dropDB(conf, dbName); @@ -207,7 +207,7 @@ public void testTxnBatchConsumption() throws Exception { } private void checkRecordCountInTable(int expectedCount) - throws CommandNeedRetryException, IOException { + throws IOException { int count = TestUtil.listRecordsInTable(driver, dbName, tblName).size(); Assert.assertEquals(expectedCount, count); } diff --git a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestUtil.java b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestUtil.java index 1fcb4eb46f..a3a8a2042d 100644 --- a/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestUtil.java +++ b/flume-ng-sinks/flume-hive-sink/src/test/java/org/apache/flume/sink/hive/TestUtil.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.shims.ShimLoader; @@ -58,6 +57,17 @@ public static void setConfValues(HiveConf conf) { conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, txnMgr); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true); conf.set("fs.raw.impl", RawFileSystem.class.getName()); + try{ + conf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false ); + conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:;databaseName=metastore_db;create=true"); + conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "org.apache.derby.jdbc.EmbeddedDriver"); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL, true); + conf.setIntVar(HiveConf.ConfVars.METASTORE_SERVER_PORT, 0); + conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("java.io.tmpdir")); + }catch (Throwable t){ + t.printStackTrace(); + } + } public static void createDbAndTable(Driver driver, String databaseName, @@ -148,7 +158,7 @@ private static String getTablePartsStr2(String[] partNames, List partVal } public static ArrayList listRecordsInTable(Driver driver, String dbName, String tblName) - throws CommandNeedRetryException, IOException { + throws IOException { driver.run("select * from " + dbName + "." + tblName); ArrayList res = new ArrayList(); driver.getResults(res); @@ -158,7 +168,7 @@ public static ArrayList listRecordsInTable(Driver driver, String dbName, public static ArrayList listRecordsInPartition(Driver driver, String dbName, String tblName, String continent, String country) - throws CommandNeedRetryException, IOException { + throws IOException { driver.run("select * from " + dbName + "." + tblName + " where continent='" + continent + "' and country='" + country + "'"); ArrayList res = new ArrayList(); @@ -217,15 +227,8 @@ public FileStatus getFileStatus(Path path) throws IOException { private static boolean runDDL(Driver driver, String sql) throws QueryFailedException { int retryCount = 1; // # of times to retry if first attempt fails for (int attempt = 0; attempt <= retryCount; ++attempt) { - try { driver.run(sql); - return true; - } catch (CommandNeedRetryException e) { - if (attempt == retryCount) { - throw new QueryFailedException(sql, e); - } continue; - } } // for return false; }