From bf7584889b31fb0853253e3202bc5dbf644c2e23 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Thu, 16 Jan 2025 17:21:21 +0100 Subject: [PATCH] fix UT failures and javadoc problems --- pom.xml | 5 +++++ .../java/org/apache/hadoop/hive/ql/DriverContext.java | 10 ++++++---- .../hadoop/hive/ql/exec/tez/TezRuntimeContext.java | 2 +- .../hive/ql/queryhistory/FromDriverContextUpdater.java | 3 +++ service/pom.xml | 4 ++++ .../hive/service/servlet/QueriesRESTfulAPIServlet.java | 4 ++++ .../apache/hive/service/server/TestHS2HttpServer.java | 1 + .../hive/service/server/TestHS2HttpServerLDAP.java | 3 +++ .../hive/service/server/TestHS2HttpServerPam.java | 3 +++ 9 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 0b275498bcc9..8319812d85f8 100644 --- a/pom.xml +++ b/pom.xml @@ -756,6 +756,11 @@ pom import + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + org.codehaus.jettison jettison diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java index ea501e772797..0fc9f21265d0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java @@ -287,13 +287,15 @@ public long getQueryStartTime() { * 1. general semantic exception * 2. transaction manager validation (throwin exception) * What a user expects here is something like "QUERY", "DDL", "DML", so this magic will do its best to tell. - * any kind of "analyze" ==> STATS - * DML operations (INSERT, UPDATE, DELETE, MERGE) ==> DML - * MAPRED ==> QUERY, DML (depending on QueryProperties achieved in compile time) - * FETCH ==> QUERY: a simple fetch task is a QUERY + * any kind of "analyze": STATS + * DML operations (INSERT, UPDATE, DELETE, MERGE): DML + * MAPRED: QUERY, DML (depending on QueryProperties achieved in compile time) + * FETCH: QUERY: a simple fetch task is a QUERY * UNKNOWN: if we can't determine the type of the query: * e.g. when ParseException happens, we won't do further magic, * even if it's obvious by reading the sql statement that user wanted to run e.g. a select query + * @param sem the semantic analyzer which already analyzed the query + * @param tree the root ASTNode of the query */ public void setQueryType(BaseSemanticAnalyzer sem, ASTNode tree) { List> rootTasks = sem.getAllRootTasks(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezRuntimeContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezRuntimeContext.java index d119f850a709..3c70a8b50588 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezRuntimeContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezRuntimeContext.java @@ -99,7 +99,7 @@ public void setMonitor(TezJobMonitor monitor) { } public long getCounter(String groupName, String counterName) { - CounterGroup group = getCounters().getGroup(groupName); + CounterGroup group = counters == null ? null : counters.getGroup(groupName); if (group == null) { return 0; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/queryhistory/FromDriverContextUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/queryhistory/FromDriverContextUpdater.java index ec9b26c12fe6..b8ae6cecbe0a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/queryhistory/FromDriverContextUpdater.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/queryhistory/FromDriverContextUpdater.java @@ -53,6 +53,9 @@ private void updateFromQueryState(QueryState queryState, QueryHistoryRecord reco } private void updateFromQueryInfo(QueryInfo queryInfo, QueryHistoryRecord record) { + if (queryInfo == null) { + return; + } // this state is the same as "state" displayed in the JSON returned by QueriesRESTfulAPIServlet record.setOperationId(queryInfo.getOperationId()); record.setExecutionEngine(queryInfo.getExecutionEngine()); diff --git a/service/pom.xml b/service/pom.xml index 307050618aec..bf258395b531 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -240,6 +240,10 @@ org.jamon jamon-runtime + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + org.apache.hive diff --git a/service/src/java/org/apache/hive/service/servlet/QueriesRESTfulAPIServlet.java b/service/src/java/org/apache/hive/service/servlet/QueriesRESTfulAPIServlet.java index 22c40b254e68..62089a539b9c 100644 --- a/service/src/java/org/apache/hive/service/servlet/QueriesRESTfulAPIServlet.java +++ b/service/src/java/org/apache/hive/service/servlet/QueriesRESTfulAPIServlet.java @@ -32,6 +32,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + import javax.servlet.ServletContext; import javax.servlet.ServletException; @@ -144,6 +146,8 @@ private void sendAsJson( SimpleModule module = new SimpleModule("CustomSessionModule", new Version(1, 0, 0, null, null, null)); module.addSerializer(HiveSession.class, new HiveSessionSerializer()); mapper.registerModule(module); + // support java.time.Instant from QueryInfo + mapper.registerModule(new JavaTimeModule()); try { PrintWriter out = response.getWriter(); diff --git a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java index 586db64bad0a..990464662f1d 100644 --- a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java +++ b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java @@ -78,6 +78,7 @@ public static void beforeTests() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); // query history adds no value to this test, it would just bring iceberg handler dependency, which isn't worth + // this should be handled with HiveConfForTests when it's used here too hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_QUERY_HISTORY_SERVICE_ENABLED, false); startHiveServer2WithConf(hiveConf); diff --git a/service/src/test/org/apache/hive/service/server/TestHS2HttpServerLDAP.java b/service/src/test/org/apache/hive/service/server/TestHS2HttpServerLDAP.java index 2eb09b1907d0..b911a082d204 100644 --- a/service/src/test/org/apache/hive/service/server/TestHS2HttpServerLDAP.java +++ b/service/src/test/org/apache/hive/service/server/TestHS2HttpServerLDAP.java @@ -70,6 +70,9 @@ public static void beforeTests() throws Exception { hiveConf.set(ConfVars.METASTORE_PWD.varname, METASTORE_PASSWD); hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + // query history adds no value to this test, it would just bring iceberg handler dependency, which isn't worth + // this should be handled with HiveConfForTests when it's used here too + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_QUERY_HISTORY_SERVICE_ENABLED, false); PasswdAuthenticationProvider authenticationProvider = new DummyLdapAuthenticationProviderImpl(); hiveServer2 = new HiveServer2(authenticationProvider); hiveServer2.init(hiveConf); diff --git a/service/src/test/org/apache/hive/service/server/TestHS2HttpServerPam.java b/service/src/test/org/apache/hive/service/server/TestHS2HttpServerPam.java index 92706af7033a..6f919edb7481 100644 --- a/service/src/test/org/apache/hive/service/server/TestHS2HttpServerPam.java +++ b/service/src/test/org/apache/hive/service/server/TestHS2HttpServerPam.java @@ -64,6 +64,9 @@ public static void beforeTests() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_PAM_SERVICES, "sshd"); hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_WEBUI_USE_PAM, true); hiveConf.setBoolVar(ConfVars.HIVE_IN_TEST, true); + // query history adds no value to this test, it would just bring iceberg handler dependency, which isn't worth + // this should be handled with HiveConfForTests when it's used here too + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_QUERY_HISTORY_SERVICE_ENABLED, false); hiveServer2 = new HiveServer2(new TestPamAuthenticator(hiveConf)); hiveServer2.init(hiveConf); hiveServer2.start();