SUBMARINE-1261. Put configurations into configmap and remove submarin…

…e-site

dev-support/mini-submarine/conf/bootstrap.sh

@@ -36,15 +36,15 @@ for f in slaves core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml contai
   fi
 done
 
-# Copy submarine config
-for f in slaves submarine-site.xml submarine-env.sh; do
-  if [[ -e ${CONFIG_DIR}/$f ]]; then
-    cp ${CONFIG_DIR}/$f /opt/submarine-current/conf/$f
-  else
-    echo "ERROR: Could not find $f in $CONFIG_DIR"
-    exit 1
-  fi
-done
+# # Copy submarine config
+# for f in slaves submarine-site.xml submarine-env.sh; do
+#   if [[ -e ${CONFIG_DIR}/$f ]]; then
+#     cp ${CONFIG_DIR}/$f /opt/submarine-current/conf/$f
+#   else
+#     echo "ERROR: Could not find $f in $CONFIG_DIR"
+#     exit 1
+#   fi
+# done
 
 # create cgroups
 mkdir -p /sys/fs/cgroup/cpu/hadoop-yarn

submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java

@@ -77,46 +77,63 @@ public enum ConfVars {
     FLOAT_TEST("float.test", -123.456f),
     SUBMARINE_CONF_DIR("submarine.conf.dir", "conf"),
     SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB(
-        "submarine.localization.max-allowed-file-size-mb", 2048L),
-    SUBMARINE_SERVER_ADDR("submarine.server.addr", "0.0.0.0"),
-    SUBMARINE_SERVER_PORT("submarine.server.port", 8080),
-    SUBMARINE_SERVER_SSL("submarine.server.ssl", false),
-    SUBMARINE_SERVER_SSL_PORT("submarine.server.ssl.port", 8443),
-    SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX("submarine.server.jetty.thread.pool.max", 400),
-    SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN("submarine.server.jetty.thread.pool.min", 8),
-    SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT("submarine.server.jetty.thread.pool.timeout", 30),
-    SUBMARINE_SERVER_JETTY_REQUEST_HEADER_SIZE("submarine.server.jetty.request.header.size", 8192),
-    SUBMARINE_SERVER_SSL_CLIENT_AUTH("submarine.server.ssl.client.auth", false),
-    SUBMARINE_SERVER_SSL_KEYSTORE_PATH("submarine.server.ssl.keystore.path", "keystore"),
-    SUBMARINE_SERVER_SSL_KEYSTORE_TYPE("submarine.server.ssl.keystore.type", "JKS"),
-    SUBMARINE_SERVER_SSL_KEYSTORE_PASSWORD("submarine.server.ssl.keystore.password", ""),
-    SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD("submarine.server.ssl.key.manager.password", null),
-    SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH("submarine.server.ssl.truststore.path", null),
-    SUBMARINE_SERVER_SSL_TRUSTSTORE_TYPE("submarine.server.ssl.truststore.type", null),
-    SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD("submarine.server.ssl.truststore.password", null),
-    SUBMARINE_CLUSTER_ADDR("submarine.cluster.addr", ""),
+        "submarine.localization.max-allowed-file-size-mb",
+        Long.parseLong(System.getenv("SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB"))),
+    SUBMARINE_SERVER_ADDR("submarine.server.addr", System.getenv("SUBMARINE_SERVER_ADDR")),
+    SUBMARINE_SERVER_PORT("submarine.server.port", Integer.valueOf(System.getenv("SUBMARINE_SERVER_PORT"))),
+    SUBMARINE_SERVER_SSL("submarine.server.ssl", Boolean.parseBoolean(System.getenv("SUBMARINE_SERVER_SSL"))),
+    SUBMARINE_SERVER_SSL_PORT("submarine.server.ssl.port",
+        Integer.valueOf(System.getenv("SUBMARINE_SERVER_SSL_PORT"))),
+    SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX("submarine.server.jetty.thread.pool.max",
+        Integer.valueOf(System.getenv("SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX"))),
+    SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN("submarine.server.jetty.thread.pool.min",
+        Integer.valueOf(System.getenv("SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN"))),
+    SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT("submarine.server.jetty.thread.pool.timeout",
+        Integer.valueOf(System.getenv("SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT"))),
+    SUBMARINE_SERVER_JETTY_REQUEST_HEADER_SIZE("submarine.server.jetty.request.header.size",
+        Integer.valueOf(System.getenv("SUBMARINE_SERVER_JETTY_REQUEST_HEADER_SIZE"))),
+    SUBMARINE_SERVER_SSL_CLIENT_AUTH("submarine.server.ssl.client.auth",
+        Boolean.parseBoolean(System.getenv("SUBMARINE_SERVER_SSL_CLIENT_AUTH"))),
+    SUBMARINE_SERVER_SSL_KEYSTORE_PATH("submarine.server.ssl.keystore.path",
+        System.getenv("SUBMARINE_SERVER_SSL_KEYSTORE_PATH")),
+    SUBMARINE_SERVER_SSL_KEYSTORE_TYPE("submarine.server.ssl.keystore.type",
+        System.getenv("SUBMARINE_SERVER_SSL_KEYSTORE_TYPE")),
+    SUBMARINE_SERVER_SSL_KEYSTORE_PASSWORD("submarine.server.ssl.keystore.password",
+        System.getenv("SUBMARINE_SERVER_SSL_KEYSTORE_PASSWORD")),
+    SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD("submarine.server.ssl.key.manager.password",
+        System.getenv("SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD") == "" ? null :
+        System.getenv("SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD")),
+    SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH("submarine.server.ssl.truststore.path",
+        System.getenv("SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH") == "" ? null :
+        System.getenv("SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH")),
+    SUBMARINE_SERVER_SSL_TRUSTSTORE_TYPE("submarine.server.ssl.truststore.type",
+        System.getenv("SUBMARINE_SERVER_SSL_TRUSTST_TYPE") == "" ? null :
+        System.getenv("SUBMARINE_SERVER_SSL_TRUSTST_TYPE")),
+    SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD("submarine.server.ssl.truststore.password",
+        System.getenv("SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD") == "" ? null :
+        System.getenv("SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD")),
+    SUBMARINE_CLUSTER_ADDR("submarine.cluster.addr",
+        System.getenv("SUBMARINE_CLUSTER_ADDR")),
     SUBMARINE_SERVER_RPC_ENABLED(
-        "submarine.server.rpc.enabled", false),
+        "submarine.server.rpc.enabled",
+        Boolean.parseBoolean(System.getenv("SUBMARINE_SERVER_RPC_ENABLED"))),
     SUBMARINE_SERVER_RPC_PORT(
         "submarine.server.rpc.port", 8980),
-    CLUSTER_HEARTBEAT_INTERVAL("cluster.heartbeat.interval", 3000),
-    CLUSTER_HEARTBEAT_TIMEOUT("cluster.heartbeat.timeout", 9000),
+    CLUSTER_HEARTBEAT_INTERVAL("cluster.heartbeat.interval",
+        Integer.valueOf(System.getenv("CLUSTER_HEARTBEAT_INTERVAL"))),
+    CLUSTER_HEARTBEAT_TIMEOUT("cluster.heartbeat.timeout",
+        Integer.valueOf(System.getenv("CLUSTER_HEARTBEAT_TIMEOUT"))),
 
-    JDBC_DRIVERCLASSNAME("jdbc.driverClassName", "com.mysql.jdbc.Driver"),
-    JDBC_URL("jdbc.url", "jdbc:mysql://127.0.0.1:3306/submarine" +
-        "?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&allowMultiQueries=true&" +
-        "failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false&" +
-        // use timezone for dateformat, current default database timezone is utc
-        "serverTimezone=UTC&useTimezone=true&useLegacyDatetimeCode=true"),
-    JDBC_USERNAME("jdbc.username", "submarine"),
-    JDBC_PASSWORD("jdbc.password", "password"),
-    METASTORE_JDBC_URL("metastore.jdbc.url", "jdbc:mysql://127.0.0.1:3306/metastore" +
-        "?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" +
-        "failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"),
-    METASTORE_JDBC_USERNAME("metastore.jdbc.username", "metastore"),
-    METASTORE_JDBC_PASSWORD("metastore.jdbc.password", "password"),
+    JDBC_DRIVERCLASSNAME("jdbc.driverClassName", System.getenv("JDBC_DRIVERCLASSNAME")),
+    JDBC_URL("jdbc.url", System.getenv("JDBC_URL")),
+    JDBC_USERNAME("jdbc.username", System.getenv("JDBC_USERNAME")),
+    JDBC_PASSWORD("jdbc.password", System.getenv("JDBC_PASSWORD")),
+    METASTORE_JDBC_URL("metastore.jdbc.url", System.getenv("METASTORE_JDBC_URL")),
+    METASTORE_JDBC_USERNAME("metastore.jdbc.username", System.getenv("METASTORE_JDBC_USERNAME")),
+    METASTORE_JDBC_PASSWORD("metastore.jdbc.password", System.getenv("METASTORE_JDBC_PASSWORD")),
 
-    SUBMARINE_NOTEBOOK_DEFAULT_OVERWRITE_JSON("submarine.notebook.default.overwrite_json", ""),
+    SUBMARINE_NOTEBOOK_DEFAULT_OVERWRITE_JSON("submarine.notebook.default.overwrite_json",
+        System.getenv("SUBMARINE_NOTEBOOK_DEFAULT_OVERWRITE_JSON")),
 
     WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE(
         "workbench.websocket.max.text.message.size", "1024000"),

submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java

@@ -19,17 +19,13 @@
 package org.apache.submarine.commons.utils;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.XMLConfiguration;
-import org.apache.commons.configuration.tree.ConfigurationNode;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.net.URL;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 public class SubmarineConfiguration extends XMLConfiguration {
@@ -42,26 +38,6 @@ public class SubmarineConfiguration extends XMLConfiguration {
 
   private Map<String, String> properties = new HashMap<>();
 
-  private SubmarineConfiguration(URL url) throws ConfigurationException {
-    setDelimiterParsingDisabled(true);
-    load(url);
-    initProperties();
-  }
-
-  private void initProperties() {
-    List<ConfigurationNode> nodes = getRootNode().getChildren();
-    if (nodes == null || nodes.isEmpty()) {
-      return;
-    }
-    for (ConfigurationNode p : nodes) {
-      String name = (String) p.getChildren("name").get(0).getValue();
-      String value = (String) p.getChildren("value").get(0).getValue();
-      if (!StringUtils.isEmpty(name)) {
-        properties.put(name, value);
-      }
-    }
-  }
-
   private SubmarineConfiguration() {
     SubmarineConfVars.ConfVars[] vars = SubmarineConfVars.ConfVars.values();
     for (SubmarineConfVars.ConfVars v : vars) {
@@ -129,10 +105,6 @@ public static SubmarineConfiguration getInstance() {
   //   return submarineConfig;
   // }
 
-  // Debug
-  public int getTestConfig() {
-    return getInt(SubmarineConfVars.ConfVars.TEST_CONFIG);
-  }
 
   public String getServerAddress() {
     return getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_ADDR);

submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java

@@ -80,9 +80,6 @@ public static long getServerTimeStamp() {
 
   public static void main(String[] args) throws InterruptedException {
     PropertyConfigurator.configure(ClassLoader.getSystemResource("log4j.properties"));
-
-    // Debug
-    LOG.info("Test config message:" + conf.getTestConfig());
 
     LOG.info("Submarine server Host: " + conf.getServerAddress());
     if (!conf.useSsl()) {