diff --git a/java/pom.xml b/java/pom.xml new file mode 100644 index 0000000..0e34eee --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,170 @@ + + 4.0.0 + com._4paradigm + benchmark + 0.0.1 + jar + + + + org.testng + testng + 6.9.4 + test + + + mysql + mysql-connector-java + 5.1.49 + + + org.voltdb + voltdbclient + 10.1.1 + + + com.4paradigm.hybridsql + fedb-jdbc + 2.2.2 + + + redis.clients + jedis + 3.6.0 + + + junit + junit + 4.10 + test + + + com.google.guava + guava + 19.0 + + + + org.slf4j + slf4j-api + 1.7.21 + + + + + joda-time + joda-time + 2.10 + + + + ch.qos.logback + logback-core + 1.1.2 + test + + + ch.qos.logback + logback-classic + 1.1.2 + test + + + slf4j-api + org.slf4j + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + UTF-8 + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.5.0 + +/rtidb/thirdparty/bin/protoc + + + + + compile + test-compile + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.0 + + once + -Dfile.encoding=UTF-8 + + + junit + false + + + 1 + + + + org.apache.maven.surefire + surefire-junit47 + 2.22.0 + + + org.apache.maven.surefire + surefire-testng + 2.22.0 + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4.1 + + + + jar-with-dependencies + + + + + com._4paradigm.benchmark.memory.FedbTest + + + + + + + make-assembly + + package + + single + + + + + + + + + + diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java new file mode 100644 index 0000000..2924110 --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/FedbTest.java @@ -0,0 +1,94 @@ +package com._4paradigm.benchmark.memory; + +import com._4paradigm.hybridsql.fedb.sdk.SqlExecutor; +import com._4paradigm.hybridsql.fedb.sdk.SdkOption; +import com._4paradigm.hybridsql.fedb.sdk.impl.SqlClusterExecutor; +import com._4paradigm.hybridsql.fedb.SQLInsertRow; +import com._4paradigm.hybridsql.fedb.SQLInsertRows; + +import java.util.Properties; +import java.util.Random; + +public class FedbTest implements Test { + private static String zkCluster; + private static String zkRootPath; + private static String tableName; + private static String dbName; + private static int pkCnt; + private static int tsCnt; + private static String baseKey; + private static boolean needCreate; + + private SqlExecutor executor; + private String createDDL = "create table " + tableName + " (col1 string, col2 bigint, " + + "col3 float," + + "col4 double," + + "col5 string," + + "index(key=(col1),ts=col2)) partitionnum=4;"; + private String dropDDL = "drop table " + tableName + " ;"; + private String format = "insert into " + tableName + " values(?, ?, 100.0, 200.0, 'hello world');"; + private Random random = new Random(System.currentTimeMillis()); + + static { + try { + Properties prop = new Properties(); + prop.load(FedbTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + zkCluster = prop.getProperty("zk_cluster"); + zkRootPath = prop.getProperty("zk_root_path"); + tableName = prop.getProperty("table_name"); + dbName = prop.getProperty("db_name"); + baseKey = prop.getProperty("base_key"); + pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); + tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); + needCreate = Boolean.parseBoolean(prop.getProperty("need_create", "true")); + } catch (Exception e) { + e.printStackTrace(); + } + } + @Override + public boolean init() { + SdkOption sdkOption = new SdkOption(); + sdkOption.setSessionTimeout(30000); + sdkOption.setZkCluster(zkCluster); + sdkOption.setZkPath(zkRootPath); + sdkOption.setEnableDebug(false); + try { + executor = new SqlClusterExecutor(sdkOption); + } catch (Exception e) { + e.printStackTrace(); + return false; + } + /* if (!executor.createDB(dbName)) { + return false; + }*/ + if (needCreate) { + if (!executor.executeDDL(dbName, createDDL)) { + return false; + } + } + return true; + } + + @Override + public void put() { + while(true) { + SQLInsertRows rows = executor.getInsertRows(dbName, format); + int num = random.nextInt(pkCnt) + pkCnt; + String key = baseKey + String.valueOf(num); + for (int i = 0; i < tsCnt; i++) { + SQLInsertRow row = rows.NewRow(); + row.Init(key.length()); + row.AppendString(key); + row.AppendInt64(System.currentTimeMillis()); + row.delete(); + } + try { + executor.executeInsert(dbName, format, rows); + } catch (Exception e) { + e.printStackTrace(); + } finally { + rows.delete(); + } + } + } +} diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java new file mode 100644 index 0000000..1ec4414 --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/JDBCSqlTest.java @@ -0,0 +1,87 @@ +package com._4paradigm.benchmark.memory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.Statement; +import java.util.Properties; +import java.util.Random; + +public class JDBCSqlTest implements Test { + private static String connectURL; + private static String tableName; + private static String dbName; + private static int pkCnt; + private static int tsCnt; + private static String baseKey; + private static String method; + private static boolean needCreate; + private String createDDL = "create table " + tableName + " (col1 varchar(20), col2 bigint, " + + "col3 float," + + "col4 float," + + "col5 varchar(12)," + + "PRIMARY KEY (col1, col2));"; + private String sql = "insert into " + tableName + " values(?, ?, 100.0, 200.0, 'hello world');"; + private Random random = new Random(System.currentTimeMillis()); + static { + try { + Properties prop = new Properties(); + prop.load(JDBCSqlTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + connectURL = prop.getProperty("connect_url"); + tableName = prop.getProperty("table_name"); + dbName = prop.getProperty("db_name"); + baseKey = prop.getProperty("base_key"); + pkCnt = Integer.parseInt(prop.getProperty("pk_cnt", "1")); + tsCnt = Integer.parseInt(prop.getProperty("ts_cnt", "1")); + method = prop.getProperty("method"); + needCreate = Boolean.parseBoolean(prop.getProperty("need_create")); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public boolean init() { + try { + if (method.equals("voltdb")) { + Class.forName("org.voltdb.jdbc.Driver"); + } + if (needCreate) { + Connection cnn = DriverManager.getConnection(connectURL); + Statement st = cnn.createStatement(); + st.execute(createDDL); + st.close(); + cnn.close(); + } + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public void put() { + Connection cnn = null; + try { + cnn = DriverManager.getConnection(connectURL); + } catch (Exception e) { + e.printStackTrace(); + return; + } + while(true) { + int num = random.nextInt(pkCnt) + pkCnt; + String key = baseKey + String.valueOf(num); + long ts = System.currentTimeMillis(); + for (int i = 0; i < tsCnt; i++) { + try { + PreparedStatement st = cnn.prepareStatement(sql); + st.setString(1, key); + st.setLong(2, ts - i); + st.executeUpdate(); + st.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java new file mode 100644 index 0000000..018f925 --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/MemoryBenchmark.java @@ -0,0 +1,49 @@ +package com._4paradigm.benchmark.memory; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +public class MemoryBenchmark { + private ExecutorService executorService; + private int threadNum; + private String method; + public MemoryBenchmark() { + try { + Properties prop = new Properties(); + prop.load(MemoryBenchmark.class.getClassLoader().getResourceAsStream("benchmark.properties")); + threadNum = Integer.parseInt(prop.getProperty("thread_num", "1")); + method = prop.getProperty("method"); + } catch (Exception e) { + e.printStackTrace(); + } + executorService = Executors.newFixedThreadPool(threadNum); + } + public void run() { + Test tmp = null; + if (method.equals("rtidb")) { + tmp = new FedbTest(); + } else if (method.equals("redis")) { + tmp = new RedisTest(); + } else { + tmp = new JDBCSqlTest(); + } + if (!tmp.init()) { + System.out.println("init failed"); + return; + } + final Test test = tmp; + for (int i = 0; i < threadNum; i++) { + System.out.println("create thread " + String.valueOf(i)); + executorService.submit(new Runnable() { + @Override + public void run() { + test.put(); + } + }); + } + } + + public static void main(String[] args) { + MemoryBenchmark benchmark = new MemoryBenchmark(); + benchmark.run(); + } +} diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/RedisTest.java b/java/src/main/java/com/_4paradigm/benchmark/memory/RedisTest.java new file mode 100644 index 0000000..8a60858 --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/RedisTest.java @@ -0,0 +1,56 @@ +package com._4paradigm.benchmark.memory; +import redis.clients.jedis.*; + +import java.util.Properties; +import java.util.Random; +import java.util.Set; +import java.util.LinkedHashSet; +import java.util.concurrent.atomic.AtomicLong; + +public class RedisTest implements Test{ + private JedisPoolConfig poolConfig; + private JedisCluster cluster; + static private String endpoints; + + private static String baseKey; + private long base; + private AtomicLong deta = new AtomicLong(0); + private Random random = new Random(System.currentTimeMillis()); + static { + try { + Properties prop = new Properties(); + prop.load(FedbTest.class.getClassLoader().getResourceAsStream("benchmark.properties")); + endpoints = prop.getProperty("redis_addr"); + baseKey = prop.getProperty("base_key"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public boolean init() { + poolConfig = new JedisPoolConfig(); + poolConfig.setMaxTotal(128); + Set nodes = new LinkedHashSet(); + + for (String endpoint : endpoints.trim().split(",")) { + String[] arr = endpoint.split(":"); + if (arr.length != 2) { + continue; + } + System.out.println("add endpoint " + endpoint); + nodes.add(new HostAndPort(arr[0], Integer.valueOf(arr[1]))); + } + base = Long.valueOf(baseKey); + cluster = new JedisCluster(nodes, poolConfig); + return true; + } + + @Override + public void put() { + while (true) { + String key = baseKey + (base + deta.getAndIncrement()); + cluster.set(key, key); + } + } +} diff --git a/java/src/main/java/com/_4paradigm/benchmark/memory/Test.java b/java/src/main/java/com/_4paradigm/benchmark/memory/Test.java new file mode 100644 index 0000000..9b76a5d --- /dev/null +++ b/java/src/main/java/com/_4paradigm/benchmark/memory/Test.java @@ -0,0 +1,6 @@ +package com._4paradigm.benchmark.memory; + +public interface Test { + public abstract boolean init(); + public abstract void put(); +} diff --git a/java/src/main/resources/benchmark.properties b/java/src/main/resources/benchmark.properties new file mode 100644 index 0000000..5438953 --- /dev/null +++ b/java/src/main/resources/benchmark.properties @@ -0,0 +1,13 @@ +db_name=memory +table_name=perf +#connect_url=jdbc:mysql://172.24.4.40:3306/memory?user=root&password=root +connect_url=jdbc:voltdb://172.24.4.55:21212 +thread_num=2 +zk_cluster=172.24.4.55:12200 +zk_root_path=/rtidb_scan +redis_addr=172.24.4.40:6001,172.24.4.40:6002,172.24.4.40:6003 +base_key=10000000000 +pk_cnt=1000000 +ts_cnt=10 +need_create=false +method=redis \ No newline at end of file diff --git a/java/src/main/resources/log4j.properties b/java/src/main/resources/log4j.properties new file mode 100644 index 0000000..d9bb827 --- /dev/null +++ b/java/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout, file + +# Redirect log messages to console +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.ayout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file