diff --git a/docs/interactive_engine/benchmark_tool.md b/docs/interactive_engine/benchmark_tool.md index c077f9491896..c8daf1f6f311 100644 --- a/docs/interactive_engine/benchmark_tool.md +++ b/docs/interactive_engine/benchmark_tool.md @@ -25,7 +25,7 @@ The program uses a round-robin strategy to iterate all the **enabled** queries w - bin - bench.sh // script for running benchmark for queries - collect.sh // script for collecting benchmark results -- config +- config - interactive-benchmark.properties // configurations for running benchmark - data - substitution_parameters // query parameter files using to fill the query templates @@ -43,7 +43,7 @@ and the queries with the prefix _job_ are the implementation of JOB Benchmark. The gremlin queries should be with suffix _.gremlin_, and cypher queries should be with suffix _.cypher_. The corresponding parameters (factor 1) for LDBC queries are generated by [LDBC official tools](http://github.com/ldbc/ldbc_snb_datagen). -### Building +### Building the benchmark Build benchmark program using Maven: @@ -56,8 +56,13 @@ and you can use deploy the package to anywhere could connect to the gremlin endp ### Running the benchmark +You can unzip builded _target/benchmark-0.0.1-SNAPSHOT-dist.tar.gz_, and run the benchmark. + ```bash -./bin/bench.sh # run the benchmark program with the provided properties +cd target +tar -xvf gaia-benchmark-0.0.1-SNAPSHOT-dist.tar.gz +cd gaia-benchmark-0.0.1-SNAPSHOT +./bin/bench.sh # run the benchmark program. You can also modify running configurations in config/interactive-benchmark.properties ``` With the example configuration file ``example/job_benchmark.properties``, which compares GraphScope-GIE and KuzuDB while executing the JOB Benchmark, the example of results are as follows: @@ -88,7 +93,7 @@ System: KuzuDB; query count: 35; execute time(ms): xxx qps: xxx ./bin/collect.sh # run the result collection program to collect the results and generate a performance comparison table ``` -Based on the benchmark results, the collected data and the final performance comparison table are as follows: +Furthermore, based on the benchmark results, the collected data and the final performance comparison table are as follows: | QueryName | GIE Avg | GIE P50 | GIE P90 | GIE P95 | GIE P99 | GIE Count | KuzuDb Avg | KuzuDb P50 | KuzuDb P90 | KuzuDb P95 | KuzuDb P99 | KuzuDb Count | diff --git a/interactive_engine/benchmark/README.md b/interactive_engine/benchmark/README.md index 853c6c26dec7..4df817b5d0f3 100644 --- a/interactive_engine/benchmark/README.md +++ b/interactive_engine/benchmark/README.md @@ -38,7 +38,10 @@ and you can use deploy the package to anywhere could connect to the endpoint (wh ### Running the benchmark ```bash -./bin/bench.sh # run the benchmark program with the provided properties +cd target +tar -xvf gaia-benchmark-0.0.1-SNAPSHOT-dist.tar.gz +cd gaia-benchmark-0.0.1-SNAPSHOT +./bin/bench.sh # run the benchmark program. You can also modify running configurations in config/interactive-benchmark.properties ``` With the example configuration file ``example/job_benchmark.properties``, which compares GraphScope-GIE and KuzuDB while executing the JOB Benchmark, the results are as follows: ``` @@ -66,7 +69,7 @@ System: KuzuDB; query count: 35; execute time(ms): xxx qps: xxx ```bash ./bin/collect.sh # run the result collection program to collect the results and generate a performance comparison table ``` -Based on the benchmark results, the collected data and the final performance comparison table are as follows: +Furthermore, based on the benchmark results, the collected data and the final performance comparison table are as follows: And the comparison result after collection is as follows: | QueryName | GIE Avg | GIE P50 | GIE P90 | GIE P95 | GIE P99 | GIE Count | KuzuDb Avg | KuzuDb P50 | KuzuDb P90 | KuzuDb P95 | KuzuDb P99 | KuzuDb Count | diff --git a/interactive_engine/benchmark/bin/bench.sh b/interactive_engine/benchmark/bin/bench.sh index 959a339905f4..ecf2ec022af6 100755 --- a/interactive_engine/benchmark/bin/bench.sh +++ b/interactive_engine/benchmark/bin/bench.sh @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -CURR_DIR=$(pwd) +SCRIPT_DIR=$(dirname $(readlink -f $0)) +CURR_DIR=${SCRIPT_DIR}/../ CONF_DIR=$CURR_DIR/config/interactive-benchmark.properties java -cp $CURR_DIR:lib/* com.alibaba.graphscope.gaia.benchmark.InteractiveBenchmark $CONF_DIR cd $CURR_DIR \ No newline at end of file diff --git a/interactive_engine/benchmark/bin/collect.sh b/interactive_engine/benchmark/bin/collect.sh index a393290f811d..6b5d41381b28 100755 --- a/interactive_engine/benchmark/bin/collect.sh +++ b/interactive_engine/benchmark/bin/collect.sh @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -CURR_DIR=$(pwd) +SCRIPT_DIR=$(dirname $(readlink -f $0)) +CURR_DIR=${SCRIPT_DIR}/../ CONF_DIR=$CURR_DIR/config/interactive-benchmark.properties java -cp $CURR_DIR:lib/* com.alibaba.graphscope.gaia.benchmark.CollectResult $CONF_DIR cd $CURR_DIR \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_11.cypher b/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_11.cypher index 30f176e6959e..0bad88558760 100644 --- a/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_11.cypher +++ b/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_11.cypher @@ -1,6 +1,6 @@ -MATCH (person:PERSON {id: $personId})-[:KNOWS*1..3]-(friend:PERSON)-[workAt:WORKAT]->(company:COMPANY)-[:ISLOCATEDIN]->(:COUNTRY {name: '$countryName'}) +MATCH (person:PERSON {id: $personId})-[:KNOWS*1..3]-(friend:PERSON)-[workAt:WORKAT]->(company:ORGANISATION)-[:ISLOCATEDIN]->(:PLACE {name: '$countryName'}) WHERE person <> friend - and workAt.workFrom < $workFromYear + and workAt.workFrom < $workFromYear RETURN DISTINCT friend.id AS personId, friend.firstName AS personFirstName, diff --git a/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_3.cypher b/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_3.cypher index ed3a181f6600..57d3f6876491 100644 --- a/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_3.cypher +++ b/interactive_engine/benchmark/queries/cypher_queries/ic/gie/ldbc_query_3.cypher @@ -1,5 +1,5 @@ -MATCH (countryX:COUNTRY {name: '$countryXName'})<-[:ISLOCATEDIN]-(messageX)-[:HASCREATOR]->(otherP:PERSON), - (countryY:COUNTRY {name: '$countryYName'})<-[:ISLOCATEDIN]-(messageY)-[:HASCREATOR]->(otherP:PERSON), +MATCH (countryX:PLACE {name: '$countryXName'})<-[:ISLOCATEDIN]-(messageX)-[:HASCREATOR]->(otherP:PERSON), + (countryY:PLACE {name: '$countryYName'})<-[:ISLOCATEDIN]-(messageY)-[:HASCREATOR]->(otherP:PERSON), (otherP)-[:ISLOCATEDIN]->(city)-[:ISPARTOF]->(countryCity), (person:PERSON {id:$personId})-[:KNOWS*1..3]-(otherP) WHERE messageX.creationDate >= $startDate and messageX.creationDate < $endDate diff --git a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/clients/kuzu/KuzuGraphClient.java b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/clients/kuzu/KuzuGraphClient.java index 7a24739c2da8..d2779cb7f389 100644 --- a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/clients/kuzu/KuzuGraphClient.java +++ b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/clients/kuzu/KuzuGraphClient.java @@ -38,6 +38,14 @@ public KuzuGraphClient(String dbPath) { } } + public void setMaxNumThreadForExec(long parallelism) { + try { + conn.setMaxNumThreadForExec(parallelism); + } catch (Exception e) { + e.printStackTrace(); + } + } + @Override public GraphResultSet submit(String query) { try { diff --git a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/AbstractLdbcWithSubQuery.java b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/AbstractLdbcWithSubQuery.java index 207739df77f4..e95e6dba083a 100644 --- a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/AbstractLdbcWithSubQuery.java +++ b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/AbstractLdbcWithSubQuery.java @@ -21,11 +21,15 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.tinkerpop.gremlin.driver.Result; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; public abstract class AbstractLdbcWithSubQuery extends CommonQuery { + private static Logger logger = LoggerFactory.getLogger(AbstractLdbcWithSubQuery.class); + public AbstractLdbcWithSubQuery(String queryName, String queryFile, String parameterFile) throws Exception { super(queryName, queryFile, parameterFile); @@ -63,13 +67,13 @@ public void processGraphQuery( if (printResult) { printInfo = String.format("%s Result: { %s }", printInfo, resultStr); } - System.out.println(printInfo); + logger.info(printInfo); } if (!comparator.isEmpty()) { comparator.compareResults(queryName, result.getRight()); } } catch (Exception e) { - System.out.println( + logger.error( String.format( "Timeout or failed: QueryName[%s], Parameter[%s].", queryName, singleParameter.toString())); diff --git a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/BenchmarkSystemUtil.java b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/BenchmarkSystemUtil.java index 5f5b4b6db21f..f17a6673f68f 100644 --- a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/BenchmarkSystemUtil.java +++ b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/BenchmarkSystemUtil.java @@ -63,6 +63,9 @@ public static List initSystems(Configuration configuration) throws case "kuzu": if (path.isPresent()) { client = new KuzuGraphClient(path.get()); + long parallelism = + configuration.getInt("system." + systemCount + ".parallelism", 1); + ((KuzuGraphClient) client).setMaxNumThreadForExec(parallelism); } else { throw new IllegalArgumentException( "Kuzu client must have path to access the database"); diff --git a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/QueryUtil.java b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/QueryUtil.java index f53eb622f629..c882871cc9cb 100644 --- a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/QueryUtil.java +++ b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/QueryUtil.java @@ -80,7 +80,7 @@ public static List initQueryList(Configuration configuration) throw String queryDir = configuration.getString(Configuration.QUERY_DIR); String parameterDir = configuration.getString(Configuration.QUERY_PARAMETER_DIR, null); List queryList = new ArrayList<>(); - String suffix = "." + configuration.getString(Configuration.QUERY_FILE_SUFFIX); + String suffix = configuration.getString(Configuration.QUERY_FILE_SUFFIX); if (configuration.getBoolean(Configuration.QUERY_ALL_ENABLE, false)) { // Automatically add all queries in the specified directory @@ -89,7 +89,7 @@ public static List initQueryList(Configuration configuration) throw for (File file : dir.listFiles()) { if (file.isFile() && file.getName().endsWith(suffix)) { // assume the query name is the file name without the suffix - String queryName = file.getName().replace(suffix, ""); + String queryName = file.getName().replace("." + suffix, ""); String queryFilePath = file.getAbsolutePath(); String parameterFilePath = null; if (parameterDir != null && !parameterDir.isEmpty()) { diff --git a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/ResultComparator.java b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/ResultComparator.java index 69229dbaf0dc..e5a1b39ca349 100644 --- a/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/ResultComparator.java +++ b/interactive_engine/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/ResultComparator.java @@ -18,6 +18,9 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.Reader; import java.lang.reflect.Type; import java.nio.file.Files; @@ -26,6 +29,7 @@ public class ResultComparator { private Map expectedResults; + private static Logger logger = LoggerFactory.getLogger(ResultComparator.class); public ResultComparator(String expectedResultsPath) { if (expectedResultsPath != null && !expectedResultsPath.isEmpty()) { @@ -47,15 +51,14 @@ public void compareResults(String queryName, String actualResult) { String expectedResult = expectedResults.getOrDefault(queryName, "").trim(); if (!expectedResult.isEmpty()) { if (normalizeString(expectedResult).equals(normalizeString(actualResult))) { - System.out.println(queryName + ": Query result matches the expected result."); + logger.info(queryName + ": Query result matches the expected result."); } else { - System.err.println( - queryName + ": Query result does not match the expected result."); - System.err.println("Expected: " + expectedResult); - System.err.println("Actual : " + actualResult); + logger.error(queryName + ": Query result does not match the expected result."); + logger.error("Expected: " + expectedResult); + logger.error("Actual : " + actualResult); } } else { - System.err.println(queryName + ": No expected result found for comparison."); + logger.error(queryName + ": No expected result found for comparison."); } }