refactor(interactive): Support shortestpath in Cypher Queries (#4295)

Co-authored-by: BingqingLyu <[email protected]>

docs/interactive_engine/neo4j/supported_cypher.md

@@ -115,6 +115,8 @@ Note that some Aggregator operators, such as `max()`, we listed here are impleme
 | User Defined Functions | get start node from an edge | startNode(edge) | gs.function.startNode(edge) | <input type="checkbox" disabled checked /> |   |
 | User Defined Functions | get end node from an edge | endNode(edge) | gs.function.endNode(edge) | <input type="checkbox" disabled checked /> |   |
 | User Defined Functions | convert integer value to datetime | datetime(1287230400000) | gs.function.datetime(1287230400000) | <input type="checkbox" disabled checked /> |   |
+| Path Modifier | get any shortest path between two endpoints | SHORTEST | SHORTESTPATH | <input type="checkbox" disabled checked /> |   |
+| Path Modifier | get all shortest paths between two endpoints | ALL SHORTEST | ALL SHORTESTPATH | <input type="checkbox" disabled checked /> |
 
 ## Clause
 A notable limitation for now is that we do not

interactive_engine/compiler/Makefile

@@ -53,7 +53,7 @@ gremlin_calcite_test:
 
 run:
 	cd $(CUR_DIR) && $(java) \
-	  -cp ".:./target/compiler-0.0.1-SNAPSHOT.jar:./target/libs/*" \
+	  -cp ".:./target/libs/*:./target/compiler-0.0.1-SNAPSHOT.jar" \
 	  -Djna.library.path=../executor/ir/target/release \
 	  -Dgraph.schema=${graph.schema} \
 	  -Dgraph.store=${graph.store} \

interactive_engine/compiler/src/main/antlr4/CypherGS.g4

@@ -76,12 +76,15 @@ oC_PatternPart
        ;
 
 oC_AnonymousPatternPart
-                    :  oC_PatternElement ;
+                    :  oC_ShortestPathOption? oC_PatternElement ;
 
 oC_PatternElement
-              :  ( oC_NodePattern ( SP? oC_PatternElementChain )* )
-                  | ( '(' oC_PatternElement ')' )
-                  ;
+              : ( oC_NodePattern ( SP? oC_PatternElementChain )* )
+              | ( '(' oC_PatternElement ')' )
+              ;
+
+oC_ShortestPathOption
+            : ( ALL SP? )? SHORTESTPATH ;
 
 oC_With
     :  WITH oC_ProjectionBody ( SP? oC_Where )? ;

interactive_engine/compiler/src/main/antlr4/ExprGS.g4

@@ -365,6 +365,10 @@ oC_SymbolicName
                 | oC_ReservedWord
                 ;
 
+ALL : ( 'A' | 'a' ) ( 'L' | 'l' ) ( 'L' | 'l' ) ;
+
+SHORTESTPATH : ( 'S' | 's' ) ( 'H' | 'h' ) ( 'O' | 'o' ) ( 'R' | 'r' ) ( 'T' | 't' ) ( 'E' | 'e' ) ( 'S' | 's' ) ( 'T' | 't' ) ( 'P' | 'p' ) ( 'A' | 'a' ) ( 'T' | 't' ) ( 'H' | 'h' );
+
 oC_ReservedWord
             : LABELS
             | ELEMENTID
@@ -382,6 +386,8 @@ oC_ReservedWord
             | MEAN
             | 'd' | 'D' | 'f' | 'F' | 'l' | 'L'
             | 'id' // lexer rule for ID conflicts with parser rule definition in gremlin grammar, include 'id' as reserved word so it can be used to denote a symbolic name
+            | ALL
+            | SHORTESTPATH
             ;
 
 UnescapedSymbolicName

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java

@@ -374,7 +374,8 @@ private PatternEdge visitAndAddPxdEdge(
                                             new PathExpandRange(offset, fetch),
                                             innerGetVTypes,
                                             pxd.getResultOpt(),
-                                            pxd.getPathOpt());
+                                            pxd.getPathOpt(),
+                                            expandEdge.getElementDetails().isOptional());
                             expandEdge =
                                     (expandEdge instanceof SinglePatternEdge)
                                             ? new SinglePatternEdge(

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/RexToProtoConverter.java

@@ -31,8 +31,11 @@
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.type.IntervalSqlType;
+import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.util.Sarg;
 
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -365,6 +368,88 @@ private OuterExpression.Expression visitUnaryOperator(RexCall call) {
                                 OuterExpression.ExprOpr.newBuilder()
                                         .setBrace(OuterExpression.ExprOpr.Brace.RIGHT_BRACE))
                         .build();
+                // if the operand of MINUS_PREFIX is a literal, we can convert it to a negative
+                // value
+            case MINUS_PREFIX:
+                if (operand.getKind() == SqlKind.LITERAL) {
+                    RexLiteral literal = (RexLiteral) operand;
+                    switch (literal.getType().getSqlTypeName()) {
+                        case INTEGER:
+                            BigInteger negative =
+                                    BigInteger.valueOf(literal.getValueAs(Number.class).intValue())
+                                            .negate();
+                            if (negative.compareTo(BigInteger.valueOf(Integer.MIN_VALUE)) >= 0
+                                    && negative.compareTo(BigInteger.valueOf(Integer.MAX_VALUE))
+                                            <= 0) {
+                                return OuterExpression.Expression.newBuilder()
+                                        .addOperators(
+                                                OuterExpression.ExprOpr.newBuilder()
+                                                        .setConst(
+                                                                Common.Value.newBuilder()
+                                                                        .setI32(
+                                                                                negative
+                                                                                        .intValue()))
+                                                        .setNodeType(
+                                                                Utils.protoIrDataType(
+                                                                        call.getType(),
+                                                                        isColumnId)))
+                                        .build();
+                            }
+                        case BIGINT:
+                            BigInteger negative2 =
+                                    BigInteger.valueOf(literal.getValueAs(Number.class).longValue())
+                                            .negate();
+                            if (negative2.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) >= 0
+                                    && negative2.compareTo(BigInteger.valueOf(Long.MAX_VALUE))
+                                            <= 0) {
+                                return OuterExpression.Expression.newBuilder()
+                                        .addOperators(
+                                                OuterExpression.ExprOpr.newBuilder()
+                                                        .setConst(
+                                                                Common.Value.newBuilder()
+                                                                        .setI64(
+                                                                                negative2
+                                                                                        .longValue()))
+                                                        .setNodeType(
+                                                                Utils.protoIrDataType(
+                                                                        rexBuilder
+                                                                                .getTypeFactory()
+                                                                                .createSqlType(
+                                                                                        SqlTypeName
+                                                                                                .BIGINT),
+                                                                        isColumnId)))
+                                        .build();
+                            } else {
+                                throw new IllegalArgumentException(
+                                        "negation of value ["
+                                                + negative2
+                                                + "] is out of range of BIGINT");
+                            }
+                        case FLOAT:
+                        case DOUBLE:
+                            BigDecimal negative3 =
+                                    BigDecimal.valueOf(
+                                                    literal.getValueAs(Number.class).doubleValue())
+                                            .negate();
+                            return OuterExpression.Expression.newBuilder()
+                                    .addOperators(
+                                            OuterExpression.ExprOpr.newBuilder()
+                                                    .setConst(
+                                                            Common.Value.newBuilder()
+                                                                    .setF64(
+                                                                            negative3
+                                                                                    .doubleValue()))
+                                                    .setNodeType(
+                                                            Utils.protoIrDataType(
+                                                                    rexBuilder
+                                                                            .getTypeFactory()
+                                                                            .createSqlType(
+                                                                                    SqlTypeName
+                                                                                            .DOUBLE),
+                                                                    isColumnId)))
+                                    .build();
+                    }
+                }
             case IS_NULL:
             case NOT:
             default:

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java

@@ -565,6 +565,10 @@ public static final GraphAlgebraPhysical.PathExpand.PathOpt protoPathOpt(
                 return GraphAlgebraPhysical.PathExpand.PathOpt.SIMPLE;
             case TRAIL:
                 return GraphAlgebraPhysical.PathExpand.PathOpt.TRAIL;
+            case ANY_SHORTEST:
+                return GraphAlgebraPhysical.PathExpand.PathOpt.ANY_SHORTEST;
+            case ALL_SHORTEST:
+                return GraphAlgebraPhysical.PathExpand.PathOpt.ALL_SHORTEST;
             default:
                 throw new UnsupportedOperationException(
                         "opt " + opt + " in path is unsupported yet");

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/config/GraphOpt.java

@@ -60,7 +60,9 @@ public enum Match {
     public enum PathExpandPath {
         ARBITRARY,
         SIMPLE,
-        TRAIL
+        TRAIL,
+        ANY_SHORTEST,
+        ALL_SHORTEST
     }
 
     public enum PathExpandResult {

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java

@@ -20,6 +20,8 @@
 import com.alibaba.graphscope.common.antlr4.ExprVisitorResult;
 import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate;
 import com.alibaba.graphscope.common.ir.rel.GraphProcedureCall;
+import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalGetV;
+import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalPathExpand;
 import com.alibaba.graphscope.common.ir.rel.type.group.GraphAggCall;
 import com.alibaba.graphscope.common.ir.rex.RexTmpVariableConverter;
 import com.alibaba.graphscope.common.ir.rex.RexVariableAliasCollector;
@@ -30,8 +32,10 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
+import org.apache.calcite.plan.GraphOptCluster;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelNode;
@@ -123,6 +127,45 @@ public GraphBuilder visitOC_Match(CypherGSParser.OC_MatchContext ctx) {
         return (ctx.oC_Where() != null) ? visitOC_Where(ctx.oC_Where()) : builder;
     }
 
+    @Override
+    public GraphBuilder visitOC_AnonymousPatternPart(
+            CypherGSParser.OC_AnonymousPatternPartContext ctx) {
+        visitOC_PatternElement(ctx.oC_PatternElement());
+        if (ctx.oC_ShortestPathOption() == null) return builder;
+        if (builder.size() > 0) {
+            RelNode top = builder.peek();
+            if (top instanceof GraphLogicalGetV
+                    && top.getInputs().size() == 1
+                    && top.getInput(0) instanceof GraphLogicalPathExpand) {
+                GraphLogicalGetV getV = (GraphLogicalGetV) top;
+                GraphLogicalPathExpand path = (GraphLogicalPathExpand) getV.getInput(0);
+                boolean isAll = ctx.oC_ShortestPathOption().ALL() != null;
+                GraphLogicalPathExpand shortestPath =
+                        GraphLogicalPathExpand.create(
+                                (GraphOptCluster) path.getCluster(),
+                                ImmutableList.of(),
+                                path.getInput(),
+                                path.getExpand(),
+                                path.getGetV(),
+                                path.getOffset(),
+                                path.getFetch(),
+                                path.getResultOpt(),
+                                isAll
+                                        ? GraphOpt.PathExpandPath.ALL_SHORTEST
+                                        : GraphOpt.PathExpandPath.ANY_SHORTEST,
+                                path.getUntilCondition(),
+                                path.getAliasName(),
+                                path.getStartAlias(),
+                                path.isOptional());
+                GraphLogicalGetV shortestGetV =
+                        getV.copy(getV.getTraitSet(), ImmutableList.of(shortestPath));
+                builder.build();
+                builder.push(shortestGetV);
+            }
+        }
+        return builder;
+    }
+
     @Override
     public GraphBuilder visitOC_PatternElementChain(
             CypherGSParser.OC_PatternElementChainContext ctx) {

interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java

@@ -611,7 +611,7 @@ public void udf_function_test() {
     }
 
     @Test
-    public void shortest_path_test() {
+    public void procedure_shortest_path_test() {
         GraphBuilder builder =
                 com.alibaba.graphscope.common.ir.Utils.mockGraphBuilder(optimizer, irMeta);
         LogicalPlanVisitor logicalPlanVisitor = new LogicalPlanVisitor(builder, irMeta);
@@ -649,4 +649,68 @@ public void shortest_path_test() {
                     + " alias=[person1], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])",
                 after.explain().trim());
     }
+
+    @Test
+    public void shortest_path_test() {
+        // convert 'shortestpath' modifier to 'path_opt=[ANY_SHORTEST]' in IR, and 'all
+        // shortestpath' to 'path_opt=[ALL_SHORTEST]'
+        RelNode rel =
+                Utils.eval(
+                                "MATCH"
+                                    + " shortestPath((person1:person)-[path:knows*1..5]->(person2:person))"
+                                    + " Return count(person1)")
+                        .build();
+        Assert.assertEquals(
+                "GraphLogicalAggregate(keys=[{variables=[], aliases=[]}],"
+                    + " values=[[{operands=[person1], aggFunction=COUNT, alias='$f0',"
+                    + " distinct=false}]])\n"
+                    + "  GraphLogicalSingleMatch(input=[null],"
+                    + " sentence=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}],"
+                    + " alias=[person2], opt=[END])\n"
+                    + "  GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false,"
+                    + " tables=[knows]}], alias=[_], opt=[OUT])\n"
+                    + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}],"
+                    + " alias=[_], opt=[END])\n"
+                    + "], offset=[1], fetch=[4], path_opt=[ANY_SHORTEST], result_opt=[ALL_V_E],"
+                    + " alias=[path])\n"
+                    + "    GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}],"
+                    + " alias=[person1], opt=[VERTEX])\n"
+                    + "], matchOpt=[INNER])",
+                rel.explain().trim());
+    }
+
+    @Test
+    public void optional_shortest_path_test() {
+        GraphBuilder builder =
+                com.alibaba.graphscope.common.ir.Utils.mockGraphBuilder(optimizer, irMeta);
+        RelNode node =
+                Utils.eval(
+                                "Match (p1: person {id: $id1})\n"
+                                        + "Optional Match shortestPath((p1:person {id:"
+                                        + " $id1})-[k:knows*1..5]->(p2:person {id: $id2}))\n"
+                                        + "WITH\n"
+                                        + "CASE WHEN k is null then -1\n"
+                                        + "ELSE length(k)\n"
+                                        + "END as len\n"
+                                        + "RETURN len;",
+                                builder)
+                        .build();
+        RelNode after = optimizer.optimize(node, new GraphIOProcessor(builder, irMeta));
+        Assert.assertEquals(
+                "GraphLogicalProject(len=[len], isAppend=[false])\n"
+                    + "  GraphLogicalProject(len=[CASE(IS NULL(k), -(1), k.~len)],"
+                    + " isAppend=[false])\n"
+                    + "    GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}],"
+                    + " alias=[p2], fusedFilter=[[=(_.id, ?1)]], opt=[END])\n"
+                    + "     "
+                    + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false,"
+                    + " tables=[knows]}], alias=[_], opt=[OUT])\n"
+                    + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[person]}],"
+                    + " alias=[_], opt=[END])\n"
+                    + "], offset=[1], fetch=[4], path_opt=[ANY_SHORTEST], result_opt=[ALL_V_E],"
+                    + " alias=[k], start_alias=[p1], optional=[true])\n"
+                    + "        GraphLogicalSource(tableConfig=[{isAll=false, tables=[person]}],"
+                    + " alias=[p1], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])",
+                after.explain().trim());
+    }
 }

interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs

@@ -96,8 +96,11 @@ pub enum GraphPath {
 impl GraphPath {
     pub fn new<E: Into<VertexOrEdge>>(
         entry: E, path_opt: pb::path_expand::PathOpt, result_opt: pb::path_expand::ResultOpt,
-    ) -> Self {
-        match result_opt {
+    ) -> Result<Self, ParsePbError> {
+        if let pb::path_expand::PathOpt::AnyShortest | pb::path_expand::PathOpt::AllShortest = path_opt {
+            return Err(ParsePbError::Unsupported("unsupported path type of shortest path".to_string()));
+        }
+        let path = match result_opt {
             pb::path_expand::ResultOpt::EndV => match path_opt {
                 pb::path_expand::PathOpt::Arbitrary => GraphPath::EndV((entry.into(), 1)),
                 pb::path_expand::PathOpt::Simple => {
@@ -106,13 +109,20 @@ impl GraphPath {
                     GraphPath::SimpleEndV((entry, vec![id], 1))
                 }
                 pb::path_expand::PathOpt::Trail => GraphPath::TrailAllPath(vec![entry.into()]),
+                pb::path_expand::PathOpt::AnyShortest | pb::path_expand::PathOpt::AllShortest => {
+                    unreachable!()
+                }
             },
             pb::path_expand::ResultOpt::AllV | pb::path_expand::ResultOpt::AllVE => match path_opt {
                 pb::path_expand::PathOpt::Arbitrary => GraphPath::AllPath(vec![entry.into()]),
                 pb::path_expand::PathOpt::Simple => GraphPath::SimpleAllPath(vec![entry.into()]),
                 pb::path_expand::PathOpt::Trail => GraphPath::TrailAllPath(vec![entry.into()]),
+                pb::path_expand::PathOpt::AnyShortest | pb::path_expand::PathOpt::AllShortest => {
+                    unreachable!()
+                }
             },
-        }
+        };
+        Ok(path)
     }
 
     // append an entry and return the flag of whether the entry has been appended or not.