✨ Program Structure and Neighbour Methods (#11)

* ⬆️ Upgraded TigerGraph version

* ✨ Implemented program structure calls in driver

* ✨ Added getWholeGraph

* ✨ Implemented TigerGraph driver method bodies

* 👌 Improved coverage

* ⚡ Leveraging Travis Graviton VMs

.travis.yml

@@ -1,5 +1,7 @@
 language: java
 os: linux
+arch: arm64-graviton2
+virt: lxd
 
 services:
   - docker

codecov.yml

@@ -5,6 +5,13 @@ coverage:
   precision: 2
   round: down
   range: "70...100"
+  status:
+    patch: false
+    project: {
+      default: {
+        threshold: 1
+      }
+    }
 
 parsers:
   gcov:

src/main/kotlin/za/ac/sun/plume/drivers/GremlinDriver.kt

@@ -18,10 +18,11 @@ import za.ac.sun.plume.domain.mappers.VertexMapper.Companion.checkSchemaConstrai
 import za.ac.sun.plume.domain.mappers.VertexMapper.Companion.vertexToMap
 import za.ac.sun.plume.domain.models.PlumeGraph
 import za.ac.sun.plume.domain.models.PlumeVertex
+import za.ac.sun.plume.domain.models.vertices.FileVertex
+import za.ac.sun.plume.domain.models.vertices.MetaDataVertex
 import za.ac.sun.plume.domain.models.vertices.MethodVertex
 import java.util.*
-import kotlin.collections.HashMap
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.`__` as underscore
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.`__` as un
 
 /**
  * The driver used by remote Gremlin connections.
@@ -128,7 +129,7 @@ abstract class GremlinDriver : IDriver {
             if (!findVertexTraversal(fromV).hasNext() || !findVertexTraversal(toV).hasNext()) return false
             val a = findVertexTraversal(fromV).next()
             val b = findVertexTraversal(toV).next()
-            return g.V(a).outE(edge.name).filter(underscore.inV().`is`(b)).hasLabel(edge.name).hasNext()
+            return g.V(a).outE(edge.name).filter(un.inV().`is`(b)).hasLabel(edge.name).hasNext()
         } finally {
             if (transactionOpen) closeTx()
         }
@@ -209,50 +210,84 @@ abstract class GremlinDriver : IDriver {
         }
     }
 
+    override fun getWholeGraph(): PlumeGraph {
+        if (!transactionOpen) openTx()
+        val plumeGraph = gremlinToPlume(g)
+        if (transactionOpen) closeTx()
+        return plumeGraph
+    }
+
     override fun getMethod(fullName: String, signature: String): PlumeGraph {
         if (!transactionOpen) openTx()
         val methodSubgraph = g.V().hasLabel(MethodVertex.LABEL.toString())
                 .has("fullName", fullName).has("signature", signature)
-                .repeat(underscore.outE(EdgeLabel.AST.toString()).inV()).emit()
+                .repeat(un.outE(EdgeLabel.AST.toString()).inV()).emit()
                 .inE()
                 .subgraph("sg")
                 .cap<Graph>("sg")
                 .next()
-        val result = gremlinToPlume(methodSubgraph)
-        methodSubgraph.traversal().io<Any>("/tmp/plume/yay.xml").write().iterate()
+        val result = gremlinToPlume(methodSubgraph.traversal())
+        if (transactionOpen) closeTx()
+        return result
+    }
+
+    override fun getProgramStructure(): PlumeGraph {
+        if (!transactionOpen) openTx()
+        val programStructureSubGraph = g.V().hasLabel(FileVertex.LABEL.toString())
+                .repeat(un.outE(EdgeLabel.AST.toString()).inV()).emit()
+                .inE()
+                .subgraph("sg")
+                .cap<Graph>("sg")
+                .next()
+        val result = gremlinToPlume(programStructureSubGraph.traversal())
+        if (transactionOpen) closeTx()
+        return result
+    }
+
+    override fun getNeighbours(v: PlumeVertex): PlumeGraph {
+        if (v is MetaDataVertex) return PlumeGraph().apply { addVertex(v) }
+        if (!transactionOpen) openTx()
+        val neighbourSubgraph = findVertexTraversal(v)
+                .repeat(un.outE(EdgeLabel.AST.toString()).bothV())
+                .times(1)
+                .inE()
+                .subgraph("sg")
+                .cap<Graph>("sg")
+                .next()
+        val result = gremlinToPlume(neighbourSubgraph.traversal())
         if (transactionOpen) closeTx()
         return result
     }
 
     /**
-     * Converts a [Graph] instance to a [PlumeGraph] instance.
+     * Converts a [GraphTraversalSource] instance to a [PlumeGraph] instance.
      *
-     * @param graph A Gremlin graph.
+     * @param g A [GraphTraversalSource] from the subgraph to convert.
      * @return The resulting [PlumeGraph].
      */
-    private fun gremlinToPlume(graph: Graph): PlumeGraph {
-        val g = graph.traversal()
+    private fun gremlinToPlume(g: GraphTraversalSource): PlumeGraph {
         val plumeGraph = PlumeGraph()
-        g.V().toSet().map(this::gremlinToPlume).forEach { plumeGraph.addVertex(it) }
-        g.E().valueMap<String>().with(WithOptions.tokens).forEachRemaining {
-            val edgeLabel = EdgeLabel.valueOf(it[T.label].toString())
-            val plumeSrc = gremlinToPlume(g.E(it[T.id]).outV().next())
-            val plumeTgt = gremlinToPlume(g.E(it[T.id]).inV().next())
-            plumeGraph.addEdge(plumeSrc, plumeTgt, edgeLabel)
+        val f = { gt: GraphTraversal<Edge, Vertex> ->
+            gt.valueMap<String>()
+                    .by(un.unfold<Any>())
+                    .with(WithOptions.tokens)
+                    .next()
+                    .mapKeys { k -> k.key.toString() }
         }
+        g.V().valueMap<Any>()
+                .with(WithOptions.tokens)
+                .by(un.unfold<Any>()).toStream()
+                .map { props -> VertexMapper.mapToVertex(props.mapKeys { it.key.toString() }) }
+                .forEach { plumeGraph.addVertex(it) }
+        g.E().barrier().valueMap<String>()
+                .with(WithOptions.tokens)
+                .by(un.unfold<Any>())
+                .forEach {
+                    val edgeLabel = EdgeLabel.valueOf(it[T.label].toString())
+                    val plumeSrc = VertexMapper.mapToVertex(f(g.E(it[T.id]).outV()))
+                    val plumeTgt = VertexMapper.mapToVertex(f(g.E(it[T.id]).inV()))
+                    plumeGraph.addEdge(plumeSrc, plumeTgt, edgeLabel)
+                }
         return plumeGraph
     }
-
-    /**
-     * Converts a [Vertex] instance to a [PlumeVertex] instance.
-     *
-     * @param v A Gremlin vertex.
-     * @return The resulting [PlumeVertex].
-     */
-    private fun gremlinToPlume(v: Vertex): PlumeVertex {
-        val map = HashMap<String, Any>()
-        map["label"] = v.label()
-        v.properties<Any>().forEach { map[it.key()] = it.value() }
-        return VertexMapper.mapToVertex(map)
-    }
 }

src/main/kotlin/za/ac/sun/plume/drivers/IDriver.kt

@@ -63,6 +63,14 @@ interface IDriver : AutoCloseable {
      */
     fun clearGraph(): IDriver
 
+    /**
+     * Returns the whole CPG as a [PlumeGraph] object. Depending on the size of the CPG, this may be very memory
+     * intensive.
+     *
+     * @return The whole CPG in the graph database.
+     */
+    fun getWholeGraph(): PlumeGraph
+
     /**
      * Given the full signature of a method, returns the subgraph of the method body.
      *
@@ -71,4 +79,19 @@ interface IDriver : AutoCloseable {
      * @return The [PlumeGraph] containing the method graph.
      */
     fun getMethod(fullName: String, signature: String): PlumeGraph
+
+    /**
+     * Obtains all program structure related vertices.
+     *
+     * @return The [PlumeGraph] containing the program structure related sub-graphs.
+     */
+    fun getProgramStructure(): PlumeGraph
+
+    /**
+     * Given a vertex, returns a [PlumeGraph] representation of neighbouring vertices.
+     *
+     * @param v The source vertex.
+     * @return The [PlumeGraph] representation of the source vertex and its neighbouring vertices.
+     */
+    fun getNeighbours(v: PlumeVertex): PlumeGraph
 }

src/main/kotlin/za/ac/sun/plume/drivers/TigerGraphDriver.kt

@@ -188,18 +188,38 @@ class TigerGraphDriver : IDriver {
         )
     }
 
+    override fun getWholeGraph(): PlumeGraph {
+        val result = get("query/$GRAPH_NAME/showAll")
+        return graphPayloadToPlumeGraph(result)
+    }
+
     override fun getMethod(fullName: String, signature: String): PlumeGraph {
         var methodHash = MethodVertex::class.java.hashCode()
         methodHash = 31 * methodHash + fullName.hashCode()
         methodHash = 31 * methodHash + signature.hashCode()
         val result = get("query/$GRAPH_NAME/getMethod", mapOf("methodHash" to methodHash.toString()))
+        return graphPayloadToPlumeGraph(result)
+    }
+
+    override fun getProgramStructure(): PlumeGraph {
+        val result = get("query/$GRAPH_NAME/getProgramStructure")
+        return graphPayloadToPlumeGraph(result)
+    }
+
+    override fun getNeighbours(v: PlumeVertex): PlumeGraph {
+        if (v is MetaDataVertex) return PlumeGraph().apply { addVertex(v) }
+        val result = get("query/$GRAPH_NAME/getNeighbours", mapOf("source" to v.hashCode().toString()))
+        return graphPayloadToPlumeGraph(result)
+    }
+
+    private fun graphPayloadToPlumeGraph(a: JSONArray): PlumeGraph {
         val plumeGraph = PlumeGraph()
-        result[0]?.let { res ->
+        a[0]?.let { res ->
             val o = res as JSONObject
             val vertices = o["allVert"] as JSONArray
-            vertices.map { gsqlToPlume(it as JSONObject) }.forEach { plumeGraph.addVertex(it) }
+            vertices.map { vertexPayloadToPlumeGraph(it as JSONObject) }.forEach { plumeGraph.addVertex(it) }
         }
-        result[1]?.let { res ->
+        a[1]?.let { res ->
             val o = res as JSONObject
             val edges = o["@@edges"] as JSONArray
             edges.forEach { connectEdgeResult(plumeGraph, it as JSONObject) }
@@ -216,7 +236,7 @@ class TigerGraphDriver : IDriver {
         }
     }
 
-    private fun gsqlToPlume(o: JSONObject): PlumeVertex {
+    private fun vertexPayloadToPlumeGraph(o: JSONObject): PlumeVertex {
         val attributes = o["attributes"] as JSONObject
         val vertexMap = HashMap<String, Any>()
         attributes.keySet().filter { attributes[it] != "" }