✨ Support Scala Lists (#119)

* Added list mapper

* Refactored old hacky list handling with new one

* Fixed failing test

CHANGELOG.md

@@ -8,33 +8,41 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## [0.3.6] - [Unreleased]
 
 ### Added
+
 - `CONTAINS` edges are generated for `METHOD` to body vertices.
+- `ListMapper` to process Scala lists to a serialized string and back. More formally processing Scala lists to and from
+  OverflowDB node objects.
 
 ### Changed
+
 - `BaseCpgPass` now uses a local cache for method body nodes instead of relying solely on `GlobalCache`
 
 ## [0.3.5] - 2021-03-17
 
 ### Added
+
 - `IDriver::getVerticesOfType` to aid in caching from existing database vertices.
 - External methods signatures are parsed to figure out their method parameters.
-- `MethodStubPass` and `BaseCPGPass` now includes `METHOD_PARAM_IN` and `METHOD_PARAM_OUT` and connects them to their 
+- `MethodStubPass` and `BaseCPGPass` now includes `METHOD_PARAM_IN` and `METHOD_PARAM_OUT` and connects them to their
   type.
 - Field accesses are now constructed as a `Call` vertex.
 - Plume now has a new logo and branding.
 - Better logging for loaded files.
 
 ### Changed
+
 - Many of the `nodeCache` uses in `IProgramPass` passes were converted to using the `GlobalCache` instead.
 - `MethodStubPass` now runs in parallel if possible.
 
 ## [0.3.4] - 2021-03-15
 
 ### Changed
+
 - Upped the default chunk size
 - `DeltaGraph::toOverflowDb` can now take in an optional `overflowdb.Graph` object to write to
 
 ### Fixed
+
 - Memory leak where thread pools weren't getting shutdown
 
 ## [0.3.3] - 2021-03-15

plume/src/main/kotlin/io/github/plume/oss/domain/mappers/ListMapper.kt

@@ -0,0 +1,30 @@
+package io.github.plume.oss.domain.mappers
+
+import scala.jdk.CollectionConverters
+
+/**
+ * Contains functions to serialize and deserialize Scala lists.
+ */
+object ListMapper {
+
+    /**
+     * Converts a given Scala list to a string for serialization.
+     *
+     * @param list The list to convert.
+     * @return a comma-delimited string.
+     */
+    fun scalaListToString(list: scala.collection.immutable.List<*>): String =
+        CollectionConverters.IterableHasAsJava(list).asJava().joinToString(separator = ",")
+
+    /**
+     * Converts a serialized list to a Scala list.
+     *
+     * @param string The string to convert.
+     * @return a Scala list of the given type.
+     */
+    fun stringToScalaList(string: String): scala.collection.immutable.List<String> {
+        val list = if (string.isBlank()) emptyList() else string.split(',').toList()
+        return CollectionConverters.ListHasAsScala(list).asScala().toList()
+    }
+
+}

plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt

@@ -1,7 +1,6 @@
 package io.github.plume.oss.domain.mappers
 
 import io.github.plume.oss.util.ExtractorConst.UNKNOWN
-import io.github.plume.oss.util.SootToPlumeUtil.createScalaList
 import io.shiftleft.codepropertygraph.generated.NodeKeyNames.*
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import org.apache.logging.log4j.LogManager
@@ -11,8 +10,6 @@ import scala.Option
 import scala.collection.immutable.`$colon$colon`
 import scala.collection.immutable.`Nil$`
 import scala.jdk.CollectionConverters
-import java.util.*
-import kotlin.collections.HashMap
 
 
 /**
@@ -28,7 +25,7 @@ object VertexMapper {
      * @return a [NewNodeBuilder] represented by the information in the givennode.
      */
     fun mapToVertex(v: Node): NewNodeBuilder {
-        val map = v.propertyMap() + mapOf<String, Any>("id" to v.id(), "label" to v.label())
+        val map = prepareListsInMap(v.propertyMap()) + mapOf<String, Any>("id" to v.id(), "label" to v.label())
         return mapToVertex(map)
     }
 
@@ -39,7 +36,8 @@ object VertexMapper {
      * @return a [NewNodeBuilder] represented by the information in the givennode.
      */
     fun mapToVertex(v: NewNode): NewNodeBuilder {
-        val map = CollectionConverters.MapHasAsJava(v.properties()).asJava() + mapOf<String, Any>("label" to v.label())
+        val map = prepareListsInMap(CollectionConverters.MapHasAsJava(v.properties()).asJava()) +
+                mapOf<String, Any>("label" to v.label())
         return mapToVertex(map)
     }
 
@@ -158,13 +156,7 @@ object VertexMapper {
                 .signature(map[SIGNATURE] as String)
                 .dispatchType(map[DISPATCH_TYPE] as String)
                 .methodFullName(map[METHOD_FULL_NAME] as String)
-                .dynamicTypeHintFullName(
-                    when (map[DYNAMIC_TYPE_HINT_FULL_NAME]) {
-                        is String -> createScalaList(map[DYNAMIC_TYPE_HINT_FULL_NAME] as String)
-                        is `$colon$colon`<*> -> createScalaList((map[DYNAMIC_TYPE_HINT_FULL_NAME] as `$colon$colon`<*>).head() as String)
-                        else -> createScalaList("")
-                    }
-                )
+                .dynamicTypeHintFullName(ListMapper.stringToScalaList(map[DYNAMIC_TYPE_HINT_FULL_NAME] as String))
             Local.Label() -> NewLocalBuilder()
                 .typeFullName(map[TYPE_FULL_NAME] as String)
                 .code(map[CODE] as String)
@@ -210,13 +202,7 @@ object VertexMapper {
                 .argumentIndex(map[ARGUMENT_INDEX] as Int)
             TypeRef.Label() -> NewTypeRefBuilder()
                 .typeFullName(map[TYPE_FULL_NAME] as String)
-                .dynamicTypeHintFullName(
-                    when (map[DYNAMIC_TYPE_HINT_FULL_NAME]) {
-                        is String -> createScalaList(map[DYNAMIC_TYPE_HINT_FULL_NAME] as String)
-                        is `$colon$colon`<*> -> createScalaList((map[DYNAMIC_TYPE_HINT_FULL_NAME] as `$colon$colon`<*>).head() as String)
-                        else -> createScalaList("")
-                    }
-                )
+                .dynamicTypeHintFullName(ListMapper.stringToScalaList(map[DYNAMIC_TYPE_HINT_FULL_NAME] as String))
                 .code(map[CODE] as String)
                 .order(map[ORDER] as Int)
                 .lineNumber(Option.apply(map[LINE_NUMBER] as Int))
@@ -335,26 +321,17 @@ object VertexMapper {
         return outRule && toRule
     }
 
-    // TODO: This may be linked to this bug https://github.com/plume-oss/plume/issues/88
-    // TODO: The fix may be to get onto https://github.com/plume-oss/plume/issues/86
-    fun extractAttributesFromMap(propertyMap: MutableMap<String, Any>): MutableMap<String, Any> {
+    /**
+     * All Scala sequences or lists are converted to strings in this map.
+     */
+    fun prepareListsInMap(propertyMap: MutableMap<String, Any>): MutableMap<String, Any> {
         val attributes = mutableMapOf<String, Any>()
-        propertyMap.computeIfPresent(DYNAMIC_TYPE_HINT_FULL_NAME) { _, value ->
+        propertyMap.forEach { (key, value) ->
             when (value) {
-                is `$colon$colon`<*> -> value.head()
-                is `Nil$` -> ""
-                else -> value
-            }
-        }
-        propertyMap.forEach {
-            val key: Optional<String> = when (it.key) {
-                PARSER_TYPE_NAME -> Optional.empty()
-                POLICY_DIRECTORIES -> Optional.empty()
-                INHERITS_FROM_TYPE_FULL_NAME -> Optional.empty()
-                OVERLAYS -> Optional.empty()
-                else -> Optional.of(it.key)
+                is `$colon$colon`<*> -> attributes[key] = ListMapper.scalaListToString(value)
+                is `Nil$` -> attributes[key] = ListMapper.scalaListToString(value)
+                else -> attributes[key] = value
             }
-            if (key.isPresent) attributes[key.get()] = it.value
         }
         return attributes
     }

plume/src/main/kotlin/io/github/plume/oss/drivers/Neo4jDriver.kt

@@ -127,7 +127,7 @@ class Neo4jDriver internal constructor() : IDriver {
     }
 
     private fun extractAttributesFromMap(propertyMap: MutableMap<String, Any>): MutableMap<String, Any> {
-        val attributes = VertexMapper.extractAttributesFromMap(propertyMap)
+        val attributes = VertexMapper.prepareListsInMap(propertyMap)
         propertyMap.forEach { e ->
             if (attributes[e.key] is Int) attributes[e.key] = (attributes[e.key] as Int).toLong()
         }

plume/src/main/kotlin/io/github/plume/oss/drivers/TigerGraphDriver.kt

@@ -236,7 +236,7 @@ class TigerGraphDriver internal constructor() : IOverridenIdDriver, ISchemaSafeD
     }
 
     private fun extractAttributesFromMap(propertyMap: MutableMap<String, Any>): MutableMap<String, Any> =
-        VertexMapper.extractAttributesFromMap(propertyMap)
+        VertexMapper.prepareListsInMap(propertyMap)
             .mapKeys { if (it.key != "label") "_${it.key}" else it.key }
             .mapValues { mapOf("value" to it.value) }
             .toMutableMap()

plume/src/main/kotlin/io/github/plume/oss/graphio/GraphMLWriter.kt

@@ -75,7 +75,7 @@ object GraphMLWriter {
     private fun writeVertices(fw: OutputStreamWriter, vertices: MutableIterator<Node>) {
         vertices.forEach { v ->
             fw.write("<node id=\"${v.id()}\">")
-            VertexMapper.extractAttributesFromMap(v.propertyMap())
+            VertexMapper.prepareListsInMap(v.propertyMap())
                 .apply { fw.write("<data key=\"labelV\">${v.label()}</data>") }
                 .forEach { (t, u) -> fw.write("<data key=\"$t\">${escape(u)}</data>") }
             fw.write("</node>")

plume/src/main/kotlin/io/github/plume/oss/graphio/GraphSONWriter.kt

@@ -1,6 +1,6 @@
 package io.github.plume.oss.graphio
 
-import io.github.plume.oss.domain.mappers.VertexMapper.extractAttributesFromMap
+import io.github.plume.oss.domain.mappers.VertexMapper.prepareListsInMap
 import overflowdb.Graph
 import overflowdb.Node
 import java.io.OutputStreamWriter
@@ -34,7 +34,7 @@ object GraphSONWriter {
 
     private fun vertexToJSON(v: Node, graph: Graph): String {
         val sb = StringBuilder()
-        val properties = extractAttributesFromMap(v.propertyMap())
+        val properties = prepareListsInMap(v.propertyMap())
         sb.append("{")
         sb.append("\"id\":{\"@type\":\"g:Int64\",\"@value\":${v.id()}},")
         sb.append("\"label\":\"${v.label()}\",")
@@ -92,7 +92,7 @@ object GraphSONWriter {
         val sb = StringBuilder()
         sb.append("{")
         var i = 0
-        extractAttributesFromMap(properties).forEach { (k, v) ->
+        prepareListsInMap(properties).forEach { (k, v) ->
             sb.append("\"$k\":[{")
             sb.append("\"id\":{\"@type\":\"g:Int64\",\"@value\":${propertyId++}}")
             when (v) {

plume/src/main/kotlin/io/github/plume/oss/passes/graph/BaseCPGPass.kt

@@ -1,6 +1,7 @@
 package io.github.plume.oss.passes.graph
 
 import io.github.plume.oss.GlobalCache
+import io.github.plume.oss.domain.mappers.ListMapper
 import io.github.plume.oss.domain.model.DeltaGraph
 import io.github.plume.oss.util.ExtractorConst
 import io.github.plume.oss.util.SootParserUtil
@@ -319,7 +320,7 @@ class BaseCPGPass(private val g: BriefUnitGraph) {
             .signature(signature)
             .code(code)
             .order(childIdx)
-            .dynamicTypeHintFullName(SootToPlumeUtil.createScalaList(unit.methodRef.returnType.toQuotedString()))
+            .dynamicTypeHintFullName(ListMapper.stringToScalaList(unit.methodRef.returnType.toQuotedString()))
             .lineNumber(Option.apply(currentLine))
             .columnNumber(Option.apply(currentCol))
             .argumentIndex(childIdx)
@@ -513,7 +514,7 @@ class BaseCPGPass(private val g: BriefUnitGraph) {
             .signature("${leftOp.type} = ${rightOp.type}")
             .methodFullName(Operators.assignment)
             .dispatchType(DispatchTypes.STATIC_DISPATCH)
-            .dynamicTypeHintFullName(SootToPlumeUtil.createScalaList(unit.rightOp.type.toQuotedString()))
+            .dynamicTypeHintFullName(ListMapper.stringToScalaList(unit.rightOp.type.toQuotedString()))
             .order(childIdx)
             .argumentIndex(childIdx)
             .typeFullName(leftOp.type.toQuotedString())
@@ -573,7 +574,7 @@ class BaseCPGPass(private val g: BriefUnitGraph) {
             .signature("${expr.op1.type.toQuotedString()}${expr.symbol}${expr.op2.type.toQuotedString()}")
             .methodFullName(binOpExpr)
             .dispatchType(DispatchTypes.STATIC_DISPATCH)
-            .dynamicTypeHintFullName(SootToPlumeUtil.createScalaList(expr.op2.type.toQuotedString()))
+            .dynamicTypeHintFullName(ListMapper.stringToScalaList(expr.op2.type.toQuotedString()))
             .order(childIdx)
             .argumentIndex(childIdx)
             .typeFullName(expr.type.toQuotedString())
@@ -611,7 +612,7 @@ class BaseCPGPass(private val g: BriefUnitGraph) {
             .typeFullName(expr.type.toQuotedString())
             .lineNumber(Option.apply(currentLine))
             .columnNumber(Option.apply(currentCol))
-            .dynamicTypeHintFullName(SootToPlumeUtil.createScalaList(expr.op2.type.toQuotedString()))
+            .dynamicTypeHintFullName(ListMapper.stringToScalaList(expr.op2.type.toQuotedString()))
             .apply { conditionVertices.add(this) }
         projectOp(expr.op1, 1)?.let {
             builder.addEdge(binOpBlock, it, AST)
@@ -637,7 +638,7 @@ class BaseCPGPass(private val g: BriefUnitGraph) {
             .signature("(${expr.castType.toQuotedString()}) ${expr.op.type.toQuotedString()}")
             .methodFullName(Operators.cast)
             .dispatchType(DispatchTypes.STATIC_DISPATCH)
-            .dynamicTypeHintFullName(SootToPlumeUtil.createScalaList(expr.op.type.toQuotedString()))
+            .dynamicTypeHintFullName(ListMapper.stringToScalaList(expr.op.type.toQuotedString()))
             .order(childIdx)
             .argumentIndex(childIdx)
             .typeFullName(expr.type.toQuotedString())
@@ -695,7 +696,7 @@ class BaseCPGPass(private val g: BriefUnitGraph) {
             .signature("")
             .methodFullName(Operators.fieldAccess)
             .dispatchType(if (fieldRef.fieldRef.isStatic) DispatchTypes.STATIC_DISPATCH else DispatchTypes.DYNAMIC_DISPATCH)
-            .dynamicTypeHintFullName(SootToPlumeUtil.createScalaList(leftOp.toQuotedString()))
+            .dynamicTypeHintFullName(ListMapper.stringToScalaList(leftOp.toQuotedString()))
             .order(childIdx)
             .argumentIndex(childIdx)
             .typeFullName(leftOp.toQuotedString())

plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt

@@ -16,6 +16,7 @@
 package io.github.plume.oss.util
 
 import io.github.plume.oss.GlobalCache
+import io.github.plume.oss.domain.mappers.ListMapper
 import io.github.plume.oss.domain.mappers.VertexMapper.mapToVertex
 import io.github.plume.oss.drivers.IDriver
 import io.github.plume.oss.util.SootParserUtil.determineEvaluationStrategy
@@ -176,6 +177,7 @@ object SootToPlumeUtil {
                     .code(type.toString())
                     .order(childIdx)
                     .argumentIndex(childIdx)
+                    .dynamicTypeHintFullName(ListMapper.stringToScalaList(type.toQuotedString()))
                     .typeFullName(type.toQuotedString())
                     .lineNumber(Option.apply(currentLine))
                     .columnNumber(Option.apply(currentCol))
@@ -234,11 +236,6 @@ object SootToPlumeUtil {
             .columnNumber(Option.apply(currentCol))
             .order(childIdx)
 
-    fun <T> createScalaList(vararg item: T): scala.collection.immutable.List<T> {
-        val list = listOf(*item)
-        return CollectionConverters.ListHasAsScala(list).asScala().toList() as scala.collection.immutable.List<T>
-    }
-
     fun parseBinopExpr(op: BinopExpr): String = parseBinopExpr(op.symbol.trim())
 
     fun parseBinopExpr(sym: String): String {

plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt

@@ -1,7 +1,7 @@
 package io.github.plume.oss
 
+import io.github.plume.oss.domain.mappers.ListMapper
 import io.github.plume.oss.drivers.IDriver
-import io.github.plume.oss.util.SootToPlumeUtil.createScalaList
 import io.shiftleft.codepropertygraph.generated.DispatchTypes.DYNAMIC_DISPATCH
 import io.shiftleft.codepropertygraph.generated.DispatchTypes.STATIC_DISPATCH
 import io.shiftleft.codepropertygraph.generated.EdgeTypes.*
@@ -34,7 +34,7 @@ class TestDomainResources {
                 .lineNumber(Option.apply(INT_1)).columnNumber(Option.apply(INT_1)),
             NewCallBuilder().methodFullName(STRING_1).argumentIndex(INT_1).dispatchType(DISPATCH_1)
                 .typeFullName(STRING_1)
-                .dynamicTypeHintFullName(createScalaList(STRING_1))
+                .dynamicTypeHintFullName(ListMapper.stringToScalaList(STRING_1))
                 .name(STRING_1).signature(STRING_1).code(STRING_1).order(INT_1).lineNumber(Option.apply(INT_1))
                 .columnNumber(Option.apply(INT_1)),
             NewControlStructureBuilder().controlStructureType(STRING_2).code(STRING_1).lineNumber(Option.apply(INT_1))
@@ -68,7 +68,7 @@ class TestDomainResources {
             NewTypeDeclBuilder().name(STRING_1).fullName(STRING_1).order(INT_1).isExternal(BOOL_1),
             NewTypeParameterBuilder().name(STRING_1).order(INT_1),
             NewTypeRefBuilder().typeFullName(STRING_1)
-                .dynamicTypeHintFullName(createScalaList(STRING_1))
+                .dynamicTypeHintFullName(ListMapper.stringToScalaList(STRING_1))
                 .code(STRING_1).argumentIndex(INT_1).order(INT_1).lineNumber(Option.apply(INT_1))
                 .columnNumber(Option.apply(INT_1)),
             NewTypeBuilder().name(STRING_1).fullName(STRING_1).typeDeclFullName(STRING_1),
@@ -78,7 +78,8 @@ class TestDomainResources {
 
         val methodVertex: NewMethodBuilder =
             NewMethodBuilder().code(STRING_1).name(STRING_1).fullName(STRING_1).order(INT_1)
-                .lineNumber(Option.apply(INT_1)).columnNumber(Option.apply(INT_1)).signature(STRING_2).filename(STRING_1)
+                .lineNumber(Option.apply(INT_1)).columnNumber(Option.apply(INT_1)).signature(STRING_2)
+                .filename(STRING_1)
                 .astParentFullName(STRING_1).astParentType(STRING_2).isExternal(BOOL_1)
         val mtdParamInVertex: NewMethodParameterInBuilder =
             NewMethodParameterInBuilder().code(STRING_1).evaluationStrategy(EVAL_1).typeFullName(STRING_1)
@@ -89,7 +90,7 @@ class TestDomainResources {
         val callVertex: NewCallBuilder =
             NewCallBuilder().methodFullName(STRING_1).argumentIndex(INT_1).dispatchType(DISPATCH_1)
                 .typeFullName(STRING_1)
-                .dynamicTypeHintFullName(createScalaList(STRING_1))
+                .dynamicTypeHintFullName(ListMapper.stringToScalaList(STRING_1))
                 .name(STRING_1).signature(STRING_1).code(STRING_1).order(INT_1).lineNumber(Option.apply(INT_1))
                 .columnNumber(Option.apply(INT_1))
         val localVertex: NewLocalBuilder =
@@ -131,7 +132,7 @@ class TestDomainResources {
             NewMethodRefBuilder().methodInstFullName(Option.apply(STRING_1)).methodFullName(STRING_1).code(STRING_1)
                 .order(INT_1).argumentIndex(INT_1).lineNumber(Option.apply(INT_1)).columnNumber(Option.apply(INT_1))
         val typeRefVertex: NewTypeRefBuilder = NewTypeRefBuilder().typeFullName(STRING_1)
-            .dynamicTypeHintFullName(createScalaList(STRING_1))
+            .dynamicTypeHintFullName(ListMapper.stringToScalaList(STRING_1))
             .code(STRING_1).argumentIndex(INT_1).order(INT_1).lineNumber(Option.apply(INT_1))
             .columnNumber(Option.apply(INT_1))
         val unknownVertex: NewUnknownBuilder =