Array/Multi-array Support (#221)

* Upgraded SBT and trying to write array tests

* Tests working

* Array initializer now supported

* Array order bug fixed

* Multi and single arrays are initializing better

* Upgraded Joern version

* Array tests working

* Code format

* Cleaned code up and fixed forking tests

* Formatted code

build.sbt

@@ -1,18 +1,29 @@
 name := "Plume"
-organization := "com.github.plume-oss"
-version := "1.0.3"
 
-scalaVersion := "2.13.6"
+inThisBuild(
+  List(
+    organization := "com.github.plume-oss",
+    version := "1.0.3",
+    scalaVersion := "2.13.7",
+    crossScalaVersions := Seq("2.13.7", "3.1.0"),
+    resolvers ++= Seq(
+      Resolver.mavenLocal,
+      Resolver.mavenCentral,
+      Resolver.JCenterRepository
+    )
+  )
+)
 
-val cpgVersion         = "1.3.477"
-val joernVersion       = "1.1.412"
+val cpgVersion         = "1.3.493"
+val joernVersion       = "1.1.488"
 val sootVersion        = "4.2.1"
 val tinkerGraphVersion = "3.4.8"
 val neo4jVersion       = "4.4.2"
 val tigerGraphVersion  = "3.1.0"
 val sttpVersion        = "3.3.17"
+val scalajHttpVersion  = "2.4.2"
 val lz4Version         = "1.8.0"
-val slf4jVersion       = "1.7.32"
+val slf4jVersion       = "1.7.33"
 val scalatestVersion   = "3.2.9"
 val circeVersion       = "0.14.1"
 
@@ -21,29 +32,22 @@ lazy val NeoIntTest        = config("neoTest") extend Test
 lazy val TigerGraphIntTest = config("tgTest") extend Test
 lazy val NeptuneIntTest    = config("nepTest") extend Test
 
-fork := true
-
-resolvers ++= Seq(
-  Resolver.mavenLocal,
-  Resolver.mavenCentral,
-  Resolver.JCenterRepository
-)
-
 trapExit := false
+Test / fork := true
 
 libraryDependencies ++= Seq(
   "io.shiftleft"                  %% "codepropertygraph"   % cpgVersion,
   "io.shiftleft"                  %% "semanticcpg"         % cpgVersion,
   "io.joern"                      %% "dataflowengineoss"   % joernVersion,
-  "io.shiftleft"                  %% "semanticcpg-tests"   % "1.3.405"        % Test classifier "tests",
+  "io.shiftleft"                  %% "semanticcpg"         % cpgVersion       % Test classifier "tests",
   "org.soot-oss"                   % "soot"                % sootVersion,
   "org.apache.tinkerpop"           % "tinkergraph-gremlin" % tinkerGraphVersion,
   "org.apache.tinkerpop"           % "gremlin-driver"      % tinkerGraphVersion,
   "org.neo4j.driver"               % "neo4j-java-driver"   % neo4jVersion,
   "com.tigergraph.client"          % "gsql_client"         % tigerGraphVersion,
   "com.softwaremill.sttp.client3" %% "core"                % sttpVersion,
   "com.softwaremill.sttp.client3" %% "circe"               % sttpVersion,
-  "org.scalaj"                     % "scalaj-http_2.13"    % "2.4.2",
+  "org.scalaj"                     % "scalaj-http_2.13"    % scalajHttpVersion,
   "org.lz4"                        % "lz4-java"            % lz4Version,
   "org.slf4j"                      % "slf4j-api"           % slf4jVersion,
   "org.slf4j"                      % "slf4j-simple"        % slf4jVersion,

project/build.properties

@@ -1 +1 @@
-sbt.version=1.5.7
+sbt.version=1.6.1

project/plugins.sbt

@@ -1,8 +1,8 @@
-addSbtPlugin("org.scalameta"     % "sbt-scalafmt"        % "2.4.2")
+addSbtPlugin("org.scalameta"     % "sbt-scalafmt"        % "2.4.6")
 addSbtPlugin("com.typesafe.sbt"  % "sbt-git"             % "1.0.0")
 addSbtPlugin("com.typesafe.sbt"  % "sbt-native-packager" % "1.4.1")
 addSbtPlugin("com.typesafe.sbt"  % "sbt-site"            % "1.4.1")
 addSbtPlugin("com.eed3si9n"      % "sbt-assembly"        % "0.14.5")
 addSbtPlugin("com.eed3si9n"      % "sbt-buildinfo"       % "0.8.0")
-addSbtPlugin("org.scoverage"     % "sbt-scoverage"       % "1.8.2")
+addSbtPlugin("org.scoverage"     % "sbt-scoverage"       % "1.9.3")
 addSbtPlugin("com.typesafe.sbt"  % "sbt-ghpages"         % "0.6.3")

src/main/scala/com/github/plume/oss/Jimple2Cpg.scala

@@ -82,18 +82,18 @@ class Jimple2Cpg {
   ): Cpg = {
     try {
       // Determine if the given path is a file or directory and sanitize accordingly
-      val rawSourceCodeFile  = new JFile(rawSourceCodePath)
-      val normSourceCodePath = rawSourceCodeFile.toPath.toAbsolutePath.normalize.toString
-      val sourceCodePath = if (rawSourceCodeFile.isDirectory) {
-        normSourceCodePath
+      val rawSourceCodeFile = new JFile(rawSourceCodePath)
+      val sourceTarget      = rawSourceCodeFile.toPath.toAbsolutePath.normalize.toString
+      val sourceCodeDir = if (rawSourceCodeFile.isDirectory) {
+        sourceTarget
       } else {
         Paths
-          .get(new JFile(normSourceCodePath).getParentFile.getAbsolutePath)
+          .get(new JFile(sourceTarget).getParentFile.getAbsolutePath)
           .normalize
           .toString
       }
 
-      configureSoot(sourceCodePath)
+      configureSoot(sourceCodeDir)
       val cpg = newEmptyCpg(outputPath)
 
       val metaDataKeyPool = new IncrementalKeyPool(1, 100, driver.idInterval(1, 100))
@@ -107,15 +107,14 @@ class Jimple2Cpg {
 
       val sourceFileExtensions  = Set(".class", ".jimple")
       val archiveFileExtensions = Set(".jar", ".war")
-      // Unpack any archives on the path onto the source code path as project root
-      val archives: List[String] =
-        if (rawSourceCodeFile.isDirectory)
-          SourceFiles.determine(Set(sourceCodePath), archiveFileExtensions)
-        else if (normSourceCodePath.endsWith(archiveFileExtensions))
-          List(normSourceCodePath)
-        else List()
       // Load source files and unpack archives if necessary
-      val sourceFileNames = loadSourceFiles(sourceCodePath, sourceFileExtensions, archives)
+      val sourceFileNames = if (sourceTarget == sourceCodeDir) {
+        // Load all source files in a directory
+        loadSourceFiles(sourceCodeDir, sourceFileExtensions, archiveFileExtensions)
+      } else {
+        // Load single file that was specified
+        loadSourceFiles(sourceTarget, sourceFileExtensions, archiveFileExtensions)
+      }
 
       val codeToProcess = new PlumeDiffPass(sourceFileNames, driver).createAndApply()
       // After the diff pass any changed types are removed. Remaining types should be black listed to avoid duplicates
@@ -134,9 +133,9 @@ class Jimple2Cpg {
         .createAndApply(driver)
 
       // Load classes into Soot
-      loadClassesIntoSoot(sourceCodePath, sourceFileNames)
+      loadClassesIntoSoot(sourceCodeDir, sourceFileNames)
       // Project Soot classes
-      val astCreator = new AstCreationPass(sourceCodePath, codeToProcess.toList, cpg, methodKeyPool)
+      val astCreator = new AstCreationPass(sourceCodeDir, codeToProcess.toList, cpg, methodKeyPool)
       astCreator.createAndApply(driver)
       // Clear classes from Soot
       closeSoot()
@@ -159,31 +158,45 @@ class Jimple2Cpg {
     }
   }
 
+  /** Retrieve parseable files from archive types.
+    */
+  private def extractSourceFilesFromArchive(
+      sourceCodeDir: String,
+      archiveFileExtensions: Set[String]
+  ): List[String] = {
+    val archives = if (new JFile(sourceCodeDir).isFile) {
+      List(sourceCodeDir)
+    } else {
+      SourceFiles.determine(Set(sourceCodeDir), archiveFileExtensions)
+    }
+    archives.flatMap { x =>
+      unzipArchive(new ZipFile(x), sourceCodeDir) match {
+        case Failure(e) =>
+          throw new RuntimeException(s"Error extracting files from archive at $x", e)
+        case Success(files) => files
+      }
+    }
+  }
+
+  /** Load all source files from archive and/or source file types.
+    */
   private def loadSourceFiles(
       sourceCodePath: String,
       sourceFileExtensions: Set[String],
-      archives: List[String]
-  ) = {
-    (archives
-      .map(new ZipFile(_))
-      .flatMap(x => {
-        unzipArchive(x, sourceCodePath) match {
-          case Failure(e) =>
-            logger.error(s"Error extracting files from archive at ${x.getName}", e); null
-          case Success(value) => value
-        }
-      })
-      .map(_.getAbsolutePath) ++ SourceFiles.determine(
-      Set(sourceCodePath),
-      sourceFileExtensions
-    )).distinct
+      archiveFileExtensions: Set[String]
+  ): List[String] = {
+    (
+      extractSourceFilesFromArchive(sourceCodePath, archiveFileExtensions) ++
+        SourceFiles.determine(Set(sourceCodePath), sourceFileExtensions)
+    ).distinct
   }
 
   private def loadClassesIntoSoot(sourceCodePath: String, sourceFileNames: List[String]): Unit = {
     sourceFileNames
-      .map(getQualifiedClassPath(sourceCodePath, _))
-      .map { x =>
-        Scene.v().addBasicClass(x, SootClass.BODIES); x
+      .map { fName =>
+        val cp = getQualifiedClassPath(sourceCodePath, fName)
+        Scene.v().addBasicClass(cp, SootClass.BODIES)
+        cp
       }
       .foreach(Scene.v().loadClassAndSupport(_).setApplicationClass())
     Scene.v().loadNecessaryClasses()
@@ -253,7 +266,7 @@ class Jimple2Cpg {
       zip
         .entries()
         .asScala
-        .filter(x => !x.isDirectory && x.getName.contains(".class"))
+        .filter(f => !f.isDirectory && f.getName.contains(".class"))
         .flatMap(entry => {
           val sourceCodePathFile = new JFile(sourceCodePath)
           // Handle the case if the input source code path is an archive itself
@@ -275,7 +288,7 @@ class Jimple2Cpg {
               Files.copy(input, destFile.toPath)
             }
             destFile.deleteOnExit()
-            Option(destFile)
+            Option(destFile.getAbsolutePath)
           } catch {
             case e: Exception =>
               logger.warn(

src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreator.scala

@@ -307,7 +307,7 @@ class PlumeAstCreator(filename: String, global: Global) {
       .typeFullName(registerType(arrRef.getType.toQuotedString))
 
     val astChildren =
-      astsForValue(arrRef.getBase, 0, parentUnit) ++ astsForValue(arrRef.getIndex, 1, parentUnit)
+      astsForValue(arrRef.getBase, 1, parentUnit) ++ astsForValue(arrRef.getIndex, 2, parentUnit)
     Ast(indexAccess)
       .withChildren(astChildren)
       .withArgEdges(indexAccess, astChildren.flatMap(_.root))
@@ -380,15 +380,44 @@ class PlumeAstCreator(filename: String, global: Global) {
   }
 
   private def astForNewExpr(x: AnyNewExpr, order: Int, parentUnit: soot.Unit): Ast = {
-    Ast(
-      NewUnknown()
-        .typeFullName(registerType(x.getType.toQuotedString))
-        .code("new")
-        .order(order)
-        .argumentIndex(order)
-        .lineNumber(line(parentUnit))
-        .columnNumber(column(parentUnit))
-    )
+    x match {
+      case u: NewArrayExpr =>
+        astForArrayInitializeExpr(x, List(u.getSize), order, parentUnit)
+      case u: NewMultiArrayExpr =>
+        astForArrayInitializeExpr(x, u.getSizes.asScala, order, parentUnit)
+      case _ =>
+        Ast(
+          NewUnknown()
+            .typeFullName(registerType(x.getType.toQuotedString))
+            .code("new")
+            .order(order)
+            .argumentIndex(order)
+            .lineNumber(line(parentUnit))
+            .columnNumber(column(parentUnit))
+        )
+    }
+  }
+
+  private def astForArrayInitializeExpr(
+      arrayInitExpr: Expr,
+      sizes: Iterable[Value],
+      order: Int,
+      parentUnit: soot.Unit
+  ): Ast = {
+    val callBlock = NewCall()
+      .name(Operators.arrayInitializer)
+      .methodFullName(Operators.arrayInitializer)
+      .code(arrayInitExpr.toString())
+      .dispatchType(DispatchTypes.STATIC_DISPATCH)
+      .order(order)
+      .typeFullName(registerType(arrayInitExpr.getType.toQuotedString))
+      .argumentIndex(order)
+      .lineNumber(line(parentUnit))
+      .columnNumber(column(parentUnit))
+    val valueAsts = withOrder(sizes) { (s, o) => astsForValue(s, o, parentUnit) }.flatten
+    Ast(callBlock)
+      .withChildren(valueAsts)
+      .withArgEdges(callBlock, valueAsts.flatMap(_.root))
   }
 
   private def astForUnaryExpr(

src/test/scala/com/github/plume/oss/JavaCompiler.scala

@@ -2,7 +2,7 @@ package com.github.plume.oss
 
 import java.io.File
 import java.util.Collections
-import javax.tools.{JavaCompiler, JavaFileObject, StandardLocation, ToolProvider}
+import javax.tools.{JavaCompiler => Javac, JavaFileObject, StandardLocation, ToolProvider}
 import scala.jdk.CollectionConverters
 
 /** Compiles a given source file.
@@ -39,7 +39,7 @@ object JavaCompiler {
 
   /** Programmatically obtains the system Java compiler.
     */
-  def getJavaCompiler: JavaCompiler = {
+  def getJavaCompiler: Javac = {
     Option(ToolProvider.getSystemJavaCompiler) match {
       case Some(javac) => javac
       case None        => throw new RuntimeException("Unable to find a Java compiler on the system!")