Fixed issues with parallel file reading/CPG writing (#58)

This problem came in as soon we applied the new X2CPG file reading.
ShiftLeftSecurity · Jan 5, 2022 · 1ea3f44 · 1ea3f44
1 parent 79311f6
commit 1ea3f44
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 21 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,4 +1,4 @@
-val cpgVersion   = "1.3.474"
+val cpgVersion   = "1.3.477"
 val joernVersion = "1.1.407"
 
 val gitCommitString = SettingKey[String]("gitSha")

diff --git a/src/main/scala/io/shiftleft/js2cpg/cpg/passes/AstCreationPass.scala b/src/main/scala/io/shiftleft/js2cpg/cpg/passes/AstCreationPass.scala
@@ -1,7 +1,6 @@
 package io.shiftleft.js2cpg.cpg.passes
 
 import java.nio.file.Path
-
 import better.files.File
 import com.oracle.js.parser.Source
 import com.oracle.js.parser.ir.FunctionNode
@@ -10,7 +9,7 @@ import io.shiftleft.js2cpg.core.Report
 import io.shiftleft.js2cpg.cpg.passes.astcreation.AstCreator
 import io.shiftleft.js2cpg.io.{FileUtils, JsFileChecks, TimeUtils}
 import io.shiftleft.js2cpg.parser.{JavaScriptParser, JsSource}
-import io.shiftleft.passes.{DiffGraph, IntervalKeyPool, ParallelCpgPass}
+import io.shiftleft.passes.{DiffGraph, IntervalKeyPool, ConcurrentWriterCpgPass}
 import org.slf4j.LoggerFactory
 import io.shiftleft.js2cpg.util.SourceWrapper._
 
@@ -25,19 +24,16 @@ class AstCreationPass(srcDir: File,
                       cpg: Cpg,
                       keyPool: IntervalKeyPool,
                       report: Report)
-    extends ParallelCpgPass[(Path, Path)](cpg, keyPools = Some(keyPool.split(filenames.size))) {
+    extends ConcurrentWriterCpgPass[(Path, Path)](cpg, keyPool = Some(keyPool)) {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
   private case class ParseResult(file: File, jsSource: JsSource, ast: FunctionNode)
 
-  override def partIterator: Iterator[(Path, Path)] = filenames.iterator
-
-  override def runOnPart(filename: (Path, Path)): Iterator[DiffGraph] = {
-    val diffGraph = DiffGraph.newBuilder
+  override def generateParts(): Array[(Path, Path)] = filenames.toArray
 
-    val file     = filename._1
-    val fileRoot = filename._2
+  override def runOnPart(diffGraph: DiffGraph.Builder, filename: (Path, Path)): Unit = {
+    val (file, fileRoot) = filename
 
     val parseResult = parse(file, fileRoot) match {
       case Failure(parseException) =>
@@ -47,36 +43,33 @@ class AstCreationPass(srcDir: File,
         Some((parseResult, preAnalyze(parseResult)))
     }
 
-    parseResult match {
-      case Some((parseResult, usedIdentNodes)) =>
+    parseResult.map {
+      case (parseResult, usedIdentNodes) =>
         val (result, duration) = {
-          TimeUtils.time(generateCpg(parseResult, diffGraph, usedIdentNodes))
+          TimeUtils.time(generateCpg(parseResult, DiffGraph.newBuilder, usedIdentNodes))
         }
         val path = parseResult.jsSource.originalFilePath
         result match {
           case Failure(exception) =>
             logger.warn(s"Failed to generate CPG for '$path'!", exception)
-            Iterator.empty
-          case Success(cpg) =>
+          case Success(localDiff) =>
             logger.info(s"Processed file '$path'")
             report.updateReportDuration(path, duration)
-            Iterator(cpg)
+            diffGraph.moveFrom(localDiff)
         }
-      case None =>
-        Iterator.empty
     }
   }
 
   private def generateCpg(parseResult: ParseResult,
                           diffGraph: DiffGraph.Builder,
-                          usedIdentNodes: Set[String]): Try[DiffGraph] = {
+                          usedIdentNodes: Set[String]): Try[DiffGraph.Builder] = {
     Try {
       val source = parseResult.jsSource
       val ast    = parseResult.ast
       logger.debug(s"Generating CPG for file '${source.originalFilePath}'.")
       val astBuilderPass = new AstCreator(diffGraph, source, usedIdentNodes)
       astBuilderPass.convert(ast)
-      diffGraph.build()
+      diffGraph
     }
   }
 
@@ -100,7 +93,7 @@ class AstCreationPass(srcDir: File,
     Try(JavaScriptParser.parseFromSource(jsSource)) match {
       case Failure(exception) =>
         report.addReportInfo(jsSource.originalFilePath, fileStatistics.linesOfCode)
-        throw exception
+        Failure(exception)
       case Success((ast, jsSource)) =>
         report.addReportInfo(jsSource.originalFilePath, fileStatistics.linesOfCode, parsed = true)
         Success(ParseResult(File(path), jsSource, ast))