Use yarn as default if available (#42)

Yarn has a much better performance (especially under Windows) compared to the plain old npm. We use this as default now if available. Also: * applied caching to PackageJsonParser * refactored TranspilingEnvironment
ShiftLeftSecurity · Dec 6, 2021 · 4e04458 · 4e04458
1 parent f47cb5c
commit 4e04458
Show file tree

Hide file tree

Showing 12 changed files with 127 additions and 109 deletions.
diff --git a/src/main/scala/io/shiftleft/js2cpg/cpg/passes/DependenciesPass.scala b/src/main/scala/io/shiftleft/js2cpg/cpg/passes/DependenciesPass.scala
@@ -22,7 +22,7 @@ class DependenciesPass(cpg: Cpg, config: Config, keyPool: KeyPool)
         config.createPathForPackageJson()).toSet
 
     val dependencies: Map[String, String] =
-      packagesJsons.flatMap(p => new PackageJsonParser(p).dependencies()).toMap
+      packagesJsons.flatMap(p => PackageJsonParser.dependencies(p)).toMap
 
     dependencies.foreach {
       case (name, version) =>

diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala
@@ -1,11 +1,11 @@
 package io.shiftleft.js2cpg.parser
 
 import java.nio.file.{Path, Paths}
-
 import io.shiftleft.js2cpg.io.FileUtils
 import org.slf4j.LoggerFactory
 import play.api.libs.json.Json
 
+import scala.collection.concurrent.TrieMap
 import scala.util.Using
 
 object PackageJsonParser {
@@ -20,56 +20,56 @@ object PackageJsonParser {
     "peerDependencies",
     "optionalDependencies"
   )
-}
 
-class PackageJsonParser(packageJsonPath: Path) {
-  import PackageJsonParser._
+  private val cachedDependencies: TrieMap[Path, Map[String, String]] = TrieMap.empty
 
-  def dependencies(): Map[String, String] = {
-    val depsPath     = packageJsonPath
-    val lockDepsPath = packageJsonPath.resolveSibling(Paths.get(PACKAGE_JSON_LOCK_FILENAME))
+  def dependencies(packageJsonPath: Path): Map[String, String] =
+    cachedDependencies.getOrElseUpdate(
+      packageJsonPath, {
+        val depsPath     = packageJsonPath
+        val lockDepsPath = packageJsonPath.resolveSibling(Paths.get(PACKAGE_JSON_LOCK_FILENAME))
 
-    val lockDeps = Using(FileUtils.bufferedSourceFromFile(lockDepsPath)) { bufferedSource =>
-      val content     = FileUtils.contentFromBufferedSource(bufferedSource)
-      val packageJson = Json.parse(content)
+        val lockDeps = Using(FileUtils.bufferedSourceFromFile(lockDepsPath)) { bufferedSource =>
+          val content     = FileUtils.contentFromBufferedSource(bufferedSource)
+          val packageJson = Json.parse(content)
 
-      (packageJson \ "dependencies")
-        .asOpt[Map[String, Map[String, String]]]
-        .map { versions =>
-          versions.map {
-            case (depName, entry) => depName -> entry("version")
-          }
-        }
-        .getOrElse(Map.empty)
-    }.toOption
+          (packageJson \ "dependencies")
+            .asOpt[Map[String, Map[String, String]]]
+            .map { versions =>
+              versions.map {
+                case (depName, entry) => depName -> entry("version")
+              }
+            }
+            .getOrElse(Map.empty)
+        }.toOption
 
-    // lazy val because we only evaluate this in case no package lock file is available.
-    lazy val deps = Using(FileUtils.bufferedSourceFromFile(depsPath)) { bufferedSource =>
-      val content     = FileUtils.contentFromBufferedSource(bufferedSource)
-      val packageJson = Json.parse(content)
+        // lazy val because we only evaluate this in case no package lock file is available.
+        lazy val deps = Using(FileUtils.bufferedSourceFromFile(depsPath)) { bufferedSource =>
+          val content     = FileUtils.contentFromBufferedSource(bufferedSource)
+          val packageJson = Json.parse(content)
 
-      projectDependencies
-        .flatMap { dependency =>
-          (packageJson \ dependency).asOpt[Map[String, String]]
-        }
-        .flatten
-        .toMap
-    }.toOption
+          projectDependencies
+            .flatMap { dependency =>
+              (packageJson \ dependency).asOpt[Map[String, String]]
+            }
+            .flatten
+            .toMap
+        }.toOption
 
-    if (lockDeps.isDefined && lockDeps.get.nonEmpty) {
-      logger.debug(s"Loaded dependencies from '$lockDepsPath'.")
-      lockDeps.get
-    } else {
-      if (deps.isDefined && deps.get.nonEmpty) {
-        logger.debug(s"Loaded dependencies from '$depsPath'.")
-        deps.get
-      } else {
-        logger.debug(
-          s"No project dependencies found in $PACKAGE_JSON_FILENAME or $PACKAGE_JSON_LOCK_FILENAME at '${depsPath.getParent}'.")
-        Map.empty
+        if (lockDeps.isDefined && lockDeps.get.nonEmpty) {
+          logger.debug(s"Loaded dependencies from '$lockDepsPath'.")
+          lockDeps.get
+        } else {
+          if (deps.isDefined && deps.get.nonEmpty) {
+            logger.debug(s"Loaded dependencies from '$depsPath'.")
+            deps.get
+          } else {
+            logger.debug(
+              s"No project dependencies found in $PACKAGE_JSON_FILENAME or $PACKAGE_JSON_LOCK_FILENAME at '${depsPath.getParent}'.")
+            Map.empty
+          }
+        }
       }
-    }
-
-  }
+    )
 
 }
diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala
@@ -13,12 +13,12 @@ class BabelTranspiler(override val config: Config,
                       override val projectPath: Path,
                       subDir: Option[Path] = None,
                       inDir: Option[Path] = None)
-    extends Transpiler
-    with NpmEnvironment {
+    extends Transpiler {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
-  override def shouldRun(): Boolean = config.babelTranspiling && !isVueProject
+  override def shouldRun(): Boolean =
+    config.babelTranspiling && !VueTranspiler.isVueProject(config, projectPath)
 
   private def constructIgnoreDirArgs: String = {
     val ignores = if (config.ignoreTests) {

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala
@@ -38,16 +38,15 @@ object NuxtTranspiler {
 }
 
 class NuxtTranspiler(override val config: Config, override val projectPath: Path)
-    extends Transpiler
-    with NpmEnvironment {
+    extends Transpiler {
 
   import NuxtTranspiler._
 
   private val logger = LoggerFactory.getLogger(getClass)
 
   private def isNuxtProject: Boolean =
-    new PackageJsonParser((File(projectPath) / PackageJsonParser.PACKAGE_JSON_FILENAME).path)
-      .dependencies()
+    PackageJsonParser
+      .dependencies((File(projectPath) / PackageJsonParser.PACKAGE_JSON_FILENAME).path)
       .contains("nuxt")
 
   override def shouldRun(): Boolean = config.nuxtTranspiling && isNuxtProject

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala
@@ -10,8 +10,7 @@ import java.nio.file.{Path, Paths}
 import scala.util.{Failure, Success}
 
 class PugTranspiler(override val config: Config, override val projectPath: Path)
-    extends Transpiler
-    with NpmEnvironment {
+    extends Transpiler {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
@@ -21,10 +20,10 @@ class PugTranspiler(override val config: Config, override val projectPath: Path)
   override def shouldRun(): Boolean = config.templateTranspiling && hasPugFiles
 
   private def installPugPlugins(): Boolean = {
-    val command = if ((File(projectPath) / "yarn.lock").exists) {
-      s"yarn add pug-cli --dev && ${NpmEnvironment.YARN_INSTALL}"
+    val command = if (yarnAvailable()) {
+      s"yarn add pug-cli --dev && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"npm install --save-dev pug-cli && ${NpmEnvironment.NPM_INSTALL}"
+      s"npm install --save-dev pug-cli && ${TranspilingEnvironment.NPM_INSTALL}"
     }
     logger.debug(s"\t+ Installing Pug plugins ...")
     ExternalCommand.run(command, projectPath.toString) match {

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/Transpiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/Transpiler.scala
@@ -8,7 +8,7 @@ import io.shiftleft.js2cpg.parser.PackageJsonParser
 
 import java.nio.file.Path
 
-trait Transpiler {
+trait Transpiler extends TranspilingEnvironment {
 
   protected val DEFAULT_IGNORED_DIRS: List[String] = List(
     "build",
@@ -37,17 +37,6 @@ trait Transpiler {
   protected val config: Config
   protected val projectPath: Path
 
-  private def hasVueFiles: Boolean =
-    FileUtils.getFileTree(projectPath, config, List(VUE_SUFFIX)).nonEmpty
-
-  protected def isVueProject: Boolean = {
-    val hasVueDep =
-      new PackageJsonParser((File(projectPath) / PackageJsonParser.PACKAGE_JSON_FILENAME).path)
-        .dependencies()
-        .contains("vue")
-    hasVueDep || hasVueFiles
-  }
-
   def shouldRun(): Boolean
 
   def validEnvironment(): Boolean

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala
@@ -28,23 +28,11 @@ case class TranspilerGroup(override val config: Config,
       "@babel/plugin-proposal-nullish-coalescing-operator " +
       "@babel/plugin-transform-property-mutators"
 
-  private def isYarnAvailable: Boolean = {
-    logger.debug("\t+ Checking yarn ...")
-    ExternalCommand.run("yarn -v", projectPath.toString) match {
-      case Success(result) =>
-        logger.debug(s"\t+ yarn is available: $result")
-        true
-      case Failure(_) =>
-        logger.error("\t- yarn is not installed. Transpiling sources will not be available.")
-        false
-    }
-  }
-
   private def installPlugins(): Boolean = {
-    val command = if ((File(projectPath) / "yarn.lock").exists && isYarnAvailable) {
-      s"yarn add $BABEL_PLUGINS --dev -W && ${NpmEnvironment.YARN_INSTALL}"
+    val command = if (yarnAvailable()) {
+      s"yarn add $BABEL_PLUGINS --dev -W && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"npm install --save-dev $BABEL_PLUGINS && ${NpmEnvironment.NPM_INSTALL}"
+      s"npm install --save-dev $BABEL_PLUGINS && ${TranspilingEnvironment.NPM_INSTALL}"
     }
     logger.info("Installing project dependencies and plugins. This might take a while.")
     logger.debug("\t+ Installing plugins ...")

diff --git a/...js2cpg/preprocessing/NpmEnvironment.scala → ...reprocessing/TranspilingEnvironment.scala b/...js2cpg/preprocessing/NpmEnvironment.scala → ...reprocessing/TranspilingEnvironment.scala
@@ -5,23 +5,37 @@ import org.slf4j.LoggerFactory
 
 import scala.util.{Failure, Success}
 
-object NpmEnvironment {
-  // This is in a singleton object because we want to check the environment only once
-  // even if multiple transpilers require this specific environment.
-  private var isValid: Option[Boolean] = None
+object TranspilingEnvironment {
+  // These are singleton objects because we want to check the environment only once
+  // even if multiple transpilers require this specific environment:
+  private var isValid: Option[Boolean]         = None
+  private var isYarnAvailable: Option[Boolean] = None
+  private var isNpmAvailable: Option[Boolean]  = None
 
   val YARN_INSTALL = "yarn install --prefer-offline --ignore-scripts"
   val NPM_INSTALL  = "npm install --prefer-offline --no-audit --progress=false --ignore-scripts"
 }
 
-trait NpmEnvironment {
+trait TranspilingEnvironment {
   self: Transpiler =>
 
-  import NpmEnvironment._
+  import TranspilingEnvironment._
 
   private val logger = LoggerFactory.getLogger(getClass)
 
-  private def isNpmAvailable: Boolean = {
+  private def checkForYarn(): Boolean = {
+    logger.debug("\t+ Checking yarn ...")
+    ExternalCommand.run("yarn -v", projectPath.toString) match {
+      case Success(result) =>
+        logger.debug(s"\t+ yarn is available: $result")
+        true
+      case Failure(_) =>
+        logger.error("\t- yarn is not installed. Transpiling sources will not be available.")
+        false
+    }
+  }
+
+  private def checkForNpm(): Boolean = {
     logger.debug(s"\t+ Checking npm ...")
     ExternalCommand.run("npm -v", projectPath.toString) match {
       case Success(result) =>
@@ -61,8 +75,24 @@ trait NpmEnvironment {
     case Some(value) =>
       value
     case None =>
-      isValid = Some(isNpmAvailable && setNpmPython())
+      isValid = Some((yarnAvailable() || npmAvailable()) && setNpmPython())
       isValid.get
   }
 
+  protected def yarnAvailable(): Boolean = isYarnAvailable match {
+    case Some(value) =>
+      value
+    case None =>
+      isYarnAvailable = Some(checkForYarn())
+      isYarnAvailable.get
+  }
+
+  protected def npmAvailable(): Boolean = isNpmAvailable match {
+    case Some(value) =>
+      value
+    case None =>
+      isNpmAvailable = Some(checkForNpm())
+      isNpmAvailable.get
+  }
+
 }
diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala
@@ -31,8 +31,7 @@ object TypescriptTranspiler {
 class TypescriptTranspiler(override val config: Config,
                            override val projectPath: Path,
                            subDir: Option[Path] = None)
-    extends Transpiler
-    with NpmEnvironment {
+    extends Transpiler {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
@@ -44,7 +43,10 @@ class TypescriptTranspiler(override val config: Config,
     FileUtils.getFileTree(projectPath, config, List(TS_SUFFIX)).nonEmpty
 
   override def shouldRun(): Boolean =
-    config.tsTranspiling && (File(projectPath) / "tsconfig.json").exists && hasTsFiles && !isVueProject
+    config.tsTranspiling &&
+      (File(projectPath) / "tsconfig.json").exists &&
+      hasTsFiles &&
+      !VueTranspiler.isVueProject(config, projectPath)
 
   private def moveIgnoredDirs(from: File, to: File): Unit = {
     val ignores = if (config.ignoreTests) {

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala
@@ -3,20 +3,38 @@ package io.shiftleft.js2cpg.preprocessing
 import better.files.File
 import io.shiftleft.js2cpg.core.Config
 import io.shiftleft.js2cpg.io.ExternalCommand
+import io.shiftleft.js2cpg.io.FileDefaults.VUE_SUFFIX
+import io.shiftleft.js2cpg.io.FileUtils
+import io.shiftleft.js2cpg.parser.PackageJsonParser
 import org.slf4j.LoggerFactory
 
 import java.nio.file.{Path, Paths}
 import scala.util.{Failure, Success}
 
+object VueTranspiler {
+
+  private def hasVueFiles(config: Config, projectPath: Path): Boolean =
+    FileUtils.getFileTree(projectPath, config, List(VUE_SUFFIX)).nonEmpty
+
+  def isVueProject(config: Config, projectPath: Path): Boolean = {
+    val hasVueDep =
+      PackageJsonParser
+        .dependencies((File(projectPath) / PackageJsonParser.PACKAGE_JSON_FILENAME).path)
+        .contains("vue")
+    hasVueDep || hasVueFiles(config, projectPath)
+  }
+}
+
 class VueTranspiler(override val config: Config, override val projectPath: Path)
-    extends Transpiler
-    with NpmEnvironment {
+    extends Transpiler {
+
+  import VueTranspiler.isVueProject
 
   private val logger = LoggerFactory.getLogger(getClass)
 
   private lazy val NODE_OPTIONS: Map[String, String] = nodeOptions()
 
-  override def shouldRun(): Boolean = config.vueTranspiling && isVueProject
+  override def shouldRun(): Boolean = config.vueTranspiling && isVueProject(config, projectPath)
 
   private def nodeOptions(): Map[String, String] = {
     // TODO: keep this until https://github.com/webpack/webpack/issues/14532 is fixed
@@ -28,10 +46,10 @@ class VueTranspiler(override val config: Config, override val projectPath: Path)
   }
 
   private def installVuePlugins(): Boolean = {
-    val command = if ((File(projectPath) / "yarn.lock").exists) {
-      s"yarn add @vue/cli-service-global --dev && ${NpmEnvironment.YARN_INSTALL}"
+    val command = if (yarnAvailable()) {
+      s"yarn add @vue/cli-service-global --dev && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"npm install --save-dev @vue/cli-service-global && ${NpmEnvironment.NPM_INSTALL}"
+      s"npm install --save-dev @vue/cli-service-global && ${TranspilingEnvironment.NPM_INSTALL}"
     }
     logger.debug(s"\t+ Installing Vue.js plugins ...")
     ExternalCommand.run(command, projectPath.toString, extraEnv = NODE_OPTIONS) match {

diff --git a/src/test/scala/io/shiftleft/js2cpg/io/FileUtilsTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/FileUtilsTest.scala
@@ -26,10 +26,8 @@ class FileUtilsTest extends AnyWordSpec with Matchers {
         (sourceDir / ".folder" / "e.js").createIfNotExists(createParents = true)
 
         File.usingTemporaryDirectory() { targetDir: File =>
-          val copiedDir = FileUtils.copyToDirectory(sourceDir, targetDir, Config())
-
+          val copiedDir  = FileUtils.copyToDirectory(sourceDir, targetDir, Config())
           val dirContent = FileUtils.getFileTree(copiedDir.path, Config(), List(JS_SUFFIX))
-
           dirContent shouldBe empty
         }
       }