Fix OS specific external command execution (#55)

* We do not need to run them through cmd or sh if we create an OS-specific command beforehand. This removes several layers of indirection (e.g., Git bash -> WSL1 or WSL2 -> cmd on Windows) and should improve the speed a lot. * We need typescript explicitly for subprojects.
ShiftLeftSecurity · Dec 28, 2021 · 2d3802f · 2d3802f
1 parent d57fdc9
commit 2d3802f
Show file tree

Hide file tree

Showing 12 changed files with 110 additions and 92 deletions.
diff --git a/src/main/scala/io/shiftleft/js2cpg/io/ExternalCommand.scala b/src/main/scala/io/shiftleft/js2cpg/io/ExternalCommand.scala
@@ -2,32 +2,25 @@ package io.shiftleft.js2cpg.io
 
 import java.io
 
-import org.slf4j.LoggerFactory
-
 import scala.collection.mutable
 import scala.sys.process.{Process, ProcessLogger}
 import scala.util.{Failure, Success, Try}
 
 object ExternalCommand {
-  private val logger              = LoggerFactory.getLogger(ExternalCommand.getClass)
-  private val windowsSystemPrefix = "Windows"
-  private val osNameProperty      = "os.name"
+
+  private val COMMAND_AND: String = " && "
+  private val IS_WIN: Boolean     = scala.util.Properties.isWin
+
+  def toOSCommand(command: String): String = if (IS_WIN) command + ".cmd" else command
 
   def run(command: String,
           inDir: String = ".",
           extraEnv: Map[String, String] = Map.empty): Try[String] = {
     val result                      = mutable.ArrayBuffer.empty[String]
-    val lineHandler: String => Unit = line => result.addOne(line)
-    val systemString                = System.getProperty(osNameProperty)
-    val shellPrefix =
-      if (systemString != null && systemString.startsWith(windowsSystemPrefix)) {
-        "cmd" :: "/c" :: Nil
-      } else {
-        "sh" :: "-c" :: Nil
-      }
-
-    Process(shellPrefix :+ command, new io.File(inDir), extraEnv.toList: _*)
-      .!(ProcessLogger(lineHandler, lineHandler)) match {
+    val lineHandler: String => Unit = line => result += line
+    val logger                      = ProcessLogger(lineHandler, lineHandler)
+    val commands                    = command.split(COMMAND_AND).toSeq
+    commands.map(Process(_, new io.File(inDir), extraEnv.toList: _*).!(logger)).sum match {
       case 0 =>
         Success(result.mkString(System.lineSeparator()))
       case _ =>

diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala
@@ -13,7 +13,7 @@ object TsConfigJsonParser {
   private val logger = LoggerFactory.getLogger(TsConfigJsonParser.getClass)
 
   def module(projectPath: Path, tsc: String): String = {
-    ExternalCommand.run(s"$tsc --showConfig", projectPath.toString) match {
+    ExternalCommand.run(s"${ExternalCommand.toOSCommand(tsc)} --showConfig", projectPath.toString) match {
       case Success(tsConfig) =>
         val json = Json.parse(tsConfig)
         val moduleOption = (json \ "compilerOptions" \ "module")
@@ -31,7 +31,8 @@ object TsConfigJsonParser {
 
   def isSolutionTsConfig(projectPath: Path, tsc: String): Boolean = {
     // a solution tsconfig is one with 0 files and at least one reference, see https://angular.io/config/solution-tsconfig
-    ExternalCommand.run(s"$tsc --listFilesOnly", projectPath.toString) match {
+    ExternalCommand.run(s"${ExternalCommand.toOSCommand(tsc)} --listFilesOnly",
+                        projectPath.toString) match {
       case Success(files) =>
         files.isEmpty
       case Failure(exception) =>
@@ -42,7 +43,7 @@ object TsConfigJsonParser {
   }
 
   def subprojects(projectPath: Path, tsc: String): List[String] = {
-    ExternalCommand.run(s"$tsc --showConfig", projectPath.toString) match {
+    ExternalCommand.run(s"${ExternalCommand.toOSCommand(tsc)} --showConfig", projectPath.toString) match {
       case Success(config) =>
         val json = Json.parse(config)
 

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala
@@ -34,8 +34,8 @@ class BabelTranspiler(override val config: Config,
     val outDir =
       subDir.map(s => File(tmpTranspileDir.toString, s.toString)).getOrElse(File(tmpTranspileDir))
 
-    val babel = Paths.get(projectPath.toString, "node_modules", ".bin", "babel")
-    val command = s"$babel . " +
+    val babel = Paths.get(projectPath.toString, "node_modules", ".bin", "babel").toString
+    val command = s"${ExternalCommand.toOSCommand(babel)} . " +
       "--no-babelrc " +
       s"--source-root '${in.toString}' " +
       "--source-maps true " +

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala
@@ -52,8 +52,8 @@ class NuxtTranspiler(override val config: Config, override val projectPath: Path
   override def shouldRun(): Boolean = config.nuxtTranspiling && isNuxtProject
 
   override protected def transpile(tmpTranspileDir: Path): Boolean = {
-    val nuxt    = Paths.get(projectPath.toString, "node_modules", ".bin", "nuxt")
-    val command = s"$nuxt --force-exit"
+    val nuxt    = Paths.get(projectPath.toString, "node_modules", ".bin", "nuxt").toString
+    val command = s"${ExternalCommand.toOSCommand(nuxt)} --force-exit"
     logger.debug(s"\t+ Nuxt.js transpiling $projectPath")
     ExternalCommand.run(command, projectPath.toString) match {
       case Success(_) =>

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala
@@ -21,9 +21,9 @@ class PugTranspiler(override val config: Config, override val projectPath: Path)
 
   private def installPugPlugins(): Boolean = {
     val command = if (yarnAvailable()) {
-      s"yarn add pug-cli --dev --legacy-peer-deps && ${TranspilingEnvironment.YARN_INSTALL}"
+      s"${TranspilingEnvironment.YARN_ADD} pug-cli --dev && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"npm install --save-dev pug-cli --legacy-peer-deps && ${TranspilingEnvironment.NPM_INSTALL}"
+      s"${TranspilingEnvironment.NPM_INSTALL} --save-dev pug-cli && ${TranspilingEnvironment.NPM_INSTALL}"
     }
     logger.info("Installing Pug dependencies and plugins. That will take a while.")
     logger.debug(s"\t+ Installing Pug plugins with command '$command' in path '$projectPath'")
@@ -39,8 +39,9 @@ class PugTranspiler(override val config: Config, override val projectPath: Path)
 
   override protected def transpile(tmpTranspileDir: Path): Boolean = {
     if (installPugPlugins()) {
-      val pug     = Paths.get(projectPath.toString, "node_modules", ".bin", "pug")
-      val command = s"$pug --client --no-debug --out $tmpTranspileDir ."
+      val pug = Paths.get(projectPath.toString, "node_modules", ".bin", "pug").toString
+      val command =
+        s"${ExternalCommand.toOSCommand(pug)} --client --no-debug --out $tmpTranspileDir ."
       logger.debug(s"\t+ transpiling Pug templates in $projectPath to $tmpTranspileDir")
       ExternalCommand.run(command, projectPath.toString) match {
         case Success(_) =>

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala
@@ -30,9 +30,9 @@ case class TranspilerGroup(override val config: Config,
 
   private def installPlugins(): Boolean = {
     val command = if (yarnAvailable()) {
-      s"yarn add $BABEL_PLUGINS --dev -W --legacy-peer-deps && ${TranspilingEnvironment.YARN_INSTALL}"
+      s"${TranspilingEnvironment.YARN_ADD} $BABEL_PLUGINS --dev -W && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"npm install --save-dev $BABEL_PLUGINS --legacy-peer-deps && ${TranspilingEnvironment.NPM_INSTALL}"
+      s"${TranspilingEnvironment.NPM_INSTALL} --save-dev $BABEL_PLUGINS && ${TranspilingEnvironment.NPM_INSTALL}"
     }
     logger.info("Installing project dependencies and plugins. That will take a while.")
     logger.debug(s"\t+ Installing plugins with command '$command' in path '$projectPath'")

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala
@@ -13,9 +13,15 @@ object TranspilingEnvironment {
   private var isYarnAvailable: Option[Boolean] = None
   private var isNpmAvailable: Option[Boolean]  = None
 
-  val YARN_INSTALL = "yarn install --prefer-offline --ignore-scripts --legacy-peer-deps"
-  val NPM_INSTALL =
-    "npm install --prefer-offline --no-audit --progress=false --ignore-scripts --legacy-peer-deps"
+  val YARN: String = ExternalCommand.toOSCommand("yarn")
+  val NPM: String  = ExternalCommand.toOSCommand("npm")
+
+  val YARN_ADD: String =
+    s"$YARN} add --prefer-offline --ignore-scripts --legacy-peer-deps"
+  val YARN_INSTALL: String =
+    s"$YARN} install --prefer-offline --ignore-scripts --legacy-peer-deps"
+  val NPM_INSTALL: String =
+    s"$NPM install --prefer-offline --no-audit --progress=false --ignore-scripts --legacy-peer-deps"
 }
 
 trait TranspilingEnvironment {
@@ -27,7 +33,7 @@ trait TranspilingEnvironment {
 
   private def checkForYarn(): Boolean = {
     logger.debug("\t+ Checking yarn ...")
-    ExternalCommand.run("yarn -v", projectPath.toString) match {
+    ExternalCommand.run(s"${TranspilingEnvironment.YARN} -v", projectPath.toString) match {
       case Success(result) =>
         logger.debug(s"\t+ yarn is available: $result")
         true
@@ -39,7 +45,7 @@ trait TranspilingEnvironment {
 
   private def checkForNpm(): Boolean = {
     logger.debug(s"\t+ Checking npm ...")
-    ExternalCommand.run("npm -v", projectPath.toString) match {
+    ExternalCommand.run(s"${TranspilingEnvironment.NPM} -v", projectPath.toString) match {
       case Success(result) =>
         logger.debug(s"\t+ npm is available: $result")
         true
@@ -51,7 +57,8 @@ trait TranspilingEnvironment {
 
   private def setNpmPython(): Boolean = {
     logger.debug("\t+ Setting npm config ...")
-    ExternalCommand.run("npm config set python python2.7", projectPath.toString) match {
+    ExternalCommand.run(s"${TranspilingEnvironment.NPM} config set python python2.7",
+                        projectPath.toString) match {
       case Success(_) =>
         logger.debug("\t+ Set successfully")
         true

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala
@@ -36,7 +36,7 @@ class TypescriptTranspiler(override val config: Config,
 
   private val NODE_OPTIONS: Map[String, String] = Map("NODE_OPTIONS" -> "--max_old_space_size=4096")
 
-  private val tsc = Paths.get(projectPath.toString, "node_modules", ".bin", "tsc")
+  private val tsc = Paths.get(projectPath.toString, "node_modules", ".bin", "tsc").toString
 
   private def hasTsFiles: Boolean =
     FileUtils.getFileTree(projectPath, config, List(TS_SUFFIX)).nonEmpty
@@ -96,57 +96,80 @@ class TypescriptTranspiler(override val config: Config,
     }
   }
 
+  private def installTsPlugins(): Boolean = {
+    val command = if (yarnAvailable()) {
+      s"${TranspilingEnvironment.YARN_ADD} typescript --dev"
+    } else {
+      s"${TranspilingEnvironment.NPM_INSTALL} --save-dev typescript"
+    }
+    logger.info("Installing TypeScript dependencies and plugins. That will take a while.")
+    logger.debug(
+      s"\t+ Installing Typescript plugins with command '$command' in path '$projectPath'")
+    ExternalCommand.run(command, projectPath.toString, extraEnv = NODE_OPTIONS) match {
+      case Success(_) =>
+        logger.info("\t+ TypeScript plugins installed")
+        true
+      case Failure(exception) =>
+        logger.error("\t- Failed to install TypeScript plugins", exception)
+        false
+    }
+  }
+
   override protected def transpile(tmpTranspileDir: Path): Boolean = {
-    File.usingTemporaryDirectory() { tmpForIgnoredDirs =>
-      // Sadly, tsc does not allow to exclude folders when being run from cli.
-      // Hence, we have to move ignored folders to a temporary folder ...
-      moveIgnoredDirs(File(projectPath), tmpForIgnoredDirs)
-
-      val isSolutionTsConfig = TsConfigJsonParser.isSolutionTsConfig(projectPath, tsc.toString)
-      val projects = if (isSolutionTsConfig) {
-        TsConfigJsonParser.subprojects(projectPath, tsc.toString)
-      } else {
-        "" :: Nil
-      }
+    if (installTsPlugins()) {
+      File.usingTemporaryDirectory() { tmpForIgnoredDirs =>
+        // Sadly, tsc does not allow to exclude folders when being run from cli.
+        // Hence, we have to move ignored folders to a temporary folder ...
+        moveIgnoredDirs(File(projectPath), tmpForIgnoredDirs)
+
+        val isSolutionTsConfig = TsConfigJsonParser.isSolutionTsConfig(projectPath, tsc)
+        val projects = if (isSolutionTsConfig) {
+          TsConfigJsonParser.subprojects(projectPath, tsc)
+        } else {
+          "" :: Nil
+        }
 
-      val module = config.moduleMode.getOrElse(TsConfigJsonParser.module(projectPath, tsc.toString))
-      val outDir =
-        subDir.map(s => File(tmpTranspileDir.toString, s.toString)).getOrElse(File(tmpTranspileDir))
+        val module = config.moduleMode.getOrElse(TsConfigJsonParser.module(projectPath, tsc))
+        val outDir =
+          subDir
+            .map(s => File(tmpTranspileDir.toString, s.toString))
+            .getOrElse(File(tmpTranspileDir))
+
+        for (proj <- projects) {
+          val projCommand = if (proj.nonEmpty) {
+            s"--project $proj"
+          } else {
+            // for the root project we try to create a custom tsconfig file to ignore settings that may be there
+            // and that we sadly cannot override with tsc directly:
+            createCustomTsConfigFile() match {
+              case Failure(f) =>
+                logger.debug("\t- Creating a custom TS config failed", f)
+                ""
+              case Success(customTsConfigFile) => s"--project $customTsConfigFile"
+            }
+          }
 
-      for (proj <- projects) {
-        val projCommand = if (proj.nonEmpty) {
-          s"--project $proj"
-        } else {
-          // for the root project we try to create a custom tsconfig file to ignore settings that may be there
-          // and that we sadly cannot override with tsc directly:
-          createCustomTsConfigFile() match {
-            case Failure(f) =>
-              logger.debug("\t- Creating a custom TS config failed", f)
-              ""
-            case Success(customTsConfigFile) => s"--project $customTsConfigFile"
+          val projOutDir =
+            if (proj.nonEmpty) outDir / proj.substring(0, proj.lastIndexOf("/")) else outDir
+          val sourceRoot =
+            if (proj.nonEmpty)
+              s"--sourceRoot ${File(projectPath) / proj.substring(0, proj.lastIndexOf("/"))}"
+            else ""
+
+          val command =
+            s"${ExternalCommand.toOSCommand(tsc)} -sourcemap $sourceRoot --outDir $projOutDir -t ES2015 -m $module --jsx react --noEmit false $projCommand"
+          logger.debug(
+            s"\t+ TypeScript compiling $projectPath $projCommand to $projOutDir (using $module style modules)")
+
+          ExternalCommand.run(command, projectPath.toString, extraEnv = NODE_OPTIONS) match {
+            case Success(_)         => logger.debug("\t+ TypeScript compiling finished")
+            case Failure(exception) => logger.debug("\t- TypeScript compiling failed", exception)
           }
         }
 
-        val projOutDir =
-          if (proj.nonEmpty) outDir / proj.substring(0, proj.lastIndexOf("/")) else outDir
-        val sourceRoot =
-          if (proj.nonEmpty)
-            s"--sourceRoot ${File(projectPath) / proj.substring(0, proj.lastIndexOf("/"))}"
-          else ""
-
-        val command =
-          s"$tsc -sourcemap $sourceRoot --outDir $projOutDir -t ES2015 -m $module --jsx react --noEmit false $projCommand"
-        logger.debug(
-          s"\t+ TypeScript compiling $projectPath $projCommand to $projOutDir (using $module style modules)")
-
-        ExternalCommand.run(command, projectPath.toString, extraEnv = NODE_OPTIONS) match {
-          case Success(_)         => logger.debug("\t+ TypeScript compiling finished")
-          case Failure(exception) => logger.debug("\t- TypeScript compiling failed", exception)
-        }
+        // ... and copy them back afterward.
+        moveIgnoredDirs(tmpForIgnoredDirs, File(projectPath))
       }
-
-      // ... and copy them back afterward.
-      moveIgnoredDirs(tmpForIgnoredDirs, File(projectPath))
     }
     true
   }

diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala
@@ -47,9 +47,9 @@ class VueTranspiler(override val config: Config, override val projectPath: Path)
 
   private def installVuePlugins(): Boolean = {
     val command = if (yarnAvailable()) {
-      s"yarn add @vue/cli-service-global --dev --legacy-peer-deps && ${TranspilingEnvironment.YARN_INSTALL}"
+      s"${TranspilingEnvironment.YARN_ADD} @vue/cli-service-global --dev && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"npm install --save-dev @vue/cli-service-global --legacy-peer-deps && ${TranspilingEnvironment.NPM_INSTALL}"
+      s"${TranspilingEnvironment.NPM_INSTALL} --save-dev @vue/cli-service-global && ${TranspilingEnvironment.NPM_INSTALL}"
     }
     logger.info("Installing Vue.js dependencies and plugins. That will take a while.")
     logger.debug(s"\t+ Installing Vue.js plugins with command '$command' in path '$projectPath'")
@@ -65,8 +65,9 @@ class VueTranspiler(override val config: Config, override val projectPath: Path)
 
   override protected def transpile(tmpTranspileDir: Path): Boolean = {
     if (installVuePlugins()) {
-      val vue     = Paths.get(projectPath.toString, "node_modules", ".bin", "vue-cli-service")
-      val command = s"$vue build --dest $tmpTranspileDir --mode development --no-clean"
+      val vue = Paths.get(projectPath.toString, "node_modules", ".bin", "vue-cli-service").toString
+      val command =
+        s"${ExternalCommand.toOSCommand(vue)} build --dest $tmpTranspileDir --mode development --no-clean"
       logger.debug(s"\t+ Vue.js transpiling $projectPath to $tmpTranspileDir")
       ExternalCommand.run(command, projectPath.toString, extraEnv = NODE_OPTIONS) match {
         case Success(_)         => logger.debug("\t+ Vue.js transpiling finished")

diff --git a/src/test/resources/vue2/package.json b/src/test/resources/vue2/package.json
@@ -3,9 +3,5 @@
   "version": "0.1.0",
   "dependencies": {
     "vue": "^2.6.14"
-  },
-  "devDependencies": {
-    "@vue/cli": "^4.5.13",
-    "vue-template-compiler": "^2.6.14"
   }
 }
diff --git a/src/test/resources/vue3/babel.config.js b/src/test/resources/vue3/babel.config.js
diff --git a/src/test/resources/vue3/vue.config.js b/src/test/resources/vue3/vue.config.js
@@ -1,4 +1,3 @@
-// eslint-disable-next-line @typescript-eslint/no-var-requires
 const { defineConfig } = require('@vue/cli-service');
 module.exports = defineConfig({
   transpileDependencies: true,