From 723396b8953b17bf669531116f283947c38d5da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:05:58 +0100 Subject: [PATCH] Do not match the projects root folder name with ignored folders (#263) Also: ignore eslint related dependencies and config. For: https://shiftleftinc.atlassian.net/browse/SEN-670 --- .../scala/io/shiftleft/js2cpg/io/PathFilter.scala | 1 + .../js2cpg/parser/PackageJsonParser.scala | 4 +++- .../js2cpg/preprocessing/TranspilationRunner.scala | 9 ++++++--- .../preprocessing/TypescriptTranspiler.scala | 14 ++++++++------ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala b/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala index fe661f9c8..3c45694d3 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala @@ -29,6 +29,7 @@ case class PathFilter( private def filterDir(dir: Path): FilterResult = { val relDir = rootPath.relativize(dir) Paths.get(dir.toString.replace(rootPath.toString, projectDir)) match { + case dirPath if dirPath.toString == projectDir => Accepted() case dirPath if IGNORED_FOLDERS_REGEX.exists(_.matches(File(dirPath).name)) && !acceptFromNodeModulesFolder(dirPath) => diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala index 4c4b006a9..380cb83d0 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala @@ -27,6 +27,7 @@ object PackageJsonParser { val YARN_LOCK_FILENAME: String = "yarn.lock" val YARN_LOCK_FILENAME_BAK: String = "yarn.lock.bak" val WEBPACK_CONFIG_FILENAME: String = "webpack.config.js" + val ES_LINT_RC_FILENAME: String = ".eslintrc.js" val PROJECT_CONFIG_FILES: List[String] = List( JSON_LOCK_FILENAME, @@ -36,7 +37,8 @@ object PackageJsonParser { PNPM_WS_FILENAME, NPM_SHRINKWRAP_FILENAME, WEBPACK_CONFIG_FILENAME, - ANGULAR_JSON_FILENAME + ANGULAR_JSON_FILENAME, + ES_LINT_RC_FILENAME ) val PROJECT_DEPENDENCIES: Seq[String] = Seq( diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala index 6c5deb52f..338751f40 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala @@ -24,7 +24,7 @@ class TranspilationRunner(projectPath: Path, tmpTranspileDir: Path, config: Conf private val transpilers: Seq[Transpiler] = createTranspilers() - private val DEPS_TO_KEEP: List[String] = List("@vue", "vue", "nuxt", "eslint", "@typescript-eslint") + private val DEPS_TO_KEEP: List[String] = List("@vue", "vue", "nuxt") private def createTranspilers(): Seq[Transpiler] = { // We always run the following transpilers by default when not stated otherwise in the Config. @@ -115,6 +115,9 @@ class TranspilationRunner(projectPath: Path, tmpTranspileDir: Path, config: Conf } } + private def shouldKeepDependency(dep: String): Boolean = + DEPS_TO_KEEP.exists(dep.startsWith) && !dep.contains("eslint") + private def withTemporaryPackageJson(workUnit: () => Unit): Unit = { val packageJson = File(projectPath) / PackageJsonParser.PACKAGE_JSON_FILENAME if (config.optimizeDependencies && packageJson.exists) { @@ -138,11 +141,11 @@ class TranspilationRunner(projectPath: Path, tmpTranspileDir: Path, config: Conf .fieldNames() .asScala .toSet - .filterNot(f => DEPS_TO_KEEP.exists(f.startsWith)) + .filterNot(shouldKeepDependency) fieldsToRemove.foreach(depNode.remove) case Some(depNode: ArrayNode) => val allFields = depNode.elements().asScala.toSet - val fieldsToRemove = allFields.filterNot(f => DEPS_TO_KEEP.exists(f.asText().startsWith)) + val fieldsToRemove = allFields.filterNot(f => shouldKeepDependency(f.asText())) val remainingElements = allFields -- fieldsToRemove depNode.removeAll() remainingElements.foreach(depNode.add) diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala index d87b764f0..955b3f713 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala @@ -72,9 +72,9 @@ class TypescriptTranspiler(override val config: Config, override val projectPath } private def createCustomTsConfigFile(): Try[File] = { - val customTsConfigFilePath = (File(projectPath) / "tsconfig.json").path + val tsConfigFilePath = (File(projectPath) / "tsconfig.json").path Try { - val content = IOUtils.readLinesInFile(customTsConfigFilePath).mkString("\n") + val content = IOUtils.readLinesInFile(tsConfigFilePath).mkString("\n") val mapper = new ObjectMapper() val json = mapper.readTree(PackageJsonParser.removeComments(content)) Option(json.get("compilerOptions")).foreach { options => @@ -84,10 +84,7 @@ class TypescriptTranspiler(override val config: Config, override val projectPath } // --include is not available as tsc CLI argument; we set it manually: json.asInstanceOf[ObjectNode].putArray("include").add("**/*") - val customTsConfigFile = - File - .newTemporaryFile("js2cpgTsConfig", ".json", parent = Some(projectPath)) - .deleteOnExit(swallowIOExceptions = true) + val customTsConfigFile = File.newTemporaryFile("js2cpgTsConfig", ".json", parent = Some(projectPath)) customTsConfigFile.writeText(mapper.writeValueAsString(json)) } } @@ -178,6 +175,11 @@ class TypescriptTranspiler(override val config: Config, override val projectPath // ... and copy them back afterward. moveIgnoredDirs(tmpForIgnoredDirs, File(projectPath)) + // ... and remove the temporary tsconfig.json + File(projectPath) + .walk(maxDepth = 1) + .find(_.toString().contains("js2cpgTsConfig")) + .foreach(_.delete(swallowIOExceptions = true)) } } true