diff --git a/build.sbt b/build.sbt index 0a1b0b84..a19c1dd0 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "chen" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "2.1.7" +ThisBuild / version := "2.1.8" ThisBuild / scalaVersion := "3.5.0" val cpgVersion = "1.0.0" diff --git a/codemeta.json b/codemeta.json index 1f7a0c66..07433bce 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,7 +7,7 @@ "downloadUrl": "https://github.com/AppThreat/chen", "issueTracker": "https://github.com/AppThreat/chen/issues", "name": "chen", - "version": "2.1.7", + "version": "2.1.8", "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.", "applicationCategory": "code-analysis", "keywords": [ diff --git a/meta.yaml b/meta.yaml index c5db20cd..1a951dc0 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,4 +1,4 @@ -{% set version = "2.1.7" %} +{% set version = "2.1.8" %} package: name: chen diff --git a/platform/frontends/jssrc2cpg/build.sbt b/platform/frontends/jssrc2cpg/build.sbt index 7ee51cf5..0d63a5b8 100644 --- a/platform/frontends/jssrc2cpg/build.sbt +++ b/platform/frontends/jssrc2cpg/build.sbt @@ -20,7 +20,6 @@ astGenVersion := appProperties.value.getString("jssrc2cpg.astgen_version") libraryDependencies ++= Seq( "io.appthreat" %% "cpg2" % Versions.cpg, "com.lihaoyi" %% "upickle" % Versions.upickle, - "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.2", "com.typesafe" % "config" % "1.4.3", "com.michaelpollmeier" % "versionsort" % "1.0.13", "org.scalatest" %% "scalatest" % Versions.scalatest % Test diff --git a/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/JsSrc2Cpg.scala b/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/JsSrc2Cpg.scala index b4fe6d70..1e95b72c 100644 --- a/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/JsSrc2Cpg.scala +++ b/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/JsSrc2Cpg.scala @@ -9,7 +9,6 @@ import io.appthreat.jssrc2cpg.passes.{ BuiltinTypesPass, ConfigPass, ConstClosurePass, - DependenciesPass, ImportResolverPass, ImportsPass, JavaScriptInheritanceNamePass, @@ -49,7 +48,6 @@ class JsSrc2Cpg extends X2CpgFrontend[Config]: new TypeNodePass(astCreationPass.allUsedTypes(), cpg).createAndApply() new JsMetaDataPass(cpg, hash, config.inputPath).createAndApply() new BuiltinTypesPass(cpg).createAndApply() - new DependenciesPass(cpg, config).createAndApply() new ConfigPass(cpg, config, report).createAndApply() new PrivateKeyFilePass(cpg, config, report).createAndApply() new ImportsPass(cpg).createAndApply() diff --git a/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/DependenciesPass.scala b/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/DependenciesPass.scala deleted file mode 100644 index 77b20e7b..00000000 --- a/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/passes/DependenciesPass.scala +++ /dev/null @@ -1,36 +0,0 @@ -package io.appthreat.jssrc2cpg.passes - -import io.appthreat.jssrc2cpg.Config -import io.appthreat.jssrc2cpg.utils.PackageJsonParser -import io.appthreat.x2cpg.SourceFiles -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.NewDependency -import io.shiftleft.passes.CpgPass - -import java.nio.file.Paths - -/** Creation of DEPENDENCY nodes from "package.json" files. - */ -class DependenciesPass(cpg: Cpg, config: Config) extends CpgPass(cpg): - - override def run(diffGraph: DiffGraphBuilder): Unit = - val packagesJsons = SourceFiles - .determine(config.inputPath, Set(".json")) - .filterNot(_.contains(Defines.NodeModulesFolder)) - .filter(f => - f.endsWith(PackageJsonParser.PackageJsonFilename) || f.endsWith( - PackageJsonParser.PackageJsonLockFilename - ) - ) - - val dependencies: Map[String, String] = - packagesJsons.flatMap(p => PackageJsonParser.dependencies(Paths.get(p))).toMap - - dependencies.foreach { case (name, version) => - val dep = NewDependency() - .name(name) - .version(version) - diffGraph.addNode(dep) - } - end run -end DependenciesPass diff --git a/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/utils/PackageJsonParser.scala b/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/utils/PackageJsonParser.scala deleted file mode 100644 index 4befda43..00000000 --- a/platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/utils/PackageJsonParser.scala +++ /dev/null @@ -1,93 +0,0 @@ -package io.appthreat.jssrc2cpg.utils - -import java.nio.file.{Path, Paths} -import org.slf4j.LoggerFactory -import com.fasterxml.jackson.databind.ObjectMapper -import io.shiftleft.utils.IOUtils -import org.apache.commons.lang.StringUtils - -import scala.collection.concurrent.TrieMap -import scala.util.Try -import scala.jdk.CollectionConverters.* -import scala.util.Failure -import scala.util.Success - -object PackageJsonParser: - private val logger = LoggerFactory.getLogger(PackageJsonParser.getClass) - - val PackageJsonFilename = "package.json" - val PackageJsonLockFilename = "package-lock.json" - - private val ProjectDependencies = - Seq("dependencies", "devDependencies", "peerDependencies", "optionalDependencies") - - private val cachedDependencies: TrieMap[Path, Map[String, String]] = TrieMap.empty - - def isValidProjectPackageJson(packageJsonPath: Path): Boolean = - if packageJsonPath.toString.endsWith(PackageJsonParser.PackageJsonFilename) then - val isNotEmpty = Try(IOUtils.readLinesInFile(packageJsonPath)) match - case Success(content) => - content.forall(l => StringUtils.isNotBlank(StringUtils.normalizeSpace(l))) - case Failure(_) => false - isNotEmpty && dependencies(packageJsonPath).nonEmpty - else - false - - def dependencies(packageJsonPath: Path): Map[String, String] = - cachedDependencies.getOrElseUpdate( - packageJsonPath, { - val depsPath = packageJsonPath - val lockDepsPath = packageJsonPath.resolveSibling(Paths.get(PackageJsonLockFilename)) - - val lockDeps = Try { - val content = IOUtils.readEntireFile(lockDepsPath) - val objectMapper = new ObjectMapper - val packageJson = objectMapper.readTree(content) - - var depToVersion = Map.empty[String, String] - val dependencyIt = Option(packageJson.get("dependencies")) - .map(_.fields().asScala) - .getOrElse(Iterator.empty) - dependencyIt.foreach { entry => - val depName = entry.getKey - val versionNode = entry.getValue.get("version") - if versionNode != null then - depToVersion = depToVersion.updated(depName, versionNode.asText()) - } - depToVersion - }.toOption - - // lazy val because we only evaluate this in case no package lock file is available. - lazy val deps = Try { - val content = IOUtils.readEntireFile(depsPath) - val objectMapper = new ObjectMapper - val packageJson = objectMapper.readTree(content) - - var depToVersion = Map.empty[String, String] - ProjectDependencies - .foreach { dependency => - val dependencyIt = Option(packageJson.get(dependency)) - .map(_.fields().asScala) - .getOrElse(Iterator.empty) - dependencyIt.foreach { entry => - depToVersion = - depToVersion.updated(entry.getKey, entry.getValue.asText()) - } - } - depToVersion - }.toOption - - if lockDeps.isDefined && lockDeps.get.nonEmpty then - logger.debug(s"Loaded dependencies from '$lockDepsPath'.") - lockDeps.get - else if deps.isDefined && deps.get.nonEmpty then - logger.debug(s"Loaded dependencies from '$depsPath'.") - deps.get - else - logger.debug( - s"No project dependencies found in $PackageJsonFilename or $PackageJsonLockFilename at '${depsPath.getParent}'." - ) - Map.empty - } - ) -end PackageJsonParser diff --git a/platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/DependenciesPassTest.scala b/platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/DependenciesPassTest.scala deleted file mode 100644 index b94e30d3..00000000 --- a/platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/passes/DependenciesPassTest.scala +++ /dev/null @@ -1,151 +0,0 @@ -package io.appthreat.jssrc2cpg.passes - -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.semanticcpg.language._ -import better.files.File -import io.appthreat.jssrc2cpg.Config -import io.appthreat.jssrc2cpg.utils.PackageJsonParser -import io.appthreat.x2cpg.X2Cpg.newEmptyCpg - -class DependenciesPassTest extends AbstractPassTest { - - "DependenciesPass" should { - - "ignore empty package.json" in { - File.usingTemporaryDirectory("jssrc2cpgTest") { dir => - val json = dir / PackageJsonParser.PackageJsonFilename - json.write("") - PackageJsonParser.isValidProjectPackageJson(json.path) shouldBe false - } - } - - "ignore package.json without any useful content" in { - File.usingTemporaryDirectory("jssrc2cpgTest") { dir => - val json = dir / PackageJsonParser.PackageJsonFilename - json.write(""" - |{ - | "name": "something", - | "version": "0.1.0", - | "description": "foobar", - | "main": "./target_node/index.js", - | "private": true - |} - |""".stripMargin) - PackageJsonParser.isValidProjectPackageJson(json.path) shouldBe false - } - } - - "ignore package.json without dependencies" in { - File.usingTemporaryDirectory("jssrc2cpgTest") { dir => - val json = dir / PackageJsonParser.PackageJsonFilename - json.write("{}") - PackageJsonParser.isValidProjectPackageJson(json.path) shouldBe false - } - } - - "generate dependency nodes correctly (no dependencies at all)" in DependencyFixture("", "{}") { cpg => - cpg.dependency.size shouldBe 0 - } - - "generate dependency nodes correctly (empty dependency)" in DependencyFixture( - "", - """ - |{ - | "dependencies": { - | } - |} - |""".stripMargin - ) { cpg => - cpg.dependency.size shouldBe 0 - } - - "generate dependency nodes correctly (simple lock dependencies)" in DependencyFixture( - code = "", - packageJsonContent = """ - |{ - | "dependencies": { - | "dep1": { - | "version": "0.1" - | }, - | "dep2": { - | "version": "0.2" - | } - | } - |} - |""".stripMargin, - packageJsonName = PackageJsonParser.PackageJsonLockFilename - ) { cpg => - val List(depA, depB) = cpg.dependency.l - depA.name shouldBe "dep1" - depA.version shouldBe "0.1" - depB.name shouldBe "dep2" - depB.version shouldBe "0.2" - } - - "generate dependency nodes correctly (simple dependency)" in DependencyFixture( - code = "", - packageJsonContent = """ - |{ - | "dependencies": { - | "dep1": "0.1" - | } - |} - |""".stripMargin - ) { cpg => - val List(depA) = cpg.dependency.l - depA.name shouldBe "dep1" - depA.version shouldBe "0.1" - } - - "generate dependency nodes correctly (different types of dependencies)" in DependencyFixture( - code = "", - packageJsonContent = """ - { - "dependencies": { - "dep1": "0.1" - }, - "devDependencies": { - "dep2": "0.2" - }, - "peerDependencies": { - "dep3": "0.3" - }, - "optionalDependencies": { - "dep4": "0.4" - } - } - """.stripMargin - ) { cpg => - val List(depA, depB, depC, depD) = cpg.dependency.l - depA.name shouldBe "dep1" - depA.version shouldBe "0.1" - depB.name shouldBe "dep2" - depB.version shouldBe "0.2" - depC.name shouldBe "dep3" - depC.version shouldBe "0.3" - depD.name shouldBe "dep4" - depD.version shouldBe "0.4" - } - - } - - private object DependencyFixture extends Fixture { - def apply( - code: String, - packageJsonContent: String, - packageJsonName: String = PackageJsonParser.PackageJsonFilename - )(f: Cpg => Unit): Unit = { - File.usingTemporaryDirectory("jssrc2cpgTest") { dir => - val file = dir / "file.js" - val json = dir / packageJsonName - file.write(code) - json.write(packageJsonContent) - val cpg = newEmptyCpg() - val config = Config().withInputPath(dir.toString()) - new DependenciesPass(cpg, config).createAndApply() - f(cpg) - } - } - } - -} diff --git a/pyproject.toml b/pyproject.toml index 781a2e8d..84807c60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appthreat-chen" -version = "2.1.7" +version = "2.1.8" description = "Code Hierarchy Exploration Net (chen)" authors = ["Team AppThreat "] license = "Apache-2.0"