From dcd9916e2ff070760d89e511c0a2367c64cc5044 Mon Sep 17 00:00:00 2001 From: Markus Lottmann Date: Thu, 6 Jan 2022 19:25:34 +0100 Subject: [PATCH] Replace play json library with jackson library. Reason: No Scala 3 support for play json in sight any time soon. --- build.sbt | 3 +- .../js2cpg/parser/PackageJsonParser.scala | 49 +++++++++++++------ .../js2cpg/parser/TsConfigJsonParser.scala | 23 +++++---- .../preprocessing/TypescriptTranspiler.scala | 21 +++----- 4 files changed, 54 insertions(+), 42 deletions(-) diff --git a/build.sbt b/build.sbt index 56a0df9c9..ef0043b0b 100644 --- a/build.sbt +++ b/build.sbt @@ -88,8 +88,7 @@ lazy val commonSettings = Seq( "com.github.pathikrit" %% "better-files" % "3.9.1", "org.slf4j" % "slf4j-api" % "1.7.32", "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.17.0" % Runtime, - "com.typesafe.play" %% "play-json" % "2.9.2", - "com.fasterxml.jackson" % "jackson-base" % "2.13.1", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.1", "com.atlassian.sourcemap" % "sourcemap" % "2.0.0", "commons-io" % "commons-io" % "2.11.0", "io.shiftleft" %% "semanticcpg" % cpgVersion % Test classifier "tests", diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala index cbe394506..f36b45768 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala @@ -3,11 +3,14 @@ package io.shiftleft.js2cpg.parser import java.nio.file.{Path, Paths} import io.shiftleft.js2cpg.io.FileUtils import org.slf4j.LoggerFactory -import play.api.libs.json.Json +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.JsonNode import scala.collection.concurrent.TrieMap import scala.util.Try import scala.util.Using +import scala.jdk.CollectionConverters._ object PackageJsonParser { private val logger = LoggerFactory.getLogger(PackageJsonParser.getClass) @@ -31,27 +34,43 @@ object PackageJsonParser { val lockDepsPath = packageJsonPath.resolveSibling(Paths.get(PACKAGE_JSON_LOCK_FILENAME)) val lockDeps = Try { - val content = FileUtils.readLinesInFile(lockDepsPath).mkString("\n") - (Json.parse(content) \ "dependencies") - .asOpt[Map[String, Map[String, String]]] - .map { versions => - versions.map { - case (depName, entry) => depName -> entry("version") + val content = FileUtils.readLinesInFile(lockDepsPath).mkString("\n") + val objectMapper = new ObjectMapper + val packageJson = objectMapper.readTree(content) + + var depToVersion = Map.empty[String, String] + val dependencyIt = Option(packageJson.get("dependencies")) + .map(_.fields().asScala) + .getOrElse(Iterator.empty) + dependencyIt.foreach { + case entry: java.util.Map.Entry[String, JsonNode] => + val depName = entry.getKey + val versionNode = entry.getValue.get("version") + if (versionNode != null) { + depToVersion = depToVersion.updated(depName, versionNode.asText()) } - } - .getOrElse(Map.empty) + } + depToVersion }.toOption // lazy val because we only evaluate this in case no package lock file is available. lazy val deps = Try { - val content = FileUtils.readLinesInFile(depsPath).mkString("\n") - val packageJson = Json.parse(content) + val content = FileUtils.readLinesInFile(depsPath).mkString("\n") + val objectMapper = new ObjectMapper + val packageJson = objectMapper.readTree(content) + + var depToVersion = Map.empty[String, String] projectDependencies - .flatMap { dependency => - (packageJson \ dependency).asOpt[Map[String, String]] + .foreach { dependency => + val dependencyIt = Option(packageJson.get(dependency)) + .map(_.fields().asScala) + .getOrElse(Iterator.empty) + dependencyIt.foreach { + case entry: java.util.Map.Entry[String, JsonNode] => + depToVersion = depToVersion.updated(entry.getKey, entry.getValue.asText()) + } } - .flatten - .toMap + depToVersion }.toOption if (lockDeps.isDefined && lockDeps.get.nonEmpty) { diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala index 51e5a1dae..0bc0bd309 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala @@ -1,12 +1,13 @@ package io.shiftleft.js2cpg.parser +import com.fasterxml.jackson.databind.ObjectMapper import io.shiftleft.js2cpg.io.ExternalCommand import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler._ import org.slf4j.LoggerFactory -import play.api.libs.json.Json import java.nio.file.Path import scala.util.{Failure, Success} +import scala.jdk.CollectionConverters._ object TsConfigJsonParser { @@ -15,9 +16,11 @@ object TsConfigJsonParser { def module(projectPath: Path, tsc: String): String = { ExternalCommand.run(s"${ExternalCommand.toOSCommand(tsc)} --showConfig", projectPath.toString) match { case Success(tsConfig) => - val json = Json.parse(tsConfig) - val moduleOption = (json \ "compilerOptions" \ "module") - .asOpt[String] + val json = new ObjectMapper().readTree(tsConfig) + val moduleOption = + Option(json.get("compilerOptions")) + .flatMap(compOption => Option(compOption.get("module"))) + .map(_.asText()) moduleOption match { case Some(module) if module == ESNEXT || module == ES2020 => ESNEXT case _ => DEFAULT_MODULE @@ -45,12 +48,12 @@ object TsConfigJsonParser { def subprojects(projectPath: Path, tsc: String): List[String] = { ExternalCommand.run(s"${ExternalCommand.toOSCommand(tsc)} --showConfig", projectPath.toString) match { case Success(config) => - val json = Json.parse(config) - - (json \ "references") - .asOpt[List[Map[String, String]]] - .getOrElse(Nil) - .flatMap(_.get("path")) + val json = new ObjectMapper().readTree(config) + val referenceIt = + Option(json.get("references")).map(_.elements().asScala).getOrElse(Iterator.empty) + referenceIt.flatMap { reference => + Option(reference.get("path")).map(_.asText) + }.toList case Failure(exception) => logger.debug( diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala index 904e0f868..81d616b4c 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala @@ -3,16 +3,13 @@ package io.shiftleft.js2cpg.preprocessing import better.files.File import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode import io.shiftleft.js2cpg.core.Config import io.shiftleft.js2cpg.io.FileDefaults.TS_SUFFIX import io.shiftleft.js2cpg.io.{ExternalCommand, FileUtils} import io.shiftleft.js2cpg.parser.TsConfigJsonParser import org.slf4j.LoggerFactory import org.apache.commons.io.{FileUtils => CommonsFileUtils} -import play.api.libs.json.JsArray -import play.api.libs.json.JsObject -import play.api.libs.json.Json -import play.api.libs.json.JsString import java.nio.file.{Path, Paths} import scala.util.{Failure, Success, Try} @@ -77,22 +74,16 @@ class TypescriptTranspiler(override val config: Config, val customTsConfigFilePath = (File(projectPath) / "tsconfig.json").path Try { val content = FileUtils.readLinesInFile(customTsConfigFilePath).mkString("\n") - val json = Json.parse(removeComments(content)) - val compilerOptions = - json - .as[JsObject] - .value - .get("compilerOptions") - .map(_.as[JsObject] - "sourceRoot") - .getOrElse(JsObject.empty) + val mapper = new ObjectMapper() + val json = mapper.readTree(removeComments(content)) // --include is not available as tsc CLI argument; we set it manually: - val jsonCleaned = json - .as[JsObject] + ("include" -> JsArray(Array(JsString("**/*")))) + ("compilerOptions" -> compilerOptions) + Option(json.get("compilerOptions")).foreach(_.asInstanceOf[ObjectNode].remove("sourceRoot")) + json.asInstanceOf[ObjectNode].putArray("include").add("**/*") val customTsConfigFile = File .newTemporaryFile("js2cpgTsConfig", ".json", parent = Some(projectPath)) .deleteOnExit(swallowIOExceptions = true) - customTsConfigFile.writeText(Json.stringify(jsonCleaned)) + customTsConfigFile.writeText(mapper.writeValueAsString(json)) } }