Skip to content

Commit

Permalink
Merge pull request #64 from ShiftLeftSecurity/markus/replacePlayJson
Browse files Browse the repository at this point in the history 
Replace play json library with Jackson java library.
  • Loading branch information
@ml86
ml86 authored Jan 7, 2022
2 parents 620a344 + dcd9916 commit 576aa24
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 42 deletions.
3 changes: 1 addition & 2 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ lazy val commonSettings = Seq(
"com.github.pathikrit" %% "better-files" % "3.9.1",
"org.slf4j" % "slf4j-api" % "1.7.32",
"org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.17.0" % Runtime,
"com.typesafe.play" %% "play-json" % "2.9.2",
"com.fasterxml.jackson" % "jackson-base" % "2.13.1",
"com.fasterxml.jackson.core" % "jackson-databind" % "2.13.1",
"com.atlassian.sourcemap" % "sourcemap" % "2.0.0",
"commons-io" % "commons-io" % "2.11.0",
"io.shiftleft" %% "semanticcpg" % cpgVersion % Test classifier "tests",
Expand Down
49 changes: 34 additions & 15 deletions src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package io.shiftleft.js2cpg.parser
import java.nio.file.{Path, Paths}
import io.shiftleft.js2cpg.io.FileUtils
import org.slf4j.LoggerFactory
import play.api.libs.json.Json
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.JsonNode

import scala.collection.concurrent.TrieMap
import scala.util.Try
import scala.util.Using
import scala.jdk.CollectionConverters._

object PackageJsonParser {
private val logger = LoggerFactory.getLogger(PackageJsonParser.getClass)
Expand All @@ -31,27 +34,43 @@ object PackageJsonParser {
val lockDepsPath = packageJsonPath.resolveSibling(Paths.get(PACKAGE_JSON_LOCK_FILENAME))

val lockDeps = Try {
val content = FileUtils.readLinesInFile(lockDepsPath).mkString("\n")
(Json.parse(content) \ "dependencies")
.asOpt[Map[String, Map[String, String]]]
.map { versions =>
versions.map {
case (depName, entry) => depName -> entry("version")
val content = FileUtils.readLinesInFile(lockDepsPath).mkString("\n")
val objectMapper = new ObjectMapper
val packageJson = objectMapper.readTree(content)

var depToVersion = Map.empty[String, String]
val dependencyIt = Option(packageJson.get("dependencies"))
.map(_.fields().asScala)
.getOrElse(Iterator.empty)
dependencyIt.foreach {
case entry: java.util.Map.Entry[String, JsonNode] =>
val depName = entry.getKey
val versionNode = entry.getValue.get("version")
if (versionNode != null) {
depToVersion = depToVersion.updated(depName, versionNode.asText())
}
}
.getOrElse(Map.empty)
}
depToVersion
}.toOption

// lazy val because we only evaluate this in case no package lock file is available.
lazy val deps = Try {
val content = FileUtils.readLinesInFile(depsPath).mkString("\n")
val packageJson = Json.parse(content)
val content = FileUtils.readLinesInFile(depsPath).mkString("\n")
val objectMapper = new ObjectMapper
val packageJson = objectMapper.readTree(content)

var depToVersion = Map.empty[String, String]
projectDependencies
.flatMap { dependency =>
(packageJson \ dependency).asOpt[Map[String, String]]
.foreach { dependency =>
val dependencyIt = Option(packageJson.get(dependency))
.map(_.fields().asScala)
.getOrElse(Iterator.empty)
dependencyIt.foreach {
case entry: java.util.Map.Entry[String, JsonNode] =>
depToVersion = depToVersion.updated(entry.getKey, entry.getValue.asText())
}
}
.flatten
.toMap
depToVersion
}.toOption

if (lockDeps.isDefined && lockDeps.get.nonEmpty) {
Expand Down
23 changes: 13 additions & 10 deletions src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package io.shiftleft.js2cpg.parser

import com.fasterxml.jackson.databind.ObjectMapper
import io.shiftleft.js2cpg.io.ExternalCommand
import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler._
import org.slf4j.LoggerFactory
import play.api.libs.json.Json

import java.nio.file.Path
import scala.util.{Failure, Success}
import scala.jdk.CollectionConverters._

object TsConfigJsonParser {

Expand All @@ -15,9 +16,11 @@ object TsConfigJsonParser {
def module(projectPath: Path, tsc: String): String = {
ExternalCommand.run(s"${ExternalCommand.toOSCommand(tsc)} --showConfig", projectPath.toString) match {
case Success(tsConfig) =>
val json = Json.parse(tsConfig)
val moduleOption = (json \ "compilerOptions" \ "module")
.asOpt[String]
val json = new ObjectMapper().readTree(tsConfig)
val moduleOption =
Option(json.get("compilerOptions"))
.flatMap(compOption => Option(compOption.get("module")))
.map(_.asText())
moduleOption match {
case Some(module) if module == ESNEXT || module == ES2020 => ESNEXT
case _ => DEFAULT_MODULE
Expand Down Expand Up @@ -45,12 +48,12 @@ object TsConfigJsonParser {
def subprojects(projectPath: Path, tsc: String): List[String] = {
ExternalCommand.run(s"${ExternalCommand.toOSCommand(tsc)} --showConfig", projectPath.toString) match {
case Success(config) =>
val json = Json.parse(config)

(json \ "references")
.asOpt[List[Map[String, String]]]
.getOrElse(Nil)
.flatMap(_.get("path"))
val json = new ObjectMapper().readTree(config)
val referenceIt =
Option(json.get("references")).map(_.elements().asScala).getOrElse(Iterator.empty)
referenceIt.flatMap { reference =>
Option(reference.get("path")).map(_.asText)
}.toList

case Failure(exception) =>
logger.debug(
Expand Down
21 changes: 6 additions & 15 deletions src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,13 @@ package io.shiftleft.js2cpg.preprocessing
import better.files.File
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ObjectNode
import io.shiftleft.js2cpg.core.Config
import io.shiftleft.js2cpg.io.FileDefaults.TS_SUFFIX
import io.shiftleft.js2cpg.io.{ExternalCommand, FileUtils}
import io.shiftleft.js2cpg.parser.TsConfigJsonParser
import org.slf4j.LoggerFactory
import org.apache.commons.io.{FileUtils => CommonsFileUtils}
import play.api.libs.json.JsArray
import play.api.libs.json.JsObject
import play.api.libs.json.Json
import play.api.libs.json.JsString

import java.nio.file.{Path, Paths}
import scala.util.{Failure, Success, Try}
Expand Down Expand Up @@ -77,22 +74,16 @@ class TypescriptTranspiler(override val config: Config,
val customTsConfigFilePath = (File(projectPath) / "tsconfig.json").path
Try {
val content = FileUtils.readLinesInFile(customTsConfigFilePath).mkString("\n")
val json = Json.parse(removeComments(content))
val compilerOptions =
json
.as[JsObject]
.value
.get("compilerOptions")
.map(_.as[JsObject] - "sourceRoot")
.getOrElse(JsObject.empty)
val mapper = new ObjectMapper()
val json = mapper.readTree(removeComments(content))
// --include is not available as tsc CLI argument; we set it manually:
val jsonCleaned = json
.as[JsObject] + ("include" -> JsArray(Array(JsString("**/*")))) + ("compilerOptions" -> compilerOptions)
Option(json.get("compilerOptions")).foreach(_.asInstanceOf[ObjectNode].remove("sourceRoot"))
json.asInstanceOf[ObjectNode].putArray("include").add("**/*")
val customTsConfigFile =
File
.newTemporaryFile("js2cpgTsConfig", ".json", parent = Some(projectPath))
.deleteOnExit(swallowIOExceptions = true)
customTsConfigFile.writeText(Json.stringify(jsonCleaned))
customTsConfigFile.writeText(mapper.writeValueAsString(json))
}
}

Expand Down

0 comments on commit 576aa24

Please sign in to comment.