Skip to content

Commit

Permalink
reduce classpath dependencies from frontends: jssrc2cpg (#4682)
Browse files Browse the repository at this point in the history
* reduce classpath dependencies from frontends: jssrc2cpg

similar to #4672, with one
additional refactor: reuse the scopt parser rather than manually
fiddling with the commandline args

* refactor

* reuse scopt parser for javasrc as well

* inline ParameterNames.TypePropagationIterations etc. again

* fixup

* add back old api as @deprecated to help users migrate
  • Loading branch information
mpollmeier authored Jun 20, 2024
1 parent 821c865 commit 30265f3
Show file tree
Hide file tree
Showing 47 changed files with 157 additions and 110 deletions.
1 change: 0 additions & 1 deletion console/build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ enablePlugins(JavaAppPackaging)
dependsOn(
Projects.semanticcpg,
Projects.macros,
Projects.jssrc2cpg,
Projects.php2cpg,
Projects.pysrc2cpg,
Projects.rubysrc2cpg,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,32 @@ package io.joern.console.cpgcreation

import io.joern.console.FrontendConfig
import io.joern.x2cpg.frontendspecific.javasrc2cpg
import io.joern.x2cpg.passes.frontend.XTypeRecovery
import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig
import io.shiftleft.codepropertygraph.generated.Cpg

import java.nio.file.Path
import scala.compiletime.uninitialized
import scala.util.Try

/** Source-based front-end for Java
*/
case class JavaSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator {
private lazy val command: Path = if (isWin) rootPath.resolve("javasrc2cpg.bat") else rootPath.resolve("javasrc2cpg")
private var enableTypeRecovery = false
private var disableDummyTypes = false
private var typeRecoveryConfig: XTypeRecoveryConfig = uninitialized

/** Generate a CPG for the given input path. Returns the output path, or None, if no CPG was generated.
*/
override def generate(inputPath: String, outputPath: String = "cpg.bin"): Try[String] = {
val arguments = config.cmdLineParams.toSeq ++ Seq(inputPath, "--output", outputPath)
enableTypeRecovery = arguments.exists(_ == s"--${javasrc2cpg.ParameterNames.EnableTypeRecovery}")
disableDummyTypes = arguments.exists(_ == s"--${XTypeRecovery.ParameterNames.NoDummyTypes}")
if (enableTypeRecovery) typeRecoveryConfig = XTypeRecoveryConfig.parse(arguments)
runShellCommand(command.toString, arguments).map(_ => outputPath)
}

override def applyPostProcessingPasses(cpg: Cpg): Cpg = {
if (enableTypeRecovery)
javasrc2cpg.typeRecoveryPasses(cpg, disableDummyTypes).foreach(_.createAndApply())
javasrc2cpg.typeRecoveryPasses(cpg, typeRecoveryConfig).foreach(_.createAndApply())
super.applyPostProcessingPasses(cpg)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,23 @@ package io.joern.console.cpgcreation

import better.files.File
import io.joern.console.FrontendConfig
import io.joern.jssrc2cpg.{Config, Frontend, JsSrc2Cpg}
import io.joern.x2cpg.X2Cpg
import io.joern.x2cpg.frontendspecific.jssrc2cpg
import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig
import io.shiftleft.codepropertygraph.generated.Cpg

import java.nio.file.Path
import scala.compiletime.uninitialized
import scala.util.Try

case class JsSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator {
private lazy val command: Path = if (isWin) rootPath.resolve("jssrc2cpg.bat") else rootPath.resolve("jssrc2cpg.sh")
private var jsConfig: Option[Config] = None
private var typeRecoveryConfig: XTypeRecoveryConfig = uninitialized

/** Generate a CPG for the given input path. Returns the output path, or None, if no CPG was generated.
*/
override def generate(inputPath: String, outputPath: String = "cpg.bin.zip"): Try[String] = {
typeRecoveryConfig = XTypeRecoveryConfig.parse(config.cmdLineParams.toSeq)

if (File(inputPath).isDirectory) {
invoke(inputPath, outputPath)
} else {
Expand All @@ -27,15 +30,14 @@ case class JsSrcCpgGenerator(config: FrontendConfig, rootPath: Path) extends Cpg

private def invoke(inputPath: String, outputPath: String): Try[String] = {
val arguments = Seq(inputPath, "--output", outputPath) ++ config.cmdLineParams
jsConfig = X2Cpg.parseCommandLine(arguments.toArray, Frontend.cmdLineParser, Config())
runShellCommand(command.toString, arguments).map(_ => outputPath)
}

override def isAvailable: Boolean =
command.toFile.exists

override def applyPostProcessingPasses(cpg: Cpg): Cpg = {
JsSrc2Cpg.postProcessingPasses(cpg, jsConfig).foreach(_.createAndApply())
jssrc2cpg.postProcessingPasses(cpg, typeRecoveryConfig).foreach(_.createAndApply())
cpg
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package io.joern.csharpsrc2cpg

import io.joern.csharpsrc2cpg.Frontend.{cmdLineParser, defaultConfig}
import io.joern.x2cpg.astgen.AstGenConfig
import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery}
import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig}
import io.joern.x2cpg.utils.Environment
import io.joern.x2cpg.{DependencyDownloadConfig, X2CpgConfig, X2CpgMain}
import org.slf4j.LoggerFactory
Expand Down Expand Up @@ -31,7 +31,11 @@ object Frontend {
val cmdLineParser: OParser[Unit, Config] = {
val builder = OParser.builder[Config]
import builder.*
OParser.sequence(programName("csharpsrc2cpg"), DependencyDownloadConfig.parserOptions, XTypeRecovery.parserOptions)
OParser.sequence(
programName("csharpsrc2cpg"),
DependencyDownloadConfig.parserOptions,
XTypeRecoveryConfig.parserOptionsForParserConfig
)
}

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import io.joern.javasrc2cpg.Frontend.*
import io.joern.javasrc2cpg.jpastprinter.JavaParserAstPrinter
import io.joern.x2cpg.frontendspecific.javasrc2cpg
import io.joern.x2cpg.{X2CpgConfig, X2CpgMain}
import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery}
import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig}
import scopt.OParser

/** Command line configuration parameters
Expand Down Expand Up @@ -104,7 +104,7 @@ private object Frontend {
.hidden()
.action((_, c) => c.withEnableTypeRecovery(true))
.text("enable generic type recovery"),
XTypeRecovery.parserOptions,
XTypeRecoveryConfig.parserOptionsForParserConfig,
opt[String]("jdk-path")
.action((path, c) => c.withJdkPath(path))
.text("JDK used for resolving builtin Java types. If not set, current classpath will be used"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import io.joern.dataflowengineoss.testfixtures.{SemanticCpgTestFixture, Semantic
import io.joern.javasrc2cpg.{Config, JavaSrc2Cpg}
import io.joern.x2cpg.X2Cpg
import io.joern.x2cpg.frontendspecific.javasrc2cpg
import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig
import io.joern.x2cpg.testfixtures.{Code2CpgFixture, DefaultTestCpg, LanguageFrontend, TestCpg}
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.codepropertygraph.generated.nodes.{Expression, Literal}
Expand Down Expand Up @@ -33,7 +34,7 @@ class JavaSrcTestCpg(enableTypeRecovery: Boolean = false)
override protected def applyPasses(): Unit = {
super.applyPasses()
if (enableTypeRecovery)
javasrc2cpg.typeRecoveryPasses(this, disableDummyTypes = false).foreach(_.createAndApply())
javasrc2cpg.typeRecoveryPasses(this, XTypeRecoveryConfig(enabledDummyTypes = true)).foreach(_.createAndApply())
applyOssDataFlow()
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ package io.joern.jssrc2cpg

import better.files.File
import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions}
import io.joern.jssrc2cpg.JsSrc2Cpg.postProcessingPasses
import io.joern.jssrc2cpg.passes.*
import io.joern.jssrc2cpg.utils.AstGenRunner
import io.joern.x2cpg.X2Cpg.withNewEmptyCpg
import io.joern.x2cpg.X2CpgFrontend
import io.joern.x2cpg.frontendspecific.jssrc2cpg.postProcessingPasses
import io.joern.x2cpg.passes.callgraph.NaiveCallLinker
import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig
import io.joern.x2cpg.utils.{HashUtil, Report}
Expand Down Expand Up @@ -46,23 +46,10 @@ class JsSrc2Cpg extends X2CpgFrontend[Config] {
val maybeCpg = createCpgWithOverlays(config)
maybeCpg.map { cpg =>
new OssDataFlow(new OssDataFlowOptions()).run(new LayerCreatorContext(cpg))
postProcessingPasses(cpg, Option(config)).foreach(_.createAndApply())
val typeRecoveryConfig = XTypeRecoveryConfig(config.typePropagationIterations, !config.disableDummyTypes)
postProcessingPasses(cpg, typeRecoveryConfig).foreach(_.createAndApply())
cpg
}
}

}

object JsSrc2Cpg {

def postProcessingPasses(cpg: Cpg, config: Option[Config] = None): List[CpgPassBase] = {
val typeRecoveryConfig = config
.map(c => XTypeRecoveryConfig(c.typePropagationIterations, !c.disableDummyTypes))
.getOrElse(XTypeRecoveryConfig())
List(new JavaScriptInheritanceNamePass(cpg), new ConstClosurePass(cpg), new JavaScriptImportResolverPass(cpg))
++
new JavaScriptTypeRecoveryPassGenerator(cpg, typeRecoveryConfig).generate() ++
List(new JavaScriptTypeHintCallLinker(cpg), ObjectPropertyCallLinker(cpg), new NaiveCallLinker(cpg))
}

}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.joern.jssrc2cpg

import io.joern.jssrc2cpg.Frontend.*
import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery}
import io.joern.x2cpg.passes.frontend.{TypeRecoveryParserConfig, XTypeRecovery, XTypeRecoveryConfig}
import io.joern.x2cpg.utils.Environment
import io.joern.x2cpg.{X2CpgConfig, X2CpgMain}
import scopt.OParser
Expand All @@ -28,7 +28,7 @@ object Frontend {
.hidden()
.action((_, c) => c.withTsTypes(false))
.text("disable generation of types via Typescript"),
XTypeRecovery.parserOptions
XTypeRecoveryConfig.parserOptionsForParserConfig
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import io.joern.jssrc2cpg.datastructures.{MethodScope, Scope}
import io.joern.jssrc2cpg.parser.BabelAst.*
import io.joern.jssrc2cpg.parser.BabelJsonParser.ParseResult
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.datastructures.Stack.*
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.NodeBuilders.{newMethodReturnNode, newModifierNode}
import io.joern.x2cpg.{Ast, AstCreatorBase, ValidationMode, AstNodeBuilder as X2CpgAstNodeBuilder}
import io.joern.x2cpg.datastructures.Global
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package io.joern.jssrc2cpg.astcreation
import io.joern.jssrc2cpg.datastructures.*
import io.joern.jssrc2cpg.parser.BabelAst.*
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.NodeBuilders.{newClosureBindingNode, newLocalNode}
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.nodes.*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package io.joern.jssrc2cpg.astcreation
import io.joern.jssrc2cpg.datastructures.{BlockScope, MethodScope, ScopeType}
import io.joern.jssrc2cpg.parser.BabelAst.*
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.{Ast, ValidationMode}
import io.joern.x2cpg.datastructures.Stack.*
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.NodeBuilders.newDependencyNode
import io.shiftleft.codepropertygraph.generated.nodes.{NewCall, NewImport}
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ package io.joern.jssrc2cpg.astcreation

import io.joern.jssrc2cpg.parser.BabelAst.*
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines.OperatorsNew
import io.joern.jssrc2cpg.passes.{Defines, EcmaBuiltins, GlobalBuiltins}
import io.joern.jssrc2cpg.passes.EcmaBuiltins
import io.joern.x2cpg.frontendspecific.jssrc2cpg.{Defines, GlobalBuiltins}
import io.joern.x2cpg.{Ast, ValidationMode}
import io.joern.x2cpg.datastructures.Stack.*
import io.shiftleft.codepropertygraph.generated.nodes.NewNode
Expand Down Expand Up @@ -130,7 +130,7 @@ trait AstForExpressionsCreator(implicit withSchemaValidation: ValidationMode) {

val receiverNode = astForNodeWithFunctionReference(callee)

val callAst = handleCallNodeArgs(newExpr, receiverNode, tmpAllocNode2, OperatorsNew)
val callAst = handleCallNodeArgs(newExpr, receiverNode, tmpAllocNode2, Defines.OperatorsNew)

val tmpAllocReturnNode = Ast(identifierNode(newExpr, tmpAllocName))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package io.joern.jssrc2cpg.astcreation
import io.joern.jssrc2cpg.datastructures.{BlockScope, MethodScope}
import io.joern.jssrc2cpg.parser.BabelAst.*
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.datastructures.Stack.*
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.NodeBuilders.{newBindingNode, newModifierNode}
import io.joern.x2cpg.{Ast, ValidationMode}
import io.shiftleft.codepropertygraph.generated.nodes.{Identifier as _, *}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.joern.jssrc2cpg.astcreation

import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.{Ast, ValidationMode}
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, Operators}

trait AstForPrimitivesCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ package io.joern.jssrc2cpg.astcreation

import io.joern.jssrc2cpg.parser.BabelAst.*
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.Ast
import io.joern.x2cpg.ValidationMode
import io.joern.x2cpg.datastructures.Stack.*
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.shiftleft.codepropertygraph.generated.ControlStructureTypes
import io.shiftleft.codepropertygraph.generated.DispatchTypes
import io.shiftleft.codepropertygraph.generated.EdgeTypes
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package io.joern.jssrc2cpg.astcreation
import io.joern.jssrc2cpg.datastructures.BlockScope
import io.joern.jssrc2cpg.parser.BabelAst.*
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.{Ast, ValidationMode}
import io.joern.x2cpg.datastructures.Stack.*
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.NodeBuilders.newBindingNode
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, ModifierTypes, Operators}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package io.joern.jssrc2cpg.astcreation

import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg
import io.joern.x2cpg.{Ast, ValidationMode}
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.NodeBuilders.newMethodReturnNode
import io.shiftleft.codepropertygraph.generated.nodes.*
import io.shiftleft.codepropertygraph.generated.DispatchTypes
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package io.joern.jssrc2cpg.astcreation

import io.joern.jssrc2cpg.parser.BabelAst._
import io.joern.jssrc2cpg.parser.BabelNodeInfo
import io.joern.jssrc2cpg.passes.Defines
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines

import java.util.regex.Pattern

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import io.joern.jssrc2cpg.parser.BabelJsonParser
import io.joern.jssrc2cpg.utils.AstGenRunner.AstGenRunnerResult
import io.joern.x2cpg.ValidationMode
import io.joern.x2cpg.datastructures.Global
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.{Report, TimeUtils}
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.passes.ConcurrentWriterCpgPass
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package io.joern.jssrc2cpg.passes
import better.files.File
import io.joern.jssrc2cpg.Config
import io.joern.x2cpg.SourceFiles
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.utils.{Report, TimeUtils}
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package io.joern.jssrc2cpg.passes
import io.joern.jssrc2cpg.Config
import io.joern.jssrc2cpg.utils.PackageJsonParser
import io.joern.x2cpg.SourceFiles
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.codepropertygraph.generated.nodes.NewDependency
import io.shiftleft.passes.CpgPass
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.joern.jssrc2cpg.passes

import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.joern.x2cpg.passes.frontend.TypeNodePass
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.semanticcpg.language.*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package io.joern.jssrc2cpg.passes

import better.files.File
import io.joern.jssrc2cpg.Config
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.shiftleft.codepropertygraph.generated.Cpg
import io.shiftleft.semanticcpg.language._
import org.scalatest.matchers.should.Matchers
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package io.joern.jssrc2cpg.passes
import io.joern.jssrc2cpg.JsSrc2Cpg
import io.joern.jssrc2cpg.testfixtures.JsSrc2CpgFrontend
import io.joern.x2cpg.X2Cpg
import io.joern.x2cpg.frontendspecific.jssrc2cpg
import io.joern.x2cpg.passes.frontend.XTypeRecoveryConfig
import io.joern.x2cpg.testfixtures.{Code2CpgFixture, TestCpg}
import io.shiftleft.semanticcpg.language._
import io.shiftleft.semanticcpg.language.*

class ImportsPassTests extends Code2CpgFixture(() => new TestCpgWithoutDataFlow()) {

Expand Down Expand Up @@ -47,6 +49,6 @@ class TestCpgWithoutDataFlow extends TestCpg with JsSrc2CpgFrontend {
override val fileSuffix: String = ".js"
override def applyPasses(): Unit = {
X2Cpg.applyDefaultOverlays(this)
JsSrc2Cpg.postProcessingPasses(this).foreach(_.createAndApply())
jssrc2cpg.postProcessingPasses(this, XTypeRecoveryConfig()).foreach(_.createAndApply())
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.joern.jssrc2cpg.passes

import io.joern.jssrc2cpg.passes.Defines.OperatorsNew
import io.joern.jssrc2cpg.testfixtures.DataFlowCodeToCpgSuite
import io.joern.x2cpg.frontendspecific.jssrc2cpg.Defines
import io.shiftleft.codepropertygraph.generated.Operators
import io.shiftleft.semanticcpg.language.importresolver.*
import io.shiftleft.semanticcpg.language.*
Expand Down Expand Up @@ -474,7 +474,7 @@ class TypeRecoveryPassTests extends DataFlowCodeToCpgSuite {
|new Print("Hello")
|""".stripMargin)

cpg.call.nameExact(OperatorsNew).methodFullName.head shouldBe "Test0.js::program:Print"
cpg.call.nameExact(Defines.OperatorsNew).methodFullName.head shouldBe "Test0.js::program:Print"
}

"A function assigned to a member should have it's full name resolved" in {
Expand Down
Loading

0 comments on commit 30265f3

Please sign in to comment.