Skip filename properties in cdxpass. upstream fixes (#22)

* Skip properties with file names Signed-off-by: Prabhu Subramanian <[email protected]> * Upstream fixes Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Oct 24, 2023 · 4690502 · 4690502
1 parent 596c85e
commit 4690502
Show file tree

Hide file tree

Showing 13 changed files with 105 additions and 90 deletions.
diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "0.0.20"
+ThisBuild / version      := "0.0.21"
 ThisBuild / scalaVersion := "3.3.1"
 
 val cpgVersion = "1.4.22"

diff --git a/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala b/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/Jimple2Cpg.scala
@@ -1,7 +1,7 @@
 package io.appthreat.jimple2cpg
 
 import better.files.File
-import io.appthreat.jimple2cpg.passes.{AstCreationPass, ConfigFileCreationPass, SootAstCreationPass}
+import io.appthreat.jimple2cpg.passes.{AstCreationPass, ConfigFileCreationPass, DeclarationRefPass, SootAstCreationPass}
 import io.appthreat.jimple2cpg.util.ProgramHandlingUtil.ClassFile
 import io.appthreat.jimple2cpg.util.ProgramHandlingUtil.{ClassFile, extractClassesInPackageLayout}
 import io.appthreat.x2cpg.X2Cpg.withNewEmptyCpg
@@ -110,8 +110,6 @@ class Jimple2Cpg extends X2CpgFrontend[Config] {
           astCreator.global
         }
     }
-    new ConfigFileCreationPass(cpg).createAndApply()
-    new CdxPass(cpg).createAndApply()
     logger.debug("Loading classes to soot")
     Scene.v().loadNecessaryClasses()
     logger.debug(s"Loaded ${Scene.v().getApplicationClasses.size()} classes")
@@ -120,6 +118,9 @@ class Jimple2Cpg extends X2CpgFrontend[Config] {
     TypeNodePass
       .withRegisteredTypes(global.usedTypes.keys().asScala.toList, cpg)
       .createAndApply()
+    DeclarationRefPass(cpg).createAndApply()
+    new ConfigFileCreationPass(cpg).createAndApply()
+    new CdxPass(cpg).createAndApply()
   }
 
   override def createCpg(config: Config): Try[Cpg] =

diff --git a/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/passes/AstCreator.scala b/platform/frontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/passes/AstCreator.scala
@@ -173,6 +173,7 @@ class AstCreator(filename: String, cls: SootClass, global: Global)(implicit with
           .withChildren(astsForModifiers(methodDeclaration))
           .withChildren(parameterAsts)
           .withChildren(astsForHostTags(methodDeclaration))
+          .withChild(Ast(NewBlock()))
           .withChild(astForMethodReturn(methodDeclaration))
       } else {
         val lastOrder = 2 + methodDeclaration.getParameterCount

diff --git a/...ontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/passes/DeclarationRefPass.scala b/...ontends/jimple2cpg/src/main/scala/io/appthreat/jimple2cpg/passes/DeclarationRefPass.scala
@@ -0,0 +1,22 @@
+package io.appthreat.jimple2cpg.passes
+
+import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.codepropertygraph.generated.EdgeTypes
+import io.shiftleft.codepropertygraph.generated.nodes.{Declaration, Method}
+import io.shiftleft.passes.ConcurrentWriterCpgPass
+import io.shiftleft.semanticcpg.language.*
+
+/** Links declarations to their identifier nodes. Due to the flat AST of bytecode, we don't need to account for varying
+  * scope.
+  */
+class DeclarationRefPass(atom: Cpg) extends ConcurrentWriterCpgPass[Method](atom) {
+
+  override def generateParts(): Array[Method] = atom.method.toArray
+
+  override def runOnPart(builder: DiffGraphBuilder, part: Method): Unit = {
+    val identifiers  = part.ast.isIdentifier.toList
+    val declarations = (part.parameter ++ part.block.astChildren.isLocal).collectAll[Declaration].l
+    declarations.foreach(d => identifiers.nameExact(d.name).foreach(builder.addEdge(_, d, EdgeTypes.REF)))
+  }
+
+}
diff --git a/...orm/frontends/jimple2cpg/src/test/scala/io/appthreat/jimple2cpg/querying/LocalTests.scala b/...orm/frontends/jimple2cpg/src/test/scala/io/appthreat/jimple2cpg/querying/LocalTests.scala
@@ -34,4 +34,14 @@ class LocalTests extends JimpleCode2CpgFixture {
     y.typeFullName shouldBe "java.lang.Integer"
     y.order shouldBe 2
   }
+
+  "should allow traversing from local to identifier" in {
+    val ys = cpg.local.nameExact("y").referencingIdentifiers.l
+    ys.size shouldBe 2
+    ys.head.name shouldBe "y"
+    val xs = cpg.local.nameExact("$stack3").referencingIdentifiers.l
+    xs.size shouldBe 3
+    xs.head.name shouldBe "$stack3"
+  }
+
 }
diff --git a/...nds/jimple2cpg/src/test/scala/io/appthreat/jimple2cpg/querying/MethodParameterTests.scala b/...nds/jimple2cpg/src/test/scala/io/appthreat/jimple2cpg/querying/MethodParameterTests.scala
@@ -10,7 +10,7 @@ class MethodParameterTests extends JimpleCode2CpgFixture {
   val cpg: Cpg = code("""package a;
       |class Foo {
       | int foo(int param1, Object param2) {
-      |  return 0;
+      |  return param1;
       | }
       |}
       """.stripMargin).cpg
@@ -47,4 +47,7 @@ class MethodParameterTests extends JimpleCode2CpgFixture {
     cpg.parameter.name("param1").method.filter(_.isExternal == false).name.l shouldBe List("foo")
   }
 
+  "should allow traversing from parameter to identifier" in {
+    cpg.parameter.name("param1").referencingIdentifiers.name.l shouldBe List("param1")
+  }
 }
diff --git a/...rm/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/parser/BabelJsonParser.scala b/...rm/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/parser/BabelJsonParser.scala
@@ -27,7 +27,7 @@ object BabelJsonParser {
     }
 
     val jsonContent       = IOUtils.readEntireFile(file)
-    val json              = ujson.transform(jsonContent, JsValueVisitor)
+    val json              = ujson.read(jsonContent)
     val filename          = json("relativeName").str
     val fullPath          = Paths.get(rootPath.toString, filename)
     val sourceFileContent = IOUtils.readEntireFile(fullPath)

diff --git a/...orm/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/parser/JsValueVisitor.scala b/...orm/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/parser/JsValueVisitor.scala
diff --git a/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/ImportResolverPass.scala b/platform/frontends/pysrc2cpg/src/main/scala/io/appthreat/pysrc2cpg/ImportResolverPass.scala
@@ -1,17 +1,19 @@
 package io.appthreat.pysrc2cpg
 
-import better.files.{File => BFile}
-import io.appthreat.x2cpg.passes.frontend.ImportsPass._
+import better.files.File as BFile
+import io.appthreat.x2cpg.passes.frontend.ImportsPass.*
 import io.appthreat.x2cpg.passes.frontend.XImportResolverPass
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes._
-import io.shiftleft.semanticcpg.language._
+import io.shiftleft.codepropertygraph.generated.nodes.*
+import io.shiftleft.semanticcpg.language.*
 
-import java.io.{File => JFile}
+import java.io.File as JFile
 import java.util.regex.{Matcher, Pattern}
 
 class ImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg) {
 
+  private lazy val root = cpg.metaData.root.headOption.getOrElse("").stripSuffix(JFile.separator)
+
   override protected def optionalResolveImport(
     fileName: String,
     importCall: Call,
@@ -24,7 +26,13 @@ class ImportResolverPass(cpg: Cpg) extends XImportResolverPass(cpg) {
       val namespace = importedEntity.stripSuffix(s".${splitName.last}")
       (relativizeNamespace(namespace, fileName), splitName.last)
     } else {
-      ("", importedEntity)
+      val currDir = BFile(root) / fileName match
+        case x if x.isDirectory => x
+        case x                  => x.parent
+
+      val relCurrDir = currDir.pathAsString.stripPrefix(root).stripPrefix(JFile.separator)
+
+      (relCurrDir, importedEntity)
     }
 
     resolveEntities(namespace, entityName, importedAs).foreach(x => resolvedImportToTag(x, importCall, diffGraph))

diff --git a/...ntends/pysrc2cpg/src/test/scala/io/appthreat/pysrc2cpg/passes/TypeRecoveryPassTests.scala b/...ntends/pysrc2cpg/src/test/scala/io/appthreat/pysrc2cpg/passes/TypeRecoveryPassTests.scala
@@ -1121,4 +1121,47 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) {
 
   }
 
+  "Imports from two module neighbours" should {
+
+    lazy val cpg = code(
+      """
+        |from fastapi import FastAPI
+        |import itemsrouter
+        |app = FastAPI()
+        |app.include_router(
+        |    itemsrouter.router,
+        |    prefix="/items",
+        |    tags=["items"],
+        |    responses={404: {"description": "Not found"}},
+        |)
+        |""".stripMargin,
+      Seq("code", "main.py").mkString(File.separator)
+    ).moreCode(
+      """
+        |from fastapi import APIRouter
+        |
+        |router = APIRouter()
+        |fake_items_db = {"plumbus": {"name": "Plumbus"}, "gun": {"name": "Portal Gun"}}
+        |@router.get("/")
+        |async def read_items():
+        |    return fake_items_db
+        |
+        |""".stripMargin,
+      Seq("code", "itemsrouter.py").mkString(File.separator)
+    )
+
+    "preserve the filename path relative to the root and not themselves" in {
+      val itemsrouter = cpg.identifier.where(_.typeFullName(".*itemsrouter.py:<module>")).l
+      itemsrouter.forall(
+        _.typeFullName == Seq("code", "itemsrouter.py:<module>").mkString(File.separator)
+      ) shouldBe true
+    }
+
+    "correctly infer the `fastapi` types" in {
+      cpg.identifier("fastapi").forall(_.typeFullName == "fastapi.py:<module>.APIRouter") shouldBe true
+      cpg.identifier("app").forall(_.typeFullName == "fastapi.py:<module>.FastAPI") shouldBe true
+    }
+
+  }
+
 }
diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/CdxPass.scala
@@ -70,6 +70,8 @@ class CdxPass(atom: Cpg) extends CpgPass(atom) {
             .filterNot(_.startsWith("com.sun"))
             .filterNot(_.contains("test"))
             .filterNot(_.contains("mock"))
+            .filterNot(_.endsWith(".lock"))
+            .filterNot(_.endsWith(".json"))
             .foreach { (pkg: String) =>
               var bpkg = pkg.takeWhile(_ != '$')
               if (language == Languages.JAVA || language == Languages.JAVASRC)

diff --git a/project/Versions.scala b/project/Versions.scala
@@ -10,8 +10,8 @@ object Versions {
   val gradleTooling = "8.3"
   val circe         = "0.14.5"
   val requests      = "0.8.0"
-  val upickle       = "3.1.2"
-  val scalaReplPP   = "0.1.70"
+  val upickle       = "3.1.3"
+  val scalaReplPP   = "0.1.72"
 
   private def parseVersion(key: String): String = {
     val versionRegexp = s""".*val $key[ ]+=[ ]?"(.*?)"""".r

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "appthreat-chen"
-version = "0.0.20"
+version = "0.0.21"
 description = "Code Hierarchy Exploration Net (chen)"
 authors = ["Team AppThreat <[email protected]>"]
 license = "Apache-2.0"