From e33a47753f12360d6f64143cb1dab032cd07c3eb Mon Sep 17 00:00:00 2001 From: Prabhu Subramanian Date: Fri, 5 Jul 2024 21:55:13 +0100 Subject: [PATCH 1/4] Improve windows compat Signed-off-by: Prabhu Subramanian --- README.md | 6 +++--- build.sbt | 2 +- codemeta.json | 2 +- meta.yaml | 2 +- .../scala/io/appthreat/c2cpg/passes/AstCreationPass.scala | 4 ++-- .../io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala | 3 ++- pyproject.toml | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 14ba73d1..cb88568f 100644 --- a/README.md +++ b/README.md @@ -190,9 +190,9 @@ This error is mostly due to missing python .so (linux), .dll (windows) or .dylib chen is a fork of the popular [joern](https://github.com/joernio/joern) project. We deviate from the joern project in the following ways: +- Keep the CPG implementation at 1.0 based on the original paper. - Make code analysis accessible by adding first-class integration with Python and frameworks such as NetworkX and PyTorch. -- Enable broader hierarchical analysis (Application + Dependency + Container + OS layer) -- By creating a welcoming community more appropriate for beginner users with great enterprise support. We want to democratize code analysis. +- Enable broader hierarchical analysis (Application + Dependency + Container + OS layer + Cloud + beyond) We don't intend for bug-to-bug compatibility and often rewrite patches to suit our needs. We also do not bring features and passes that do not add value for hierarchical analysis. @@ -202,7 +202,7 @@ Apache-2.0 ## Enterprise support -Enterprise support including custom language development and integration services is available via AppThreat Ltd. Free community support is also available via [Discord](https://discord.gg/UD4sHgbXYr). +Enterprise support including custom language development and integration services is available via AppThreat Ltd. ## Sponsors diff --git a/build.sbt b/build.sbt index 0a08734b..e53c54e9 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "chen" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "2.1.1" +ThisBuild / version := "2.1.2" ThisBuild / scalaVersion := "3.4.2" val cpgVersion = "1.0.0" diff --git a/codemeta.json b/codemeta.json index 332b070e..38db431a 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,7 +7,7 @@ "downloadUrl": "https://github.com/AppThreat/chen", "issueTracker": "https://github.com/AppThreat/chen/issues", "name": "chen", - "version": "2.1.1", + "version": "2.1.2", "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.", "applicationCategory": "code-analysis", "keywords": [ diff --git a/meta.yaml b/meta.yaml index f4346683..977e8f25 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,4 +1,4 @@ -{% set version = "2.1.1" %} +{% set version = "2.1.2" %} package: name: chen diff --git a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala index c97dc17d..9ad26848 100644 --- a/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala +++ b/platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/passes/AstCreationPass.scala @@ -3,10 +3,10 @@ package io.appthreat.c2cpg.passes import io.appthreat.c2cpg.Config import io.appthreat.c2cpg.astcreation.AstCreator import io.appthreat.c2cpg.parser.{CdtParser, FileDefaults} -import io.appthreat.c2cpg.utils.{Report, TimeUtils} +import io.appthreat.c2cpg.utils.Report +import io.appthreat.x2cpg.SourceFiles import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.passes.ConcurrentWriterCpgPass -import io.appthreat.x2cpg.SourceFiles import java.nio.file.Paths import java.util.concurrent.ConcurrentHashMap diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala index 0282c49b..9e80a791 100644 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala +++ b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala @@ -18,10 +18,11 @@ class ChennaiTagsPass(atom: Cpg) extends CpgPass(atom): private val FRAMEWORK_ROUTE = "framework-route" private val FRAMEWORK_INPUT = "framework-input" private val FRAMEWORK_OUTPUT = "framework-output" + private val FileSeparator = java.io.File.separator private val PYTHON_ROUTES_CALL_REGEXES = Array( - "django/(conf/)?urls.py:.(path|re_path|url).*", + s"django$FileSeparator(conf$FileSeparator)?urls.py:.(path|re_path|url).*", ".*(route|web\\.|add_resource).*" ) diff --git a/pyproject.toml b/pyproject.toml index 77c028d8..f1aa4b44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appthreat-chen" -version = "2.1.1" +version = "2.1.2" description = "Code Hierarchy Exploration Net (chen)" authors = ["Team AppThreat "] license = "Apache-2.0" From c843a1338bda68c92c1536db4943ef8a0c290d85 Mon Sep 17 00:00:00 2001 From: Prabhu Subramanian Date: Sat, 6 Jul 2024 05:58:42 +0100 Subject: [PATCH 2/4] Improve windows compat Signed-off-by: Prabhu Subramanian --- .../x2cpg/passes/taggers/ChennaiTagsPass.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala index 9e80a791..ac53a00c 100644 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala +++ b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/passes/taggers/ChennaiTagsPass.scala @@ -14,16 +14,16 @@ import java.util.regex.Pattern */ class ChennaiTagsPass(atom: Cpg) extends CpgPass(atom): - val language: String = atom.metaData.language.head - private val FRAMEWORK_ROUTE = "framework-route" - private val FRAMEWORK_INPUT = "framework-input" - private val FRAMEWORK_OUTPUT = "framework-output" - private val FileSeparator = java.io.File.separator + val language: String = atom.metaData.language.head + private val FRAMEWORK_ROUTE = "framework-route" + private val FRAMEWORK_INPUT = "framework-input" + private val FRAMEWORK_OUTPUT = "framework-output" + private val EscapedFileSeparator = Pattern.quote(java.io.File.separator) private val PYTHON_ROUTES_CALL_REGEXES = Array( - s"django$FileSeparator(conf$FileSeparator)?urls.py:.(path|re_path|url).*", - ".*(route|web\\.|add_resource).*" + s"django$EscapedFileSeparator(conf$EscapedFileSeparator)?urls.py:.(path|re_path|url).*".r, + ".*(route|web\\.|add_resource).*".r ) private def C_ROUTES_CALL_REGEXES = Array( @@ -70,7 +70,7 @@ class ChennaiTagsPass(atom: Cpg) extends CpgPass(atom): private def tagPythonRoutes(dstGraph: DiffGraphBuilder): Unit = PYTHON_ROUTES_CALL_REGEXES.foreach { r => atom.call - .where(_.methodFullName(r)) + .where(_.methodFullName(r.toString())) .argument .isLiteral .newTagNode(FRAMEWORK_ROUTE) From 63e48cf2dc44e2323666a7aa2501f89d36df08cd Mon Sep 17 00:00:00 2001 From: Prabhu Subramanian Date: Sat, 6 Jul 2024 06:26:19 +0100 Subject: [PATCH 3/4] Updated couple of packages Signed-off-by: Prabhu Subramanian --- platform/frontends/c2cpg/build.sbt | 2 +- platform/frontends/jssrc2cpg/build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/frontends/c2cpg/build.sbt b/platform/frontends/c2cpg/build.sbt index e230a670..25e94c56 100644 --- a/platform/frontends/c2cpg/build.sbt +++ b/platform/frontends/c2cpg/build.sbt @@ -10,7 +10,7 @@ libraryDependencies ++= Seq( ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface"), ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface.text") ), - "org.jline" % "jline" % "3.26.1", + "org.jline" % "jline" % "3.26.2", "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/platform/frontends/jssrc2cpg/build.sbt b/platform/frontends/jssrc2cpg/build.sbt index 62e3b675..8c9a9b3b 100644 --- a/platform/frontends/jssrc2cpg/build.sbt +++ b/platform/frontends/jssrc2cpg/build.sbt @@ -20,7 +20,7 @@ astGenVersion := appProperties.value.getString("jssrc2cpg.astgen_version") libraryDependencies ++= Seq( "io.appthreat" %% "cpg2" % Versions.cpg, "com.lihaoyi" %% "upickle" % Versions.upickle, - "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.1", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.17.2", "com.typesafe" % "config" % "1.4.3", "com.michaelpollmeier" % "versionsort" % "1.0.11", "org.scalatest" %% "scalatest" % Versions.scalatest % Test From 5d23099f5cdb53297629ce3a705d26c11b73e3ec Mon Sep 17 00:00:00 2001 From: Prabhu Subramanian Date: Sat, 6 Jul 2024 06:40:18 +0100 Subject: [PATCH 4/4] Tweaks Signed-off-by: Prabhu Subramanian --- .github/workflows/release.yml | 2 +- console/src/main/scala/io/appthreat/console/Console.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 802e69db..a42c6d12 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,7 +86,7 @@ jobs: - name: Release Anaconda if: startsWith(github.ref, 'refs/tags/') run: | - anaconda upload --label main -u appthreat /tmp/conda-out/noarch/*.tar.bz2 + $CONDA/bin/anaconda upload --label main -u appthreat /tmp/conda-out/noarch/*.tar.bz2 env: ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_TOKEN }} continue-on-error: true diff --git a/console/src/main/scala/io/appthreat/console/Console.scala b/console/src/main/scala/io/appthreat/console/Console.scala index a6f6783d..87971fe4 100644 --- a/console/src/main/scala/io/appthreat/console/Console.scala +++ b/console/src/main/scala/io/appthreat/console/Console.scala @@ -528,7 +528,7 @@ class Console[T <: Project]( tree: Boolean = false, as_text: Boolean = false ): String = - if tree then + if tree || includeCalls then val rootTree = richTreeLib.Tree(title, highlight = true) atom.file.whereNot(_.name("<(unknown|includes)>")).foreach { f => val childTree = richTreeLib.Tree(f.name, highlight = true) @@ -652,7 +652,7 @@ class Console[T <: Project]( val dec: Set[Declaration] = (f.assignment.argument(1).filterNot( _.code == "this" - ).isIdentifier.refsTo ++ f.method.parameter + ).isIdentifier.nameNot("tmp[0-9]+$").refsTo ++ f.method.parameter .filterNot(_.code == "this") .filter(_.typeFullName != "ANY")).toSet table.add_row(f.name, dec.name.toSet.mkString("\n"))