Update packages. Python crypto tagging improvements (#71)

Signed-off-by: Prabhu Subramanian <[email protected]>

.github/workflows/containers.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'zulu'
+          distribution: 'temurin'
           java-version: '21'
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"

.github/workflows/master.yml

@@ -14,8 +14,8 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'zulu'
-          java-version: '21'
+          distribution: 'temurin'
+          java-version: '22'
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
@@ -41,12 +41,10 @@ jobs:
         if: runner.os == 'Linux'
       - name: Install graphviz-dev
         run: |
-          brew install graphviz
-          python -m pip install \
-              --global-option=build_ext \
-              --global-option="-I$(brew --prefix graphviz)/include/" \
-              --global-option="-L$(brew --prefix graphviz)/lib/" \
-              pygraphviz
+          brew install graphviz sbt
+          export C_INCLUDE_PATH="$(brew --prefix graphviz)/include/"
+          export LIBRARY_PATH="$(brew --prefix graphviz)/lib/"
+          pip install --use-pep517 --config-setting="--global-option=build_ext" pygraphviz
         if: runner.os == 'macOS'
       - name: Install and test
         run: |

.github/workflows/pr.yml

@@ -6,16 +6,17 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        jvm: ['21']
+        jvm: ['21', '22']
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'zulu'
+          distribution: 'temurin'
           java-version: ${{ matrix.jvm }}
+          cache: "sbt"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
@@ -41,12 +42,10 @@ jobs:
         if: runner.os == 'Linux'
       - name: Install graphviz-dev
         run: |
-          brew install graphviz
-          python -m pip install \
-              --global-option=build_ext \
-              --global-option="-I$(brew --prefix graphviz)/include/" \
-              --global-option="-L$(brew --prefix graphviz)/lib/" \
-              pygraphviz
+          brew install graphviz sbt
+          export C_INCLUDE_PATH="$(brew --prefix graphviz)/include/"
+          export LIBRARY_PATH="$(brew --prefix graphviz)/lib/"
+          pip install --use-pep517 --config-setting="--global-option=build_ext" pygraphviz
         if: runner.os == 'macOS'
       - name: Install and test
         run: |

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          distribution: 'zulu'
+          distribution: 'temurin'
           java-version: '21'
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = 3.7.15
+version = 3.8.1
 runner.dialect = scala3
 preset = IntelliJ
 maxColumn = 100
@@ -12,3 +12,6 @@ rewrite.scala3.removeOptionalBraces = yes
 rewrite.scala3.insertEndMarkerMinLines = 20
 rewrite.scala3.removeEndMarkerMaxLines = 18
 
+rewrite.rules = [AvoidInfix]
+rewrite.avoidInfix.excludeFilters."+" = [ "map" ]
+rewrite.avoidInfix.excludePlaceholderArg = false

build.sbt

@@ -1,7 +1,7 @@
 name                     := "chen"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.0.8"
-ThisBuild / scalaVersion := "3.3.1"
+ThisBuild / version      := "2.0.9"
+ThisBuild / scalaVersion := "3.4.1"
 
 val cpgVersion = "1.0.0"
 
@@ -34,8 +34,8 @@ lazy val aggregatedProjects: Seq[ProjectReference] = Seq(
 )
 
 ThisBuild / libraryDependencies ++= Seq(
-  "org.slf4j"                % "slf4j-api"         % "2.0.12",
-  "org.slf4j"                % "slf4j-nop"         % "2.0.12" % Optional,
+  "org.slf4j"                % "slf4j-api"         % "2.0.13",
+  "org.slf4j"                % "slf4j-nop"         % "2.0.13" % Optional,
 )
 
 ThisBuild / compile / javacOptions ++= Seq(

ci/Dockerfile

@@ -11,9 +11,9 @@ LABEL maintainer="appthreat" \
       org.opencontainers.image.description="Container image for AppThreat chen code analysis platform" \
       org.opencontainers.docker.cmd="docker run --rm -v /tmp:/tmp -v $HOME:$HOME -v $(pwd):/app:rw -it ghcr.io/appthreat/chen chennai"
 
-ARG JAVA_VERSION=21.0.2-graalce
+ARG JAVA_VERSION=22.0.1-graalce
 ARG MAVEN_VERSION=3.9.6
-ARG GRADLE_VERSION=8.5
+ARG GRADLE_VERSION=8.7
 
 ENV JAVA_VERSION=$JAVA_VERSION \
     MAVEN_VERSION=$MAVEN_VERSION \

codemeta.json

@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/chen",
   "issueTracker": "https://github.com/AppThreat/chen/issues",
   "name": "chen",
-  "version": "2.0.8",
+  "version": "2.0.9",
   "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.",
   "applicationCategory": "code-analysis",
   "keywords": [

console/build.sbt

@@ -24,11 +24,11 @@ libraryDependencies ++= Seq(
   "io.circe"             %% "circe-generic"     % CirceVersion,
   "io.circe"             %% "circe-parser"      % CirceVersion,
   "org.zeroturnaround"    % "zt-zip"            % ZeroturnaroundVersion,
-  "com.lihaoyi"          %% "os-lib"            % "0.9.3",
-  "com.lihaoyi"          %% "pprint"            % "0.8.1",
+  "com.lihaoyi"          %% "os-lib"            % "0.10.0",
+  "com.lihaoyi"          %% "pprint"            % "0.9.0",
   "com.lihaoyi"          %% "cask"              % CaskVersion,
   "dev.scalapy"          %% "scalapy-core"      % "0.5.3",
-  "org.scala-lang.modules" % "scala-asm"        % "9.6.0-scala-1",
+  "org.scala-lang.modules" % "scala-asm"        % "9.7.0-scala-2",
   "org.scalatest"        %% "scalatest"         % Versions.scalatest % Test
 )
 

console/src/main/scala/io/appthreat/console/Console.scala

@@ -33,7 +33,7 @@ class Console[T <: Project](
     def config: ConsoleConfig = _config
     def console: Console[T]   = this
 
-    protected var workspaceManager: WorkspaceManager[T] = _
+    protected var workspaceManager: WorkspaceManager[T] = scala.compiletime.uninitialized
     switchWorkspace(baseDir.path.resolve("workspace").toString)
     protected def workspacePathName: String = workspaceManager.getPath
 
@@ -797,7 +797,7 @@ class Console[T <: Project](
                 lastCallerMethod = methodName
                 dashCount += 1
             if dashCount < depth then
-                calleeList foreach { c =>
+                calleeList.foreach { c =>
                     tree += s"${printDashes(dashCount)}${c.fullName}~~${c.location.filename}#${c.lineNumber.getOrElse(0)}"
                     findCallee(c.fullName, tree)
                 }

console/src/test/scala/io/appthreat/console/ConsoleTests.scala

@@ -289,7 +289,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
       )
       val numOverlayFilesBefore = console.project.path.resolve("overlays").toFile.list().length
       numOverlayFilesBefore shouldBe 0
-      console._runAnalyzer(defaultOverlayCreators(): _*)
+      console._runAnalyzer(defaultOverlayCreators()*)
       console.project.appliedOverlays shouldBe List(
         Base.overlayName,
         ControlFlow.overlayName,

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/language/ExtendedCfgNode.scala

@@ -39,7 +39,7 @@ class ExtendedCfgNode(val traversal: Iterator[CfgNode]) extends AnyVal:
       sourceTrav: IterableOnce[NodeType],
       sourceTravs: IterableOnce[NodeType]*
     )(implicit context: EngineContext): Iterator[NodeType] =
-        val sources = sourceTravsToStartingPoints(sourceTrav +: sourceTravs: _*)
+        val sources = sourceTravsToStartingPoints(sourceTrav +: sourceTravs*)
         val reachedSources =
             reachableByInternal(sources).map(_.path.head.node)
         reachedSources.cast[NodeType]
@@ -51,7 +51,7 @@ class ExtendedCfgNode(val traversal: Iterator[CfgNode]) extends AnyVal:
     def reachableByFlows[A](sourceTrav: IterableOnce[A], sourceTravs: IterableOnce[A]*)(implicit
       context: EngineContext
     ): Iterator[Path] =
-        val sources        = sourceTravsToStartingPoints(sourceTrav +: sourceTravs: _*)
+        val sources        = sourceTravsToStartingPoints(sourceTrav +: sourceTravs*)
         val startingPoints = sources.map(_.startingPoint)
         val paths = reachableByInternal(sources).par
             .map { result =>
@@ -79,7 +79,7 @@ class ExtendedCfgNode(val traversal: Iterator[CfgNode]) extends AnyVal:
       sourceTravs: Iterator[NodeType]*
     )(implicit context: EngineContext): Vector[TableEntry] =
         val sources =
-            SourcesToStartingPoints.sourceTravsToStartingPoints(sourceTrav +: sourceTravs: _*)
+            SourcesToStartingPoints.sourceTravsToStartingPoints(sourceTrav +: sourceTravs*)
         reachableByInternal(sources)
 
     private def removeConsecutiveDuplicates[T](l: Vector[T]): List[T] =

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/passes/reachingdef/DataFlowSolver.scala

@@ -8,7 +8,7 @@ class DataFlowSolver:
       * given by two maps: `in` and `out`. These maps associate all CFG nodes with the set of
       * definitions at node entry and node exit respectively.
       */
-    def calculateMopSolutionForwards[Node, T <: Iterable[_]](problem: DataFlowProblem[Node, T])
+    def calculateMopSolutionForwards[Node, T <: Iterable[?]](problem: DataFlowProblem[Node, T])
       : Solution[Node, T] =
         var out: Map[Node, T] = problem.inOutInit.initOut
         var in                = problem.inOutInit.initIn
@@ -43,7 +43,7 @@ class DataFlowSolver:
       * given by two maps: `in` and `out`. These maps associate all CFG nodes with the set of
       * definitions at node entry and node exit respectively.
       */
-    def calculateMopSolutionBackwards[Node, T <: Iterable[_]](problem: DataFlowProblem[Node, T])
+    def calculateMopSolutionBackwards[Node, T <: Iterable[?]](problem: DataFlowProblem[Node, T])
       : Solution[Node, T] =
         var out: Map[Node, T] = problem.inOutInit.initOut
         var in                = problem.inOutInit.initIn

dataflowengineoss/src/main/scala/io/appthreat/dataflowengineoss/queryengine/HeldTaskCompletion.scala

@@ -113,7 +113,6 @@ class HeldTaskCompletion(
                 (parentTask, TableEntry(newPath))
             }
             .filter { case (_, tableEntry) => containsCycle(tableEntry) }
-    end createResultsForHeldTaskAndTableResult
 
     private def containsCycle(tableEntry: TableEntry): Boolean =
         val pathSeq =

meta.yaml

@@ -1,4 +1,4 @@
-{% set version = "2.0.8" %}
+{% set version = "2.0.9" %}
 
 package:
   name: chen
@@ -19,17 +19,17 @@ requirements:
     - setuptools
     - poetry
   run:
-    - python>=3.8.1,<3.12
-    - conda-forge::httpx>=0.24.1,<0.25.0
-    - conda-forge::websockets>=11.0.2,<12.0.0
-    - conda-forge::orjson>=3.9.0,<4.0.0
-    - conda-forge::rich>=13.4.1,<14.0.0
+    - python>=3.10,<3.12
+    - conda-forge::httpx>=0.27.0
+    - conda-forge::websockets>=12.0
+    - conda-forge::orjson>=3.10.1,<4.0.0
+    - conda-forge::rich>=13.7.1,<14.0.0
     - conda-forge::oras-py==0.1.26
     - conda-forge::appdirs>=1.4.4,<2.0.0
-    - conda-forge::psutil>=5.9.5,<6.0.0
+    - conda-forge::psutil>=5.9.8,<6.0.0
     - conda-forge::packageurl-python>=0.11.2,<0.12.0
-    - conda-forge::gitpython>=3.1.37,<4.0.0
-    - conda-forge::networkx>=3.1
+    - conda-forge::gitpython>=3.1.43,<4.0.0
+    - conda-forge::networkx>=3.3
     - conda-forge::numpy>=1.26
     - conda-forge::scipy>=1.11.3
 build:

platform/frontends/c2cpg/build.sbt

@@ -4,13 +4,13 @@ dependsOn(Projects.semanticcpg, Projects.dataflowengineoss % Test, Projects.x2cp
 
 libraryDependencies ++= Seq(
   "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4",
-  "org.eclipse.platform"    % "org.eclipse.equinox.common"       % "3.18.200",
-  "org.eclipse.platform"    % "org.eclipse.core.resources"       % "3.20.0" excludeAll(
+  "org.eclipse.platform"    % "org.eclipse.equinox.common"       % "3.19.0",
+  "org.eclipse.platform"    % "org.eclipse.core.resources"       % "3.20.100" excludeAll(
     ExclusionRule(organization = "com.ibm.icu", name = "icu4j"),
     ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface"),
     ExclusionRule(organization = "org.eclipse.platform", name = "org.eclipse.jface.text")
   ),
-  "org.jline"               % "jline"                      % "3.25.1",
+  "org.jline"               % "jline"                      % "3.26.1",
   "org.scalatest"          %% "scalatest"                  % Versions.scalatest % Test
 )
 

platform/frontends/c2cpg/src/main/scala/io/appthreat/c2cpg/parser/HeaderFileFinder.scala

@@ -24,4 +24,3 @@ class HeaderFileFinder(root: String):
         val matches = nameToPathMap.getOrElse(name, List())
         matches.map(_.toString).sortBy(x => Levenshtein.distance(x, path)).headOption
     }
-end HeaderFileFinder

platform/frontends/javasrc2cpg/build.sbt

@@ -4,12 +4,12 @@ dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->t
 
 libraryDependencies ++= Seq(
   "io.appthreat"           %% "cpg2"             % Versions.cpg,
-  "com.github.javaparser"   % "javaparser-symbol-solver-core" % "3.25.8",
+  "com.github.javaparser"   % "javaparser-symbol-solver-core" % "3.25.10",
   "org.gradle"              % "gradle-tooling-api"            % Versions.gradleTooling,
   "org.scalatest"          %% "scalatest"                     % Versions.scalatest % Test,
-  "org.projectlombok"       % "lombok"                        % "1.18.30",
+  "org.projectlombok"       % "lombok"                        % "1.18.32",
   "org.scala-lang.modules" %% "scala-parallel-collections"    % "1.0.4",
-  "org.scala-lang.modules" %% "scala-parser-combinators"      % "2.3.0",
+  "org.scala-lang.modules" %% "scala-parser-combinators"      % "2.4.0",
   "net.lingala.zip4j"       % "zip4j"                         % "2.11.5"
 )
 

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/typesolvers/EagerSourceTypeSolver.scala

@@ -21,14 +21,14 @@ class EagerSourceTypeSolver(
 ) extends TypeSolver:
 
     private val logger             = LoggerFactory.getLogger(this.getClass)
-    private var parent: TypeSolver = _
+    private var parent: TypeSolver = scala.compiletime.uninitialized
 
     private val foundTypes: Map[String, SymbolReference[ResolvedReferenceTypeDeclaration]] =
         filenames
             .flatMap(sourceParser.parseTypesFile)
             .flatMap { cu =>
                 symbolSolver.inject(cu)
-                cu.findAll(classOf[TypeDeclaration[_]])
+                cu.findAll(classOf[TypeDeclaration[?]])
                     .asScala
                     .map { typeDeclaration =>
                         val name = typeDeclaration.getFullyQualifiedName.toScala match

platform/frontends/javasrc2cpg/src/main/scala/io/appthreat/javasrc2cpg/typesolvers/SimpleCombinedTypeSolver.scala

@@ -14,7 +14,7 @@ import scala.jdk.OptionConverters.RichOptional
 class SimpleCombinedTypeSolver extends TypeSolver:
 
     private val logger             = LoggerFactory.getLogger(this.getClass)
-    private var parent: TypeSolver = _
+    private var parent: TypeSolver = scala.compiletime.uninitialized
     // Ideally all types would be cached in the SimpleCombinedTypeSolver to avoid unnecessary unresolved types
     // from being cached. The EagerSourceTypeSolver preloads all types, however, so separating caching and
     // non-caching solvers avoids caching types twice.

platform/frontends/jimple2cpg/build.sbt

@@ -3,13 +3,13 @@ name := "jimple2cpg"
 dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
-  "io.appthreat"  %% "cpg2" % Versions.cpg,
-  "org.soot-oss"   % "soot"              % "4.4.1",
-  "org.scala-lang.modules" % "scala-asm" % "9.6.0-scala-1",
-  "org.ow2.asm"            % "asm"       % "9.6",
-  "org.ow2.asm"            % "asm-analysis"       % "9.6",
-  "org.ow2.asm"            % "asm-util"           % "9.6",
-  "org.ow2.asm"            % "asm-tree"           % "9.6",
+  "io.appthreat"           %% "cpg2" % Versions.cpg,
+  "org.soot-oss"           % "soot"              % "4.5.0",
+  "org.scala-lang.modules" % "scala-asm" % "9.7.0-scala-2",
+  "org.ow2.asm"            % "asm"       % "9.7",
+  "org.ow2.asm"            % "asm-analysis"       % "9.7",
+  "org.ow2.asm"            % "asm-util"           % "9.7",
+  "org.ow2.asm"            % "asm-tree"           % "9.7",
   "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 

platform/frontends/jimple2cpg/src/test/scala/io/appthreat/jimple2cpg/unpacking/JarUnpackingTests.scala

@@ -16,8 +16,8 @@ import scala.util.{Failure, Success, Try}
 
 class JarUnpackingTests extends AnyWordSpec with Matchers with BeforeAndAfterAll {
 
-  var validCpgs: Map[String, Cpg] = _
-  var slippyCpg: Cpg              = _
+  var validCpgs: Map[String, Cpg] = scala.compiletime.uninitialized
+  var slippyCpg: Cpg              = scala.compiletime.uninitialized
 
   override protected def beforeAll(): Unit = {
     super.beforeAll()

platform/frontends/jssrc2cpg/build.sbt

@@ -20,7 +20,7 @@ astGenVersion := appProperties.value.getString("jssrc2cpg.astgen_version")
 libraryDependencies ++= Seq(
   "io.appthreat"              %% "cpg2" % Versions.cpg,
   "com.lihaoyi"               %% "upickle"           % Versions.upickle,
-  "com.fasterxml.jackson.core" % "jackson-databind"  % "2.16.1",
+  "com.fasterxml.jackson.core" % "jackson-databind"  % "2.17.0",
   "com.typesafe"               % "config"            % "1.4.3",
   "com.michaelpollmeier"       % "versionsort"       % "1.0.11",
   "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test

platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/astcreation/AstForFunctionsCreator.scala

@@ -255,7 +255,7 @@ trait AstForFunctionsCreator(implicit withSchemaValidation: ValidationMode):
                                             case i: NewIdentifier if localNode.name == i.name => i
                                         }
                                         .map { i => assignmentCallAst.withRefEdge(i, localNode) }
-                                        .reduce(_ merge _)
+                                        .reduce(_.merge(_))
                                 case RestElement =>
                                     handleRestInParameters(elementNodeInfo, nodeInfo, paramName)
                                 case _ => astForNodeWithFunctionReference(elementNodeInfo.json)

platform/frontends/jssrc2cpg/src/main/scala/io/appthreat/jssrc2cpg/datastructures/Scope.scala

@@ -53,7 +53,7 @@ class Scope:
         addVariable(stack, variableName, variableNode, scopeType)
 
     def addVariableReference(variableName: String, referenceNode: NewNode): Unit =
-        pendingReferences prepend PendingReference(variableName, referenceNode, stack)
+        pendingReferences.prepend(PendingReference(variableName, referenceNode, stack))
 
     def resolve(unresolvedHandler: (NewNode, String) => (NewNode, ScopeType))
       : Iterator[ResolvedReference] =

platform/frontends/php2atom/src/main/scala/io/appthreat/php2atom/passes/PhpSetKnownTypes.scala

@@ -50,7 +50,7 @@ class PhpSetKnownTypesPass(cpg: Cpg, knownTypesFile: Option[JFile] = None)
         val builtinMethod = cpg.method.fullNameExact(part.name).l
         builtinMethod.foreach(mNode =>
             setTypes(builder, mNode.methodReturn, part.rTypes)
-            (mNode.parameter.l zip part.pTypes).map((p, pTypes) => setTypes(builder, p, pTypes))
+            (mNode.parameter.l.zip(part.pTypes)).map((p, pTypes) => setTypes(builder, p, pTypes))
         )
 
     def createKnownFunctionFromLine(line: String): Option[KnownFunction] =