scala 3.4.1 migration. Update chen to bring python cbom improvements. (…

…#138) Signed-off-by: Prabhu Subramanian <[email protected]>
AppThreat · Apr 29, 2024 · 059d5ca · 059d5ca
1 parent 66afc38
commit 059d5ca
Show file tree

Hide file tree

Showing 17 changed files with 57 additions and 46 deletions.
diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
@@ -29,7 +29,7 @@ jobs:
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '21'
+          java-version: '22'
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:

diff --git a/.github/workflows/nodejstests.yml b/.github/workflows/nodejstests.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        java-version: ['21']
+        java-version: ['21', '22']
         node-version: ['21.x']
     runs-on: ${{ matrix.os }}
     steps:
@@ -74,7 +74,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        java-version: ['21']
+        java-version: ['21', '22']
         node-version: ['21.x']
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -38,7 +38,7 @@ jobs:
         uses: graalvm/setup-graalvm@v1
         with:
           distribution: 'graalvm-community'
-          java-version: '21'
+          java-version: '22'
           github-token: ${{ secrets.GITHUB_TOKEN }}
           native-image-job-reports: 'true'
           cache: 'sbt'

diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,4 +1,4 @@
-version = 3.7.15
+version = 3.8.1
 runner.dialect = scala3
 preset = IntelliJ
 maxColumn = 100
@@ -12,3 +12,6 @@ rewrite.scala3.removeOptionalBraces = yes
 rewrite.scala3.insertEndMarkerMinLines = 20
 rewrite.scala3.removeEndMarkerMaxLines = 18
 
+rewrite.rules = [AvoidInfix]
+rewrite.avoidInfix.excludeFilters."+" = [ "map" ]
+rewrite.avoidInfix.excludePlaceholderArg = false
diff --git a/build.sbt b/build.sbt
@@ -1,9 +1,9 @@
 name                     := "atom"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.0.9"
-ThisBuild / scalaVersion := "3.3.1"
+ThisBuild / version      := "2.0.10"
+ThisBuild / scalaVersion := "3.4.1"
 
-val chenVersion      = "2.0.8"
+val chenVersion      = "2.0.9"
 
 lazy val atom = Projects.atom
 
@@ -12,7 +12,7 @@ val astGenVersion = "3.5.0"
 libraryDependencies ++= Seq(
   "com.github.pathikrit"    %% "better-files"      % "3.9.2",
   "com.github.scopt"        %% "scopt"             % "4.1.0",
-  "org.slf4j"                % "slf4j-nop"         % "2.0.12" % Optional,
+  "org.slf4j"                % "slf4j-nop"         % "2.0.13" % Optional,
   "io.appthreat"                %% "c2cpg"             % Versions.chen excludeAll (
     ExclusionRule(organization = "com.ibm.icu", name = "icu4j"),
     ExclusionRule(organization = "org.jline", name = "jline"),
@@ -24,19 +24,19 @@ libraryDependencies ++= Seq(
   "io.appthreat"      %% "javasrc2cpg"       % Versions.chen,
   "io.appthreat"      %% "jssrc2cpg"         % Versions.chen,
   "io.appthreat"      %% "jimple2cpg"        % Versions.chen,
-  "io.appthreat"      %% "php2atom"        % Versions.chen,
+  "io.appthreat"      %% "php2atom"          % Versions.chen,
   "io.appthreat"      %% "semanticcpg"       % Versions.chen % Test classifier "tests",
   "io.appthreat"      %% "x2cpg"             % Versions.chen % Test classifier "tests",
   "io.appthreat"      %% "pysrc2cpg"         % Versions.chen % Test classifier "tests",
-  "org.scalatest" %% "scalatest"         % "3.2.17"       % Test
+  "org.scalatest"     %% "scalatest"         % "3.2.18"       % Test
 )
 
 Compile / doc / scalacOptions ++= Seq("-doc-title", "atom apidocs", "-doc-version", version.value)
 
 ThisBuild / scalacOptions ++= Seq(
   "-deprecation", // Emit warning and location for usages of deprecated APIs.
   "--release",
-  "21",
+  "21"
 )
 
 ThisBuild / compile / javacOptions ++= Seq(

diff --git a/ci/Dockerfile.sle b/ci/Dockerfile.sle
@@ -37,6 +37,7 @@ RUN set -e; \
         *) echo >&2 "error: unsupported architecture: '$ARCH_NAME'"; exit 1 ;; \
     esac; \
     zypper --non-interactive install -l --no-recommends php8 php8-cli php8-curl php8-zip php8-bcmath php8-pear php8-mbstring php8-devel \
+    && source /root/.nvm/nvm.sh \
     && python3 --version \
     && source /root/.nvm/nvm.sh \
     && node --version \

diff --git a/codemeta.json b/codemeta.json
@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/atom",
   "issueTracker": "https://github.com/AppThreat/atom/issues",
   "name": "atom",
-  "version": "2.0.9",
+  "version": "2.0.10",
   "description": "Atom is a novel intermediate representation for next-generation code analysis.",
   "applicationCategory": "code-analysis",
   "keywords": [

diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,7 +1,7 @@
 ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
 
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
-addSbtPlugin("com.github.sbt"  % "sbt-native-packager"   % "1.9.16")
+addSbtPlugin("com.github.sbt"  % "sbt-native-packager"   % "1.10.0")
 addSbtPlugin("com.codecommit" % "sbt-github-packages" % "0.5.3")
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5")
-addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.1.5")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.2.0")
+addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.1.6")
diff --git a/src/main/scala/io/appthreat/atom/Atom.scala b/src/main/scala/io/appthreat/atom/Atom.scala
@@ -536,7 +536,7 @@ object Atom:
                               .withOutputPath(outputAtomFile)
                               .withDefaultIgnoredFilesRegex(List("\\..*".r))
                               .withIgnoredFilesRegex(
-                                ".*(samples|examples|test|tests|unittests|docs|virtualenvs|venv|benchmarks|tutorials|noxfile).*"
+                                ".*(samples|test|tests|unittests|docs|virtualenvs|venv|benchmarks|tutorials|noxfile).*"
                               )
                         )
                         .map { ag =>

diff --git a/src/main/scala/io/appthreat/atom/dataflows/DataFlowGraph.scala b/src/main/scala/io/appthreat/atom/dataflows/DataFlowGraph.scala
@@ -24,7 +24,7 @@ private class DataFlowGraph(nodes: Set[Option[DFNode]]):
     private def isSubList[A](short: List[A], long: List[A]): Boolean =
         val sLong  = long.to(LazyList)
         val sShort = short.to(LazyList)
-        sLong.tails exists (_.startsWith(sShort))
+        sLong.tails.exists(_.startsWith(sShort))
 
     private def isSubList[A](lst: List[A])(implicit finalSet: mutable.Set[Path]): Boolean =
         finalSet.filterNot(_.size < lst.size).exists(xs => isSubList(lst, xs))

diff --git a/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala b/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala
@@ -63,7 +63,7 @@ object PythonDependencyParser extends XDependencyParser:
                     ModuleWithVersion(
                       name,
                       exactVersion.stripPrefix("=="),
-                      (versions diff Seq(exactVersion)).mkString(",")
+                      (versions.diff(Seq(exactVersion))).mkString(",")
                     )
                 case requirementsPattern(name, versionSpecifiers, _) =>
                     ModuleWithVersion(name, versionSpecifiers = versionSpecifiers)

diff --git a/src/main/scala/io/appthreat/atom/parsedeps/package.scala b/src/main/scala/io/appthreat/atom/parsedeps/package.scala
@@ -65,7 +65,7 @@ package object parsedeps:
                     ModuleWithVersion(
                       name,
                       exactVersion.stripPrefix("=="),
-                      (vs diff Set(exactVersion)).mkString(","),
+                      (vs.diff(Set(exactVersion))).mkString(","),
                       importedSymbols = is
                     )
                 case None => ModuleWithVersion(

diff --git a/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/DataFlowSlicing.scala
@@ -17,7 +17,7 @@ class DataFlowSlicing:
         Executors.newVirtualThreadPerTaskExecutor()
     private val excludeOperatorCalls     = new AtomicBoolean(true)
     private val nodeCache                = new TrieMap[Long, SliceNode]()
-    private var language: Option[String] = _
+    private var language: Option[String] = scala.compiletime.uninitialized
 
     def calculateDataFlowSlice(atom: Cpg, config: DataFlowConfig): Option[DataFlowSlice] =
         language = atom.metaData.language.headOption

diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala
@@ -42,11 +42,15 @@ object ReachableSlicing:
             atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(
               toSlice
             ).toList
-        if config.includeCryptoFlows && (language == Languages.JAVA || language == Languages.JAVASRC)
-        then
-            flowsList ++= atom.tag.name(CRYPTO_GENERATE_TAG).call.reachableByFlows(
-              atom.tag.name(CRYPTO_ALGORITHM_TAG).literal
-            ).map(toSlice).toList
+        if config.includeCryptoFlows then
+            if language == Languages.JAVA || language == Languages.JAVASRC then
+                flowsList ++= atom.tag.name(CRYPTO_GENERATE_TAG).call.reachableByFlows(
+                  atom.tag.name(CRYPTO_ALGORITHM_TAG).literal
+                ).map(toSlice).toList
+            else if language == Languages.PYTHON || language == Languages.PYTHONSRC then
+                flowsList ++= atom.tag.name(CRYPTO_GENERATE_TAG).call.reachableByFlows(
+                  atom.tag.name(CRYPTO_ALGORITHM_TAG).call
+                ).map(toSlice).toList
         // For JavaScript and Python, we need flows between arguments of call nodes to track callbacks and middlewares
         if
             language == Languages.JSSRC || language == Languages.JAVASCRIPT || language == Languages.PYTHON || language == Languages.PYTHONSRC
@@ -74,9 +78,14 @@ object ReachableSlicing:
                 .reachableByFlows(sourceI, dynFrameworkIdentifier)
                 .map(toSlice)
                 .toList
-            flowsList ++= atom.tag.name("pkg.*").identifier.reachableByFlows(
-              atom.tag.name(CLI_SOURCE_TAG).call
-            ).map(toSlice).toList
+            if language == Languages.PYTHON || language == Languages.PYTHONSRC then
+                flowsList ++= atom.tag.name("pkg.*").identifier.reachableByFlows(
+                  atom.tag.name(CLI_SOURCE_TAG).identifier
+                ).map(toSlice).toList
+            else
+                flowsList ++= atom.tag.name("pkg.*").identifier.reachableByFlows(
+                  atom.tag.name(CLI_SOURCE_TAG).call
+                ).map(toSlice).toList
         end if
         if language == Languages.PHP
         then

diff --git a/src/main/scala/io/appthreat/atom/slicing/package.scala b/src/main/scala/io/appthreat/atom/slicing/package.scala
@@ -225,7 +225,6 @@ package object slicing:
                 s"inv: [${invokedCalls.mkString(",")}], " +
                 s"argsTo: [${argToCalls.mkString(",")}]" +
                 s"}"
-    end ObjectUsageSlice
 
     implicit val decodeObjectUsageSlice: Decoder[ObjectUsageSlice] =
         (c: HCursor) =>
@@ -402,7 +401,7 @@ package object slicing:
           Decoder[CallDef].widen,
           Decoder[ParamDef].widen,
           Decoder[UnknownDef].widen
-        ).reduceLeft(_ or _)
+        ).reduceLeft(_.or(_))
 
     object DefComponent:
 
@@ -550,7 +549,6 @@ package object slicing:
     ):
         override def toString: String =
             s"$callName(${paramTypes.mkString(",")}):$returnType"
-    end UsedCall
 
     /** Details related to an observed call.
       */
@@ -686,7 +684,7 @@ package object slicing:
         List[Decoder[UsedCall]](
           Decoder[ObservedCall].widen,
           Decoder[ObservedCallWithArgPos].widen
-        ).reduceLeft(_ or _)
+        ).reduceLeft(_.or(_))
 
     /** Describes types defined within the application.
       *

diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json
diff --git a/wrapper/nodejs/package.json b/wrapper/nodejs/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom",
-  "version": "2.0.9",
+  "version": "2.0.10",
   "description": "Create atom (⚛) representation for your application, packages and libraries",
   "exports": "./index.js",
   "type": "module",
@@ -9,8 +9,8 @@
     "lint": "eslint *.mjs *.js"
   },
   "dependencies": {
-    "@babel/parser": "^7.24.0",
-    "typescript": "^5.4.2",
+    "@babel/parser": "^7.24.4",
+    "typescript": "^5.4.5",
     "yargs": "^17.7.2"
   },
   "devDependencies": {