diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d8b474bd..80fef042 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - jvm: ['21', '22'] + jvm: ['21', '22', '23'] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/win_compat.yml b/.github/workflows/win_compat.yml index 8bffb232..897ea4e7 100644 --- a/.github/workflows/win_compat.yml +++ b/.github/workflows/win_compat.yml @@ -8,7 +8,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python-version: ['3.8','3.9','3.10','3.11','3.12'] + python-version: ['3.10','3.11','3.12'] with-science: ["--download", "--download --with-science"] fail-fast: false diff --git a/build.sbt b/build.sbt index f10bf43d..0a1b0b84 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "chen" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "2.1.6" +ThisBuild / version := "2.1.7" ThisBuild / scalaVersion := "3.5.0" val cpgVersion = "1.0.0" diff --git a/codemeta.json b/codemeta.json index 8de30648..1f7a0c66 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,7 +7,7 @@ "downloadUrl": "https://github.com/AppThreat/chen", "issueTracker": "https://github.com/AppThreat/chen/issues", "name": "chen", - "version": "2.1.6", + "version": "2.1.7", "description": "Code Hierarchy Exploration Net (chen) is an advanced exploration toolkit for your application source code and its dependency hierarchy.", "applicationCategory": "code-analysis", "keywords": [ diff --git a/meta.yaml b/meta.yaml index 0d8dadc4..c5db20cd 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,4 +1,4 @@ -{% set version = "2.1.6" %} +{% set version = "2.1.7" %} package: name: chen diff --git a/platform/frontends/javasrc2cpg/build.sbt b/platform/frontends/javasrc2cpg/build.sbt index 86c37a04..8130a2ad 100644 --- a/platform/frontends/javasrc2cpg/build.sbt +++ b/platform/frontends/javasrc2cpg/build.sbt @@ -5,7 +5,6 @@ dependsOn(Projects.dataflowengineoss, Projects.x2cpg % "compile->compile;test->t libraryDependencies ++= Seq( "io.appthreat" %% "cpg2" % Versions.cpg, "com.github.javaparser" % "javaparser-symbol-solver-core" % "3.26.2", - "org.gradle" % "gradle-tooling-api" % Versions.gradleTooling, "org.scalatest" %% "scalatest" % Versions.scalatest % Test, "org.projectlombok" % "lombok" % "1.18.34", "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", diff --git a/platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/types/TSTypesTest.scala b/platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/types/TSTypesTest.scala index 81c03992..14e3d124 100644 --- a/platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/types/TSTypesTest.scala +++ b/platform/frontends/jssrc2cpg/src/test/scala/io/appthreat/jssrc2cpg/types/TSTypesTest.scala @@ -46,7 +46,7 @@ class TSTypesTest extends AbstractPassTest { args.name shouldBe "args" args.code shouldBe "...args" args.isVariadic shouldBe true - args.typeFullName shouldBe Defines.Any + args.typeFullName shouldBe "Array" } "have return types for arrow functions" in AstFixture("const foo = () => 42;", tsTypes = true) { cpg => @@ -138,7 +138,7 @@ class TSTypesTest extends AbstractPassTest { inside(cpg.identifier.l) { case List(x) => x.name shouldBe "x" x.code shouldBe "x" - x.typeFullName shouldBe Defines.String // we can actually follow type intrinsics + x.typeFullName shouldBe "ModifiedNickName" } } diff --git a/platform/frontends/x2cpg/build.sbt b/platform/frontends/x2cpg/build.sbt index a0769c29..e9587457 100644 --- a/platform/frontends/x2cpg/build.sbt +++ b/platform/frontends/x2cpg/build.sbt @@ -6,7 +6,6 @@ libraryDependencies ++= Seq( "io.circe" %% "circe-core" % Versions.circe, "io.circe" %% "circe-generic" % Versions.circe, "io.circe" %% "circe-parser" % Versions.circe, - "org.gradle" % "gradle-tooling-api" % Versions.gradleTooling % Optional, "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/DependencyResolver.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/DependencyResolver.scala index ea42850a..aae4187f 100644 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/DependencyResolver.scala +++ b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/DependencyResolver.scala @@ -3,7 +3,6 @@ package io.appthreat.x2cpg.utils.dependency import better.files.File import io.appthreat.x2cpg.utils.ExternalCommand import io.appthreat.x2cpg.utils.dependency.GradleConfigKeys.GradleConfigKey -import org.slf4j.LoggerFactory import java.nio.file.Path import scala.util.{Failure, Success} @@ -17,10 +16,7 @@ case class DependencyResolverParams( ) object DependencyResolver: - private val logger = LoggerFactory.getLogger(getClass) - private val defaultGradleProjectName = "app" - private val defaultGradleConfigurationName = "compileClasspath" - private val MaxSearchDepth: Int = 4 + private val MaxSearchDepth: Int = 4 def getCoordinates( projectDir: Path, @@ -31,9 +27,8 @@ object DependencyResolver: // TODO: implement None else if isGradleBuildFile(buildFile) then - getCoordinatesForGradleProject(buildFile.getParent, defaultGradleConfigurationName) + Nil else - logger.debug(s"Found unsupported build file $buildFile") Nil }.flatten @@ -49,16 +44,10 @@ object DependencyResolver: ) match case Success(lines) => lines case Failure(exception) => - logger.debug( - s"Could not retrieve dependencies for Gradle project at path `$projectDir`\n" + - exception.getMessage - ) Seq() val coordinates = MavenCoordinates.fromGradleOutput(lines) - logger.debug("Got {} Maven coordinates", coordinates.size) Some(coordinates) - end getCoordinatesForGradleProject def getDependencies( projectDir: Path, @@ -68,35 +57,13 @@ object DependencyResolver: if isMavenBuildFile(buildFile) then MavenDependencies.get(buildFile.getParent) else if isGradleBuildFile(buildFile) then - getDepsForGradleProject(params, buildFile.getParent) + Nil else - logger.debug(s"Found unsupported build file $buildFile") Nil }.flatten Option.when(dependencies.nonEmpty)(dependencies) - private def getDepsForGradleProject( - params: DependencyResolverParams, - projectDir: Path - ): Option[collection.Seq[String]] = - logger.debug("resolving Gradle dependencies at {}", projectDir) - val gradleProjectName = - params.forGradle.getOrElse(GradleConfigKeys.ProjectName, defaultGradleProjectName) - val gradleConfiguration = - params.forGradle.getOrElse( - GradleConfigKeys.ConfigurationName, - defaultGradleConfigurationName - ) - GradleDependencies.get(projectDir, gradleProjectName, gradleConfiguration) match - case Some(deps) => Some(deps) - case None => - logger.debug( - s"Could not download Gradle dependencies for project at path `$projectDir`" - ) - None - end getDepsForGradleProject - private def isGradleBuildFile(file: File): Boolean = val pathString = file.pathAsString pathString.endsWith(".gradle") || pathString.endsWith(".gradle.kts") @@ -106,7 +73,6 @@ object DependencyResolver: private def findSupportedBuildFiles(currentDir: File, depth: Int = 0): List[Path] = if depth >= MaxSearchDepth then - logger.debug("findSupportedBuildFiles reached max depth before finding build files") Nil else val (childDirectories, childFiles) = currentDir.children.partition(_.isDirectory) diff --git a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/GradleDependencies.scala b/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/GradleDependencies.scala deleted file mode 100644 index 9aa5a3a0..00000000 --- a/platform/frontends/x2cpg/src/main/scala/io/appthreat/x2cpg/utils/dependency/GradleDependencies.scala +++ /dev/null @@ -1,334 +0,0 @@ -package io.appthreat.x2cpg.utils.dependency - -import better.files.* -import org.gradle.tooling.{GradleConnector, ProjectConnection} -import org.gradle.tooling.model.GradleProject -import org.gradle.tooling.model.build.BuildEnvironment -import org.slf4j.LoggerFactory - -import java.io.ByteArrayOutputStream -import java.nio.file.{Files, Path} -import java.io.{File as JFile} -import java.util.stream.Collectors -import scala.jdk.CollectionConverters.* -import scala.util.{Failure, Random, Success, Try, Using} - -case class GradleProjectInfo( - gradleVersion: String, - tasks: Seq[String], - hasAndroidSubproject: Boolean = false -): - def gradleVersionMajorMinor(): (Int, Int) = - def isValidPart(part: String) = part.forall(Character.isDigit) - val parts = gradleVersion.split('.') - if parts.length == 1 && isValidPart(parts(0)) then - (parts(0).toInt, 0) - else if parts.length >= 2 && isValidPart(parts(0)) && isValidPart(parts(1)) then - (parts(0).toInt, parts(1).toInt) - else - (-1, -1) - -object Constants: - val aarFileExtension = "aar" - val gradleAndroidPropertyPrefix = "android." - val gradlePropertiesTaskName = "properties" - val jarInsideAarFileName = "classes.jar" - -case class GradleDepsInitScript(contents: String, taskName: String, destinationDir: Path) - -object GradleDependencies: - private val logger = LoggerFactory.getLogger(getClass) - private val initScriptPrefix = "x2cpg.init.gradle" - private val taskNamePrefix = "x2cpgCopyDeps" - private val tempDirPrefix = "x2cpgDependencies" - - // works with Gradle 5.1+ because the script makes use of `task.register`: - // https://docs.gradle.org/current/userguide/task_configuration_avoidance.html - private def gradle5OrLaterAndroidInitScript( - taskName: String, - destination: String, - gradleProjectName: String, - gradleConfigurationName: String - ): String = - s""" - |allprojects { - | afterEvaluate { project -> - | def taskName = "$taskName" - | def destinationDir = "$destination" - | def gradleProjectName = "$gradleProjectName" - | def gradleConfigurationName = "$gradleConfigurationName" - | - | if (project.name.equals(gradleProjectName)) { - | def compileDepsCopyTaskName = taskName + "_compileDeps" - | tasks.register(compileDepsCopyTaskName, Copy) { - | def selectedConfig = project.configurations.find { it.name.equals(gradleConfigurationName) } - | def componentIds = [] - | if (selectedConfig != null) { - | componentIds = selectedConfig.incoming.resolutionResult.allDependencies.collect { it.selected.id } - | } - | def result = dependencies.createArtifactResolutionQuery() - | .forComponents(componentIds) - | .withArtifacts(JvmLibrary, SourcesArtifact) - | .execute() - | duplicatesStrategy = 'include' - | into destinationDir - | from result.resolvedComponents.collect { it.getArtifacts(SourcesArtifact).collect { it.file } } - | } - | def androidDepsCopyTaskName = taskName + "_androidDeps" - | tasks.register(androidDepsCopyTaskName, Copy) { - | duplicatesStrategy = 'include' - | into destinationDir - | from project.configurations.find { it.name.equals("androidApis") } - | } - | tasks.register(taskName, Copy) { - | dependsOn androidDepsCopyTaskName - | dependsOn compileDepsCopyTaskName - | } - | } - | } - |} - |""".stripMargin - - // this init script _should_ work with Gradle 4-8, but has not been tested thoroughly - // TODO: add test cases for older Gradle versions - private def gradle5OrLaterInitScript(taskName: String, destination: String): String = - s""" - |allprojects { - | apply plugin: 'java' - | task $taskName(type: Copy) { - | into "$destination" - | from configurations.default - | } - |} - |""".stripMargin - - private def makeInitScript( - destinationDir: Path, - forAndroid: Boolean, - gradleProjectName: String, - gradleConfigurationName: String - ): GradleDepsInitScript = - val taskName = taskNamePrefix + "_" + (Random.alphanumeric.take(8)).toList.mkString - val content = - if forAndroid then - gradle5OrLaterAndroidInitScript( - taskName, - destinationDir.toString, - gradleProjectName, - gradleConfigurationName - ) - else - gradle5OrLaterInitScript(taskName, destinationDir.toString) - GradleDepsInitScript(content, taskName, destinationDir) - - private[dependency] def makeConnection(projectDir: JFile): ProjectConnection = - GradleConnector.newConnector().forProjectDirectory(projectDir).connect() - - private def getGradleProjectInfo( - projectDir: Path, - projectName: String - ): Option[GradleProjectInfo] = - Try(makeConnection(projectDir.toFile)) match - case Success(gradleConnection) => - Using.resource(gradleConnection) { connection => - try - val buildEnv = - connection.getModel[BuildEnvironment](classOf[BuildEnvironment]) - val project = connection.getModel[GradleProject](classOf[GradleProject]) - val hasAndroidPrefixGradleProperty = - runGradleTask(connection, Constants.gradlePropertiesTaskName) match - case Some(out) => - out.split('\n').exists( - _.startsWith(Constants.gradleAndroidPropertyPrefix) - ) - case None => false - val info = GradleProjectInfo( - buildEnv.getGradle.getGradleVersion, - project.getTasks.asScala.map(_.getName).toSeq, - hasAndroidPrefixGradleProperty - ) - if hasAndroidPrefixGradleProperty then - val validProjectNames = List( - project.getName - ) ++ project.getChildren.getAll.asScala.map(_.getName) - logger.debug( - s"Found Gradle projects: ${validProjectNames.mkString(",")}" - ) - if !validProjectNames.contains(projectName) then - val validProjectNamesStr = validProjectNames.mkString(",") - logger.debug( - s"The provided Gradle project name `$projectName` is is not part of the valid project names: `$validProjectNamesStr`" - ) - None - else - Some(info) - else - Some(info) - catch - case t: Throwable => - logger.debug( - s"Caught exception while trying use Gradle connection: ${t.getMessage}" - ) - logger.debug(s"Full exception: ", t) - None - } - case Failure(t) => - logger.debug( - s"Caught exception while trying fetch Gradle project information: ${t.getMessage}" - ) - logger.debug(s"Full exception: ", t) - None - - private def runGradleTask(connection: ProjectConnection, taskName: String): Option[String] = - Using.resource(new ByteArrayOutputStream()) { out => - Try( - connection - .newBuild() - .forTasks(taskName) - .setStandardOutput(out) - .run() - ) match - case Success(_) => Some(out.toString) - case Failure(ex) => - logger.debug( - s"Caught exception while executing Gradle task named `$taskName`:", - ex.getMessage - ) - logger.debug(s"Full exception: ", ex) - None - } - - private def runGradleTask( - connection: ProjectConnection, - initScript: GradleDepsInitScript, - initScriptPath: String - ): Option[collection.Seq[String]] = - Using.resources(new ByteArrayOutputStream, new ByteArrayOutputStream) { - case (stdoutStream, stderrStream) => - logger.debug(s"Executing gradle task '${initScript.taskName}'...") - Try( - connection - .newBuild() - .forTasks(initScript.taskName) - .withArguments("--init-script", initScriptPath) - .setStandardOutput(stdoutStream) - .setStandardError(stderrStream) - .run() - ) match - case Success(_) => - val result = - Files - .list(initScript.destinationDir) - .collect(Collectors.toList[Path]) - .asScala - .map(_.toAbsolutePath.toString) - logger.debug(s"Resolved `${result.size}` dependency files.") - Some(result) - case Failure(ex) => - logger.debug( - s"Caught exception while executing Gradle task: ${ex.getMessage}" - ) - logger.debug(s"Gradle task execution stdout: \n$stdoutStream") - logger.debug(s"Gradle task execution stderr: \n$stderrStream") - None - end match - } - - private def extractClassesJarFromAar(aar: File): Option[Path] = - val newPath = aar.path.toString.replaceFirst(Constants.aarFileExtension + "$", "jar") - val aarUnzipDirSuffix = ".unzipped" - val outDir = File(aar.path.toString + aarUnzipDirSuffix) - aar.unzipTo(outDir, _.getName == Constants.jarInsideAarFileName) - val outFile = File(newPath) - val classesJarEntries = - outDir.listRecursively - .filter(_.path.getFileName.toString == Constants.jarInsideAarFileName) - .toList - if classesJarEntries.size != 1 then - logger.debug(s"Found aar file without `classes.jar` inside at path ${aar.path}") - outDir.delete() - None - else - val classesJar = classesJarEntries.head - logger.trace(s"Copying `classes.jar` for aar at `${aar.path.toString}` into `$newPath`") - classesJar.copyTo(outFile) - outDir.delete() - aar.delete() - Some(outFile.path) - end extractClassesJarFromAar - - // fetch the gradle project information first, then invoke a newly-defined gradle task to copy the necessary jars into - // a destination directory. - private[dependency] def get( - projectDir: Path, - projectName: String, - configurationName: String - ): Option[collection.Seq[String]] = - logger.debug( - s"Fetching Gradle project information at path `$projectDir` with project name `$projectName`." - ) - getGradleProjectInfo(projectDir, projectName) match - case Some(projectInfo) if projectInfo.gradleVersionMajorMinor()._1 < 5 => - logger.debug(s"Unsupported Gradle version `${projectInfo.gradleVersion}`") - None - case Some(projectInfo) => - Try(File.newTemporaryDirectory(tempDirPrefix).deleteOnExit()) match - case Success(destinationDir) => - Try(File.newTemporaryFile(initScriptPrefix).deleteOnExit()) match - case Success(initScriptFile) => - val initScript = - makeInitScript( - destinationDir.path, - projectInfo.hasAndroidSubproject, - projectName, - configurationName - ) - initScriptFile.write(initScript.contents) - - logger.debug( - s"Downloading dependencies for configuration `$configurationName` of project `$projectName` at `$projectDir` into `$destinationDir`..." - ) - Try(makeConnection(projectDir.toFile)) match - case Success(connection) => - Using.resource(connection) { c => - runGradleTask( - c, - initScript, - initScriptFile.pathAsString - ) match - case Some(deps) => - Some(deps.map { d => - if !d.endsWith(Constants.aarFileExtension) - then d - else - extractClassesJarFromAar(File(d)) match - case Some(path) => path.toString - case None => d - }) - case None => None - } - case Failure(ex) => - logger.debug( - s"Caught exception while trying to establish a Gradle connection: ${ex.getMessage}" - ) - logger.debug(s"Full exception: ", ex) - None - end match - case Failure(ex) => - logger.debug( - s"Could not create temporary file for Gradle init script: ${ex.getMessage}" - ) - logger.debug(s"Full exception: ", ex) - None - case Failure(ex) => - logger.debug( - s"Could not create temporary directory for saving dependency files: ${ex.getMessage}" - ) - logger.debug("Full exception: ", ex) - None - case None => - logger.debug("Could not fetch Gradle project information") - None - end match - end get -end GradleDependencies diff --git a/pyproject.toml b/pyproject.toml index efb92db9..781a2e8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "appthreat-chen" -version = "2.1.6" +version = "2.1.7" description = "Code Hierarchy Exploration Net (chen)" authors = ["Team AppThreat "] license = "Apache-2.0"