Find lombok uses by scanning source code instead of relying on fetche…

…d dependencies (#4358)

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/jpastprinter/JavaParserAstPrinter.scala

@@ -11,8 +11,7 @@ import java.nio.file.Path
 
 object JavaParserAstPrinter {
   def printJpAsts(config: Config): Unit = {
-
-    val sourceParser = SourceParser(config, false, None)
+    val sourceParser = SourceParser(config, None)
     val printer      = new YamlPrinter(true)
 
     SourceFiles

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/passes/AstCreationPass.scala

@@ -60,7 +60,7 @@ class AstCreationPass(config: Config, cpg: Cpg, sourcesOverride: Option[List[Str
 
   private def initParserAndUtils(config: Config): (SourceParser, JavaSymbolSolver) = {
     val dependencies = getDependencyList(config.inputPath)
-    val sourceParser = SourceParser(config, dependencies.exists(_.contains("lombok")), sourcesOverride)
+    val sourceParser = SourceParser(config, sourcesOverride)
     val symbolSolver = createSymbolSolver(config, dependencies, sourceParser)
     (sourceParser, symbolSolver)
   }

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala

@@ -76,15 +76,19 @@ object Delombok {
     }
   }
 
-  def run(inputPath: Path, relativeFilenames: List[Path], analysisJavaHome: Option[String]): DelombokRunResult = {
+  def run(
+    inputPath: Path,
+    fileInfo: List[SourceParser.FileInfo],
+    analysisJavaHome: Option[String]
+  ): DelombokRunResult = {
     Try(File.newTemporaryDirectory(prefix = "delombok").deleteOnExit()) match {
       case Failure(_) =>
         logger.warn(s"Could not create temporary directory for delombok output. Scanning original sources instead")
         DelombokRunResult(inputPath, false)
 
       case Success(tempDir) =>
         PackageRootFinder
-          .packageRootsFromFiles(inputPath, relativeFilenames)
+          .packageRootsFromFiles(inputPath, fileInfo)
           .foreach(delombokPackageRoot(inputPath, _, tempDir, analysisJavaHome))
         DelombokRunResult(tempDir.path, true)
     }

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/PackageRootFinder.scala

@@ -8,50 +8,43 @@ import scala.util.matching.Regex
 
 object PackageRootFinder {
 
-  private val PackageNameRegex: Regex = raw"package\s+([a-zA-Z$$_.]+)\s*;".r
-
-  def packageRootsFromFiles(inputPath: Path, relativeFilenames: List[Path]): List[Path] = {
+  def packageRootsFromFiles(inputPath: Path, fileInfos: List[SourceParser.FileInfo]): List[Path] = {
     // Files are sorted by path length to efficiently handle the non-standard package structure case described
     // below. If the `parentDirectory` in that case is closest to the input path, then the largest number of
     // files will be removed from the `filesToCheck` list.
-    var filesToCheck                              = mutable.ListBuffer.from(relativeFilenames).sortBy(_.getNameCount)
+    var filesToCheck = mutable.ListBuffer.from(fileInfos).sortBy(_.relativePath.getNameCount)
     val discoveredRoots: mutable.ListBuffer[Path] = mutable.ListBuffer.empty
 
     while (filesToCheck.nonEmpty) {
-      val filePath = filesToCheck.remove(0)
+      val fileInfo = filesToCheck.remove(0)
       // If filePath.getParent is null, then parent is actually the inputPath, but in this case
       // we don't want to "break out" of the specified input directory, so just use the relative
       // path `.`, which is equivalent to the inputPath since it is relative to the input path
       val dotPath         = Path.of(".")
-      val parentDirectory = Option(filePath.getParent).getOrElse(dotPath)
-
-      SourceParser.fileIfExists(inputPath.resolve(filePath)).foreach { file =>
-        val discoveredRoot = packageNameForFile(file) match {
-          case Some(packageName) =>
-            val packageDirs = Path.of(packageName.replaceAll(raw"\.", "/"))
-
-            if (parentDirectory.endsWith(packageDirs))
-              parentDirectory.subpath(0, parentDirectory.getNameCount - packageDirs.getNameCount)
-            else
-              // In this case, the package name doesn't match the given directory structure, either
-              // because the project doesn't follow convention or because the given inputPath is
-              // already a subdirectory of the package tree (for example, `projectRoot/src/main/java/com/`)
-              filePath
-
-          case None => filePath
-        }
-
-        discoveredRoots.addOne(discoveredRoot)
-        if (discoveredRoot != dotPath) {
-          filesToCheck = filesToCheck.filterNot(path => path.startsWith(discoveredRoot))
-        }
+      val parentDirectory = Option(fileInfo.relativePath.getParent).getOrElse(dotPath)
+
+      val discoveredRoot = fileInfo.packageName match {
+        case Some(packageName) =>
+          val packageDirs = Path.of(packageName.replaceAll(raw"\.", "/"))
+
+          if (parentDirectory.endsWith(packageDirs))
+            parentDirectory.subpath(0, parentDirectory.getNameCount - packageDirs.getNameCount)
+          else
+            // In this case, the package name doesn't match the given directory structure, either
+            // because the project doesn't follow convention or because the given inputPath is
+            // already a subdirectory of the package tree (for example, `projectRoot/src/main/java/com/`)
+            fileInfo.relativePath
+
+        case None => fileInfo.relativePath
+      }
+
+      discoveredRoots.addOne(discoveredRoot)
+      if (discoveredRoot != dotPath) {
+        filesToCheck = filesToCheck.filterNot(info => info.relativePath.startsWith(discoveredRoot))
       }
     }
 
     discoveredRoots.toList
   }
 
-  private def packageNameForFile(file: File): Option[String] = {
-    PackageNameRegex.findFirstMatchIn(file.contentAsString).map(_.group(1))
-  }
 }

joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/SourceParser.scala

@@ -17,6 +17,7 @@ import java.nio.file.Path
 import scala.jdk.CollectionConverters.*
 import scala.jdk.OptionConverters.RichOptional
 import scala.util.Try
+import scala.util.matching.Regex
 
 class SourceParser(
   val relativeFilenames: List[String],
@@ -97,6 +98,26 @@ class SourceParser(
 }
 
 object SourceParser {
+  case class FileInfo(relativePath: Path, packageName: Option[String], usesLombok: Boolean)
+
+  object FileInfo {
+
+    private val PackageNameRegex: Regex = raw"package\s+([a-zA-Z$$_.]+)\s*;".r
+
+    def getFileInfo(inputDir: Path, filename: String): Option[FileInfo] = {
+      fileIfExists(filename).map { file =>
+        val relativePath = inputDir.relativize(Path.of(filename))
+        val content      = file.contentAsString
+
+        val packageName = PackageNameRegex.findFirstMatchIn(content).map(_.group(1))
+
+        val usesLombok = content.contains("lombok")
+
+        new FileInfo(relativePath, packageName, usesLombok)
+      }
+    }
+  }
+
   private val logger = LoggerFactory.getLogger(this.getClass)
 
   private def checkExists(file: File): Option[File] = {
@@ -117,11 +138,10 @@ object SourceParser {
     checkExists(file)
   }
 
-  /** TODO: figure out delombok dependency here
-    */
-  def apply(config: Config, hasLombokDependency: Boolean, filenamesOverride: Option[List[String]]): SourceParser = {
+  def apply(config: Config, filenamesOverride: Option[List[String]]): SourceParser = {
     val inputPath = Path.of(config.inputPath)
-    val relativeFilenames = filenamesOverride
+
+    val fileInfo = filenamesOverride
       .getOrElse(
         SourceFiles.determine(
           config.inputPath,
@@ -131,17 +151,19 @@ object SourceParser {
           ignoredFilesPath = Option(config.ignoredFiles)
         )
       )
-      .map(filename => inputPath.relativize(Path.of(filename)))
+      .flatMap(FileInfo.getFileInfo(inputPath, _))
+
+    val usesLombok = fileInfo.exists(_.usesLombok)
 
     var dirToDelete: Option[Path] = None
-    lazy val delombokResult       = Delombok.run(inputPath, relativeFilenames, config.delombokJavaHome)
+    lazy val delombokResult       = Delombok.run(inputPath, fileInfo, config.delombokJavaHome)
     lazy val delombokDir = {
       dirToDelete = Option.when(delombokResult.isDelombokedPath)(delombokResult.path)
       delombokResult.path
     }
 
     val (analysisRoot, typesRoot) = Delombok.parseDelombokModeOption(config.delombokMode) match {
-      case DelombokMode.Default if hasLombokDependency =>
+      case DelombokMode.Default if usesLombok =>
         logger.info(s"Analysing delomboked code as lombok dependency was found.")
         (delombokDir, delombokDir)
 
@@ -154,6 +176,6 @@ object SourceParser {
       case DelombokMode.RunDelombok => (delombokDir, delombokDir)
     }
 
-    new SourceParser(relativeFilenames.map(_.toString), analysisRoot, typesRoot, dirToDelete)
+    new SourceParser(fileInfo.map(_.relativePath.toString), analysisRoot, typesRoot, dirToDelete)
   }
 }

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/querying/LombokTests.scala

@@ -7,7 +7,7 @@ import io.shiftleft.semanticcpg.language._
 import io.joern.javasrc2cpg.Config
 
 class LombokTests extends JavaSrcCode2CpgFixture {
-  val config = Config().withDelombokMode("run-delombok")
+  private val config = Config().withDelombokMode("default")
 
   "source in a mixed directory structure" should {
     val cpg = code(

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/testfixtures/JavaSrcCodeToCpgFixture.scala

@@ -16,8 +16,10 @@ trait JavaSrcFrontend extends LanguageFrontend {
   override val fileSuffix: String = ".java"
 
   override def execute(sourceCodeFile: File): Cpg = {
-    val config =
-      getConfig().map(_.asInstanceOf[Config]).getOrElse(JavaSrc2Cpg.DefaultConfig).withCacheJdkTypeSolver(true)
+    val config = getConfig()
+      .map(_.asInstanceOf[Config])
+      .getOrElse(JavaSrc2Cpg.DefaultConfig.withDelombokMode("no-delombok"))
+      .withCacheJdkTypeSolver(true)
     new JavaSrc2Cpg().createCpg(sourceCodeFile.getAbsolutePath)(config).get
   }
 }

joern-cli/frontends/javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/util/PackageRootFinderTests.scala

@@ -14,15 +14,17 @@ class PackageRootFinderTests extends SourceCodeFixture {
       .foldLeft(emptyWriter) { case (writer, (code, fileName)) => writer.moreCode(code, fileName) }
       .writeCode(".java")
 
-    files.permutations.foreach { filePermutation =>
-      val inputFilePaths = filePermutation.map(input => Path.of(input._2))
-      val foundRoots     = PackageRootFinder.packageRootsFromFiles(testDir, inputFilePaths).toSet
+    val absoluteFilenames = files.map { case (_, filename) => testDir.resolve(filename) }
+
+    absoluteFilenames.permutations.foreach { filePermutation =>
+      val inputs     = filePermutation.flatMap(input => SourceParser.FileInfo.getFileInfo(testDir, input.toString))
+      val foundRoots = PackageRootFinder.packageRootsFromFiles(testDir, inputs).toSet
 
       if (foundRoots != expectedRoots.map(Path.of(_))) {
         fail(s"""Found package roots did not match expected package roots:
                 |Found roots   : ${foundRoots.toList.sorted.mkString(",")}
                 |Expected roots: ${expectedRoots.toList.sorted.mkString(",")}
-                |for input path permutation ${inputFilePaths.mkString(",")}""".stripMargin)
+                |for input path permutation ${inputs.map(_.relativePath).mkString(",")}""".stripMargin)
       }
     }
   }