✨ Added INHERITS_FROM edges (#121)

* Added inheritance edges * Modified timer information * Removed timer reset
plume-oss · Mar 18, 2021 · dd35adc · dd35adc
1 parent e1efa07
commit dd35adc
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,10 +12,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - `CONTAINS` edges are generated for `METHOD` to body vertices.
 - `ListMapper` to process Scala lists to a serialized string and back. More formally processing Scala lists to and from
   OverflowDB node objects.
+- Handle inheritance edges i.e. `TYPE_DECL -INHERITS_FROM-> TYPE`
 
 ### Changed
 
 - `BaseCpgPass` now uses a local cache for method body nodes instead of relying solely on `GlobalCache`
+- `SCPGPass` now known as `DataFlowPass` as all passes now come from `dataflowengineoss`.
+- Added `PROGRAM_STRUCTURE` to timer keys.
 
 ## [0.3.5] - 2021-03-17
 

diff --git a/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt b/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt
@@ -25,7 +25,7 @@ import io.github.plume.oss.drivers.OverflowDbDriver
 import io.github.plume.oss.metrics.ExtractorTimeKey
 import io.github.plume.oss.metrics.PlumeTimer
 import io.github.plume.oss.options.ExtractorOptions
-import io.github.plume.oss.passes.SCPGPass
+import io.github.plume.oss.passes.DataFlowPass
 import io.github.plume.oss.passes.graph.BaseCPGPass
 import io.github.plume.oss.passes.graph.CGPass
 import io.github.plume.oss.passes.method.MethodStubPass
@@ -40,6 +40,7 @@ import io.github.plume.oss.util.ResourceCompilationUtil
 import io.github.plume.oss.util.ResourceCompilationUtil.COMP_DIR
 import io.github.plume.oss.util.ResourceCompilationUtil.TEMP_DIR
 import io.github.plume.oss.util.SootToPlumeUtil
+import io.shiftleft.codepropertygraph.generated.EdgeTypes.INHERITS_FROM
 import io.shiftleft.codepropertygraph.generated.NodeKeyNames.*
 import io.shiftleft.codepropertygraph.generated.NodeTypes.*
 import io.shiftleft.codepropertygraph.generated.nodes.NewFileBuilder
@@ -79,7 +80,6 @@ class Extractor(val driver: IDriver) {
     init {
         File(COMP_DIR).let { f -> if (f.exists()) f.deleteRecursively(); f.deleteOnExit() }
         checkDriverConnection(driver)
-        PlumeTimer.reset()
     }
 
     /**
@@ -169,7 +169,7 @@ class Extractor(val driver: IDriver) {
          */
         logger.info("Building internal program structure and type information")
         val csToBuild = mutableListOf<SootClass>()
-        PlumeTimer.measure(ExtractorTimeKey.BASE_CPG_BUILDING) {
+        PlumeTimer.measure(ExtractorTimeKey.PROGRAM_STRUCTURE_BUILDING) {
             // First read the existing TYPE, TYPE_DECL, and FILEs from the driver and load it into the cache
             populateGlobalTypeCache()
             pipeline(
@@ -189,7 +189,7 @@ class Extractor(val driver: IDriver) {
             Obtain all referenced types from fields, returns, and locals
          */
         val ts = mutableListOf<Type>()
-        PlumeTimer.measure(ExtractorTimeKey.BASE_CPG_BUILDING) {
+        PlumeTimer.measure(ExtractorTimeKey.PROGRAM_STRUCTURE_BUILDING) {
             val fieldsAndRets = csToBuild.map { c -> c.fields.map { it.type } + c.methods.map { it.returnType } }
                 .flatten().toSet()
             val locals = sootUnitGraphs.map { it.body.locals + it.body.parameterLocals }
@@ -202,13 +202,13 @@ class Extractor(val driver: IDriver) {
          */
         logger.info("Building primitive type information")
         logger.debug("All referenced types: ${ts.groupBy { it.javaClass }.mapValues { it.value.size }}}")
-        PlumeTimer.measure(ExtractorTimeKey.BASE_CPG_BUILDING) {
+        PlumeTimer.measure(ExtractorTimeKey.PROGRAM_STRUCTURE_BUILDING) {
             pipeline(
                 GlobalTypePass(driver)::runPass
             ).invoke(ts)
         }
         /*
-            TODO: Handle inheritance
+            Obtain inheritance information
         */
         logger.info("Obtaining class hierarchy")
         val parentToChildCs: MutableList<Pair<SootClass, Set<SootClass>>> = mutableListOf()
@@ -232,6 +232,19 @@ class Extractor(val driver: IDriver) {
                 ExternalTypePass(driver)::runPass,
             ).invoke(filteredExtTypes.map { it.sootClass })
         }
+        /*
+            Build inheritance edges, i.e.
+            TYPE_DECL -INHERITS_FROM-> TYPE
+        */
+        PlumeTimer.measure(ExtractorTimeKey.PROGRAM_STRUCTURE_BUILDING) {
+            parentToChildCs.forEach { (c, children) ->
+                GlobalCache.getType(c.type.toQuotedString())?.let { t ->
+                    children.intersect(csToBuild)
+                        .mapNotNull { child -> GlobalCache.getTypeDecl(child.type.toQuotedString()) }
+                        .forEach { td -> driver.addEdge(td, t, INHERITS_FROM) }
+                }
+            }
+        }
         csToBuild.clear() // Done using csToBuild
         /*
             Construct the CPGs for methods
@@ -255,9 +268,11 @@ class Extractor(val driver: IDriver) {
                 .coerceAtLeast(1)
                 .coerceAtMost(Runtime.getRuntime().availableProcessors())
             val channel = Channel<DeltaGraph>()
-            logger.info("""
+            logger.info(
+                """
                 Building ${headsToBuild.size} method heads and ${bodiesToBuild.size} method bodies over $nThreads thread(s).
-            """.trimIndent())
+            """.trimIndent()
+            )
             // Create jobs in chunks and submit these jobs to a thread pool
             val threadPool = Executors.newFixedThreadPool(nThreads)
             try {
@@ -300,8 +315,8 @@ class Extractor(val driver: IDriver) {
         /*
             Method body level analysis - only done on new/updated methods
          */
-        logger.info("Running SCPG passes")
-        PlumeTimer.measure(ExtractorTimeKey.SCPG_PASSES) { SCPGPass(driver).runPass() }
+        logger.info("Running data flow passes")
+        PlumeTimer.measure(ExtractorTimeKey.DATA_FLOW_PASS) { DataFlowPass(driver).runPass() }
         GlobalCache.methodBodies.clear()
         return this
     }

diff --git a/plume/src/main/kotlin/io/github/plume/oss/metrics/PlumeTimer.kt b/plume/src/main/kotlin/io/github/plume/oss/metrics/PlumeTimer.kt
@@ -8,17 +8,21 @@ object PlumeTimer {
     private val totalTimes = mutableMapOf(
         ExtractorTimeKey.COMPILING_AND_UNPACKING to 0L,
         ExtractorTimeKey.SOOT to 0L,
+        ExtractorTimeKey.BASE_CPG_BUILDING to 0L,
+        ExtractorTimeKey.PROGRAM_STRUCTURE_BUILDING to 0L,
         ExtractorTimeKey.DATABASE_WRITE to 0L,
         ExtractorTimeKey.DATABASE_READ to 0L,
-        ExtractorTimeKey.SCPG_PASSES to 0L
+        ExtractorTimeKey.DATA_FLOW_PASS to 0L
     )
 
     private val stopwatch = mutableMapOf(
         ExtractorTimeKey.COMPILING_AND_UNPACKING to System.nanoTime(),
         ExtractorTimeKey.SOOT to System.nanoTime(),
+        ExtractorTimeKey.BASE_CPG_BUILDING to System.nanoTime(),
+        ExtractorTimeKey.PROGRAM_STRUCTURE_BUILDING to System.nanoTime(),
         ExtractorTimeKey.DATABASE_WRITE to System.nanoTime(),
         ExtractorTimeKey.DATABASE_READ to System.nanoTime(),
-        ExtractorTimeKey.SCPG_PASSES to System.nanoTime()
+        ExtractorTimeKey.DATA_FLOW_PASS to System.nanoTime()
     )
 
     /**
@@ -97,6 +101,11 @@ enum class ExtractorTimeKey {
      */
     BASE_CPG_BUILDING,
 
+    /**
+     * Wall clock time taken to build type, package, and file information.
+     */
+    PROGRAM_STRUCTURE_BUILDING,
+
     /**
      * CPU time spent on database writes.
      */
@@ -108,9 +117,8 @@ enum class ExtractorTimeKey {
     DATABASE_READ,
 
     /**
-     * Wall clock time spent running semantic code property graph passes from [io.shiftleft.semanticcpg.passes] and
-     * [io.shiftleft.dataflowengineoss.passes].
+     * Wall clock time spent running semantic code property graph passes from [io.shiftleft.dataflowengineoss.passes].
      */
-    SCPG_PASSES
+    DATA_FLOW_PASS
 }
 
diff --git a/...in/io/github/plume/oss/passes/SCPGPass.kt → ...o/github/plume/oss/passes/DataFlowPass.kt b/...in/io/github/plume/oss/passes/SCPGPass.kt → ...o/github/plume/oss/passes/DataFlowPass.kt
@@ -6,12 +6,10 @@ import io.github.plume.oss.drivers.IDriver
 import io.github.plume.oss.util.DiffGraphUtil
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.NodeTypes
-import io.shiftleft.codepropertygraph.generated.nodes.AstNode
 import io.shiftleft.codepropertygraph.generated.nodes.Method
 import io.shiftleft.dataflowengineoss.passes.reachingdef.ReachingDefPass
 import io.shiftleft.passes.DiffGraph
 import io.shiftleft.passes.ParallelCpgPass
-import io.shiftleft.semanticcpg.passes.containsedges.ContainsEdgePass
 import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
@@ -27,12 +25,12 @@ import io.shiftleft.codepropertygraph.generated.nodes.Factories as NodeFactories
 /**
  * Runs passes from [io.shiftleft.dataflowengineoss.passes] over method bodies.
  */
-class SCPGPass(private val driver: IDriver) {
+class DataFlowPass(private val driver: IDriver) {
 
-    private val logger: Logger = LogManager.getLogger(SCPGPass::javaClass)
+    private val logger: Logger = LogManager.getLogger(DataFlowPass::javaClass)
 
     /**
-     * Calls SCPG passes. Converges all methods into a local OverflowDB graph instance. This is done concurrently.
+     * Calls data flow passes. Converges all methods into a local OverflowDB graph instance. This is done concurrently.
      */
     fun runPass() {
         runBlocking {