Scala Overhaul (#216)

* Initial scala migration

* Removed gradle artifacts

* Progress commit

* Have a few more useful passes extended

* Added 3 Plume edge passes

* Soot classes now cleaned when done generating CPG from them

* Basic reviewed structure of passes working

* Improved dynamic call linker and structure

* Started testing bulk tx

* Simplified driver and added more tests

* Implemented OverflowDBDriver and added more tests

* Added Plume shortcut

* Adding Soot settings to try avoid subclass method body issues

* Have new scaladoc site

* Separated release and docs

* Added additional testing

* Fixed soot modification bug and have safe hierarchy check

* Moved files to plume structure and started with integration tests

* Have integeration tests called correctly

* Have basic gremlin driver sorted

* Added tinkergraph driver

* Moved inmemory tests to be part of unit tests

* Have working tinkergraph tests

* Have tests for graph serialization

* Some basic Neo4j driver implementation ready

* Neo4j bulk transactions complete

* Neo4j tests passing

* Neo4j int tests working

* Organized imports

* Files are now being hashed

* Fixed some tests and a type name bug

* Hash pass working

* Have a version of the linker working for ODB and Tinker

* Neo4j driver ast linking working but untested

* Ast linker for ODB and Tinker tested

* Neo4j ast linking working

* Improved linking testing and simplified linker

* test update

* Gremlin source file removal working

* All tests green for source file removal

* Have an untested diff detection system working

* Fixed gremlin child delete bug

* Fixed unhandled ArrayRef bug and loading all classes in now

* Fixed bugs and ready to test diff creation

* Some more changes to blacklisting

* Have hierarchy fix and call linker fallback

* Perform early stopping on dynamic call linker

* Fixed prop default and gremlin driver range bug

* Fixed change detection bugs

* Added diff tests

* Tests working and change detection functional

* Fixed pesky Neo4j bugs

* Basic TG schema creation working

* Some core functions working

* TG properties retrieved correctly

* bulk set node and delete work

* Added default txMax and have bulkTx for TG working

* Removed generic node + children delete

* Have much of TG driver working

* Final touchups of TG driver being done

* Have a real "is connected" method for TG

* Fixed unnormalized path bug

* Incorporated reaching defs pass

* Basic dataflow in overflowdb driver

* Upgraded CPG and have interprocedural linking/tests

* Have tests for static calls and implemented it for gremlin

* Tested and fixed up neo4j static call linker

* Can now use plume from binary with driver config

* Minor progress commit on reachable paths

* TgerGraph tests and static call linker working

* Have basic Neptune stuff going

* Pushing some neptune stuff

* Removed accidental overflow

* Let's see if comparing string ranges works

* Not nulling graph component

* Have bulk reset code ready

* Added perform reset encoder

* Specifying types

* Moved backend to parent scope

* This looks good

* Lets see if this works

* debug by erorr

* test

* Lets try form encoded

* Trying different decoder

* Trying differnt library altogether

* Lets see if this works

* Ok things should work

* Working bulk delete!

* Fixed blockign wait

* Close connection to cluster before resetting

* Lets see if this map works

* Trying the ugly way

* Handling typed IDs a bti better

* Typing edge things too

* typed exists

* Fixed boxing issue

* Fixed boxing issue

* Fixed more boxing issues

* neptune driver works!

* Enabled access to txMax and Neptune from CLI

* progress commit

* Upgrade dependencies and saving data flow paths

* Formatted project correctly

* Scoverage should be automatically detected

* Test to see if codecov can find stuff

* Testing more coverage stuff

* Seeing if config helps

* Adding coverage clause

* Moved config to build.sbt

* Trying slightly different coverage

.github/workflows/ci.yml

@@ -8,55 +8,34 @@ on:
 
 jobs:
   test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ ubuntu-20.04, macos-10.15, windows-2019 ]
     name: Test on JDK 11 and ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-20.04
 
     steps:
       - uses: actions/checkout@v2
         with:
           lfs: true
+          fetch-depth: 0
 
       - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
           java-version: 11
 
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Compile classes
-        run: ./gradlew classes
+      - name: Check styling
+        run: sbt scalafmtCheck
 
-      - name: Run unit tests
-        run: ./gradlew test
-      - name: Run extractor tests
-        run: ./gradlew extractorTest
-      - name: Run scalatest
-        run: ./gradlew scalatest
-
-      - name: Run TinkerGaph integration tests
-        run: ./gradlew tinkerGraphIntTest
-      - name: Run OverflowDb integration tests
-        run: ./gradlew overflowDbIntTest
-      - name: Run JanusGraph integration tests
-        if: runner.os == 'Linux'
-        run: ./gradlew janusGraphIntTest
-      - name: Run TigerGraph integration tests
-        if: runner.os == 'Linux'
-        run: ./gradlew tigerGraphIntTest
+      - name: Run unit and inmem database tests
+        run: sbt coverage test
       - name: Run Neo4j integration tests
-        if: runner.os == 'Linux'
-        run: ./gradlew neo4jIntTest
+        run: sbt coverage "neoTest:test"
+      - name: Run TigerGraph integration tests
+        run: sbt coverage "tgTest:test"
 
       - name: Compile coverage report
-        if: runner.os == 'Linux'
-        run: ./gradlew jacocoTestReport
+        run: sbt coverageReport
       - name: Upload coverage to Codecov
-        if: runner.os == 'Linux'
         uses: codecov/codecov-action@v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./plume/build/reports/jacoco
+          directory: ./target/scala-2.13/scoverage-report/

.github/workflows/docs.yml

@@ -0,0 +1,26 @@
+name: Documentation
+
+on:
+  push:
+    tags: [ 'v*' ] # Push events to matching v*, i.e. v1.0, v20.15.10
+
+jobs:
+  publish:
+    name: Publish Scala doc
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          lfs: true
+          fetch-depth: 0
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+
+      - name: Build code
+        run: sbt stage
+      - name: Build and deploy Scala documentation
+        run: sbt makeSite ghpagesPushSite

.github/workflows/release.yml

@@ -6,25 +6,24 @@ on:
 
 jobs:
   publish:
-    name: Publish release and KDocs
+    name: Publish release
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
         with:
           lfs: true
+          fetch-depth: 0
 
       - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
           java-version: 11
 
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
       - name: Build code
-        run: ./gradlew assemble
+        run: sbt stage
       - name: Publish code
-        run: ./gradlew publishToMavenLocal
+        run: sbt publishM2
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
@@ -35,7 +34,3 @@ jobs:
           release_name: ${{ github.ref }}
           draft: false
           prerelease: false
-      - name: Build and deploy Orchid documentation
-        run: ./gradlew :plume:orchidDeploy -PorchidEnvironment=prod
-        env:
-          ORCHID_TOKEN: ${{ secrets.ORCHID_TOKEN }}

.gitignore

@@ -25,3 +25,6 @@ gradle-app.setting
 /sootOutput/
 /plume/sootOutput/
 /testing/build/
+/.bsp/
+target
+/project/target/

.scalafmt.conf

@@ -1,18 +1,3 @@
-style = default
-
-maxColumn = 120
-
-align = some
-align.openParenCallSite = false
-align.openParenDefnSite = false
-
-continuationIndent.defnSite = 2
-
-danglingParentheses = true
-
-rewrite.rules = [
-  AvoidInfix
-  RedundantParens
-  AsciiSortImports
-  PreferCurlyFors
-]
+version = "2.7.4"
+maxColumn = 100
+align = true

build.sbt

@@ -0,0 +1,114 @@
+import java.time.{ZoneOffset, ZonedDateTime}
+
+name := "Plume"
+organization := "io.github.plume-oss"
+version := "0.1.0"
+
+scalaVersion := "2.13.6"
+
+val cpgVersion         = "1.3.477"
+val joernVersion       = "1.1.412"
+val sootVersion        = "4.2.1"
+val tinkerGraphVersion = "3.4.8"
+val neo4jVersion       = "4.4.2"
+val tigerGraphVersion  = "3.1.0"
+val sttpVersion        = "3.3.17"
+val lz4Version         = "1.8.0"
+val slf4jVersion       = "1.7.32"
+val scalatestVersion   = "3.2.9"
+val circeVersion       = "0.14.1"
+
+lazy val scalatest         = "org.scalatest" %% "scalatest" % scalatestVersion
+lazy val NeoIntTest        = config("neoTest") extend Test
+lazy val TigerGraphIntTest = config("tgTest") extend Test
+lazy val NeptuneIntTest    = config("nepTest") extend Test
+
+fork := true
+
+resolvers ++= Seq(
+  Resolver.mavenLocal,
+  Resolver.mavenCentral,
+  Resolver.JCenterRepository
+)
+
+trapExit := false
+
+libraryDependencies ++= Seq(
+  "io.shiftleft"                  %% "codepropertygraph"   % cpgVersion,
+  "io.shiftleft"                  %% "semanticcpg"         % cpgVersion,
+  "io.joern"                      %% "dataflowengineoss"   % joernVersion,
+  "io.shiftleft"                  %% "semanticcpg-tests"   % "1.3.405"        % Test classifier "tests",
+  "org.soot-oss"                   % "soot"                % sootVersion,
+  "org.apache.tinkerpop"           % "tinkergraph-gremlin" % tinkerGraphVersion,
+  "org.apache.tinkerpop"           % "gremlin-driver"      % tinkerGraphVersion,
+  "org.neo4j.driver"               % "neo4j-java-driver"   % neo4jVersion,
+  "com.tigergraph.client"          % "gsql_client"         % tigerGraphVersion,
+  "com.softwaremill.sttp.client3" %% "core"                % sttpVersion,
+  "com.softwaremill.sttp.client3" %% "circe"               % sttpVersion,
+  "org.scalaj"                     % "scalaj-http_2.13"    % "2.4.2",
+  "org.lz4"                        % "lz4-java"            % lz4Version,
+  "org.slf4j"                      % "slf4j-api"           % slf4jVersion,
+  "org.slf4j"                      % "slf4j-simple"        % slf4jVersion,
+  "org.scala-lang"                 % "scala-reflect"       % scalaVersion.value,
+  "org.scalatest"                 %% "scalatest"           % scalatestVersion % Test
+) ++ Seq(
+  "io.circe" %% "circe-core",
+  "io.circe" %% "circe-generic",
+  "io.circe" %% "circe-parser",
+  "io.circe" %% "circe-yaml"
+).map(_ % circeVersion)
+
+enablePlugins(
+  JavaAppPackaging,
+  GitVersioning,
+  BuildInfoPlugin,
+  GhpagesPlugin,
+  SiteScaladocPlugin
+)
+
+scmInfo := Some(
+  ScmInfo(url("https://github.com/plume-oss/plume"), "scm:[email protected]:/plume-oss/plume.git")
+)
+homepage := Some(url("https://github.com/plume-oss/plume/"))
+licenses := List("Apache-2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0"))
+developers := List(
+  Developer(
+    "DavidBakerEffendi",
+    "David Baker Effendi",
+    "[email protected]",
+    url("https://github.com/DavidBakerEffendi")
+  ),
+  Developer(
+    "fabsx00",
+    "Fabian Yamaguchi",
+    "[email protected]",
+    url("https://github.com/fabsx00")
+  )
+)
+
+git.remoteRepo := "[email protected]:plume-oss/plume.git"
+
+Global / onChangedBuildSource := ReloadOnSourceChanges
+
+lazy val root = (project in file("."))
+  .configs(NeoIntTest)
+  .settings(
+    inConfig(NeoIntTest)(Defaults.testSettings),
+    libraryDependencies += scalatest % NeoIntTest,
+    Test / testOptions := Seq(Tests.Filter(s => !s.endsWith("IntTests"))),
+    NeoIntTest / testOptions := Seq(Tests.Filter(s => s.contains("Neo4j")))
+  )
+  .configs(TigerGraphIntTest)
+  .settings(
+    inConfig(TigerGraphIntTest)(Defaults.testSettings),
+    libraryDependencies += scalatest % TigerGraphIntTest,
+    Test / testOptions := Seq(Tests.Filter(s => !s.endsWith("IntTests"))),
+    TigerGraphIntTest / testOptions := Seq(Tests.Filter(s => s.contains("TigerGraph")))
+  )
+  .configs(NeptuneIntTest)
+  .settings(
+    inConfig(NeptuneIntTest)(Defaults.testSettings),
+    libraryDependencies += scalatest % NeptuneIntTest,
+    Test / testOptions := Seq(Tests.Filter(s => !s.endsWith("IntTests"))),
+    NeptuneIntTest / testOptions := Seq(Tests.Filter(s => s.contains("Neptune")))
+  )