Merge pull request #2 from johannesduesing/feature/redesign

Merge redesign into develop

build.sbt

@@ -18,7 +18,7 @@ name := "delphi-crawler"
 
 version := "0.9.5-SNAPSHOT"
 
-scalaVersion := "2.12.11"
+scalaVersion := "2.12.15"
 
 lazy val crawler = (project in file(".")).
   enablePlugins(JavaAppPackaging).
@@ -38,49 +38,35 @@ lazy val crawler = (project in file(".")).
 
 scalastyleConfig := baseDirectory.value / "project" / "scalastyle-config.xml"
 
-//mainClass in(Compile, run) := Some("de.upb.cs.swt.delphi.crawler.Crawler")
-//mainClass in(Compile, packageBin) := Some("de.upb.cs.swt.delphi.crawler.Crawler")
-//mainClass in Compile := Some("de.upb.cs.swt.delphi.crawler.Crawler")
+Compile / run / mainClass := Some("de.upb.cs.swt.delphi.crawler.Crawler")
+Compile / packageBin / mainClass := Some("de.upb.cs.swt.delphi.crawler.Crawler")
+Compile / mainClass := Some("de.upb.cs.swt.delphi.crawler.Crawler")
 
-
-val akkaVersion = "2.6.4"
-
-val akkaHttpVersion = "10.1.11"
+val akkaVersion = "2.6.16"
 
 libraryDependencies ++= Seq(
   "com.typesafe.akka" %% "akka-actor" % akkaVersion,
   "com.typesafe.akka" %% "akka-testkit" % akkaVersion % Test,
   "com.typesafe.akka" %% "akka-stream" % akkaVersion,
   "com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion % Test,
   "com.typesafe.akka" %% "akka-slf4j" % akkaVersion,
-  "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion,
-  "com.typesafe.akka" %% "akka-http" % akkaHttpVersion
+  "com.typesafe.akka" %% "akka-http-spray-json" % "10.2.6",
+  "com.typesafe.akka" %% "akka-http" % "10.2.6"
 )
 
-libraryDependencies += "org.json4s" %% "json4s-jackson" % "3.6.7"
-libraryDependencies += "com.lihaoyi" %% "fansi" % "0.2.7"
-libraryDependencies += "org.fusesource.jansi" % "jansi" % "1.18"
-libraryDependencies += "com.googlecode.lanterna" % "lanterna" % "3.0.3"
-libraryDependencies += "com.pauldijou" %% "jwt-core" % "4.2.0"
+libraryDependencies += "org.json4s" %% "json4s-jackson" % "4.0.3"
 
-val elastic4sVersion = "6.7.4"
-libraryDependencies ++= Seq(
-  "com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion,
+libraryDependencies += "com.pauldijou" %% "jwt-core" % "1.0.0"
 
-  // for the http client
+libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3" % Runtime
 
-  "com.sksamuel.elastic4s" %% "elastic4s-http" % elastic4sVersion
-    //https://snyk.io/vuln/SNYK-JAVA-ORGAPACHEHTTPCOMPONENTS-31517
-    exclude("org.apache.httpcomponents", "httpclient"),
-
-  "org.apache.httpcomponents" % "httpclient" % "4.5.10",
-
-  // if you want to use reactive streams
+val elastic4sVersion = "7.14.1"
+libraryDependencies ++= Seq(
+  "com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sVersion,
   "com.sksamuel.elastic4s" %% "elastic4s-http-streams" % elastic4sVersion,
+  "com.sksamuel.elastic4s" %% "elastic4s-client-akka" % elastic4sVersion,
 
-  // testing
-  "com.sksamuel.elastic4s" %% "elastic4s-testkit" % elastic4sVersion % "test",
-  "com.sksamuel.elastic4s" %% "elastic4s-embedded" % elastic4sVersion % "test"
+  "com.sksamuel.elastic4s" %% "elastic4s-testkit" % elastic4sVersion % "test"
 )
 
 resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"
@@ -92,27 +78,17 @@ libraryDependencies ++= Seq(
   "de.opal-project" % "hermes_2.12" % opalVersion
 )
 
-val mavenVersion = "3.5.2"
-libraryDependencies ++= Seq(
+val mavenVersion = "3.8.3"
+libraryDependencies ++= Seq (
   "org.apache.maven" % "maven-core" % mavenVersion,
   "org.apache.maven" % "maven-model" % mavenVersion,
   "org.apache.maven" % "maven-repository-metadata" % mavenVersion,
   "org.apache.maven" % "maven-resolver-provider" % mavenVersion
 )
 
-libraryDependencies ++= Seq(
-  "io.get-coursier" %% "coursier" % "1.0.1",
-  "io.get-coursier" %% "coursier-cache" % "1.0.1"
-)
+libraryDependencies += "com.squareup.tools.build" % "maven-archeologist" % "0.0.10"
 
 libraryDependencies += "org.apache.maven.indexer" % "indexer-reader" % "6.0.0"
 libraryDependencies += "org.apache.maven.indexer" % "indexer-core" % "6.0.0"
 
-libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.3"
-
-// Pinning secure versions of insecure transitive libraryDependencies
-// Please update when updating dependencies above (including Play plugin)
-libraryDependencies ++= Seq(
-  "com.google.guava" % "guava" % "25.1-jre",
-  "com.fasterxml.jackson.core" % "jackson-databind" % "2.10.2"
-)
+libraryDependencies += "de.upb.cs.swt.delphi" %% "delphi-core" % "0.9.2"

project/build.properties

@@ -1 +1 @@
-sbt.version=1.3.8
+sbt.version=1.5.5

project/plugins.sbt

@@ -15,8 +15,8 @@
 // limitations under the License.
 
 // build management and packaging
-addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.7.0")
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.15")
+addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.2")
 
 // coverage
 //addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")

src/main/scala/de/upb/cs/swt/delphi/crawler/Configuration.scala

@@ -30,8 +30,7 @@ class Configuration {
 
 
 
-  lazy val elasticsearchClientUri: ElasticsearchClientUri = ElasticsearchClientUri(
-    elasticSearchInstance.host + ":" + elasticSearchInstance.portNumber)
+  lazy val elasticsearchClientUri: String = elasticSearchInstance.host + ":" + elasticSearchInstance.portNumber
 
   lazy val elasticSearchInstance : Instance = InstanceRegistry.retrieveElasticSearchInstance(this) match {
     case Success(instance) => instance
@@ -52,7 +51,7 @@ class Configuration {
   val controlServerPort : Int = 8882
 
   val defaultElasticSearchPort : Int = 9200
-  val defaultElasticSearchHost : String = "elasticsearch://localhost"
+  val defaultElasticSearchHost : String = "localhost"
 
   lazy val fallbackElasticSearchPort : Int = sys.env.get("DELPHI_ELASTIC_URI") match {
     case Some(hostString) => if(hostString.count(c => c == ':') == 2){
@@ -76,7 +75,7 @@ class Configuration {
     case None => defaultElasticSearchHost
 
   }
-  val limit : Int = 100
+  val limit : Int = 0
   val throttle : Throttle = Throttle(5, 1 second, 5, ThrottleMode.shaping)
 
   val tempFileStorage : String = "temp/"
@@ -85,6 +84,9 @@ class Configuration {
   val callGraphStreamPoolSize : Int = 4
   val hermesActorPoolSize: Int = 2
 
+  val downloadActorPoolSize: Int = 8
+  val pomReadActorPoolSize: Int = 4
+
 
 
 

src/main/scala/de/upb/cs/swt/delphi/crawler/Crawler.scala

@@ -17,17 +17,13 @@
 package de.upb.cs.swt.delphi.crawler
 
 import akka.actor.ActorSystem
-import akka.routing.{RoundRobinPool, SmallestMailboxPool}
-import akka.stream.ActorMaterializer
-import com.sksamuel.elastic4s.http.ElasticClient
+import akka.routing.RoundRobinPool
 import de.upb.cs.swt.delphi.crawler.control.{ProcessScheduler, Server}
 import de.upb.cs.swt.delphi.crawler.discovery.maven.MavenDiscoveryProcess
 import de.upb.cs.swt.delphi.crawler.instancemanagement.InstanceRegistry
-import de.upb.cs.swt.delphi.crawler.preprocessing.PreprocessingDispatchActor
-import de.upb.cs.swt.delphi.crawler.processing.{HermesActor, HermesAnalyzer, ProcessingDispatchActor}
+import de.upb.cs.swt.delphi.crawler.processing.HermesAnalyzer
 import de.upb.cs.swt.delphi.crawler.storage.ElasticActor
-import de.upb.cs.swt.delphi.crawler.tools.OPALLogAdapter
-import org.opalj.log.{GlobalLogContext, OPALLogger}
+import de.upb.cs.swt.delphi.crawler.tools.{ElasticHelper, OPALLogAdapter}
 
 import scala.concurrent.Await
 import scala.concurrent.duration._
@@ -37,12 +33,11 @@ import scala.util.{Failure, Success}
   * The starter for Delphi Crawler
   */
 object Crawler extends App with AppLogging {
-  private val configuration = new Configuration()
 
+  private val configuration = new Configuration()
   implicit val system: ActorSystem = ActorSystem("delphi-crawler")
-  implicit val materializer = ActorMaterializer()
 
-  OPALLogger.updateLogger(GlobalLogContext, OPALLogAdapter)
+  OPALLogAdapter.setOpalLoggingEnabled(false)
   HermesAnalyzer.setConfig()
 
   sys.addShutdownHook({
@@ -56,11 +51,10 @@ object Crawler extends App with AppLogging {
   Startup.logStartupInfo
   Startup.preflightCheck(configuration) match {
     case Success(c) =>
-    case Failure(e) => {
+    case Failure(e) =>
       InstanceRegistry.handleInstanceFailure(configuration)
       system.terminate()
       sys.exit(1)
-    }
   }
 
 
@@ -69,14 +63,7 @@ object Crawler extends App with AppLogging {
   new Server(configuration.controlServerPort).start()
 
   val elasticPool = system.actorOf(RoundRobinPool(configuration.elasticActorPoolSize)
-    .props(ElasticActor.props(ElasticClient(configuration.elasticsearchClientUri))))
-
-  /*
-  val hermesPool = system.actorOf(SmallestMailboxPool(configuration.hermesActorPoolSize).props(HermesActor.props()))
-  val processingDispatchActor = system.actorOf(ProcessingDispatchActor.props(hermesPool))
-
-  val preprocessingDispatchActor = system.actorOf(PreprocessingDispatchActor.props(configuration, processingDispatchActor, elasticPool))
-*/
+    .props(ElasticActor.props(ElasticHelper.buildElasticClient(configuration))))
 
   val processScheduler = system.actorOf(ProcessScheduler.props)
   processScheduler ! ProcessScheduler.Enqueue(new MavenDiscoveryProcess(configuration, elasticPool))

src/main/scala/de/upb/cs/swt/delphi/crawler/control/ProcessActor.scala

@@ -27,15 +27,14 @@ import de.upb.cs.swt.delphi.crawler.control.ProcessActor.Go
   */
 class ProcessActor(process: Process[_]) extends Actor with ActorLogging {
   override def receive: Receive = {
-    case Go => {
+    case Go =>
       val result = process.start
       sender() ! ProcessScheduler.Finalized(process, result)
-    }
   }
 }
 
 object ProcessActor {
-  def props(process: Process[_]) = Props(new ProcessActor(process))
+  def props(process: Process[_]): Props = Props(new ProcessActor(process))
 
   case object Go
 

src/main/scala/de/upb/cs/swt/delphi/crawler/control/ProcessScheduler.scala

@@ -33,28 +33,24 @@ class ProcessScheduler extends Actor with ActorLogging {
   private val queue = mutable.Queue.empty[Process[_]]
 
   override def receive: Receive = {
-    case Enqueue(process) => {
+    case Enqueue(process) =>
       queue.enqueue(process)
       self ! UpdateQueue
-    }
-    case Finalized(process, result) => {
+    case Finalized(process, result) =>
       running.find(_.equals(process)) match {
-        case Some(f) => {
+        case Some(f) =>
           // TODO: Maybe do some permanent logging here
           running.remove(process)
           self ! UpdateQueue
-        }
         case None => log.warning(s"Could not finalized process: $process")
       }
-    }
-    case UpdateQueue => {
-      if (running.size < parallelism && queue.size > 0) {
+    case UpdateQueue =>
+      if (running.size < parallelism && queue.nonEmpty) {
         val nextProcess = queue.dequeue()
         running.add(nextProcess)
         val actor = context.actorOf(ProcessActor.props(nextProcess))
         actor ! ProcessActor.Go
       }
-    }
     case x => log.warning(s"Scheduler received unknown message. This should never ever happen. Message was: $x")
   }
 }