Merge remote-tracking branch 'origin/develop' into feature/redesign

johannesduesing · Oct 21, 2021 · 1d168c2 · 1d168c2
2 parents 7fb1f48 + 3717948
commit 1d168c2
Show file tree

Hide file tree

Showing 11 changed files with 40 additions and 20 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,11 +1,21 @@
 dist: trusty
 language: scala
 scala:
-  -  2.12.4
+  -  2.12.11
 script:
   - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then sbt ++$TRAVIS_SCALA_VERSION test; fi'
-  - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then sbt ++$TRAVIS_SCALA_VERSION coverage test coverageReport coverageAggregate codacyCoverage; fi'
+#  - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then sbt ++$TRAVIS_SCALA_VERSION test; fi'
+ - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then sbt ++$TRAVIS_SCALA_VERSION coverage test coverageReport coverageAggregate codacyCoverage; fi'
 after_success:
   - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then bash <(curl -s https://codecov.io/bash); fi'
 jdk:
   - oraclejdk8
+cache:
+  directories:
+    - $HOME/.cache/coursier
+    - $HOME/.ivy2/cache
+    - $HOME/.sbt
+before_cache:
+  - rm -fv $HOME/.ivy2/.sbt.ivy.lock
+  - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
+  - find $HOME/.sbt        -name "*.lock"               -print -delete
diff --git a/README.md b/README.md
@@ -5,10 +5,10 @@ The automated crawling and processing engine for the Delphi platform.
 We are currently in pre-alpha state! There is no release and the code in
 this repository is purely experimental!
 
-|branch | status | codacy | snyk |
-| :---: | :---: | :---: | :---: |  
-| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-crawler.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-crawler) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d52f09343249401f829585f6edcf6a32)](https://www.codacy.com/app/bhermann/delphi-crawler?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=delphi-hub/delphi-crawler&amp;utm_campaign=Badge_Grade)| [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-crawler/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-crawler/?targetFile=build.sbt) |
-| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-crawler.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-crawler)  | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d52f09343249401f829585f6edcf6a32?branch=develop)](https://www.codacy.com/app/bhermann/delphi-crawler?branch=develop&amp;utm_source=github.com&amp;utm_medium=referral&amp;utm_content=delphi-hub/delphi-crawler&amp;utm_campaign=Badge_Grade) | [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-crawler/develop/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-crawler/develop/?targetFile=build.sbt) |
+|branch | status | codacy | coverage | snyk |
+| :---: | :---: | :---: | :---: |  :---: |  
+| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-crawler.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-crawler) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d52f09343249401f829585f6edcf6a32)](https://www.codacy.com/app/bhermann/delphi-crawler?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=delphi-hub/delphi-crawler&amp;utm_campaign=Badge_Grade)| [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/d52f09343249401f829585f6edcf6a32)](https://www.codacy.com/manual/delphi-hub/delphi-crawler?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-crawler&utm_campaign=Badge_Coverage) | [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-crawler/badge.svg)](https://snyk.io/test/github/delphi-hub/delphi-crawler/) |
+| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-crawler.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-crawler)  | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/d52f09343249401f829585f6edcf6a32?branch=develop)](https://www.codacy.com/app/bhermann/delphi-crawler?branch=develop&amp;utm_source=github.com&amp;utm_medium=referral&amp;utm_content=delphi-hub/delphi-crawler&amp;utm_campaign=Badge_Grade) | [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/d52f09343249401f829585f6edcf6a32?branch=develop)](https://www.codacy.com/manual/delphi-hub/delphi-crawler?branch=develop&utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-crawler&utm_campaign=Badge_Coverage) | [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-crawler/develop/badge.svg)](https://snyk.io/test/github/delphi-hub/delphi-crawler/develop/) |
 
 ## What is the crawler component?
 

diff --git a/build.sbt b/build.sbt
@@ -16,15 +16,16 @@
 
 name := "delphi-crawler"
 
-version := "1.0.0-SNAPSHOT"
+version := "0.9.5-SNAPSHOT"
 
 scalaVersion := "2.12.15"
 
 lazy val crawler = (project in file(".")).
   enablePlugins(JavaAppPackaging).
   enablePlugins(DockerPlugin).
-  settings (
-    dockerBaseImage := "delphihub/jre-alpine-openjfx"
+  settings(
+    dockerBaseImage := "delphihub/jre-alpine-openjfx",
+    dockerAlias := com.typesafe.sbt.packager.docker.DockerAlias(None, Some("delphihub"),"delphi-crawler", Some(version.value)),
   ).
   enablePlugins(ScalastylePlugin).
   enablePlugins(AshScriptPlugin).
@@ -90,4 +91,4 @@ libraryDependencies += "com.squareup.tools.build" % "maven-archeologist" % "0.0.
 libraryDependencies += "org.apache.maven.indexer" % "indexer-reader" % "6.0.0"
 libraryDependencies += "org.apache.maven.indexer" % "indexer-core" % "6.0.0"
 
-libraryDependencies += "de.upb.cs.swt.delphi" %% "delphi-core" % "0.9.2"
+libraryDependencies += "de.upb.cs.swt.delphi" %% "delphi-core" % "0.9.2"
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.5.5
+sbt.version=1.5.5
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -19,8 +19,8 @@ addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0")
 addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.2")
 
 // coverage
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
-addSbtPlugin("com.codacy" % "sbt-codacy-coverage" % "1.3.12")
+//addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
+//addSbtPlugin("com.codacy" % "sbt-codacy-coverage" % "1.3.14")
 
 // preparation for dependency checking
 addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.1")

diff --git a/src/main/scala/de/upb/cs/swt/delphi/crawler/Configuration.scala b/src/main/scala/de/upb/cs/swt/delphi/crawler/Configuration.scala
@@ -75,7 +75,7 @@ class Configuration {
     case None => defaultElasticSearchHost
 
   }
-  val limit : Int = 0
+  val limit : Int = 100
   val throttle : Throttle = Throttle(5, 1 second, 5, ThrottleMode.shaping)
 
   val tempFileStorage : String = "temp/"

diff --git a/src/main/scala/de/upb/cs/swt/delphi/crawler/Crawler.scala b/src/main/scala/de/upb/cs/swt/delphi/crawler/Crawler.scala
@@ -48,7 +48,7 @@ object Crawler extends App with AppLogging {
   })
 
 
-  Startup.showStartupInfo
+  Startup.logStartupInfo
   Startup.preflightCheck(configuration) match {
     case Success(c) =>
     case Failure(e) =>

diff --git a/src/main/scala/de/upb/cs/swt/delphi/crawler/Startup.scala b/src/main/scala/de/upb/cs/swt/delphi/crawler/Startup.scala
@@ -23,9 +23,17 @@ import scala.util.{Failure, Success, Try}
 
 object Startup extends AppLogging {
 
-  def showStartupInfo(implicit system : ActorSystem) = {
-    log.info(s"Delphi Crawler (${BuildInfo.name} ${BuildInfo.version})")
-    log.info(s"Running on Scala ${BuildInfo.scalaVersion}")
+  def logStartupInfo(implicit system : ActorSystem) = {
+    produceStartupInfo(log.info)
+  }
+
+  def printStartInfo() : Unit = {
+    produceStartupInfo(println)
+  }
+
+  private def produceStartupInfo(output : String => Unit) : Unit = {
+    output(s"Delphi Crawler (${BuildInfo.name} ${BuildInfo.version})")
+    output(s"Running on Scala ${BuildInfo.scalaVersion} using JVM ${System.getProperty("java.version")}.")
   }
 
   def preflightCheck(configuration: Configuration)(implicit system : ActorSystem) : Try[Configuration] = {

diff --git a/src/main/scala/de/upb/cs/swt/delphi/crawler/discovery/maven/HttpResourceHandler.scala b/src/main/scala/de/upb/cs/swt/delphi/crawler/discovery/maven/HttpResourceHandler.scala
@@ -34,7 +34,7 @@ class HttpResourceHandler(root : URI) extends ResourceHandler {
       val target = root.resolve(name).toURL
 
       val conn: HttpURLConnection = target.openConnection.asInstanceOf[HttpURLConnection]
-      conn.setReadTimeout(20.minutes.toMillis.toInt)
+      conn.setReadTimeout(24.hours.toMillis.toInt)
       conn.setRequestMethod("GET")
       conn.setRequestProperty("User-Agent", s"Delphi Maven-Indexer-Reader/${BuildInfo.version}" )
 

diff --git a/src/main/scala/de/upb/cs/swt/delphi/crawler/processing/HermesAnalyzer.scala b/src/main/scala/de/upb/cs/swt/delphi/crawler/processing/HermesAnalyzer.scala
@@ -32,7 +32,6 @@ import scala.collection.JavaConverters.asScalaIteratorConverter
   *
   */
 class HermesAnalyzer(project: Project[URL]) extends HermesCore {
-
   initialize(HermesAnalyzer.temporaryConfigFile())
 
 

diff --git a/src/main/scala/de/upb/cs/swt/delphi/crawler/tools/AuthProvider.scala b/src/main/scala/de/upb/cs/swt/delphi/crawler/tools/AuthProvider.scala
@@ -15,6 +15,8 @@
 // limitations under the License.
 package de.upb.cs.swt.delphi.crawler.tools
 
+import java.time.Clock
+
 import de.upb.cs.swt.delphi.crawler.Configuration
 import pdi.jwt.{Jwt, JwtAlgorithm, JwtClaim}