Improve performance of loading used names from persisted Analysis file

internal/zinc-core/src/main/scala/sbt/internal/inc/Changes.scala

@@ -88,7 +88,10 @@ final case class ModifiedNames(names: Set[UsedName]) {
     names.flatMap(n => n.scopes.asScala.map(n.name -> _))
 
   def isModified(usedName: UsedName): Boolean =
-    usedName.scopes.asScala.exists(scope => lookupMap.contains(usedName.name -> scope))
+    usedName.scopes.asScala.exists(scope => isModifiedRaw(usedName.name, scope))
+
+  def isModifiedRaw(name: String, scope: UseScope): Boolean =
+    lookupMap.contains(name -> scope)
 
   override def toString: String =
     s"ModifiedNames(changes = ${names.mkString(", ")})"

internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala

@@ -22,6 +22,7 @@ import sbt.internal.inc.JavaInterfaceUtil.EnrichOption
 import sbt.util.{ InterfaceUtil, Level, Logger }
 import sbt.util.InterfaceUtil.{ jo2o, t2 }
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 import scala.util.control.NonFatal
 import xsbti.{ FileConverter, Position, Problem, Severity, UseScope, VirtualFile, VirtualFileRef }
 import xsbt.api.{ APIUtil, HashAPI, NameHashing }
@@ -626,7 +627,7 @@ private final class AnalysisCallback(
   private[this] val objectApis = new TrieMap[String, ApiInfo]
   private[this] val classPublicNameHashes = new TrieMap[String, Array[NameHash]]
   private[this] val objectPublicNameHashes = new TrieMap[String, Array[NameHash]]
-  private[this] val usedNames = new RelationBuilder[String, UsedName]
+  private[this] val usedNames = mutable.Map.empty[String, mutable.Set[UsedName]]
   private[this] val unreporteds = new TrieMap[VirtualFileRef, ConcurrentLinkedQueue[Problem]]
   private[this] val reporteds = new TrieMap[VirtualFileRef, ConcurrentLinkedQueue[Problem]]
   private[this] val mainClasses = new TrieMap[VirtualFileRef, ConcurrentLinkedQueue[String]]
@@ -864,7 +865,8 @@ private final class AnalysisCallback(
 
   def usedName(className: String, name: String, useScopes: EnumSet[UseScope]) =
     usedNames.synchronized {
-      usedNames(className) = UsedName.make(name, useScopes)
+      usedNames.getOrElseUpdate(className, mutable.Set.empty) += UsedName.make(name, useScopes)
+      ()
     }
 
   override def enabled(): Boolean = options.enabled
@@ -905,11 +907,13 @@ private final class AnalysisCallback(
   }
 
   def getOrNil[A, B](m: collection.Map[A, Seq[B]], a: A): Seq[B] = m.get(a).toList.flatten
+
   def addCompilation(base: Analysis): Analysis =
     base.copy(compilations = base.compilations.add(compilation))
+
   def addUsedNames(base: Analysis): Analysis = {
-    assert(base.relations.names.size == 0)
-    base.copy(relations = base.relations.addUsedNames(usedNames.result()))
+    assert(base.relations.names.isEmpty)
+    base.copy(relations = base.relations.addUsedNames(UsedNames.fromMultiMap(usedNames)))
   }
 
   private def companionsWithHash(className: String): (Companions, HashAPI.Hash, HashAPI.Hash) = {

internal/zinc-core/src/main/scala/sbt/internal/inc/MemberRefInvalidator.scala

@@ -58,7 +58,7 @@ private[inc] class MemberRefInvalidator(log: Logger, logRecompileOnMacro: Boolea
   private final val NoInvalidation = (_: String) => Set.empty[String]
   def get(
       memberRef: Relation[String, String],
-      usedNames: Relation[String, UsedName],
+      usedNames: Relations.UsedNames,
       apiChange: APIChange,
       isScalaClass: String => Boolean
   ): String => Set[String] = apiChange match {
@@ -121,7 +121,7 @@ private[inc] class MemberRefInvalidator(log: Logger, logRecompileOnMacro: Boolea
   }
 
   private class NameHashFilteredInvalidator(
-      usedNames: Relation[String, UsedName],
+      usedNames: Relations.UsedNames,
       memberRef: Relation[String, String],
       modifiedNames: ModifiedNames,
       isScalaClass: String => Boolean
@@ -131,17 +131,18 @@ private[inc] class MemberRefInvalidator(log: Logger, logRecompileOnMacro: Boolea
       val dependent = memberRef.reverse(to)
       filteredDependencies(dependent)
     }
+
     private def filteredDependencies(dependent: Set[String]): Set[String] = {
       dependent.filter {
         case from if isScalaClass(from) =>
-          val affectedNames = usedNames.forward(from).filter(modifiedNames.isModified)
+          val affectedNames = usedNames.affectedNames(modifiedNames, from)
           if (affectedNames.isEmpty) {
             log.debug(
               s"None of the modified names appears in source file of $from. This dependency is not being considered for invalidation."
             )
             false
           } else {
-            log.debug(s"The following modified names cause invalidation of $from: $affectedNames")
+            log.debug(s"The following modified names cause invalidation of $from: [$affectedNames]")
             true
           }
         case from =>

internal/zinc-core/src/main/scala/sbt/internal/inc/Relations.scala

@@ -80,8 +80,6 @@ trait Relations {
   /** Internal source dependencies that depend on external source file `dep`.  This includes both direct and inherited dependencies.  */
   def usesExternal(className: String): Set[String]
 
-  private[inc] def usedNames(className: String): Set[UsedName]
-
   /**
    * Records that the file `src` generates products `products`, has internal dependencies `internalDeps`,
    * has external dependencies `externalDeps` and library dependencies `libraryDeps`.
@@ -140,7 +138,7 @@ trait Relations {
       deps: Iterable[(VirtualFileRef, String, XStamp)]
   ): Relations
 
-  private[inc] def addUsedNames(data: Relation[String, UsedName]): Relations
+  private[inc] def addUsedNames(data: Relations.UsedNames): Relations
 
   /** Concatenates the two relations. Acts naively, i.e., doesn't internalize external deps on added files. */
   def ++(o: Relations): Relations
@@ -267,7 +265,7 @@ trait Relations {
   /**
    * Relation between source files and _unqualified_ term and type names used in given source file.
    */
-  private[inc] def names: Relation[String, UsedName]
+  private[inc] def names: Relations.UsedNames
 
   private[inc] def copy(
       srcProd: Relation[VirtualFileRef, VirtualFileRef] = srcProd,
@@ -276,12 +274,13 @@ trait Relations {
       internalDependencies: InternalDependencies = internalDependencies,
       externalDependencies: ExternalDependencies = externalDependencies,
       classes: Relation[VirtualFileRef, String] = classes,
-      names: Relation[String, UsedName] = names,
+      names: Relations.UsedNames = names,
       productClassName: Relation[String, String] = productClassName,
   ): Relations
 }
 
 object Relations {
+  type UsedNames = inc.UsedNames
 
   /** Tracks internal and external source dependencies for a specific dependency type, such as direct or inherited.*/
   private[inc] final class ClassDependencies(
@@ -318,7 +317,7 @@ object Relations {
     internalDependencies = InternalDependencies.empty,
     externalDependencies = ExternalDependencies.empty,
     classes = Relation.empty,
-    names = Relation.empty,
+    names = UsedNames.fromMultiMap(Map.empty),
     productClassName = Relation.empty
   )
 
@@ -329,7 +328,7 @@ object Relations {
       internalDependencies: InternalDependencies,
       externalDependencies: ExternalDependencies,
       classes: Relation[VirtualFileRef, String],
-      names: Relation[String, UsedName],
+      names: Relations.UsedNames,
       productClassName: Relation[String, String]
   ): Relations =
     new MRelationsNameHashing(
@@ -348,6 +347,7 @@ object Relations {
       external: Relation[String, String]
   ): ClassDependencies =
     new ClassDependencies(internal, external)
+
 }
 
 private[inc] object DependencyCollection {
@@ -368,7 +368,7 @@ private[inc] object DependencyCollection {
 private[inc] object InternalDependencies {
 
   /**
-   * Constructs an empty `InteralDependencies`
+   * Constructs an empty `InternalDependencies`
    */
   def empty = InternalDependencies(Map.empty)
 }
@@ -470,7 +470,7 @@ private class MRelationsNameHashing(
     val internalDependencies: InternalDependencies,
     val externalDependencies: ExternalDependencies,
     val classes: Relation[VirtualFileRef, String],
-    val names: Relation[String, UsedName],
+    val names: Relations.UsedNames,
     val productClassName: Relation[String, String]
 ) extends Relations {
   def allSources: collection.Set[VirtualFileRef] = srcProd._1s
@@ -501,8 +501,6 @@ private class MRelationsNameHashing(
   def externalDeps(className: String): Set[String] = externalClassDep.forward(className)
   def usesExternal(className: String): Set[String] = externalClassDep.reverse(className)
 
-  private[inc] def usedNames(className: String): Set[UsedName] = names.forward(className)
-
   def addProducts(src: VirtualFileRef, products: Iterable[VirtualFileRef]): Relations =
     new MRelationsNameHashing(
       srcProd ++ products.map(p => (src, p)),
@@ -563,15 +561,15 @@ private class MRelationsNameHashing(
       productClassName,
     )
 
-  private[inc] def addUsedNames(data: Relation[String, UsedName]): Relations = {
+  private[inc] def addUsedNames(data: Relations.UsedNames): Relations = {
     new MRelationsNameHashing(
       srcProd,
       libraryDep,
       libraryClassName,
       internalDependencies,
       externalDependencies,
       classes,
-      names = if (names.forwardMap.isEmpty) data else names ++ data,
+      names = if (names.isEmpty) data else names ++ data,
       productClassName,
     )
   }
@@ -626,7 +624,7 @@ private class MRelationsNameHashing(
       internalDependencies: InternalDependencies = internalDependencies,
       externalDependencies: ExternalDependencies = externalDependencies,
       classes: Relation[VirtualFileRef, String] = classes,
-      names: Relation[String, UsedName] = names,
+      names: Relations.UsedNames = names,
       productClassName: Relation[String, String] = productClassName,
   ): Relations = new MRelationsNameHashing(
     srcProd,
@@ -657,6 +655,13 @@ private class MRelationsNameHashing(
     if (r.forwardMap.isEmpty) "Relation [ ]"
     else r.all.toSeq.map(kv => line_s(kv._1, kv._2)).sorted.mkString("Relation [\n", "", "]")
   }
+  def usedNames_s(r: Relations.UsedNames) = {
+    if (r.isEmpty) "UsedNames [ ]"
+    else {
+      val all = r.iterator.flatMap { case (a, bs) => bs.iterator.map(b => (a, b)) }
+      all.map(kv => line_s(kv._1, kv._2)).toSeq.sorted.mkString("UsedNames [\n", "", "]")
+    }
+  }
 
   override def toString: String = {
     def deps_s(m: Map[_, Relation[_, _]]) =
@@ -671,7 +676,7 @@ private class MRelationsNameHashing(
     |  internalDependencies: ${deps_s(internalDependencies.dependencies)}
     |  externalDependencies: ${deps_s(externalDependencies.dependencies)}
     |  class names: ${relation_s(classes)}
-    |  used names: ${relation_s(names)}
+    |  used names: ${usedNames_s(names)}
     |  product class names: ${relation_s(productClassName)}
     """.trim.stripMargin
   }

internal/zinc-core/src/main/scala/sbt/internal/inc/UsedName.scala

@@ -11,16 +11,18 @@
 
 package sbt.internal.inc
 
+import java.{ util => ju }
+
+import scala.{ collection => sc }
 import xsbti.compile.{ UsedName => XUsedName }
 import xsbti.UseScope
 
-case class UsedName private (name: String, scopes: java.util.EnumSet[UseScope]) extends XUsedName {
+case class UsedName private (name: String, scopes: ju.EnumSet[UseScope]) extends XUsedName {
   override def getName: String = name
-  override def getScopes: java.util.EnumSet[UseScope] = scopes
+  override def getScopes: ju.EnumSet[UseScope] = scopes
 }
 
 object UsedName {
-
   def apply(name: String, scopes: Iterable[UseScope] = Nil): UsedName = {
     val useScopes = java.util.EnumSet.noneOf(classOf[UseScope])
     scopes.foreach(useScopes.add)
@@ -39,3 +41,86 @@ object UsedName {
       name
   }
 }
+
+sealed abstract class UsedNames private {
+  def isEmpty: Boolean
+  def toMultiMap: sc.Map[String, sc.Set[UsedName]]
+
+  def ++(other: UsedNames): UsedNames
+  def --(classes: Iterable[String]): UsedNames
+  def iterator: Iterator[(String, sc.Set[UsedName])]
+
+  def affectedNames(modifiedNames: ModifiedNames, from: String): String
+}
+
+object UsedNames {
+  def fromJavaMap(map: ju.Map[String, Schema.UsedNames]) = JavaUsedNames(map)
+  def fromMultiMap(map: sc.Map[String, sc.Set[UsedName]]) = ScalaUsedNames(map)
+
+  final case class ScalaUsedNames(map: sc.Map[String, sc.Set[UsedName]]) extends UsedNames {
+    def isEmpty = map.isEmpty
+    def toMultiMap = map
+    def ++(other: UsedNames) = fromMultiMap(map ++ other.iterator)
+    def --(classes: Iterable[String]) = fromMultiMap(map -- classes)
+    def iterator = map.iterator
+    def affectedNames(modifiedNames: ModifiedNames, from: String): String =
+      map(from).iterator.filter(modifiedNames.isModified).mkString(", ")
+  }
+
+  final case class JavaUsedNames(map: ju.Map[String, Schema.UsedNames]) extends UsedNames {
+    import scala.collection.JavaConverters._
+
+    private def fromUseScope(useScope: Schema.UseScope, id: Int): UseScope = useScope match {
+      case Schema.UseScope.DEFAULT  => UseScope.Default
+      case Schema.UseScope.IMPLICIT => UseScope.Implicit
+      case Schema.UseScope.PATMAT   => UseScope.PatMatTarget
+      case Schema.UseScope.UNRECOGNIZED =>
+        sys.error(s"Unrecognized ${classOf[Schema.UseScope].getName} with value `$id`.")
+    }
+
+    private def fromUsedName(usedName: Schema.UsedName): UsedName = {
+      val name = usedName.getName.intern() // ?
+      val useScopes = ju.EnumSet.noneOf(classOf[UseScope])
+      val len = usedName.getScopesCount
+      for (i <- 0 to len - 1)
+        useScopes.add(fromUseScope(usedName.getScopes(i), usedName.getScopesValue(i)))
+      UsedName.make(name, useScopes)
+    }
+
+    private def fromUsedNamesMap(map: ju.Map[String, Schema.UsedNames]) =
+      for ((k, used) <- map.asScala)
+        yield k -> used.getUsedNamesList.asScala.iterator.map(fromUsedName).toSet
+
+    lazy val toMultiMap: sc.Map[String, sc.Set[UsedName]] = fromUsedNamesMap(map)
+    private lazy val convert: UsedNames = fromMultiMap(toMultiMap)
+
+    def isEmpty = map.isEmpty
+    def ++(other: UsedNames) = convert ++ other
+    def --(classes: Iterable[String]) = convert -- classes
+    def iterator = convert.iterator
+
+    def affectedNames(modifiedNames: ModifiedNames, from: String): String = {
+      val b = new StringBuilder()
+      val usedNames = map.get(from)
+      var first = true
+      var i = 0
+      val n = usedNames.getUsedNamesCount
+      while (i < n) {
+        val usedName = usedNames.getUsedNames(i)
+        val name = usedName.getName
+        var i2 = 0
+        val n2 = usedName.getScopesCount
+        while (i2 < n2) {
+          val scope = fromUseScope(usedName.getScopes(i2), usedName.getScopesValue(i2))
+          if (modifiedNames.isModifiedRaw(name, scope)) {
+            if (first) first = false else b.append(", ")
+            b.append(name)
+          }
+          i2 += 1
+        }
+        i += 1
+      }
+      b.toString
+    }
+  }
+}

internal/zinc-persist/src/main/scala/sbt/internal/inc/binary/converters/ProtobufReaders.scala

@@ -12,7 +12,6 @@
 package sbt.internal.inc.binary.converters
 
 import java.nio.file.{ Path, Paths }
-import java.util
 import java.util.{ List => JList, Map => JMap }
 import sbt.internal.inc.Relations.ClassDependencies
 import sbt.internal.inc._
@@ -701,33 +700,6 @@ final class ProtobufReaders(mapper: ReadMapper, currentVersion: Schema.Version)
       new ClassDependencies(internal, external)
     }
 
-    def fromUsedName(usedName: Schema.UsedName): UsedName = {
-      val name = usedName.getName.intern()
-      val useScopes = util.EnumSet.noneOf(classOf[UseScope])
-      val len = usedName.getScopesCount
-      for {
-        i <- 0 to len - 1
-      } {
-        useScopes.add(fromUseScope(usedName.getScopes(i), usedName.getScopesValue(i)))
-      }
-      UsedName.make(name, useScopes)
-    }
-
-    def fromUsedNamesMap(
-        map: java.util.Map[String, Schema.UsedNames]
-    ): Relation[String, UsedName] = {
-      val builder = new RelationBuilder[String, UsedName]
-      for ((k, used) <- map.asScala) {
-        val usedNames = used.getUsedNamesList.asScala
-        if (!usedNames.isEmpty) {
-          for (schemaUsedName <- usedNames) {
-            builder(k) = fromUsedName(schemaUsedName)
-          }
-        }
-      }
-      builder.result()
-    }
-
     def expected(msg: String) = ReadersFeedback.expected(msg, Classes.Relations)
 
     val srcProd = fromMap(relations.getSrcProdMap, stringToSource, stringToProd)
@@ -746,7 +718,7 @@ final class ProtobufReaders(mapper: ReadMapper, currentVersion: Schema.Version)
     val classes = fromMap(relations.getClassesMap, stringToSource, stringId)
     val productClassName =
       fromMap(relations.getProductClassNameMap, stringId, stringId)
-    val names = fromUsedNamesMap(relations.getNamesMap)
+    val names = UsedNames.fromJavaMap(relations.getNamesMap)
     val internal = InternalDependencies(
       Map(
         DependencyContext.DependencyByMemberRef -> memberRef.internal,