Misc import optimizations (#3930)

benchmark/src/test/scala/com/wavesplatform/lang/v1/FunctionCallCopyBenchmark.scala

@@ -0,0 +1,71 @@
+package com.wavesplatform.lang.v1
+
+import com.wavesplatform.common.utils.EitherExt2
+import com.wavesplatform.lang.v1.FunctionCallCopyBenchmark.FuncCallSt
+import com.wavesplatform.lang.v1.compiler.Terms.*
+import org.openjdk.jmh.annotations.*
+import org.openjdk.jmh.infra.Blackhole
+
+import java.util.concurrent.TimeUnit
+
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@BenchmarkMode(Array(Mode.AverageTime))
+@Threads(1)
+@Fork(1)
+@Warmup(iterations = 10, time = 1)
+@Measurement(iterations = 10, time = 1)
+class FunctionCallCopyBenchmark {
+  @Benchmark
+  def twoLevelsDeep(st: FuncCallSt, bh: Blackhole): Unit =
+    bh.consume(st.twoLevelsDeep.deepCopy.value)
+
+  @Benchmark
+  def threeLevelsDeep(st: FuncCallSt, bh: Blackhole): Unit =
+    bh.consume(st.threeLevelsDeep.deepCopy.value)
+
+  @Benchmark
+  def fourLevelsDeep(st: FuncCallSt, bh: Blackhole): Unit =
+    bh.consume(st.fourLevelsDeep.deepCopy.value)
+}
+
+object FunctionCallCopyBenchmark {
+
+  @State(Scope.Benchmark)
+  class FuncCallSt {
+    val simpleCall: FUNCTION_CALL = FUNCTION_CALL(
+      FunctionHeader.User("fooBar"),
+      List(
+        CONST_STRING("foobar").explicitGet(),
+        CONST_LONG(1000L),
+        CONST_BOOLEAN(false)
+      )
+    )
+
+    val twoLevelsDeep: FUNCTION_CALL = FUNCTION_CALL(
+      FunctionHeader.User("fooBar"),
+      List(
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(simpleCall)),
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(simpleCall)),
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(simpleCall)),
+      )
+    )
+
+    val threeLevelsDeep: FUNCTION_CALL = FUNCTION_CALL(
+      FunctionHeader.User("fooBar"),
+      List(
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(twoLevelsDeep)),
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(twoLevelsDeep)),
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(twoLevelsDeep)),
+      )
+    )
+
+    val fourLevelsDeep: FUNCTION_CALL = FUNCTION_CALL(
+      FunctionHeader.User("fooBar"),
+      List(
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(threeLevelsDeep)),
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(threeLevelsDeep)),
+        FUNCTION_CALL(FunctionHeader.User("fooBar"), List.fill(3)(threeLevelsDeep)),
+      )
+    )
+  }
+}

lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/Terms.scala

@@ -1,18 +1,19 @@
 package com.wavesplatform.lang.v1.compiler
 
-import java.nio.charset.StandardCharsets
 import cats.Eval
 import cats.instances.list.*
 import cats.syntax.traverse.*
 import com.wavesplatform.common.state.ByteStr
-import com.wavesplatform.lang.{CommonError, ExecutionError}
 import com.wavesplatform.lang.v1.ContractLimits.*
 import com.wavesplatform.lang.v1.FunctionHeader
 import com.wavesplatform.lang.v1.compiler.Types.*
 import com.wavesplatform.lang.v1.evaluator.ctx.impl.PureContext.MaxListLengthV4
+import com.wavesplatform.lang.{CommonError, ExecutionError}
 import monix.eval.Coeval
 
-import scala.annotation.nowarn
+import java.nio.charset.StandardCharsets
+import scala.annotation.{nowarn, tailrec}
+import scala.util.hashing.MurmurHash3
 
 object Terms {
   val DataTxMaxBytes: Int      = 150 * 1024     // should be the same as DataTransaction.MaxBytes
@@ -58,6 +59,87 @@ object Terms {
     def deepCopy: Eval[EXPR]
     override def toString: String = toStr()
     def isItFailed: Boolean       = false
+
+    override def hashCode(): Int = EXPR.hashCode(List(this), 7207)
+
+    override def equals(obj: Any): Boolean = obj match {
+      case e: EXPR => EXPR.equals(List(this), List(e))
+      case _       => false
+    }
+  }
+
+  object EXPR {
+    @tailrec
+    final def hashCode(stack: List[EXPR], prevHashCode: Int): Int =
+      if (stack.isEmpty) prevHashCode
+      else
+        stack.head match {
+          case FAILED_EXPR() =>
+            hashCode(stack.tail, prevHashCode * 31 + 7001)
+          case GETTER(expr, field) =>
+            hashCode(expr :: stack.tail, (prevHashCode * 31 + field.hashCode) * 31 + 7013)
+          case LET_BLOCK(let, body) =>
+            hashCode(let.value :: body :: stack.tail, (prevHashCode * 31 + let.name.hashCode) * 31 + 7019)
+          case BLOCK(dec, body) =>
+            dec match {
+              case FAILED_DEC() =>
+                hashCode(body :: stack.tail, prevHashCode * 31 + 7027)
+              case LET(name, value) =>
+                hashCode(value :: stack.tail, (prevHashCode * 31 + name.hashCode) * 31 + 7027)
+              case FUNC(name, args, body) =>
+                hashCode(body :: stack.tail, (prevHashCode * 31 + MurmurHash3.listHash(name :: args, 7027)) * 31 + 7027)
+            }
+          case IF(cond, ifTrue, ifFalse) =>
+            hashCode(cond :: ifTrue :: ifFalse :: stack.tail, prevHashCode * 31 + 7039)
+          case REF(key) =>
+            hashCode(stack.tail, (prevHashCode * 31 + key.hashCode) * 31 + 7043)
+          case FUNCTION_CALL(function, args) =>
+            hashCode(args ++ stack.tail, (prevHashCode * 31 + function.hashCode()) * 31 + 7057)
+          case evaluated: EVALUATED =>
+            hashCode(stack.tail, (prevHashCode * 31 + evaluated.hashCode()) * 31 + 7069)
+        }
+
+    @tailrec
+    final def equals(e1: List[EXPR], e2: List[EXPR]): Boolean =
+      if (e1.isEmpty && e2.isEmpty) true
+      else
+        (e1.head, e2.head) match {
+          case (FAILED_EXPR(), FAILED_EXPR()) =>
+            equals(e1.tail, e2.tail)
+          case (g1: GETTER, g2: GETTER) =>
+            if (g1.field != g2.field) false
+            else equals(g1.expr :: e1.tail, g2.expr :: e2.tail)
+          case (lb1: LET_BLOCK, lb2: LET_BLOCK) =>
+            if (lb1.let.name != lb2.let.name) false
+            else equals(lb1.let.value :: lb1.body :: e1.tail, lb2.let.value :: lb2.body :: e2.tail)
+          case (b1: BLOCK, b2: BLOCK) =>
+            (b1.dec, b2.dec) match {
+              case (FAILED_DEC(), FAILED_DEC()) =>
+                equals(b1.body :: e1.tail, b2.body :: e2.tail)
+              case (l1: LET, l2: LET) =>
+                if (l1.name != l2.name) false
+                else equals(l1.value :: b1.body :: e1.tail, l2.value :: b2.body :: e2.tail)
+              case (f1: FUNC, f2: FUNC) =>
+                if (f1.name != f2.name || f1.args != f2.args) false
+                else equals(f1.body :: b1.body :: e1.tail, f2.body :: b2.body :: e2.tail)
+              case _ => false
+            }
+          case (if1: IF, if2: IF) =>
+            equals(
+              if1.cond :: if1.ifFalse :: if1.ifTrue :: e1.tail,
+              if2.cond :: if2.ifFalse :: if2.ifTrue :: e2.tail
+            )
+          case (REF(key1), REF(key2)) =>
+            if (key1 != key2) false
+            else equals(e1.tail, e2.tail)
+          case (fc1: FUNCTION_CALL, fc2: FUNCTION_CALL) =>
+            if (fc1.function != fc2.function) false
+            else equals(fc1.args ++ e1.tail, fc2.args ++ e2.tail)
+          case (ev1: EVALUATED, ev2: EVALUATED) =>
+            if (ev1 != ev2) false
+            else equals(e1.tail, e2.tail)
+          case _ => false
+        }
   }
 
   case class FAILED_EXPR() extends EXPR {
@@ -143,7 +225,11 @@ object Terms {
       } yield "FUNCTION_CALL(" ++ function.toString ++ "," ++ e.toString ++ ")"
 
     override def deepCopy: Eval[EXPR] =
-      Eval.defer(args.traverse(_.deepCopy)).map(FUNCTION_CALL(function, _))
+      args
+        .foldLeft(Eval.now(List.empty[EXPR])) { case (prev, cur) =>
+          prev.flatMap(p => cur.deepCopy.map(_ :: p))
+        }
+        .map(reversedArgs => FUNCTION_CALL(function, reversedArgs.reverse))
   }
 
   sealed trait EVALUATED extends EXPR {
@@ -159,11 +245,21 @@ object Terms {
     override def toString: String = t.toString
     override val weight: Long     = 8L
     override val getType: REAL    = LONG
+    override def hashCode(): Int  = t.hashCode()
+    override def equals(obj: Any): Boolean = obj match {
+      case CONST_LONG(`t`) => true
+      case _               => false
+    }
   }
   case class CONST_BIGINT(t: BigInt) extends EVALUATED {
     override def toString: String = t.toString
     override val weight: Long     = 64L
     override val getType: REAL    = BIGINT
+    override def hashCode(): Int  = t.hashCode()
+    override def equals(obj: Any): Boolean = obj match {
+      case CONST_BIGINT(`t`) => true
+      case _                 => false
+    }
   }
 
   class CONST_BYTESTR private (val bs: ByteStr) extends EVALUATED {
@@ -238,6 +334,13 @@ object Terms {
     override val weight: Long     = 1L
 
     override val getType: REAL = BOOLEAN
+
+    override def hashCode(): Int = b.hashCode()
+
+    override def equals(obj: Any): Boolean = obj match {
+      case CONST_BOOLEAN(`b`) => true
+      case _                  => false
+    }
   }
 
   lazy val TRUE: CONST_BOOLEAN  = CONST_BOOLEAN(true)
@@ -255,6 +358,13 @@ object Terms {
         TUPLE(fields.toSeq.sortBy(_._1).map(_._2.getType).toList)
       else
         caseType
+
+    override def hashCode(): Int = MurmurHash3.productHash(this)
+
+    override def equals(obj: Any): Boolean = obj match {
+      case CaseObj(`caseType`, `fields`) => true
+      case _                             => false
+    }
   }
 
   object CaseObj {
@@ -276,6 +386,13 @@ object Terms {
       weight - EMPTYARR_WEIGHT - ELEM_WEIGHT * xs.size
 
     override val getType: REAL = LIST(ANY)
+
+    override def hashCode(): Int = MurmurHash3.indexedSeqHash(xs, 4517)
+
+    override def equals(obj: Any): Boolean = obj match {
+      case ARR(`xs`) => true
+      case _         => false
+    }
   }
 
   object ARR {
@@ -300,6 +417,13 @@ object Terms {
     override def toString: String = "Evaluation failed: " ++ reason
     def weight: Long              = 0
     override val getType: REAL    = NOTHING
+
+    override def hashCode(): Int = reason.hashCode * 31 + 7129
+
+    override def equals(obj: Any): Boolean = obj match {
+      case FAIL(`reason`) => true
+      case _              => false
+    }
   }
 
   val runtimeTupleType: CASETYPEREF = CASETYPEREF("Tuple", Nil)

node/src/main/scala/com/wavesplatform/Importer.scala

@@ -369,11 +369,15 @@ object Importer extends ScorexLogging {
                 blocksOffset += meta.size + 4
             }
           }
-          val snapshotsOffset = (2 to blockchainUpdater.height).map { h =>
-            database.loadTxStateSnapshots(Height(h), rdb).map(_.toByteArray.length).sum
-          }.sum
 
-          blocksOffset -> snapshotsOffset.toLong
+          var totalSize = 0L
+          rdb.db.iterateOver(KeyTags.NthTransactionStateSnapshotAtHeight) { e =>
+            totalSize += (e.getValue.length + 4)
+          }
+
+          val snapshotsOffset = totalSize
+
+          blocksOffset -> snapshotsOffset
         case _ => 0L -> 0L
       }
     val blocksInputStream = new BufferedInputStream(initFileStream(importOptions.blockchainFile, blocksFileOffset), 2 * 1024 * 1024)

node/src/main/scala/com/wavesplatform/database/Caches.scala

@@ -16,7 +16,7 @@ import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves}
 import com.wavesplatform.transaction.{Asset, DiscardedBlocks, Transaction}
 import com.wavesplatform.utils.ObservedLoadingCache
 import monix.reactive.Observer
-import org.ehcache.sizeof.SizeOf
+import org.github.jamm.MemoryMeter
 
 import java.{lang, util}
 import scala.collection.immutable.VectorMap
@@ -128,13 +128,13 @@ abstract class Caches extends Blockchain with Storage {
     VolumeAndFee(curVf.volume, curVf.fee)
   }
 
-  private val objectWeigher = SizeOf.newInstance()
+  private val memMeter = MemoryMeter.builder().build()
 
   private val scriptCache: LoadingCache[Address, Option[AccountScriptInfo]] =
     CacheBuilder
       .newBuilder()
       .maximumWeight(128 << 20)
-      .weigher((_: Address, asi: Option[AccountScriptInfo]) => asi.map(s => objectWeigher.deepSizeOf(s).toInt).getOrElse(0))
+      .weigher((_: Address, asi: Option[AccountScriptInfo]) => asi.map(s => memMeter.measureDeep(s).toInt).getOrElse(0))
       .recordStats()
       .build(new CacheLoader[Address, Option[AccountScriptInfo]] {
         override def load(key: Address): Option[AccountScriptInfo] = loadScript(key)
@@ -183,6 +183,7 @@ abstract class Caches extends Blockchain with Storage {
 
   protected def discardAccountData(addressWithKey: (Address, String)): Unit = accountDataCache.invalidate(addressWithKey)
   protected def loadAccountData(acc: Address, key: String): CurrentData
+  protected def loadEntryHeights(keys: Iterable[(Address, String)]): Map[(Address, String), Height]
 
   private[database] def addressId(address: Address): Option[AddressId] = addressIdCache.get(address)
   private[database] def addressIds(addresses: Seq[Address]): Map[Address, Option[AddressId]] =
@@ -293,17 +294,21 @@ abstract class Caches extends Blockchain with Storage {
       BalanceNode(amount, prevBalance.height)
     )
 
-    val updatedDataWithNodes = for {
+    val newEntries = for {
       (address, entries) <- snapshot.accountData
       (key, entry)       <- entries
-    } yield {
-      val entryKey = (address, key)
-      val prevData = accountDataCache.get(entryKey)
-      entryKey -> (
-        CurrentData(entry, Height(height), prevData.height),
-        DataNode(entry, prevData.height)
-      )
-    }
+    } yield ((address, key), entry)
+
+    val cachedEntries = accountDataCache.getAllPresent(newEntries.keys.asJava).asScala
+    val loadedPrevEntries = loadEntryHeights(newEntries.keys.filterNot(cachedEntries.contains))
+
+    val updatedDataWithNodes = (for {
+      (k, currentEntry) <- cachedEntries.view.mapValues(_.height) ++ loadedPrevEntries
+      newEntry          <- newEntries.get(k)
+    } yield k -> (
+      CurrentData(newEntry, Height(height), currentEntry),
+      DataNode(newEntry, currentEntry)
+    )).toMap
 
     val orderFillsWithNodes = for {
       (orderId, VolumeAndFee(volume, fee)) <- snapshot.orderFills

node/src/main/scala/com/wavesplatform/database/ReadOnlyDB.scala

@@ -15,16 +15,16 @@ class ReadOnlyDB(db: RocksDB, readOptions: ReadOptions) {
     key.parse(bytes)
   }
 
-  def multiGetOpt[V](keys: Seq[Key[Option[V]]], valBufferSize: Int): Seq[Option[V]] =
+  def multiGetOpt[V](keys: IndexedSeq[Key[Option[V]]], valBufferSize: Int): Seq[Option[V]] =
     db.multiGetOpt(readOptions, keys, valBufferSize)
 
-  def multiGet[V](keys: Seq[Key[V]], valBufferSize: Int): Seq[Option[V]] =
+  def multiGet[V](keys: IndexedSeq[Key[V]], valBufferSize: Int): Seq[Option[V]] =
     db.multiGet(readOptions, keys, valBufferSize)
 
-  def multiGetOpt[V](keys: Seq[Key[Option[V]]], valBufSizes: Seq[Int]): Seq[Option[V]] =
+  def multiGetOpt[V](keys: IndexedSeq[Key[Option[V]]], valBufSizes: IndexedSeq[Int]): Seq[Option[V]] =
     db.multiGetOpt(readOptions, keys, valBufSizes)
 
-  def multiGetInts(keys: Seq[Key[Int]]): Seq[Option[Int]] =
+  def multiGetInts(keys: IndexedSeq[Key[Int]]): Seq[Option[Int]] =
     db.multiGetInts(readOptions, keys)
 
   def has[V](key: Key[V]): Boolean = {