InvalidStateHash has computedStateHash, added a lot of debugging logs

node-it/src/test/resources/template.conf

@@ -60,7 +60,7 @@ waves {
       # These fields are ignored: timestamp, block-timestamp, signature. They are generated in integration tests.
       genesis {
         average-block-delay = 10s
-        initial-base-target = 50000
+        initial-base-target = 1282421
         initial-balance = 6400000000000000
         transactions = [
           # Initial balances are balanced (pun intended) in such way that initial block

node-it/src/test/scala/com/wavesplatform/it/Docker.scala

@@ -27,8 +27,8 @@ import pureconfig.generic.auto.*
 import java.io.{FileOutputStream, IOException}
 import java.net.{InetAddress, InetSocketAddress, URL}
 import java.nio.file.{Files, Path, Paths}
-import java.time.{LocalDateTime, Duration as JDuration}
 import java.time.format.DateTimeFormatter
+import java.time.{LocalDateTime, Duration as JDuration}
 import java.util.Collections.*
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
@@ -572,10 +572,11 @@ object Docker {
 
   val configTemplate: Config = parseResources("template.conf")
   def genesisOverride(featuresConfig: Option[Config] = None): Config = {
+    val gap             = 6.seconds // To force node mining at start, otherwise it schedules
     val genesisTs: Long = System.currentTimeMillis()
 
     val timestampOverrides = parseString(s"""waves.blockchain.custom.genesis {
-                                            |  timestamp = $genesisTs
+                                            |  timestamp = ${genesisTs - gap.toMillis}
                                             |  block-timestamp = $genesisTs
                                             |  signature = null # To calculate it in Block.genesis
                                             |}""".stripMargin)

node-it/src/test/scala/com/wavesplatform/it/sync/ContinuousMiningTestSuite.scala

@@ -0,0 +1,47 @@
+package com.wavesplatform.it.sync
+
+import com.typesafe.config.{Config, ConfigFactory}
+import com.wavesplatform.it.api.SyncHttpApi.*
+import com.wavesplatform.it.{BaseFunSuite, NodeConfigs}
+import com.wavesplatform.transaction.TxHelpers
+
+import scala.concurrent.duration.*
+
+class ContinuousMiningTestSuite extends BaseFunSuite {
+  // private val txnSendingInterval = 4.millis
+
+  override protected val nodeConfigs: Seq[Config] = Seq(
+    ConfigFactory
+      .parseString(s"""waves {
+                      |  blockchain.custom.functionality.pre-activated-features = {
+                      |    1  = 0
+                      |    16 = 0
+                      |    17 = 0
+                      |    18 = 0
+                      |    19 = 0
+                      |    20 = 0
+                      |    21 = 0
+                      |    22 = 0
+                      |    23 = 0
+                      |  }
+                      |  miner {
+                      |    quorum = 0
+                      |    micro-block-interval = 6ms
+                      |    min-micro-block-age = 10ms # 100ms
+                      |  }
+                      |}""".stripMargin)
+      .withFallback(NodeConfigs.Miners.head)
+      .resolve()
+  )
+
+  test("Can continue mining with a last micro block removing") {
+    val waitHeight = miner.height + 10
+    var i = 0
+    while (miner.height < waitHeight) {
+      if (i == 0) miner.transfer(miner.keyPair, TxHelpers.secondAddress.toString, 1, minFee)
+      else i = (i + 1) % 5
+      // Thread.sleep(txnSendingInterval.toMillis)
+    }
+  }
+}
+

node/src/main/scala/com/wavesplatform/block/Block.scala

@@ -109,7 +109,7 @@ case class Block(
 
   override def toString: String =
     s"Block(${id()},${header.reference},${header.generator.toAddress}," +
-      s"${header.timestamp},${header.featureVotes.mkString("[", ",", "]")}${if (header.rewardVote >= 0) s",${header.rewardVote}" else ""})"
+      s"${header.timestamp},${header.featureVotes.mkString("[", ",", "]")}${if (header.rewardVote >= 0) s",${header.rewardVote}" else ""}, sh=${header.stateHash})"
 }
 
 object Block {

node/src/main/scala/com/wavesplatform/block/MicroBlock.scala

@@ -30,7 +30,7 @@ case class MicroBlock(
 
   override def toString: String = s"MicroBlock(... -> ${reference.trim}, txs=${transactionData.size}"
 
-  def stringRepr(totalBlockId: ByteStr): String = s"MicroBlock(${totalBlockId.trim} -> ${reference.trim}, txs=${transactionData.size})"
+  def stringRepr(totalBlockId: ByteStr): String = s"MicroBlock(${totalBlockId.trim} -> ${reference.trim}, txs=${transactionData.size}, sh=$stateHash)"
 }
 
 object MicroBlock {

node/src/main/scala/com/wavesplatform/mining/Miner.scala

@@ -180,7 +180,8 @@ class MinerImpl(
         blockId
       }
       balance         = blockchainUpdater.generatingBalance(account.toAddress, Some(reference))
-      lastBlockHeader = blockchainUpdater.lastBlockHeader.get.header // We'll need only baseTarget, timestamp and reference
+      lastBlock       = blockchainUpdater.lastBlockHeader.get
+      lastBlockHeader = lastBlock.header // Here is a full block, we'll need only baseTarget, timestamp and reference
       validBlockDelay <- pos
         .getValidBlockDelay(height, account, lastBlockHeader.baseTarget, balance)
         .leftMap(_.toString)
@@ -192,7 +193,8 @@ class MinerImpl(
       _ <- Either.cond(
         blockTime <= currentTime + maxTimeDrift,
         log.debug(
-          s"Forging with ${account.toAddress}, balance $balance, prev block $reference at $height with target ${lastBlockHeader.baseTarget}"
+          s"Forging with ${account.toAddress}, balance $balance, prev block $reference at $height with target ${lastBlockHeader.baseTarget}, " +
+            s"lastBlock: id=${lastBlock.id()}, sh=${lastBlockHeader.stateHash}"
         ),
         s"Block time $blockTime is from the future: current time is $currentTime, MaxTimeDrift = $maxTimeDrift"
       )
@@ -202,21 +204,24 @@ class MinerImpl(
           Some(blockchainUpdater.lastStateHash(Some(reference)))
         else None
       (unconfirmed, totalConstraint, stateHash) = packTransactionsForKeyBlock(account.toAddress, reference, prevStateHash)
-      block <- Block
-        .buildAndSign(
-          version,
-          blockTime,
-          reference,
-          consensusData.baseTarget,
-          consensusData.generationSignature,
-          unconfirmed,
-          account,
-          blockFeatures(version),
-          blockRewardVote(version),
-          if (blockchainUpdater.supportsLightNodeBlockFields(height + 1)) stateHash else None,
-          None
-        )
-        .leftMap(_.err)
+      block <- {
+        log.debug(s"Forging data: prevStateHash=$prevStateHash, stateHash=$stateHash, unconfirmed=${unconfirmed.size}")
+        Block
+          .buildAndSign(
+            version,
+            blockTime,
+            reference,
+            consensusData.baseTarget,
+            consensusData.generationSignature,
+            unconfirmed,
+            account,
+            blockFeatures(version),
+            blockRewardVote(version),
+            if (blockchainUpdater.supportsLightNodeBlockFields(height + 1)) stateHash else None,
+            None
+          )
+          .leftMap(_.err)
+      }
     } yield (block, totalConstraint))
   }
 

node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala

@@ -548,9 +548,11 @@ class BlockchainUpdaterImpl(
                   MicroBlockAppendError("Invalid total block signature", microBlock)
                 )
               blockDifferResult <- {
+                val lastBlockTimestamp = rocksdb.lastBlockTimestamp
+                log.debug(s"BlockchainUpdaterImpl.processMicroBlock: rocksdb.lastBlockTimestamp=$lastBlockTimestamp, referencedComputedStateHash=$referencedComputedStateHash")
                 BlockDiffer.fromMicroBlock(
                   this,
-                  rocksdb.lastBlockTimestamp,
+                  lastBlockTimestamp,
                   referencedComputedStateHash,
                   microBlock,
                   snapshot,

node/src/main/scala/com/wavesplatform/state/NgState.scala

@@ -2,17 +2,21 @@ package com.wavesplatform.state
 
 import cats.implicits.catsSyntaxSemigroup
 import com.google.common.cache.CacheBuilder
+import com.typesafe.scalalogging.Logger
 import com.wavesplatform.block
 import com.wavesplatform.block.Block.BlockId
 import com.wavesplatform.block.{Block, MicroBlock}
 import com.wavesplatform.common.state.ByteStr
 import com.wavesplatform.state.NgState.{CachedMicroDiff, MicroBlockInfo, NgStateCaches}
 import com.wavesplatform.state.StateSnapshot.monoid
 import com.wavesplatform.transaction.{DiscardedMicroBlocks, Transaction}
+import org.slf4j.LoggerFactory
 
 import java.util.concurrent.TimeUnit
 
 object NgState {
+  val logger: Logger = Logger(LoggerFactory.getLogger(getClass.getName))
+
   case class MicroBlockInfo(totalBlockId: BlockId, microBlock: MicroBlock) {
     def idEquals(id: ByteStr): Boolean = totalBlockId == id
   }
@@ -115,6 +119,7 @@ case class NgState(
   def snapshotOf(id: BlockId): Option[(Block, StateSnapshot, Long, Long, ByteStr, DiscardedMicroBlocks)] =
     forgeBlock(id).map { case (block, discarded) =>
       val (snapshot, carry, totalFee, computedStateHash) = this.snapshotFor(id)
+      NgState.logger.debug(s"snapshotOf($id): block.header.stateHash=${block.header.stateHash}, computedStateHash=$computedStateHash")
       (block, snapshot, carry, totalFee, computedStateHash, discarded)
     }
 
@@ -199,20 +204,27 @@ case class NgState(
           )
         } else if (!microBlocksAsc.exists(_.idEquals(blockId))) None
         else {
+          var debugMessage = ""
+
           val (accumulatedTxs, maybeFound) =
             microBlocksAsc.foldLeft((Vector.empty[Transaction], Option.empty[(ByteStr, Option[ByteStr], DiscardedMicroBlocks)])) {
               case ((accumulated, Some((sig, stateHash, discarded))), MicroBlockInfo(mbId, micro)) =>
                 val discDiff = microSnapshots(mbId).snapshot
+                debugMessage = s"Discarded($mbId), " + debugMessage
                 (accumulated, Some((sig, stateHash, discarded :+ (micro -> discDiff))))
 
               case ((accumulated, None), mb) if mb.idEquals(blockId) =>
                 val found = Some((mb.microBlock.totalResBlockSig, mb.microBlock.stateHash, Seq.empty[(MicroBlock, StateSnapshot)]))
+                debugMessage = s"Found($blockId, stateHash=${mb.microBlock.stateHash}), " + debugMessage
                 (accumulated ++ mb.microBlock.transactionData, found)
 
-              case ((accumulated, None), MicroBlockInfo(_, mb)) =>
+              case ((accumulated, None), MicroBlockInfo(totalBlockId, mb)) =>
+                debugMessage = s"NotFound($totalBlockId), " + debugMessage
                 (accumulated ++ mb.transactionData, None)
             }
 
+          NgState.logger.debug(s"NgState.forgeBlock($blockId): $debugMessage")
+
           maybeFound.map { case (sig, stateHash, discarded) =>
             (Block.create(base, base.transactionData ++ accumulatedTxs, sig, stateHash), discarded)
           }

node/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala

@@ -35,9 +35,10 @@ object BlockAppender extends ScorexLogging {
       txSignParCheck: Boolean = true
   )(newBlock: Block, snapshot: Option[BlockSnapshotResponse]): Task[Either[ValidationError, BlockApplyResult]] =
     Task {
+      log.debug(s"Appending internal ${newBlock.id()}->${newBlock.header.reference}, last block id: ${blockchainUpdater.lastBlockHeader.map(_.id())}")
       if (
-        blockchainUpdater
-          .isLastBlockId(newBlock.header.reference) || blockchainUpdater.lastBlockHeader.exists(_.header.reference == newBlock.header.reference)
+        blockchainUpdater.isLastBlockId(newBlock.header.reference) ||
+        blockchainUpdater.lastBlockHeader.exists(_.header.reference == newBlock.header.reference)
       ) {
         if (newBlock.header.challengedHeader.isDefined) {
           appendChallengeBlock(blockchainUpdater, utxStorage, pos, time, log, verify, txSignParCheck)(newBlock, snapshot)

node/src/main/scala/com/wavesplatform/state/diffs/BlockDiffer.scala

@@ -2,6 +2,7 @@ package com.wavesplatform.state.diffs
 
 import cats.implicits.{catsSyntaxOption, catsSyntaxSemigroup, toFoldableOps}
 import cats.syntax.either.*
+import com.typesafe.scalalogging.Logger
 import com.wavesplatform.account.Address
 import com.wavesplatform.block.{Block, BlockSnapshot, MicroBlock, MicroBlockSnapshot}
 import com.wavesplatform.common.state.ByteStr
@@ -21,10 +22,13 @@ import com.wavesplatform.transaction.smart.script.trace.TracedResult
 import com.wavesplatform.transaction.transfer.MassTransferTransaction.ParsedTransfer
 import com.wavesplatform.transaction.transfer.{MassTransferTransaction, TransferTransaction}
 import com.wavesplatform.transaction.{Asset, Authorized, BlockchainUpdater, GenesisTransaction, PaymentTransaction, Transaction}
+import org.slf4j.LoggerFactory
 
 import scala.collection.immutable.VectorMap
 
 object BlockDiffer {
+  val logger: Logger = Logger(LoggerFactory.getLogger(getClass.getName))
+
   final case class Result(
       snapshot: StateSnapshot,
       carry: Long,
@@ -113,6 +117,7 @@ object BlockDiffer {
       enableExecutionLog: Boolean,
       txSignParCheck: Boolean
   ): TracedResult[ValidationError, Result] = {
+    logger.debug(s"BlockDiffer.fromBlockTraced(maybePrevBlock=${maybePrevBlock.map(_.id())}, block=${block.id()})")
     val stateHeight        = blockchain.height
     val heightWithNewBlock = stateHeight + 1
 
@@ -243,6 +248,8 @@ object BlockDiffer {
       verify: Boolean,
       enableExecutionLog: Boolean
   ): TracedResult[ValidationError, Result] = {
+    logger.debug(s"BlockDiffer.fromMicroBlockTraced: prevBlockTimestamp=$prevBlockTimestamp, snapshot.totalBlockId=${snapshot.map(_.totalBlockId)}, verify=$verify")
+
     for {
       // microblocks are processed within block which is next after 40-only-block which goes on top of activated height
       _ <- TracedResult(
@@ -340,6 +347,8 @@ object BlockDiffer {
     val blockGenerator  = blockchain.lastBlockHeader.get.header.generator.toAddress
     val rideV6Activated = blockchain.isFeatureActivated(BlockchainFeatures.RideV6)
 
+    logger.debug(s"BlockDiffer.apply: blockchain.lastBlockTimestamp=$timestamp, lastBlockHeader.id=${blockchain.lastBlockHeader.map(_.id())}")
+
     val txDiffer = TransactionDiffer(prevBlockTimestamp, timestamp, verify, enableExecutionLog = enableExecutionLog) _
 
     if (verify && txSignParCheck)
@@ -497,6 +506,6 @@ object BlockDiffer {
     Either.cond(
       !blockchain.supportsLightNodeBlockFields() || blockStateHash.contains(computedStateHash),
       (),
-      InvalidStateHash(blockStateHash)
+      InvalidStateHash(blockStateHash, Some(computedStateHash))
     )
 }

node/src/main/scala/com/wavesplatform/transaction/TxValidationError.scala

@@ -49,7 +49,7 @@ object TxValidationError {
     override def toString: String = s"InvalidSignature(${entity.toString + " reason: " + details})"
   }
 
-  case class InvalidStateHash(blockStateHash: Option[ByteStr]) extends ValidationError
+  case class InvalidStateHash(blockStateHash: Option[ByteStr], computedStateHash: Option[ByteStr]) extends ValidationError
 
   sealed trait WithLog extends Product with Serializable {
     def log: Log[Id]

node/tests/src/test/scala/com/wavesplatform/state/BlockChallengeTest.scala

@@ -84,13 +84,14 @@ class BlockChallengeTest
     val challengingMiner = TxHelpers.signer(3)
     withDomain(settings, balances = AddrWithBalance.enoughBalances(sender)) { d =>
       d.appendBlock()
-      val txs             = Seq(TxHelpers.transfer(sender, amount = 1), TxHelpers.transfer(sender, amount = 2))
-      val challengedBlock = d.createBlock(Block.ProtoBlockVersion, txs, generator = challengedMiner, stateHash = Some(Some(invalidStateHash)))
-      val invalidHashChallengingBlock = d.createChallengingBlock(challengingMiner, challengedBlock, stateHash = Some(Some(invalidStateHash)))
-      val missedHashChallengingBlock  = d.createChallengingBlock(challengingMiner, challengedBlock, stateHash = Some(None))
-
-      d.appendBlockE(invalidHashChallengingBlock) shouldBe Left(InvalidStateHash(Some(invalidStateHash)))
-      d.appendBlockE(missedHashChallengingBlock) shouldBe Left(InvalidStateHash(None))
+      val txs                    = Seq(TxHelpers.transfer(sender, amount = 1), TxHelpers.transfer(sender, amount = 2))
+      val invalidChallengedBlock = d.createBlock(Block.ProtoBlockVersion, txs, generator = challengedMiner, stateHash = Some(Some(invalidStateHash)))
+      val invalidHashChallengingBlock = d.createChallengingBlock(challengingMiner, invalidChallengedBlock, stateHash = Some(Some(invalidStateHash)))
+      val missedHashChallengingBlock  = d.createChallengingBlock(challengingMiner, invalidChallengedBlock, stateHash = Some(None))
+      val validChallengingBlock       = d.createChallengingBlock(challengingMiner, invalidChallengedBlock)
+
+      d.appendBlockE(invalidHashChallengingBlock) shouldBe Left(InvalidStateHash(Some(invalidStateHash), validChallengingBlock.header.stateHash))
+      d.appendBlockE(missedHashChallengingBlock) shouldBe Left(InvalidStateHash(None, validChallengingBlock.header.stateHash))
     }
   }
 

node/tests/src/test/scala/com/wavesplatform/state/LightNodeTest.scala

@@ -41,7 +41,7 @@ class LightNodeTest extends PropSpec with WithDomain {
       d.appendBlockE(
         invalidBlock,
         Some(BlockSnapshot(invalidBlock.id(), txSnapshots))
-      ) shouldBe Left(InvalidStateHash(Some(invalidStateHash)))
+      ) shouldBe Left(InvalidStateHash(Some(invalidStateHash), validBlock.header.stateHash))
       d.lastBlock shouldBe prevBlock
 
       d.appendBlockE(