From 813034cebce6bc03b59f21b1e01c14c77b964b8f Mon Sep 17 00:00:00 2001 From: Artyom Sayadyan Date: Sat, 13 Jan 2024 12:35:32 +0300 Subject: [PATCH 1/8] NODE-2642 Transaction snapshots API (#3931) --- .github/workflows/check-pr.yaml | 2 +- .../api/grpc/TransactionsApiGrpcImpl.scala | 21 +- .../grpc/test/TransactionsApiGrpcSpec.scala | 77 ++- .../wavesplatform/lang/script/Script.scala | 7 +- .../it/sync/AmountAsStringSuite.scala | 4 +- .../VRFProtobufActivationSuite.scala | 4 +- .../grpc/ExchangeTransactionGrpcSuite.scala | 6 +- .../it/sync/smartcontract/package.scala | 2 +- .../ExchangeTransactionSuite.scala | 6 +- .../FailedTransactionSuiteLike.scala | 4 +- .../SignAndBroadcastApiSuite.scala | 2 +- .../sync/transactions/TransferNFTSuite.scala | 4 +- node/src/main/resources/application.conf | 1 + .../main/resources/swagger-ui/openapi.yaml | 232 +++++++++ .../scala/com/wavesplatform/Application.scala | 3 +- .../com/wavesplatform/account/Recipient.scala | 2 + .../api/http/DebugApiRoute.scala | 24 +- .../api/http/StateSnapshotJson.scala | 120 +++++ .../api/http/TransactionJsonSerializer.scala | 7 +- .../api/http/TransactionsApiRoute.scala | 23 +- .../com/wavesplatform/database/Keys.scala | 2 - .../database/RocksDBWriter.scala | 13 +- .../settings/RestAPISettings.scala | 1 + .../wavesplatform/state/AssetStaticInfo.scala | 6 + .../wavesplatform/state/AssetVolumeInfo.scala | 2 + .../com/wavesplatform/state/Blockchain.scala | 3 +- .../state/BlockchainUpdaterImpl.scala | 5 + .../wavesplatform/state/LeaseDetails.scala | 6 + .../state/SnapshotBlockchain.scala | 7 + .../com/wavesplatform/state/Sponsorship.scala | 3 + .../com/wavesplatform/state/package.scala | 15 +- .../wavesplatform/transaction/package.scala | 1 + .../http/DebugApiRouteSpec.scala | 12 +- .../http/ProtoVersionTransactionsSpec.scala | 4 +- .../http/TransactionSnapshotsRouteSpec.scala | 450 ++++++++++++++++++ .../RestAPISettingsSpecification.scala | 2 + .../smart/predef/MatcherBlockchainTest.scala | 2 + .../snapshot/StateSnapshotStorageTest.scala | 7 +- .../ProtoVersionTransactionsSpec.scala | 4 +- .../wavesplatform/transaction/TxHelpers.scala | 12 +- .../ExchangeTransactionSpecification.scala | 4 +- .../assets/exchange/OrderSpecification.scala | 2 +- .../wavesplatform/utils/EmptyBlockchain.scala | 3 + project/Dependencies.scala | 2 +- .../blockchain/ImmutableBlockchain.scala | 4 +- .../runner/blockchain/LazyBlockchain.scala | 4 +- 46 files changed, 1041 insertions(+), 86 deletions(-) create mode 100644 node/src/main/scala/com/wavesplatform/api/http/StateSnapshotJson.scala create mode 100644 node/src/test/scala/com/wavesplatform/http/TransactionSnapshotsRouteSpec.scala diff --git a/.github/workflows/check-pr.yaml b/.github/workflows/check-pr.yaml index 3bc81c7e95..0cb67e1254 100644 --- a/.github/workflows/check-pr.yaml +++ b/.github/workflows/check-pr.yaml @@ -49,7 +49,7 @@ jobs: sbt lang/assembly git clone https://github.com/waves-exchange/neutrino-contract git clone https://github.com/waves-exchange/contracts - git clone https://oauth2:${{ secrets.DUCKS_GITHUB_TOKEN }}@github.com/akharazyan/wavesducks-public + git clone https://github.com/waves-ducks-core/wavesducks-public git clone https://oauth2:${{ secrets.SWOPFI_GITLAB_TOKEN }}@gitlabwp.wvservices.com/swopfi/swopfi-smart-contracts find neutrino-contract/script -name "*.ride" -type f -exec java -jar lang/jvm/target/file-compiler.jar {} +; find contracts/ride -name "*.ride" -type f -exec java -jar lang/jvm/target/file-compiler.jar {} +; diff --git a/grpc-server/src/main/scala/com/wavesplatform/api/grpc/TransactionsApiGrpcImpl.scala b/grpc-server/src/main/scala/com/wavesplatform/api/grpc/TransactionsApiGrpcImpl.scala index 7848170398..0bc2fd8805 100644 --- a/grpc-server/src/main/scala/com/wavesplatform/api/grpc/TransactionsApiGrpcImpl.scala +++ b/grpc-server/src/main/scala/com/wavesplatform/api/grpc/TransactionsApiGrpcImpl.scala @@ -1,6 +1,5 @@ package com.wavesplatform.api.grpc -import scala.concurrent.Future import com.wavesplatform.account.AddressScheme import com.wavesplatform.api.common.{CommonTransactionsApi, TransactionMeta} import com.wavesplatform.api.grpc.TransactionsApiGrpcImpl.applicationStatusFromTxStatus @@ -8,13 +7,15 @@ import com.wavesplatform.protobuf.* import com.wavesplatform.protobuf.transaction.* import com.wavesplatform.protobuf.utils.PBImplicitConversions.PBRecipientImplicitConversionOps import com.wavesplatform.state.{Blockchain, TxMeta, InvokeScriptResult as VISR} -import com.wavesplatform.transaction.{Authorized, EthereumTransaction} import com.wavesplatform.transaction.TxValidationError.GenericError -import io.grpc.{Status, StatusRuntimeException} +import com.wavesplatform.transaction.{Authorized, EthereumTransaction} import io.grpc.stub.StreamObserver +import io.grpc.{Status, StatusRuntimeException} import monix.execution.Scheduler import monix.reactive.Observable +import scala.concurrent.Future + class TransactionsApiGrpcImpl(blockchain: Blockchain, commonApi: CommonTransactionsApi)(implicit sc: Scheduler) extends TransactionsApiGrpc.TransactionsApi { @@ -64,6 +65,20 @@ class TransactionsApiGrpcImpl(blockchain: Blockchain, commonApi: CommonTransacti ) } + override def getTransactionSnapshots( + request: TransactionSnapshotsRequest, + responseObserver: StreamObserver[TransactionSnapshotResponse] + ): Unit = + responseObserver.interceptErrors { + val snapshots = + for { + id <- Observable.fromIterable(request.transactionIds) + (snapshot, status) <- Observable.fromIterable(blockchain.transactionSnapshot(id.toByteStr)) + pbSnapshot = PBSnapshots.toProtobuf(snapshot, status) + } yield TransactionSnapshotResponse(id, Some(pbSnapshot)) + responseObserver.completeWith(snapshots) + } + override def getUnconfirmed(request: TransactionsRequest, responseObserver: StreamObserver[TransactionResponse]): Unit = responseObserver.interceptErrors { val unconfirmedTransactions = if (!request.sender.isEmpty) { diff --git a/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala b/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala index 5d3f4311af..d283b3e650 100644 --- a/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala +++ b/grpc-server/src/test/scala/com/wavesplatform/api/grpc/test/TransactionsApiGrpcSpec.scala @@ -2,7 +2,7 @@ package com.wavesplatform.api.grpc.test import com.google.protobuf.ByteString import com.wavesplatform.account.KeyPair -import com.wavesplatform.api.grpc.{ApplicationStatus, TransactionResponse, TransactionsApiGrpcImpl, TransactionsRequest} +import com.wavesplatform.api.grpc.{ApplicationStatus, TransactionResponse, TransactionSnapshotResponse, TransactionSnapshotsRequest, TransactionsApiGrpcImpl, TransactionsRequest} import com.wavesplatform.block.Block import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 @@ -11,16 +11,21 @@ import com.wavesplatform.db.WithDomain import com.wavesplatform.db.WithState.AddrWithBalance import com.wavesplatform.history.Domain import com.wavesplatform.protobuf.transaction.{PBTransactions, Recipient} -import com.wavesplatform.state.TxMeta +import com.wavesplatform.protobuf.{ByteStrExt, PBSnapshots} +import com.wavesplatform.state.diffs.ENOUGH_AMT +import com.wavesplatform.state.{StateSnapshot, TxMeta} import com.wavesplatform.test.* import com.wavesplatform.test.DomainPresets.* import com.wavesplatform.transaction.Asset.Waves -import com.wavesplatform.transaction.{TxHelpers, TxVersion} +import com.wavesplatform.transaction.TxHelpers.* import com.wavesplatform.transaction.assets.exchange.{ExchangeTransaction, Order, OrderType} +import com.wavesplatform.transaction.{TxHelpers, TxVersion} import com.wavesplatform.utils.DiffMatchers import monix.execution.Scheduler.Implicits.global import org.scalatest.{Assertion, BeforeAndAfterAll} +import scala.collection.immutable.VectorMap + class TransactionsApiGrpcSpec extends FreeSpec with BeforeAndAfterAll with DiffMatchers with WithDomain with GrpcApiHelpers { val sender: KeyPair = TxHelpers.signer(1) @@ -69,6 +74,67 @@ class TransactionsApiGrpcSpec extends FreeSpec with BeforeAndAfterAll with DiffM } } + "GetTransactionSnapshots" in withDomain(TransactionStateSnapshot, AddrWithBalance.enoughBalances(secondSigner)) { d => + val recipient = signer(2).toAddress + val txs = Seq.fill(5)(transfer(amount = 1, fee = 100_000, from = secondSigner, to = recipient)) + + val firstThreeSnapshots = Seq( + StateSnapshot(balances = + VectorMap( + (secondAddress, Waves) -> (ENOUGH_AMT - 100_001), + (recipient, Waves) -> 1, + (defaultAddress, Waves) -> 200_040_000 // reward and 40% fee + ) + ), + StateSnapshot(balances = + VectorMap( + (secondAddress, Waves) -> (ENOUGH_AMT - 200_002), + (recipient, Waves) -> 2, + (defaultAddress, Waves) -> 200_080_000 + ) + ), + StateSnapshot(balances = + VectorMap( + (secondAddress, Waves) -> (ENOUGH_AMT - 300_003), + (recipient, Waves) -> 3, + (defaultAddress, Waves) -> 200_120_000 + ) + ) + ) + + def getSnapshots() = { + val request = TransactionSnapshotsRequest.of(txs.map(_.id().toByteString)) + val (observer, response) = createObserver[TransactionSnapshotResponse] + getGrpcApi(d).getTransactionSnapshots(request, observer) + response.runSyncUnsafe().flatMap(_.snapshot).map(PBSnapshots.fromProtobuf(_, ByteStr.empty, 0)._1) + } + + d.appendBlock(txs(0), txs(1)) + d.appendMicroBlock(txs(2)) + + // both liquid and solid state + getSnapshots() shouldBe firstThreeSnapshots + + // hardened state + d.appendBlock(txs(3), txs(4)) + getSnapshots() shouldBe firstThreeSnapshots ++ Seq( + StateSnapshot(balances = + VectorMap( + (secondAddress, Waves) -> (ENOUGH_AMT - 400_004), + (recipient, Waves) -> 4, + (defaultAddress, Waves) -> 400_340_000 // 2 blocks reward, 100% fee from previous block and 40% fee from current + ) + ), + StateSnapshot(balances = + VectorMap( + (secondAddress, Waves) -> (ENOUGH_AMT - 500_005), + (recipient, Waves) -> 5, + (defaultAddress, Waves) -> 400_380_000 + ) + ) + ) + } + "NODE-973. GetTransactions should return correct data for orders with attachment" in { def checkOrderAttachment(txResponse: TransactionResponse, expectedAttachment: ByteStr): Assertion = { PBTransactions @@ -143,7 +209,10 @@ class TransactionsApiGrpcSpec extends FreeSpec with BeforeAndAfterAll with DiffM val challengedMiner = TxHelpers.signer(2) val resender = TxHelpers.signer(3) val recipient = TxHelpers.signer(4) - withDomain(TransactionStateSnapshot.configure(_.copy(lightNodeBlockFieldsAbsenceInterval = 0)), balances = AddrWithBalance.enoughBalances(sender)) { d => + withDomain( + TransactionStateSnapshot.configure(_.copy(lightNodeBlockFieldsAbsenceInterval = 0)), + balances = AddrWithBalance.enoughBalances(sender) + ) { d => val grpcApi = getGrpcApi(d) val challengingMiner = d.wallet.generateNewAccount().get diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/script/Script.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/script/Script.scala index 50338a9303..90c24fdc6d 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/script/Script.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/script/Script.scala @@ -36,11 +36,8 @@ trait Script { } object Script { - case class ComplexityInfo(verifierComplexity: Long, callableComplexities: Map[String, Long], maxComplexity: Long) - val checksumLength = 4 - def fromBase64String(str: String): Either[ScriptParseError, Script] = for { bytes <- Base64.tryDecode(str).toEither.left.map(ex => ScriptParseError(s"Unable to decode base64: ${ex.getMessage}")) @@ -93,9 +90,7 @@ object Script { ) complexityInfo = verifierFuncOpt.fold( ComplexityInfo(0L, callableComplexities, maxComplexity) - )( - v => ComplexityInfo(callableComplexities(v.u.name), callableComplexities - v.u.name, maxComplexity) - ) + )(v => ComplexityInfo(callableComplexities(v.u.name), callableComplexities - v.u.name, maxComplexity)) } yield complexityInfo } diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/AmountAsStringSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/AmountAsStringSuite.scala index 65410f8435..1e82fb3bd2 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/AmountAsStringSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/AmountAsStringSuite.scala @@ -82,7 +82,7 @@ class AmountAsStringSuite extends BaseTransactionSuite with OverflowBlock { amount, price, ts, - ts + Order.MaxLiveTime, + ts + Order.MaxLiveTime / 2, matcherFee ) .explicitGet() @@ -95,7 +95,7 @@ class AmountAsStringSuite extends BaseTransactionSuite with OverflowBlock { amount, price, ts, - ts + Order.MaxLiveTime, + ts + Order.MaxLiveTime / 2, matcherFee ) .explicitGet() diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/activation/VRFProtobufActivationSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/activation/VRFProtobufActivationSuite.scala index 6242328308..13501bdea1 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/activation/VRFProtobufActivationSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/activation/VRFProtobufActivationSuite.scala @@ -224,7 +224,7 @@ class VRFProtobufActivationSuite extends BaseTransactionSuite { amount, price, ts, - ts + Order.MaxLiveTime, + ts + Order.MaxLiveTime / 2, matcherFee ) .explicitGet() @@ -237,7 +237,7 @@ class VRFProtobufActivationSuite extends BaseTransactionSuite { amount, price, ts, - ts + Order.MaxLiveTime, + ts + Order.MaxLiveTime / 2, matcherFee ) .explicitGet() diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/grpc/ExchangeTransactionGrpcSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/grpc/ExchangeTransactionGrpcSuite.scala index a6d19227e4..f569f4a6f7 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/grpc/ExchangeTransactionGrpcSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/grpc/ExchangeTransactionGrpcSuite.scala @@ -40,7 +40,7 @@ class ExchangeTransactionGrpcSuite extends GrpcBaseTransactionSuite with NTPTime val pair = AssetPair.createAssetPair("WAVES", exchAssetId).get for ((o1ver, o2ver, tver) <- versions) { val ts = ntpTime.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 val buy = Order.buy(o1ver, buyer, matcher.publicKey, pair, amount, price, ts, expirationTimestamp, matcherFee).explicitGet() val sell = Order.sell(o2ver, seller, matcher.publicKey, pair, amount, price, ts, expirationTimestamp, matcherFee).explicitGet() val buyerWavesBalanceBefore = sender.wavesBalance(buyerAddress).available @@ -98,7 +98,7 @@ class ExchangeTransactionGrpcSuite extends GrpcBaseTransactionSuite with NTPTime val sellerAssetBalanceBefore = sender.assetsBalance(sellerAddress, Seq(feeAssetId.toString)).getOrElse(feeAssetId.toString, 0L) val ts = ntpTime.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 val assetPair = AssetPair.createAssetPair("WAVES", feeAssetId.toString).get val buy = Order.buy(o1ver, buyer, matcher.publicKey, assetPair, amount, price, ts, expirationTimestamp, matcherFee, matcherFeeOrder1).explicitGet() @@ -133,7 +133,7 @@ class ExchangeTransactionGrpcSuite extends GrpcBaseTransactionSuite with NTPTime val assetId = exchAsset.id().toString val ts = ntpTime.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 val price = 2 * Order.PriceConstant val amount = 1 val pair = AssetPair.createAssetPair("WAVES", assetId).get diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/smartcontract/package.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/smartcontract/package.scala index 425c5facf5..afbd37a916 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/smartcontract/package.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/smartcontract/package.scala @@ -150,7 +150,7 @@ package object smartcontract { val seller = accounts.tail.head // second one val matcher = accounts.last val ts = time.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 val buyPrice = 1 * Order.PriceConstant val sellPrice = (0.50 * Order.PriceConstant).toLong val buyAmount = 2 diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/ExchangeTransactionSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/ExchangeTransactionSuite.scala index f20e4c6399..28d18fda43 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/ExchangeTransactionSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/ExchangeTransactionSuite.scala @@ -55,7 +55,7 @@ class ExchangeTransactionSuite extends BaseTransactionSuite with NTPTime { val matcher = acc2 val ts = ntpTime.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 val buyPrice = 2 * Order.PriceConstant val sellPrice = 2 * Order.PriceConstant @@ -197,7 +197,7 @@ class ExchangeTransactionSuite extends BaseTransactionSuite with NTPTime { val matcher = thirdKeyPair val ts = ntpTime.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 var assetBalanceBefore: Long = 0L if (matcherFeeOrder1 == Waves && matcherFeeOrder2 != Waves) { @@ -283,7 +283,7 @@ class ExchangeTransactionSuite extends BaseTransactionSuite with NTPTime { val matcher = thirdKeyPair val ts = ntpTime.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 val amount = 1 val nftWavesPrice = 1000 * math.pow(10, 8).toLong val nftForAssetPrice = 1 * math.pow(10, 8).toLong diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/FailedTransactionSuiteLike.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/FailedTransactionSuiteLike.scala index 1f06568310..accac7e177 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/FailedTransactionSuiteLike.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/FailedTransactionSuiteLike.scala @@ -288,7 +288,7 @@ object FailedTransactionSuiteLike { 100, 100, timestamp, - timestamp + Order.MaxLiveTime, + timestamp + Order.MaxLiveTime / 2, buyMatcherFee, Asset.fromString(Some(buyMatcherFeeAsset)) ).explicitGet() @@ -300,7 +300,7 @@ object FailedTransactionSuiteLike { 100, 100, timestamp, - timestamp + Order.MaxLiveTime, + timestamp + Order.MaxLiveTime / 2, sellMatcherFee, Asset.fromString(Some(sellMatcherFeeAsset)) ).explicitGet() diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala index e7ec0682e0..a4d7f4323b 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala @@ -426,7 +426,7 @@ class SignAndBroadcastApiSuite extends BaseTransactionSuite with NTPTime with Be val seller = secondKeyPair val matcher = thirdKeyPair val ts = ntpTime.correctedTime() - val expirationTimestamp = ts + Order.MaxLiveTime + val expirationTimestamp = ts + Order.MaxLiveTime / 2 val buyPrice = 1 * Order.PriceConstant val sellPrice = (0.50 * Order.PriceConstant).toLong val mf = 300000L diff --git a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/TransferNFTSuite.scala b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/TransferNFTSuite.scala index 838c95e581..7d090b3341 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/TransferNFTSuite.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/sync/transactions/TransferNFTSuite.scala @@ -161,7 +161,7 @@ class TransferNFTSuite extends BaseTransactionSuite with NTPTime { amount = 1, price = 1.waves, timestamp = ts, - expiration = ts + Order.MaxLiveTime, + expiration = ts + Order.MaxLiveTime / 2, matcherFee = matcherFee ) .explicitGet() @@ -174,7 +174,7 @@ class TransferNFTSuite extends BaseTransactionSuite with NTPTime { amount = 1, price = 1.waves, timestamp = ts, - expiration = ts + Order.MaxLiveTime, + expiration = ts + Order.MaxLiveTime / 2, matcherFee = matcherFee ) .explicitGet() diff --git a/node/src/main/resources/application.conf b/node/src/main/resources/application.conf index d73e17f932..e89a571e00 100644 --- a/node/src/main/resources/application.conf +++ b/node/src/main/resources/application.conf @@ -233,6 +233,7 @@ waves { # Max number of transactions # returned by /transactions/address/{address}/limit/{limit} transactions-by-address-limit = 1000 + transaction-snapshots-limit = 100 distribution-address-limit = 1000 data-keys-request-limit = 1000 asset-details-limit = 100 diff --git a/node/src/main/resources/swagger-ui/openapi.yaml b/node/src/main/resources/swagger-ui/openapi.yaml index b8fcd02eb0..2176d32dfa 100644 --- a/node/src/main/resources/swagger-ui/openapi.yaml +++ b/node/src/main/resources/swagger-ui/openapi.yaml @@ -1410,6 +1410,65 @@ paths: largeSignificandTransaction: $ref: '#/components/examples/largeSignificandTransaction' x-codegen-request-body-name: ids + '/transactions/snapshot/{id}': + get: + tags: + - transactions + summary: Transaction snapshot + description: Get transaction snapshot by transaction ID + operationId: getTxSnapshotById + parameters: + - $ref: '#/components/parameters/txId' + responses: + '200': + description: successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/TransactionSnapshot' + /transactions/snapshot: + post: + tags: + - transactions + summary: Transaction snapshots + description: >- + Get transaction snapshots by transaction IDs. + Limited by `rest-api.transaction-snapshots-limit`, 100 by default. +
Transaction snapshots in the response are in the same order as in the request. + operationId: getTxSnapshotsViaPost + requestBody: + content: + application/json: + schema: + properties: + ids: + type: array + items: + $ref: '#/components/schemas/TransactionId' + minItems: 1 + maxItems: 100 + application/x-www-form-urlencoded: + schema: + type: object + properties: + id: + type: array + items: + $ref: '#/components/schemas/TransactionId' + encoding: + id: + style: form + explode: true + responses: + '200': + description: successful operation + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/TransactionSnapshot' + x-codegen-request-body-name: ids /transactions/sign: post: tags: @@ -5431,6 +5490,179 @@ components: nullable: true state: $ref: '#/components/schemas/BlockchainOverrides' + TransactionSnapshot: + type: object + properties: + applicationStatus: + $ref: '#/components/schemas/ApplicationStatus' + balances: + type: array + items: + type: object + properties: + address: + $ref: '#/components/schemas/Address' + asset: + $ref: '#/components/schemas/AssetId' + balance: + type: integer + format: int64 + leaseBalances: + type: array + items: + type: object + properties: + address: + $ref: '#/components/schemas/Address' + in: + type: integer + format: int64 + out: + type: integer + format: int64 + assetStatics: + type: array + items: + type: object + properties: + id: + $ref: '#/components/schemas/AssetId' + source: + $ref: '#/components/schemas/TransactionId' + issuer: + $ref: '#/components/schemas/PublicKey' + decimals: + type: integer + minimum: 0 + maximum: 8 + nft: + type: boolean + assetVolumes: + type: array + items: + type: object + properties: + id: + $ref: '#/components/schemas/AssetId' + isReissuable: + type: boolean + volume: + type: integer + format: int64 + assetNamesAndDescriptions: + type: array + items: + type: object + properties: + id: + $ref: '#/components/schemas/AssetId' + name: + type: string + description: + type: string + lastUpdatedAt: + type: integer + assetScripts: + type: array + items: + type: object + properties: + id: + $ref: '#/components/schemas/AssetId' + script: + type: string + format: byte + complexity: + type: integer + sponsorships: + type: array + items: + type: object + properties: + id: + $ref: '#/components/schemas/AssetId' + minSponsoredAssetFee: + type: integer + format: int64 + newLeases: + type: array + items: + type: object + properties: + id: + $ref: '#/components/schemas/LeaseId' + sender: + $ref: '#/components/schemas/PublicKey' + recipient: + $ref: '#/components/schemas/Address' + amount: + type: integer + format: int64 + txId: + $ref: '#/components/schemas/TransactionId' + height: + $ref: '#/components/schemas/Height' + cancelledLeases: + type: array + items: + type: object + properties: + id: + $ref: '#/components/schemas/LeaseId' + txId: + $ref: '#/components/schemas/TransactionId' + height: + $ref: '#/components/schemas/Height' + aliases: + type: array + items: + type: object + properties: + address: + $ref: '#/components/schemas/Address' + alias: + type: string + orderFills: + type: array + items: + type: object + properties: + id: + type: string + format: byte + volume: + type: integer + format: int64 + fee: + type: integer + format: int64 + accountScripts: + type: array + items: + type: object + properties: + publicKey: + $ref: '#/components/schemas/PublicKey' + script: + type: string + format: byte + verifierComplexity: + type: integer + format: int64 + accountData: + type: array + items: + type: object + properties: + address: + $ref: '#/components/schemas/Address' + data: + type: array + items: + oneOf: + - $ref: '#/components/schemas/DataEntry' + - $ref: '#/components/schemas/DeleteEntry' + responses: Height: description: Block height diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index 0f6dbd5134..a9804b9340 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -431,8 +431,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con mbSyncCacheSizes, scoreStatsReporter, configRoot, - rocksDB.loadBalanceHistory, - rocksDB.loadStateHash, + rocksDB, () => utxStorage.getPriorityPool.map(_.compositeBlockchain), routeTimeout, heavyRequestScheduler diff --git a/node/src/main/scala/com/wavesplatform/account/Recipient.scala b/node/src/main/scala/com/wavesplatform/account/Recipient.scala index 29927894d5..69430d2951 100644 --- a/node/src/main/scala/com/wavesplatform/account/Recipient.scala +++ b/node/src/main/scala/com/wavesplatform/account/Recipient.scala @@ -227,4 +227,6 @@ object Alias { case Some(expected) if expected != chainId => Left(WrongChain(expected, chainId)) case _ => Right(Alias(chainId, name)) } + + implicit val writes: Writes[Alias] = Writes(a => JsString(a.toString)) } diff --git a/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala b/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala index 6aacb11294..590a6fa708 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/DebugApiRoute.scala @@ -8,19 +8,19 @@ import com.wavesplatform.Version import com.wavesplatform.account.{Address, PKKeyPair} import com.wavesplatform.api.common.{CommonAccountsApi, CommonAssetsApi, CommonTransactionsApi, TransactionMeta} import com.wavesplatform.common.state.ByteStr +import com.wavesplatform.database.RocksDBWriter import com.wavesplatform.lang.ValidationError import com.wavesplatform.mining.{Miner, MinerDebugInfo} import com.wavesplatform.network.{PeerDatabase, PeerInfo, *} import com.wavesplatform.settings.{RestAPISettings, WavesSettings} import com.wavesplatform.state.diffs.TransactionDiffer -import com.wavesplatform.state.SnapshotBlockchain -import com.wavesplatform.state.{Blockchain, Height, LeaseBalance, NG, Portfolio, StateHash, TxMeta} +import com.wavesplatform.state.{Blockchain, Height, LeaseBalance, NG, Portfolio, SnapshotBlockchain, TxMeta} import com.wavesplatform.transaction.* import com.wavesplatform.transaction.Asset.IssuedAsset import com.wavesplatform.transaction.TxValidationError.GenericError import com.wavesplatform.transaction.smart.InvokeScriptTransaction import com.wavesplatform.transaction.smart.script.trace.{InvokeScriptTrace, TracedResult} -import com.wavesplatform.utils.{ScorexLogging, Time} +import com.wavesplatform.utils.{ScorexLogging, Time, byteStrFormat} import com.wavesplatform.utx.UtxPool import com.wavesplatform.wallet.Wallet import io.netty.channel.Channel @@ -54,8 +54,7 @@ case class DebugApiRoute( mbsCacheSizesReporter: Coeval[MicroBlockSynchronizer.CacheSizes], scoreReporter: Coeval[RxScoreObserver.Stats], configRoot: ConfigObject, - loadBalanceHistory: Address => Seq[(Int, Long)], - loadStateHash: Int => Option[StateHash], + db: RocksDBWriter, priorityPoolBlockchain: () => Option[Blockchain], routeTimeout: RouteTimeout, heavyRequestScheduler: Scheduler @@ -71,7 +70,7 @@ case class DebugApiRoute( override val settings: RestAPISettings = ws.restAPISettings - private[this] val serializer = TransactionJsonSerializer(blockchain, transactionsApi) + private[this] val serializer = TransactionJsonSerializer(blockchain) override lazy val route: Route = pathPrefix("debug") { balanceHistory ~ stateHash ~ validate ~ withAuth { @@ -86,7 +85,7 @@ case class DebugApiRoute( }) def balanceHistory: Route = (path("balances" / "history" / AddrSegment) & get) { address => - complete(Json.toJson(loadBalanceHistory(address).map { case (h, b) => + complete(Json.toJson(db.loadBalanceHistory(address).map { case (h, b) => Json.obj("height" -> h, "balance" -> b) })) } @@ -259,13 +258,14 @@ case class DebugApiRoute( private def stateHashAt(height: Int): Route = { val result = for { - sh <- loadStateHash(height) + sh <- db.loadStateHash(height) h <- blockchain.blockHeader(height) } yield Json.toJson(sh).as[JsObject] ++ Json.obj( - "blockId" -> h.id().toString, - "baseTarget" -> h.header.baseTarget, - "height" -> height, - "version" -> Version.VersionString + "snapshotHash" -> db.snapshotStateHash(height), + "blockId" -> h.id().toString, + "baseTarget" -> h.header.baseTarget, + "height" -> height, + "version" -> Version.VersionString ) result match { diff --git a/node/src/main/scala/com/wavesplatform/api/http/StateSnapshotJson.scala b/node/src/main/scala/com/wavesplatform/api/http/StateSnapshotJson.scala new file mode 100644 index 0000000000..c83a281e64 --- /dev/null +++ b/node/src/main/scala/com/wavesplatform/api/http/StateSnapshotJson.scala @@ -0,0 +1,120 @@ +package com.wavesplatform.api.http +import com.wavesplatform.account.{Address, PublicKey} +import com.wavesplatform.api.http.StateSnapshotJson.* +import com.wavesplatform.common.state.ByteStr +import com.wavesplatform.lang.script.Script +import com.wavesplatform.state.* +import com.wavesplatform.transaction.Asset +import com.wavesplatform.transaction.Asset.IssuedAsset +import play.api.libs.json.* +import play.api.libs.json.Json.MacroOptions +import play.api.libs.json.JsonConfiguration.Aux +import play.api.libs.json.OptionHandlers.WritesNull + +case class StateSnapshotJson( + applicationStatus: String, + balances: Seq[BalanceJson], + leaseBalances: Seq[LeaseBalanceJson], + assetStatics: Seq[AssetStaticInfo], + assetVolumes: Seq[AssetVolumeJson], + assetNamesAndDescriptions: Seq[AssetInfoJson], + assetScripts: Seq[AssetScriptJson], + sponsorships: Seq[SponsorshipJson], + newLeases: Seq[NewLeaseJson], + cancelledLeases: Seq[CancelledLeaseJson], + aliases: Seq[AliasJson], + orderFills: Seq[OrderFillJson], + accountScripts: Seq[AccountScriptJson], + accountData: Seq[AccountDataJson] +) + +object StateSnapshotJson { + def fromSnapshot(s: StateSnapshot, txStatus: TxMeta.Status): StateSnapshotJson = + StateSnapshotJson( + TransactionJsonSerializer.applicationStatusFromTxStatus(txStatus), + s.balances.map { case ((address, asset), balance) => BalanceJson(address, asset, balance) }.toSeq, + s.leaseBalances.map { case (address, lease) => LeaseBalanceJson(address, lease.in, lease.out) }.toSeq, + s.assetStatics.map(_._2._1).toSeq, + s.assetVolumes.map { case (id, info) => AssetVolumeJson(id, info.isReissuable, info.volume) }.toSeq, + s.assetNamesAndDescriptions.map { case (id, info) => + AssetInfoJson(id, info.name.toStringUtf8, info.description.toStringUtf8, info.lastUpdatedAt) + }.toSeq, + s.assetScripts.map { case (id, info) => AssetScriptJson(id, info.script, info.complexity) }.toSeq, + s.sponsorships.map { case (id, value) => SponsorshipJson(id, value.minFee) }.toSeq, + s.newLeases.map { case (id, info) => + NewLeaseJson(id, info.sender, info.recipientAddress, info.amount.value, info.sourceId, info.height) + }.toSeq, + s.cancelledLeases.map { case (id, status) => CancelledLeaseJson(id, status.cancelTransactionId.get, status.cancelHeight.get) }.toSeq, + s.aliases.map { case (alias, address) => AliasJson(address, alias.name) }.toSeq, + s.orderFills.map { case (id, info) => OrderFillJson(id, info.volume, info.fee) }.toSeq, + s.accountScripts.map { case (pk, info) => + info.fold(AccountScriptJson(pk, None, 0))(i => AccountScriptJson(i.publicKey, Some(i.script), i.verifierComplexity)) + }.toSeq, + s.accountData.map { case (address, data) => AccountDataJson(address, data.values.toSeq) }.toSeq + ) + implicit val byteStrWrites: Writes[ByteStr] = com.wavesplatform.utils.byteStrFormat + implicit val writes: OWrites[StateSnapshotJson] = Json.writes + + case class BalanceJson(address: Address, asset: Asset, balance: Long) + object BalanceJson { + implicit val writes: OWrites[BalanceJson] = Json.writes + } + + case class LeaseBalanceJson(address: Address, in: Long, out: Long) + object LeaseBalanceJson { + implicit val writes: OWrites[LeaseBalanceJson] = Json.writes + } + + case class AssetVolumeJson(id: IssuedAsset, isReissuable: Boolean, volume: BigInt) + object AssetVolumeJson { + implicit val writes: OWrites[AssetVolumeJson] = Json.writes + } + + case class AssetInfoJson(id: IssuedAsset, name: String, description: String, lastUpdatedAt: Height) + object AssetInfoJson { + implicit val writes: OWrites[AssetInfoJson] = Json.writes + } + + case class AssetScriptJson(id: IssuedAsset, script: Script, complexity: Long) + object AssetScriptJson { + implicit val scriptWrites: Writes[Script] = Writes[Script](s => JsString(s.bytes().base64)) + implicit val writes: OWrites[AssetScriptJson] = Json.writes[AssetScriptJson] + } + + case class AccountScriptJson(publicKey: PublicKey, script: Option[Script], verifierComplexity: Long) + object AccountScriptJson { + implicit val config: Aux[MacroOptions] = JsonConfiguration(optionHandlers = WritesNull) + implicit val scriptWrites: Writes[Script] = Writes[Script](s => JsString(s.bytes().base64)) + implicit val writes: OWrites[AccountScriptJson] = Json.writes + } + + case class SponsorshipJson(id: IssuedAsset, minSponsoredAssetFee: Long) + object SponsorshipJson { + implicit val writes: OWrites[SponsorshipJson] = Json.writes + } + + case class NewLeaseJson(id: ByteStr, sender: PublicKey, recipient: Address, amount: Long, txId: ByteStr, height: Int) + object NewLeaseJson { + implicit val writes: OWrites[NewLeaseJson] = Json.writes + } + + case class CancelledLeaseJson(id: ByteStr, txId: ByteStr, height: Int) + object CancelledLeaseJson { + implicit val writes: OWrites[CancelledLeaseJson] = Json.writes + } + + case class AccountDataJson(address: Address, data: Seq[DataEntry[?]]) + object AccountDataJson { + implicit val writes: OWrites[AccountDataJson] = Json.writes + } + + case class AliasJson(address: Address, alias: String) + object AliasJson { + implicit val writes: OWrites[AliasJson] = Json.writes + } + + case class OrderFillJson(id: ByteStr, volume: Long, fee: Long) + object OrderFillJson { + implicit val writes: OWrites[OrderFillJson] = Json.writes + } +} diff --git a/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala b/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala index d2b180f326..5588765144 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala @@ -3,7 +3,7 @@ package com.wavesplatform.api.http import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.databind.{JsonSerializer, SerializerProvider} import com.wavesplatform.account.{Address, AddressOrAlias} -import com.wavesplatform.api.common.{CommonTransactionsApi, TransactionMeta} +import com.wavesplatform.api.common.TransactionMeta import com.wavesplatform.api.http.StreamSerializerUtils.* import com.wavesplatform.api.http.TransactionJsonSerializer.* import com.wavesplatform.api.http.TransactionsApiRoute.{ApplicationStatus, LeaseStatus, TxMetaEnriched} @@ -16,8 +16,7 @@ import com.wavesplatform.lang.v1.compiler.Terms.{ARR, CONST_BOOLEAN, CONST_BYTES import com.wavesplatform.lang.v1.serialization.SerdeV1 import com.wavesplatform.protobuf.transaction.PBAmounts import com.wavesplatform.state.InvokeScriptResult.{AttachedPayment, Burn, Call, ErrorMessage, Invocation, Issue, Lease, LeaseCancel, Reissue, SponsorFee} -import com.wavesplatform.state.LeaseDetails -import com.wavesplatform.state.{Blockchain, DataEntry, InvokeScriptResult, TxMeta} +import com.wavesplatform.state.{Blockchain, DataEntry, InvokeScriptResult, LeaseDetails, TxMeta} import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} import com.wavesplatform.transaction.lease.{LeaseCancelTransaction, LeaseTransaction} import com.wavesplatform.transaction.serialization.impl.InvokeScriptTxSerializer @@ -29,7 +28,7 @@ import com.wavesplatform.utils.EthEncoding import play.api.libs.json.* import play.api.libs.json.JsonConfiguration.Aux -final case class TransactionJsonSerializer(blockchain: Blockchain, commonApi: CommonTransactionsApi) { +final case class TransactionJsonSerializer(blockchain: Blockchain) { val assetSerializer: JsonSerializer[Asset] = (value: Asset, gen: JsonGenerator, serializers: SerializerProvider) => { diff --git a/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala b/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala index 282e88198a..86fc8f6e8d 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/TransactionsApiRoute.scala @@ -41,12 +41,12 @@ case class TransactionsApiRoute( with AuthRoute { import TransactionsApiRoute.* - private[this] val serializer = TransactionJsonSerializer(blockchain, commonApi) + private[this] val serializer = TransactionJsonSerializer(blockchain) private[this] implicit val transactionMetaWrites: OWrites[TransactionMeta] = OWrites[TransactionMeta](serializer.transactionWithMetaJson) override lazy val route: Route = pathPrefix("transactions") { - unconfirmed ~ addressWithLimit ~ info ~ status ~ sign ~ calculateFee ~ signedBroadcast ~ merkleProof + unconfirmed ~ addressWithLimit ~ info ~ snapshot ~ status ~ sign ~ calculateFee ~ signedBroadcast ~ merkleProof } def addressWithLimit: Route = { @@ -83,6 +83,25 @@ case class TransactionsApiRoute( } } + def snapshot: Route = pathPrefix("snapshot") { + def readSnapshot(id: ByteStr) = + blockchain + .transactionSnapshot(id) + .toRight(TransactionDoesNotExist) + .map { case (snapshot, txStatus) => StateSnapshotJson.fromSnapshot(snapshot, txStatus) } + val single = (get & path(TransactionId))(id => complete(readSnapshot(id))) + val multiple = (pathEndOrSingleSlash & anyParam("id", limit = settings.transactionSnapshotsLimit))(rawIds => + complete( + for { + _ <- Either.cond(rawIds.nonEmpty, (), InvalidTransactionId("Transaction ID was not specified")) + ids <- rawIds.toSeq.traverse(ByteStr.decodeBase58(_).toEither.leftMap(err => CustomValidationError(err.toString))) + meta <- ids.traverse(readSnapshot) + } yield meta + ) + ) + single ~ multiple + } + private[this] def loadTransactionStatus(id: ByteStr): JsObject = { import Status.* val statusJson = blockchain.transactionInfo(id) match { diff --git a/node/src/main/scala/com/wavesplatform/database/Keys.scala b/node/src/main/scala/com/wavesplatform/database/Keys.scala index 1de442ff95..05660deeab 100644 --- a/node/src/main/scala/com/wavesplatform/database/Keys.scala +++ b/node/src/main/scala/com/wavesplatform/database/Keys.scala @@ -234,8 +234,6 @@ object Keys { def nftAt(addressId: AddressId, index: Int, assetId: IssuedAsset): Key[Option[Unit]] = Key.opt(NftPossession, addressId.toByteArray ++ Longs.toByteArray(index) ++ assetId.id.arr, _ => (), _ => Array.emptyByteArray) - def bloomFilterChecksum(filterName: String): Key[Array[Byte]] = Key(KeyTags.BloomFilterChecksum, filterName.utf8Bytes, identity, identity) - def stateHash(height: Int): Key[Option[StateHash]] = Key.opt(StateHash, h(height), readStateHash, writeStateHash) diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index 388b1c5eea..caabfe52d5 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -17,7 +17,7 @@ import com.wavesplatform.database.protobuf.{StaticAssetInfo, TransactionMeta, Bl import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError import com.wavesplatform.protobuf.block.PBBlocks -import com.wavesplatform.protobuf.snapshot.{TransactionStateSnapshot, TransactionStatus as PBStatus} +import com.wavesplatform.protobuf.snapshot.TransactionStatus as PBStatus import com.wavesplatform.protobuf.{ByteStrExt, ByteStringExt, PBSnapshots} import com.wavesplatform.settings.{BlockchainSettings, DBSettings} import com.wavesplatform.state.* @@ -632,7 +632,6 @@ class RocksDBWriter( expiredKeys += Keys.carryFee(threshold - 1).keyBytes rw.put(Keys.blockStateHash(height), computedBlockStateHash) - expiredKeys += Keys.blockStateHash(threshold - 1).keyBytes if (dbSettings.storeInvokeScriptResults) snapshot.scriptResults.foreach { case (txId, result) => val (txHeight, txNum) = transactionsWithSize @@ -992,11 +991,11 @@ class RocksDBWriter( } } - def transactionSnapshot(id: ByteStr): Option[TransactionStateSnapshot] = readOnly { db => + override def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, TxMeta.Status)] = readOnly { db => for { meta <- db.get(Keys.transactionMetaById(TransactionId(id), rdb.txMetaHandle)) snapshot <- db.get(Keys.transactionStateSnapshotAt(Height(meta.height), TxNum(meta.num.toShort), rdb.txSnapshotHandle)) - } yield snapshot + } yield PBSnapshots.fromProtobuf(snapshot, id, meta.height) } override def resolveAlias(alias: Alias): Either[ValidationError, Address] = @@ -1147,7 +1146,9 @@ class RocksDBWriter( override def resolveERC20Address(address: ERC20Address): Option[IssuedAsset] = readOnly(_.get(Keys.assetStaticInfo(address)).map(assetInfo => IssuedAsset(assetInfo.id.toByteStr))) - override def lastStateHash(refId: Option[ByteStr]): ByteStr = { + override def lastStateHash(refId: Option[ByteStr]): ByteStr = + snapshotStateHash(height) + + def snapshotStateHash(height: Int): ByteStr = readOnly(_.get(Keys.blockStateHash(height))) - } } diff --git a/node/src/main/scala/com/wavesplatform/settings/RestAPISettings.scala b/node/src/main/scala/com/wavesplatform/settings/RestAPISettings.scala index c0b9f5ae46..a9f40265ec 100644 --- a/node/src/main/scala/com/wavesplatform/settings/RestAPISettings.scala +++ b/node/src/main/scala/com/wavesplatform/settings/RestAPISettings.scala @@ -7,6 +7,7 @@ case class RestAPISettings( apiKeyHash: String, corsHeaders: CorsHeaders, transactionsByAddressLimit: Int, + transactionSnapshotsLimit: Int, distributionAddressLimit: Int, dataKeysRequestLimit: Int, assetDetailsLimit: Int, diff --git a/node/src/main/scala/com/wavesplatform/state/AssetStaticInfo.scala b/node/src/main/scala/com/wavesplatform/state/AssetStaticInfo.scala index db14ea3977..542d53c4f2 100644 --- a/node/src/main/scala/com/wavesplatform/state/AssetStaticInfo.scala +++ b/node/src/main/scala/com/wavesplatform/state/AssetStaticInfo.scala @@ -2,5 +2,11 @@ package com.wavesplatform.state import com.wavesplatform.account.PublicKey import com.wavesplatform.common.state.ByteStr +import play.api.libs.json.{Format, Json, OWrites} case class AssetStaticInfo(id: ByteStr, source: TransactionId, issuer: PublicKey, decimals: Int, nft: Boolean) + +object AssetStaticInfo { + implicit val byteStrFormat: Format[ByteStr] = com.wavesplatform.utils.byteStrFormat + implicit val format: OWrites[AssetStaticInfo] = Json.writes[AssetStaticInfo] +} diff --git a/node/src/main/scala/com/wavesplatform/state/AssetVolumeInfo.scala b/node/src/main/scala/com/wavesplatform/state/AssetVolumeInfo.scala index 2b3d491c31..46c5c463ef 100644 --- a/node/src/main/scala/com/wavesplatform/state/AssetVolumeInfo.scala +++ b/node/src/main/scala/com/wavesplatform/state/AssetVolumeInfo.scala @@ -1,10 +1,12 @@ package com.wavesplatform.state import cats.kernel.Monoid +import play.api.libs.json.{Json, OFormat} case class AssetVolumeInfo(isReissuable: Boolean, volume: BigInt) object AssetVolumeInfo { + implicit val format: OFormat[AssetVolumeInfo] = Json.format implicit val assetInfoMonoid: Monoid[AssetVolumeInfo] = new Monoid[AssetVolumeInfo] { override def empty: AssetVolumeInfo = AssetVolumeInfo(isReissuable = true, 0) override def combine(x: AssetVolumeInfo, y: AssetVolumeInfo): AssetVolumeInfo = diff --git a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala index a040d36226..3e7471e6fc 100644 --- a/node/src/main/scala/com/wavesplatform/state/Blockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/Blockchain.scala @@ -12,7 +12,7 @@ import com.wavesplatform.lang.script.ContractScript import com.wavesplatform.lang.v1.ContractLimits import com.wavesplatform.lang.v1.traits.domain.Issue import com.wavesplatform.settings.BlockchainSettings -import com.wavesplatform.state.LeaseDetails +import com.wavesplatform.state.TxMeta.Status import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} import com.wavesplatform.transaction.TxValidationError.AliasDoesNotExist import com.wavesplatform.transaction.assets.IssueTransaction @@ -47,6 +47,7 @@ trait Blockchain { def transactionInfo(id: ByteStr): Option[(TxMeta, Transaction)] def transactionInfos(ids: Seq[ByteStr]): Seq[Option[(TxMeta, Transaction)]] def transactionMeta(id: ByteStr): Option[TxMeta] + def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, Status)] def containsTransaction(tx: Transaction): Boolean diff --git a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index 2c8e3a3c5e..26e694c766 100644 --- a/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/node/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -17,6 +17,7 @@ import com.wavesplatform.metrics.{TxsInBlockchainStats, *} import com.wavesplatform.mining.{Miner, MiningConstraint, MiningConstraints} import com.wavesplatform.settings.{BlockchainSettings, WavesSettings} import com.wavesplatform.state.BlockchainUpdaterImpl.BlockApplyResult.{Applied, Ignored} +import com.wavesplatform.state.TxMeta.Status import com.wavesplatform.state.diffs.BlockDiffer import com.wavesplatform.transaction.* import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} @@ -753,6 +754,10 @@ class BlockchainUpdaterImpl( snapshotBlockchain.transactionMeta(id) } + override def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, Status)] = readLock { + snapshotBlockchain.transactionSnapshot(id) + } + override def balance(address: Address, mayBeAssetId: Asset): Long = readLock { snapshotBlockchain.balance(address, mayBeAssetId) } diff --git a/node/src/main/scala/com/wavesplatform/state/LeaseDetails.scala b/node/src/main/scala/com/wavesplatform/state/LeaseDetails.scala index 20617937ce..43ee376086 100644 --- a/node/src/main/scala/com/wavesplatform/state/LeaseDetails.scala +++ b/node/src/main/scala/com/wavesplatform/state/LeaseDetails.scala @@ -3,6 +3,7 @@ package com.wavesplatform.state import com.wavesplatform.account.{Address, PublicKey} import com.wavesplatform.common.state.ByteStr import com.wavesplatform.transaction.TxPositiveAmount +import play.api.libs.json.{Format, Json, Writes} object LeaseDetails { sealed trait Status @@ -24,6 +25,11 @@ object LeaseDetails { case _ => None } } + + implicit val byteStrFormat: Format[ByteStr] = com.wavesplatform.utils.byteStrFormat + implicit val writesCancelled: Writes[Cancelled] = Json.writes + implicit val writesExpired: Writes[Expired] = Json.writes + implicit val writesInactive: Writes[Inactive] = Json.writes } } diff --git a/node/src/main/scala/com/wavesplatform/state/SnapshotBlockchain.scala b/node/src/main/scala/com/wavesplatform/state/SnapshotBlockchain.scala index 6f39f6cfff..d379243af3 100644 --- a/node/src/main/scala/com/wavesplatform/state/SnapshotBlockchain.scala +++ b/node/src/main/scala/com/wavesplatform/state/SnapshotBlockchain.scala @@ -8,6 +8,7 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.features.BlockchainFeatures.RideV6 import com.wavesplatform.lang.ValidationError import com.wavesplatform.settings.BlockchainSettings +import com.wavesplatform.state.TxMeta.Status import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} import com.wavesplatform.transaction.TxValidationError.{AliasDoesNotExist, AliasIsDisabled} import com.wavesplatform.transaction.transfer.{TransferTransaction, TransferTransactionLike} @@ -119,6 +120,12 @@ case class SnapshotBlockchain( .map(t => TxMeta(Height(this.height), t.status, t.spentComplexity)) .orElse(inner.transactionMeta(id)) + override def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, Status)] = + snapshot.transactions + .get(id) + .map(tx => (tx.snapshot, tx.status)) + .orElse(inner.transactionSnapshot(id)) + override def height: Int = inner.height + blockMeta.fold(0)(_ => 1) override def resolveAlias(alias: Alias): Either[ValidationError, Address] = inner.resolveAlias(alias) match { diff --git a/node/src/main/scala/com/wavesplatform/state/Sponsorship.scala b/node/src/main/scala/com/wavesplatform/state/Sponsorship.scala index 08de565ba8..2eb670ebf3 100644 --- a/node/src/main/scala/com/wavesplatform/state/Sponsorship.scala +++ b/node/src/main/scala/com/wavesplatform/state/Sponsorship.scala @@ -5,12 +5,15 @@ import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.state.diffs.FeeValidation import com.wavesplatform.transaction.Asset.IssuedAsset import com.wavesplatform.transaction.{Asset, Transaction} +import play.api.libs.json.{JsNumber, Writes} sealed abstract class Sponsorship case class SponsorshipValue(minFee: Long) extends Sponsorship case object SponsorshipNoInfo extends Sponsorship object Sponsorship { + implicit val writesValue: Writes[SponsorshipValue] = Writes[SponsorshipValue](v => JsNumber(v.minFee)) + implicit val sponsorshipMonoid: Monoid[Sponsorship] = new Monoid[Sponsorship] { override def empty: Sponsorship = SponsorshipNoInfo diff --git a/node/src/main/scala/com/wavesplatform/state/package.scala b/node/src/main/scala/com/wavesplatform/state/package.scala index 0c5f5f1471..5e06335ff8 100644 --- a/node/src/main/scala/com/wavesplatform/state/package.scala +++ b/node/src/main/scala/com/wavesplatform/state/package.scala @@ -41,8 +41,8 @@ package object state { implicit val dstWrites: Writes[AssetDistribution] = Writes { dst => Json - .toJson(dst.map { - case (addr, balance) => addr.toString -> balance + .toJson(dst.map { case (addr, balance) => + addr.toString -> balance }) } @@ -57,7 +57,9 @@ package object state { ) } - object Height extends TaggedType[Int] + object Height extends TaggedType[Int] { + implicit val format: Format[Height] = implicitly[Format[Int]].bimap(Height(_), identity) + } type Height = Height.Type object TxNum extends TaggedType[Short] @@ -66,6 +68,11 @@ package object state { object AssetNum extends TaggedType[Int] type AssetNum = AssetNum.Type - object TransactionId extends TaggedType[ByteStr] + object TransactionId extends TaggedType[ByteStr] { + implicit val format: Format[TransactionId] = Format[TransactionId]( + com.wavesplatform.utils.byteStrFormat.map(this(_)), + Writes(com.wavesplatform.utils.byteStrFormat.writes) + ) + } type TransactionId = TransactionId.Type } diff --git a/node/src/main/scala/com/wavesplatform/transaction/package.scala b/node/src/main/scala/com/wavesplatform/transaction/package.scala index f01a87faa7..c74409bd85 100644 --- a/node/src/main/scala/com/wavesplatform/transaction/package.scala +++ b/node/src/main/scala/com/wavesplatform/transaction/package.scala @@ -38,6 +38,7 @@ package object transaction { type TxPositiveAmount = Long Refined Positive object TxPositiveAmount extends RefinedTypeOps[TxPositiveAmount, Long] + implicit val posAmountWrites: Writes[TxPositiveAmount] = Writes(v => JsNumber(v.value)) type TxNonNegativeAmount = Long Refined NonNegative object TxNonNegativeAmount extends RefinedTypeOps[TxNonNegativeAmount, Long] { diff --git a/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala b/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala index 56109d5002..6f5453d75f 100644 --- a/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/DebugApiRouteSpec.scala @@ -89,8 +89,7 @@ class DebugApiRouteSpec null, null, configObject, - domain.rocksDBWriter.loadBalanceHistory, - domain.rocksDBWriter.loadStateHash, + domain.rocksDBWriter, () => Some(domain.blockchain), new RouteTimeout(60.seconds)(sharedScheduler), sharedScheduler @@ -174,10 +173,11 @@ class DebugApiRouteSpec val lastButOneHeader = domain.blockchain.blockHeader(lastButOneHeight).value val lastButOneStateHash = domain.rocksDBWriter.loadStateHash(lastButOneHeight).value val expectedResponse = Json.toJson(lastButOneStateHash).as[JsObject] ++ Json.obj( - "blockId" -> lastButOneHeader.id().toString, - "baseTarget" -> lastButOneHeader.header.baseTarget, - "height" -> lastButOneHeight, - "version" -> Version.VersionString + "snapshotHash" -> domain.rocksDBWriter.snapshotStateHash(lastButOneHeight), + "blockId" -> lastButOneHeader.id().toString, + "baseTarget" -> lastButOneHeader.header.baseTarget, + "height" -> lastButOneHeight, + "version" -> Version.VersionString ) Get(routePath(s"/stateHash/last")) ~> route ~> check { diff --git a/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala b/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala index bf1c804f30..42cb1f4c68 100644 --- a/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/ProtoVersionTransactionsSpec.scala @@ -191,9 +191,9 @@ class ProtoVersionTransactionsSpec val assetPair = assetPairGen.sample.get val buyOrder = - Order.buy(Order.V3, buyer, account.publicKey, assetPair, Order.MaxAmount / 2, 100L, Now, Now + Order.MaxLiveTime, MinFee * 3).explicitGet() + Order.buy(Order.V3, buyer, account.publicKey, assetPair, Order.MaxAmount / 2, 100L, Now, Now + Order.MaxLiveTime / 2, MinFee * 3).explicitGet() val sellOrder = - Order.sell(Order.V3, seller, account.publicKey, assetPair, Order.MaxAmount / 2, 100L, Now, Now + Order.MaxLiveTime, MinFee * 3).explicitGet() + Order.sell(Order.V3, seller, account.publicKey, assetPair, Order.MaxAmount / 2, 100L, Now, Now + Order.MaxLiveTime / 2, MinFee * 3).explicitGet() val exchangeTx = ExchangeTransaction diff --git a/node/src/test/scala/com/wavesplatform/http/TransactionSnapshotsRouteSpec.scala b/node/src/test/scala/com/wavesplatform/http/TransactionSnapshotsRouteSpec.scala new file mode 100644 index 0000000000..f625f82f7a --- /dev/null +++ b/node/src/test/scala/com/wavesplatform/http/TransactionSnapshotsRouteSpec.scala @@ -0,0 +1,450 @@ +package com.wavesplatform.http + +import akka.http.scaladsl.model.ContentTypes.`application/json` +import akka.http.scaladsl.model.StatusCodes.{BadRequest, NotFound} +import akka.http.scaladsl.model.{FormData, HttpEntity} +import com.wavesplatform.BlockGen +import com.wavesplatform.api.http.{RouteTimeout, TransactionsApiRoute} +import com.wavesplatform.db.WithState.AddrWithBalance +import com.wavesplatform.lang.directives.values.V8 +import com.wavesplatform.lang.v1.compiler.TestCompiler +import com.wavesplatform.state.diffs.ENOUGH_AMT +import com.wavesplatform.test.* +import com.wavesplatform.test.DomainPresets.TransactionStateSnapshot +import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} +import com.wavesplatform.transaction.Transaction +import com.wavesplatform.transaction.TxHelpers.* +import com.wavesplatform.transaction.assets.exchange.OrderType.{BUY, SELL} +import com.wavesplatform.utils.{EthHelpers, SharedSchedulerMixin} +import org.scalatest.OptionValues +import play.api.libs.json.* +import play.api.libs.json.Json.JsValueWrapper + +import scala.concurrent.Future +import scala.concurrent.duration.* + +class TransactionSnapshotsRouteSpec + extends RouteSpec("/transactions") + with RestAPISettingsHelper + with BlockGen + with OptionValues + with SharedDomain + with EthHelpers + with SharedSchedulerMixin { + + override def settings = TransactionStateSnapshot + override def genesisBalances = AddrWithBalance.enoughBalances(defaultSigner, secondSigner) + + private val transactionsApiRoute = new TransactionsApiRoute( + settings.restAPISettings, + domain.transactionsApi, + domain.wallet, + domain.blockchain, + () => domain.blockchain, + () => domain.utxPool.size, + (tx, _) => Future.successful(domain.utxPool.putIfNew(tx, forceValidate = true)), + new TestTime, + new RouteTimeout(60.seconds)(sharedScheduler) + ) + private val route = seal(transactionsApiRoute.route) + + private def multipleJson(txs: Seq[Transaction]) = + Post( + routePath("/snapshot"), + HttpEntity(`application/json`, Json.obj("ids" -> Json.arr(txs.map(_.id().toString: JsValueWrapper)*)).toString()) + ) + private def multipleFormData(txs: Seq[Transaction]) = + Post(routePath("/snapshot"), FormData(txs.map("id" -> _.id().toString)*)) + + routePath("/snapshot/{id}") - { + "all snapshot fields" in { + val script = TestCompiler(V8).compileContract( + """ + | @Callable(i) + | func default() = { + | let issue = Issue("aaaa", "bbbb", 1000, 4, true, unit, 0) + | let lease = Lease(i.caller, 123) + | [ + | lease, + | LeaseCancel(lease.calculateLeaseId()), + | issue, + | SponsorFee(issue.calculateAssetId(), 1000), + | IntegerEntry("int", 777), + | BinaryEntry("bytes", base64'abc'), + | BooleanEntry("bool", true), + | StringEntry("str", "text"), + | DeleteEntry("delete") + | ] + | } + """.stripMargin + ) + val setScriptTx = setScript(secondSigner, script) + val invokeTx = invoke(fee = 100500000) + val removeScriptTx = removeScript(secondSigner) + val issueTx = issue(script = Some(TestCompiler(V8).compileExpression("true"))) + val asset = IssuedAsset(issueTx.id()) + val aliasTx = createAlias() + val order1 = order(BUY, asset, Waves, amount = 123, price = 40_000_000, fee = 777) + val order2 = order(SELL, asset, Waves, amount = 123, price = 40_000_000, fee = 888) + val exchangeTx = exchange(order1, order2, amount = 123, price = 40_000_000, buyMatcherFee = 777, sellMatcherFee = 888) + val allTxs = Seq(setScriptTx, invokeTx, issueTx, aliasTx, removeScriptTx, exchangeTx) + + domain.appendBlock(allTxs*) + val invokeAsset = domain.liquidSnapshot.assetStatics.head._1 + val leaseId = domain.liquidSnapshot.newLeases.head._1 + + val setScriptJson = Json.parse( + s""" + | { + | "applicationStatus": "succeeded", + | "balances": [ + | { + | "address": "$secondAddress", + | "asset": null, + | "balance": ${ENOUGH_AMT - setScriptTx.fee.value} + | }, + | { + | "address": "$defaultAddress", + | "asset": null, + | "balance": ${ENOUGH_AMT + 200_000_000 + setScriptTx.fee.value * 2 / 5} + | } + | ], + | "leaseBalances": [], + | "assetStatics": [], + | "assetVolumes": [], + | "assetNamesAndDescriptions": [], + | "assetScripts": [], + | "sponsorships": [], + | "newLeases": [], + | "cancelledLeases": [], + | "aliases": [], + | "orderFills": [], + | "accountScripts": [ + | { + | "publicKey": "${secondSigner.publicKey}", + | "script": "${script.bytes().base64}", + | "verifierComplexity": 0 + | } + | ], + | "accountData": [] + | } + """.stripMargin + ) + Get(routePath(s"/snapshot/${setScriptTx.id()}")) ~> route ~> check( + responseAs[JsObject] shouldBe setScriptJson + ) + + val invokeJson = Json.parse( + s""" + | { + | "applicationStatus": "succeeded", + | "balances": [ + | { + | "address": "$defaultAddress", + | "asset": null, + | "balance": ${ENOUGH_AMT + 200_000_000 + setScriptTx.fee.value * 2 / 5 - invokeTx.fee.value * 3 / 5} + | }, + | { + | "address": "$secondAddress", + | "asset": "$invokeAsset", + | "balance": 1000 + | } + | ], + | "leaseBalances": [ + | { + | "address": "$secondAddress", + | "in": 0, + | "out": 0 + | }, + | { + | "address": "$defaultAddress", + | "in": 0, + | "out": 0 + | } + | ], + | "assetStatics": [ + | { + | "id": "$invokeAsset", + | "source": "${invokeTx.id()}", + | "issuer": "${secondSigner.publicKey}", + | "decimals": 4, + | "nft": false + | } + | ], + | "assetVolumes": [ + | { + | "id": "$invokeAsset", + | "isReissuable": true, + | "volume": 1000 + | } + | ], + | "assetNamesAndDescriptions": [ + | { + | "id": "$invokeAsset", + | "name": "aaaa", + | "description": "bbbb", + | "lastUpdatedAt": 2 + | } + | ], + | "assetScripts": [], + | "sponsorships": [ + | { + | "id": "$invokeAsset", + | "minSponsoredAssetFee": 1000 + | } + | ], + | "newLeases": [ + | { + | "id": "$leaseId", + | "sender": "${secondSigner.publicKey}", + | "recipient": "$defaultAddress", + | "amount": 123, + | "txId": "${invokeTx.id()}", + | "height": 2 + | } + | ], + | "cancelledLeases": [ + | { + | "id": "$leaseId", + | "txId": "${invokeTx.id()}", + | "height": 2 + | } + | ], + | "aliases": [], + | "orderFills": [], + | "accountScripts": [], + | "accountData": [ + | { + | "address": "$secondAddress", + | "data": [ + | { + | "key": "bool", + | "type": "boolean", + | "value": true + | }, + | { + | "key": "str", + | "type": "string", + | "value": "text" + | }, + | { + | "key": "bytes", + | "type": "binary", + | "value": "base64:abc=" + | }, + | { + | "key": "int", + | "type": "integer", + | "value": 777 + | }, + | { + | "key": "delete", + | "value": null + | } + | ] + | } + | ] + | } + """.stripMargin + ) + Get(routePath(s"/snapshot/${invokeTx.id()}")) ~> route ~> check( + responseAs[JsObject] shouldBe invokeJson + ) + + val issueJson = Json.parse( + s""" + | { + | "applicationStatus": "succeeded", + | "balances" : [ { + | "address" : "$defaultAddress", + | "asset" : "$asset", + | "balance" : ${Long.MaxValue / 100} + | }, { + | "address" : "$defaultAddress", + | "asset" : null, + | "balance" : ${ENOUGH_AMT + 200_000_000 + setScriptTx.fee.value * 2 / 5 - (invokeTx.fee.value + issueTx.fee.value) * 3 / 5} + | } ], + | "leaseBalances" : [ ], + | "assetStatics" : [ { + | "id" : "$asset", + | "source" : "${issueTx.id()}", + | "issuer" : "${defaultSigner.publicKey}", + | "decimals" : 0, + | "nft" : false + | } ], + | "assetVolumes" : [ { + | "id": "$asset", + | "isReissuable": true, + | "volume": 92233720368547758 + | } ], + | "assetNamesAndDescriptions" : [ { + | "id": "$asset", + | "name" : "test", + | "description" : "description", + | "lastUpdatedAt" : 2 + | } ], + | "assetScripts" : [ { + | "id": "$asset", + | "script" : "base64:CAEG32nosg==", + | "complexity" : 0 + | } ], + | "sponsorships" : [ ], + | "newLeases" : [ ], + | "cancelledLeases" : [ ], + | "aliases" : [ ], + | "orderFills" : [ ], + | "accountScripts" : [ ], + | "accountData" : [ ] + | } + """.stripMargin + ) + Get(routePath(s"/snapshot/${issueTx.id()}")) ~> route ~> check( + responseAs[JsObject] shouldBe issueJson + ) + + val aliasJson = Json.parse( + s""" + | { + | "applicationStatus": "succeeded", + | "balances" : [ { + | "address" : "3MtGzgmNa5fMjGCcPi5nqMTdtZkfojyWHL9", + | "asset" : null, + | "balance" : ${ENOUGH_AMT + 200_000_000 + setScriptTx.fee.value * 2 / 5 - (invokeTx.fee.value + issueTx.fee.value + aliasTx.fee.value) * 3 / 5} + | } ], + | "leaseBalances" : [ ], + | "assetStatics" : [ ], + | "assetVolumes" : [ ], + | "assetNamesAndDescriptions" : [ ], + | "assetScripts" : [ ], + | "sponsorships" : [ ], + | "newLeases" : [ ], + | "cancelledLeases" : [ ], + | "aliases" : [ { "address": "$defaultAddress", "alias": "alias" } ], + | "orderFills" : [ ], + | "accountScripts" : [ ], + | "accountData" : [ ] + | } + """.stripMargin + ) + Get(routePath(s"/snapshot/${aliasTx.id()}")) ~> route ~> check(responseAs[JsObject]) shouldBe aliasJson + + val removeScriptJson = Json.parse( + s""" + | { + | "applicationStatus": "succeeded", + | "balances": [ + | { + | "address": "$secondAddress", + | "asset": null, + | "balance": ${ENOUGH_AMT - setScriptTx.fee.value - removeScriptTx.fee.value} + | }, + | { + | "address": "$defaultAddress", + | "asset": null, + | "balance": ${ENOUGH_AMT + 200_000_000 + (setScriptTx.fee.value + removeScriptTx.fee.value) * 2 / 5 - (invokeTx.fee.value + issueTx.fee.value + aliasTx.fee.value) * 3 / 5} + | } + | ], + | "leaseBalances": [], + | "assetStatics": [], + | "assetVolumes": [], + | "assetNamesAndDescriptions": [], + | "assetScripts": [], + | "sponsorships": [], + | "newLeases": [], + | "cancelledLeases": [], + | "aliases": [], + | "orderFills": [], + | "accountScripts": [ + | { + | "publicKey": "${secondSigner.publicKey}", + | "script": null, + | "verifierComplexity": 0 + | } + | ], + | "accountData": [] + | } + """.stripMargin + ) + Get(routePath(s"/snapshot/${removeScriptTx.id()}")) ~> route ~> check( + responseAs[JsObject] shouldBe removeScriptJson + ) + + val exchangeJson = Json.parse( + s""" + | { + | "applicationStatus": "succeeded", + | "balances": [ + | { + | "address": "$defaultAddress", + | "asset": null, + | "balance": ${ENOUGH_AMT + 200_000_000 + (setScriptTx.fee.value + removeScriptTx.fee.value) * 2 / 5 - (invokeTx.fee.value + issueTx.fee.value + aliasTx.fee.value + exchangeTx.fee.value) * 3 / 5} + | } + | ], + | "leaseBalances": [], + | "assetStatics": [], + | "assetVolumes": [], + | "assetNamesAndDescriptions": [], + | "assetScripts": [], + | "sponsorships": [], + | "newLeases": [], + | "cancelledLeases": [], + | "aliases": [], + | "orderFills": [ + | { + | "id": "${exchangeTx.order1.id()}", + | "volume": 123, + | "fee": 777 + | }, + | { + | "id": "${exchangeTx.order2.id()}", + | "volume": 123, + | "fee": 888 + | } + | ], + | "accountScripts": [], + | "accountData": [] + | } + """.stripMargin + ) + Get(routePath(s"/snapshot/${exchangeTx.id()}")) ~> route ~> check( + responseAs[JsObject] shouldBe exchangeJson + ) + + val allSnapshotsJson = JsArray(Seq(setScriptJson, invokeJson, issueJson, aliasJson, removeScriptJson, exchangeJson)) + multipleJson(allTxs) ~> route ~> check(responseAs[JsArray] shouldBe allSnapshotsJson) + multipleFormData(allTxs) ~> route ~> check(responseAs[JsArray] shouldBe allSnapshotsJson) + } + + "multiple snapshots limit" in { + val transfers = (1 to 101).map(_ => transfer()) + domain.appendBlock(transfers*) + multipleJson(transfers.drop(1)) ~> route ~> check(responseAs[JsArray].value.size shouldBe 100) + multipleFormData(transfers.drop(1)) ~> route ~> check(responseAs[JsArray].value.size shouldBe 100) + + multipleJson(transfers) ~> route ~> check { + status shouldEqual BadRequest + (responseAs[JsObject] \ "message").as[String] shouldBe "Too big sequence requested: max limit is 100 entries" + } + multipleFormData(transfers) ~> route ~> check { + status shouldEqual BadRequest + (responseAs[JsObject] \ "message").as[String] shouldBe "Too big sequence requested: max limit is 100 entries" + } + } + + "unexisting id" in { + val tx = transfer() + Get(routePath(s"/snapshot/${tx.id()}")) ~> route ~> check { + status shouldEqual NotFound + (responseAs[JsObject] \ "message").as[String] shouldBe "transactions does not exist" + } + multipleJson(Seq(tx)) ~> route ~> check { + status shouldEqual NotFound + (responseAs[JsObject] \ "message").as[String] shouldBe "transactions does not exist" + } + multipleFormData(Seq(tx)) ~> route ~> check { + status shouldEqual NotFound + (responseAs[JsObject] \ "message").as[String] shouldBe "transactions does not exist" + } + } + } +} diff --git a/node/src/test/scala/com/wavesplatform/settings/RestAPISettingsSpecification.scala b/node/src/test/scala/com/wavesplatform/settings/RestAPISettingsSpecification.scala index 6a45eb43a5..eec3149053 100644 --- a/node/src/test/scala/com/wavesplatform/settings/RestAPISettingsSpecification.scala +++ b/node/src/test/scala/com/wavesplatform/settings/RestAPISettingsSpecification.scala @@ -23,6 +23,7 @@ class RestAPISettingsSpecification extends FlatSpec { | access-control-allow-credentials = yes | } | transactions-by-address-limit = 1000 + | transaction-snapshots-limit = 123 | distribution-address-limit = 1000 | data-keys-request-limit = 1000 | asset-details-limit = 100 @@ -42,6 +43,7 @@ class RestAPISettingsSpecification extends FlatSpec { settings.port should be(6869) settings.apiKeyHash should be("BASE58APIKEYHASH") settings.transactionsByAddressLimit shouldBe 1000 + settings.transactionSnapshotsLimit shouldBe 123 settings.distributionAddressLimit shouldBe 1000 settings.dataKeysRequestLimit shouldBe 1000 settings.assetDetailsLimit shouldBe 100 diff --git a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala index 151d899a55..6b28ca7876 100644 --- a/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/diffs/smart/predef/MatcherBlockchainTest.scala @@ -13,6 +13,7 @@ import com.wavesplatform.lang.v1.compiler.TestCompiler import com.wavesplatform.lang.v1.traits.domain.Recipient import com.wavesplatform.settings.BlockchainSettings import com.wavesplatform.state.* +import com.wavesplatform.state.TxMeta.Status import com.wavesplatform.test.PropSpec import com.wavesplatform.transaction.Asset.Waves import com.wavesplatform.transaction.smart.script.ScriptRunner @@ -41,6 +42,7 @@ class MatcherBlockchainTest extends PropSpec with MockFactory with WithDomain { override def transactionInfo(id: ByteStr): Option[(TxMeta, Transaction)] = ??? override def transactionInfos(ids: Seq[BlockId]): Seq[Option[(TxMeta, Transaction)]] = ??? override def transactionMeta(id: ByteStr): Option[TxMeta] = ??? + override def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, Status)] = ??? override def containsTransaction(tx: Transaction): Boolean = ??? override def assetDescription(id: Asset.IssuedAsset): Option[AssetDescription] = ??? override def resolveAlias(a: Alias): Either[ValidationError, Address] = ??? diff --git a/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala b/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala index ea0f5064d0..41e8fa8767 100644 --- a/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala +++ b/node/src/test/scala/com/wavesplatform/state/snapshot/StateSnapshotStorageTest.scala @@ -9,7 +9,6 @@ import com.wavesplatform.db.WithDomain import com.wavesplatform.lang.directives.values.V6 import com.wavesplatform.lang.v1.compiler.TestCompiler import com.wavesplatform.lang.v1.traits.domain.{Issue, Lease, Recipient} -import com.wavesplatform.protobuf.PBSnapshots import com.wavesplatform.state.* import com.wavesplatform.state.TxMeta.Status.{Failed, Succeeded} import com.wavesplatform.state.diffs.BlockDiffer.CurrentBlockFeePart @@ -46,11 +45,7 @@ class StateSnapshotStorageTest extends PropSpec with WithDomain { if (failed) d.appendAndAssertFailed(tx) else d.appendAndAssertSucceed(tx) d.appendBlock() val status = if (failed) Failed else Succeeded - PBSnapshots.fromProtobuf( - d.rocksDBWriter.transactionSnapshot(tx.id()).get, - tx.id(), - d.blockchain.height - 1 - ) shouldBe (expectedSnapshotWithMiner, status) + d.rocksDBWriter.transactionSnapshot(tx.id()).get shouldBe (expectedSnapshotWithMiner, status) } // Genesis diff --git a/node/src/test/scala/com/wavesplatform/transaction/ProtoVersionTransactionsSpec.scala b/node/src/test/scala/com/wavesplatform/transaction/ProtoVersionTransactionsSpec.scala index 6b6f37323f..b57a718889 100644 --- a/node/src/test/scala/com/wavesplatform/transaction/ProtoVersionTransactionsSpec.scala +++ b/node/src/test/scala/com/wavesplatform/transaction/ProtoVersionTransactionsSpec.scala @@ -94,9 +94,9 @@ class ProtoVersionTransactionsSpec extends FreeSpec { val assetPair = assetPairGen.sample.get val buyOrder = - Order.buy(Order.V3, buyer, Account.publicKey, assetPair, Order.MaxAmount / 2, 100, Now, Now + Order.MaxLiveTime, MinFee * 3).explicitGet() + Order.buy(Order.V3, buyer, Account.publicKey, assetPair, Order.MaxAmount / 2, 100, Now, Now + Order.MaxLiveTime / 2, MinFee * 3).explicitGet() val sellOrder = - Order.sell(Order.V3, seller, Account.publicKey, assetPair, Order.MaxAmount / 2, 100, Now, Now + Order.MaxLiveTime, MinFee * 3).explicitGet() + Order.sell(Order.V3, seller, Account.publicKey, assetPair, Order.MaxAmount / 2, 100, Now, Now + Order.MaxLiveTime / 2, MinFee * 3).explicitGet() val exchangeTx = ExchangeTransaction diff --git a/node/src/test/scala/com/wavesplatform/transaction/TxHelpers.scala b/node/src/test/scala/com/wavesplatform/transaction/TxHelpers.scala index 580f611af2..a4f99acf55 100644 --- a/node/src/test/scala/com/wavesplatform/transaction/TxHelpers.scala +++ b/node/src/test/scala/com/wavesplatform/transaction/TxHelpers.scala @@ -359,6 +359,16 @@ object TxHelpers { SetScriptTransaction.selfSigned(version, acc, Some(script), fee, timestamp, chainId).explicitGet() } + def removeScript( + acc: KeyPair, + fee: Long = FeeConstants(TransactionType.SetScript) * FeeUnit, + version: TxVersion = TxVersion.V1, + chainId: Byte = AddressScheme.current.chainId, + timestamp: TxTimestamp = timestamp + ): SetScriptTransaction = { + SetScriptTransaction.selfSigned(version, acc, None, fee, timestamp, chainId).explicitGet() + } + def setAssetScript( acc: KeyPair, asset: IssuedAsset, @@ -432,7 +442,7 @@ object TxHelpers { } def createAlias( - name: String, + name: String = "alias", sender: KeyPair = defaultSigner, fee: Long = TestValues.fee, version: TxVersion = TxVersion.V2, diff --git a/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/ExchangeTransactionSpecification.scala b/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/ExchangeTransactionSpecification.scala index 99fda8b092..03e5531d19 100644 --- a/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/ExchangeTransactionSpecification.scala +++ b/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/ExchangeTransactionSpecification.scala @@ -193,7 +193,7 @@ class ExchangeTransactionSpecification extends PropSpec with NTPTime with JsonMa forAll(preconditions) { case (sender1, sender2, matcher, pair, buyerMatcherFeeAssetId, sellerMatcherFeeAssetId, versions) => val time = ntpTime.correctedTime() - val expirationTimestamp = time + Order.MaxLiveTime + val expirationTimestamp = time + Order.MaxLiveTime / 2 val buyPrice = 60 * Order.PriceConstant val sellPrice = 50 * Order.PriceConstant @@ -350,7 +350,7 @@ class ExchangeTransactionSpecification extends PropSpec with NTPTime with JsonMa forAll(preconditions) { case (sender1, sender2, matcher, pair, buyerMatcherFeeAssetId, sellerMatcherFeeAssetId, versions) => val time = ntpTime.correctedTime() - val expirationTimestamp = time + Order.MaxLiveTime + val expirationTimestamp = time + Order.MaxLiveTime / 2 val buyPrice = 1 * Order.PriceConstant val sellPrice = (0.50 * Order.PriceConstant).toLong val matcherFee = 300000L diff --git a/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/OrderSpecification.scala b/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/OrderSpecification.scala index 4042fd31b0..f24051e6eb 100644 --- a/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/OrderSpecification.scala +++ b/node/src/test/scala/com/wavesplatform/transaction/assets/exchange/OrderSpecification.scala @@ -201,7 +201,7 @@ class OrderSpecification extends PropSpec with ValidationMatcher with NTPTime { property("Buy and Sell orders") { forAll(orderParamGen) { case (sender, matcher, pair, _, amount, price, timestamp, _, _) => - val expiration = timestamp + Order.MaxLiveTime - 1000 + val expiration = timestamp + Order.MaxLiveTime / 2 - 1000 val buy = Order .buy( Order.V1, diff --git a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala index 27f41f53e8..ecddcb139a 100644 --- a/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala +++ b/node/src/test/scala/com/wavesplatform/utils/EmptyBlockchain.scala @@ -7,6 +7,7 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.lang.ValidationError import com.wavesplatform.settings.BlockchainSettings import com.wavesplatform.state.* +import com.wavesplatform.state.TxMeta.Status import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} import com.wavesplatform.transaction.TxValidationError.GenericError import com.wavesplatform.transaction.transfer.TransferTransactionLike @@ -49,6 +50,8 @@ trait EmptyBlockchain extends Blockchain { override def transactionMeta(id: ByteStr): Option[TxMeta] = None + override def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, Status)] = None + override def containsTransaction(tx: Transaction): Boolean = false override def assetDescription(id: IssuedAsset): Option[AssetDescription] = None diff --git a/project/Dependencies.scala b/project/Dependencies.scala index afcbeca8f3..3fd849c9c0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,7 +6,7 @@ import scalapb.compiler.Version.scalapbVersion object Dependencies { // Node protobuf schemas private[this] val protoSchemasLib = - "com.wavesplatform" % "protobuf-schemas" % "1.5.1" classifier "protobuf-src" intransitive () + "com.wavesplatform" % "protobuf-schemas" % "1.5.2-86-SNAPSHOT" classifier "protobuf-src" intransitive () private def akkaModule(module: String) = "com.typesafe.akka" %% s"akka-$module" % "2.6.21" diff --git a/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/ImmutableBlockchain.scala b/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/ImmutableBlockchain.scala index 9c4a28116c..e6d202883b 100644 --- a/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/ImmutableBlockchain.scala +++ b/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/ImmutableBlockchain.scala @@ -16,7 +16,7 @@ import com.wavesplatform.lang.v1.estimator.v2.ScriptEstimatorV2 import com.wavesplatform.ride.runner.* import com.wavesplatform.ride.runner.input.RideRunnerBlockchainState import com.wavesplatform.settings.BlockchainSettings -import com.wavesplatform.state.{AccountScriptInfo, AssetDescription, AssetScriptInfo, BalanceSnapshot, DataEntry, Height, LeaseBalance, TxMeta} +import com.wavesplatform.state.{AccountScriptInfo, AssetDescription, AssetScriptInfo, BalanceSnapshot, DataEntry, Height, LeaseBalance, StateSnapshot, TxMeta} import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves} import com.wavesplatform.transaction.TxValidationError.AliasDoesNotExist import com.wavesplatform.transaction.transfer.{TransferTransaction, TransferTransactionLike} @@ -188,6 +188,8 @@ class ImmutableBlockchain(override val settings: BlockchainSettings, input: Ride override def transactionInfos(ids: Seq[BlockId]): Seq[Option[(TxMeta, Transaction)]] = ??? + override def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, TxMeta.Status)] = ??? + override def leaseBalances(addresses: Seq[Address]): Map[Address, LeaseBalance] = ??? override def balances(req: Seq[(Address, Asset)]): Map[(Address, Asset), Long] = ??? diff --git a/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/LazyBlockchain.scala b/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/LazyBlockchain.scala index e90e50d2b0..3275187dca 100644 --- a/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/LazyBlockchain.scala +++ b/ride-runner/src/main/scala/com/wavesplatform/ride/runner/blockchain/LazyBlockchain.scala @@ -25,7 +25,7 @@ import com.wavesplatform.ride.runner.estimate import com.wavesplatform.ride.runner.stats.RideRunnerStats import com.wavesplatform.ride.runner.stats.RideRunnerStats.* import com.wavesplatform.settings.BlockchainSettings -import com.wavesplatform.state.{AccountScriptInfo, AssetDescription, AssetScriptInfo, BalanceSnapshot, DataEntry, Height, LeaseBalance, TransactionId, TxMeta} +import com.wavesplatform.state.{AccountScriptInfo, AssetDescription, AssetScriptInfo, BalanceSnapshot, DataEntry, Height, LeaseBalance, StateSnapshot, TransactionId, TxMeta} import com.wavesplatform.transaction import com.wavesplatform.transaction.Asset import com.wavesplatform.transaction.Asset.IssuedAsset @@ -93,6 +93,8 @@ class LazyBlockchain[TagT] private ( override def transactionInfos(ids: Seq[BlockId]): Seq[Option[(TxMeta, transaction.Transaction)]] = ??? + override def transactionSnapshot(id: ByteStr): Option[(StateSnapshot, TxMeta.Status)] = ??? + override def leaseBalances(addresses: Seq[Address]): Map[Address, LeaseBalance] = ??? override def balances(req: Seq[(Address, Asset)]): Map[(Address, Asset), Long] = ??? From 247af0200c14cab4aa4ec255127e4bad21b44d9c Mon Sep 17 00:00:00 2001 From: Artyom Sayadyan Date: Mon, 15 Jan 2024 15:15:37 +0300 Subject: [PATCH 2/8] NODE-2643 Fixed EthereumInvokeScript serialization (#3932) Co-authored-by: Sergey Nazarov --- .../api/http/TransactionJsonSerializer.scala | 7 ++- .../http/TransactionsRouteSpec.scala | 62 ++++++++++++++++++- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala b/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala index 5588765144..11d8287c09 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/TransactionJsonSerializer.scala @@ -285,9 +285,10 @@ final case class TransactionJsonSerializer(blockchain: Blockchain) { val payments = i.payments.map(p => InvokeScriptTransaction.Payment(p.amount, PBAmounts.toVanillaAssetId(p.assetId))) gen.writeStartObject() + gen.writeNumberField("type", tx.tpe.id, numbersAsString) gen.writeStringField("id", tx.id().toString) gen.writeNumberField("fee", tx.assetFee._2, numbersAsString) - tx.assetFee._1.maybeBase58Repr.foreach(gen.writeStringField("feeAssetId", _)) + gen.writeStringField("feeAssetId", null) gen.writeNumberField("timestamp", tx.timestamp, numbersAsString) gen.writeNumberField("version", 1, numbersAsString) gen.writeNumberField("chainId", tx.chainId, numbersAsString) @@ -302,6 +303,7 @@ final case class TransactionJsonSerializer(blockchain: Blockchain) { None appStatus.foreach(s => gen.writeStringField("applicationStatus", s)) gen.writeNumberField("spentComplexity", spentComplexity, numbersAsString) + gen.writeObjectFieldStart("payload") gen.writeStringField("type", "invocation") gen.writeStringField("dApp", Address(EthEncoding.toBytes(tx.underlying.getTo)).toString) functionCallEi.fold(gen.writeNullField("call"))(fc => @@ -312,7 +314,8 @@ final case class TransactionJsonSerializer(blockchain: Blockchain) { gen.writeValueField("stateChanges")(invokeScriptResultSerializer(numbersAsString).serialize(isr, _, serializers)) ) gen.writeEndObject() - case meta @ TransactionMeta.Default(height, mtt: MassTransferTransaction, succeeded, spentComplexity) if mtt.sender.toAddress != address => + gen.writeEndObject() + case meta @ TransactionMeta.Default(_, mtt: MassTransferTransaction, _, _) if mtt.sender.toAddress != address => /** Produces compact representation for large transactions by stripping unnecessary data. Currently implemented for MassTransfer transaction * only. */ diff --git a/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala b/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala index 916fd75ac9..3b836b31dc 100644 --- a/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala +++ b/node/src/test/scala/com/wavesplatform/http/TransactionsRouteSpec.scala @@ -11,7 +11,7 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.{Base58, *} import com.wavesplatform.db.WithState.AddrWithBalance import com.wavesplatform.history.defaultSigner -import com.wavesplatform.lang.directives.values.{V5, V7} +import com.wavesplatform.lang.directives.values.{V5, V7, V8} import com.wavesplatform.lang.v1.FunctionHeader import com.wavesplatform.lang.v1.compiler.Terms.{ARR, CONST_BOOLEAN, CONST_BYTESTR, CONST_LONG, CONST_STRING, FUNCTION_CALL} import com.wavesplatform.lang.v1.compiler.TestCompiler @@ -20,7 +20,7 @@ import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state.{BinaryDataEntry, EmptyDataEntry, InvokeScriptResult, StringDataEntry} import com.wavesplatform.test.* import com.wavesplatform.transaction.Asset.Waves -import com.wavesplatform.transaction.TxHelpers.defaultAddress +import com.wavesplatform.transaction.TxHelpers.{defaultAddress, setScript, transfer} import com.wavesplatform.transaction.TxValidationError.ScriptExecutionError import com.wavesplatform.transaction.assets.exchange.{Order, OrderType} import com.wavesplatform.transaction.serialization.impl.InvokeScriptTxSerializer @@ -434,6 +434,64 @@ class TransactionsRouteSpec (result \ "timestamp").as[Long] shouldBe tx.timestamp } } + + "ethereum invocation" in { + val dApp = TestCompiler(V8).compileContract("@Callable(i)\nfunc f() = []") + val ethInvoke = EthTxGenerator.generateEthInvoke(richAccount.toEthKeyPair, richAddress, "f", Nil, Nil) + domain.appendAndAssertSucceed( + transfer(richAccount, richAccount.toEthWavesAddress), + setScript(richAccount, dApp), + ethInvoke + ) + Get(routePath(s"/address/$richAddress/limit/1")) ~> route ~> check { + val responseByAddress = responseAs[JsArray] + responseByAddress shouldBe Json.parse( + s""" [ + | [ + | { + | "type": 18, + | "id": "${ethInvoke.id()}", + | "fee": 500000, + | "feeAssetId": null, + | "timestamp": ${ethInvoke.timestamp}, + | "version": 1, + | "chainId": 84, + | "bytes": "${EthEncoding.toHexString(ethInvoke.bytes())}", + | "sender": "3MysRW4Crv73o2naQVbcVujvZoXvRXzA5Cg", + | "senderPublicKey": "AdyAnaBxRoqiuuCPMUJc2EHS6AkGcVCnj9D1y67bVP5fTa3Lb785hc8a2ccic7SsafSeskBFf2c7apsxyLs1TQo", + | "height": ${domain.blockchain.height}, + | "applicationStatus": "succeeded", + | "spentComplexity": 1, + | "payload": { + | "type": "invocation", + | "dApp": "3N7mQqVKEmpvRCefaRU4mvhmLKLvW1mjXfo", + | "call": { + | "function": "f", + | "args": [] + | }, + | "payment": [], + | "stateChanges": { + | "data": [], + | "transfers": [], + | "issues": [], + | "reissues": [], + | "burns": [], + | "sponsorFees": [], + | "leases": [], + | "leaseCancels": [], + | "invokes": [] + | } + | } + | } + | ] + | ] + """.stripMargin + ) + Get(routePath(s"/info/${ethInvoke.id()}")) ~> route ~> check { + responseAs[JsObject] shouldBe responseByAddress.head.get.head.get + } + } + } } routePath("/info/{id}") - { From f3cb2201b9ad6d18ff0c60289cbe0e19a8f5d0ca Mon Sep 17 00:00:00 2001 From: Vyatcheslav Suharnikov Date: Tue, 16 Jan 2024 09:37:40 +0400 Subject: [PATCH 3/8] NODE-2622 Delete old data (#3912) --- .../com/wavesplatform/state/DBState.scala | 14 +- .../com/wavesplatform/RollbackBenchmark.scala | 3 +- .../com/wavesplatform/state/BaseState.scala | 1 + .../state/RocksDBIteratorBenchmark.scala | 21 +- .../state/RocksDBSeekForPrevBenchmark.scala | 24 +- .../state/RocksDBWriterBenchmark.scala | 15 +- .../state/WavesEnvironmentBenchmark.scala | 10 +- .../wavesplatform/it/BaseTargetChecker.scala | 16 +- .../scala/com/wavesplatform/it/Docker.scala | 1 + .../test/BlockchainGenerator.scala | 176 +++++----- node/src/main/resources/application.conf | 15 + .../scala/com/wavesplatform/Application.scala | 1 + .../scala/com/wavesplatform/Explorer.scala | 52 +-- .../scala/com/wavesplatform/Exporter.scala | 27 +- .../scala/com/wavesplatform/Importer.scala | 3 +- .../api/common/CommonAccountsApi.scala | 27 +- .../com/wavesplatform/database/Caches.scala | 6 +- .../wavesplatform/database/KeyHelpers.scala | 15 +- .../com/wavesplatform/database/KeyTags.scala | 8 +- .../com/wavesplatform/database/Keys.scala | 29 +- .../com/wavesplatform/database/RDB.scala | 19 +- .../scala/com/wavesplatform/database/RW.scala | 7 + .../wavesplatform/database/ReadOnlyDB.scala | 38 +-- .../database/RocksDBWriter.scala | 302 ++++++++++++++++-- .../com/wavesplatform/database/package.scala | 76 +++-- .../history/StorageFactory.scala | 2 +- .../wavesplatform/settings/DBSettings.scala | 1 + .../generator/BlockchainGeneratorApp.scala | 3 +- .../generator/MinerChallengeSimulator.scala | 2 +- node/src/test/resources/application.conf | 16 +- .../api/common/CommonAccountApiSpec.scala | 15 +- .../consensus/FPPoSSelectorTest.scala | 1 + .../database/RocksDBWriterSpec.scala | 234 +++++++++++++- .../database/TestStorageFactory.scala | 10 +- .../wavesplatform/db/ScriptCacheTest.scala | 2 +- .../com/wavesplatform/db/WithState.scala | 5 +- .../mining/BlockWithMaxBaseTargetTest.scala | 1 + .../mining/MiningWithRewardSuite.scala | 28 +- .../com/wavesplatform/test/SharedDomain.scala | 3 +- .../utx/UtxPoolSpecification.scala | 25 +- project/Dependencies.scala | 3 +- .../database/rocksdb/DBResource.scala | 5 + 42 files changed, 918 insertions(+), 344 deletions(-) diff --git a/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala b/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala index 7f39682c73..8402209956 100644 --- a/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala +++ b/benchmark/src/main/scala/com/wavesplatform/state/DBState.scala @@ -22,13 +22,12 @@ abstract class DBState extends ScorexLogging { lazy val rdb: RDB = RDB.open(settings.dbSettings) - lazy val rocksDBWriter: RocksDBWriter = - new RocksDBWriter( - rdb, - settings.blockchainSettings, - settings.dbSettings.copy(maxCacheSize = 1), - settings.enableLightMode - ) + lazy val rocksDBWriter: RocksDBWriter = RocksDBWriter( + rdb, + settings.blockchainSettings, + settings.dbSettings.copy(maxCacheSize = 1), + settings.enableLightMode + ) AddressScheme.current = new AddressScheme { override val chainId: Byte = 'W' } @@ -44,6 +43,7 @@ abstract class DBState extends ScorexLogging { @TearDown def close(): Unit = { + rocksDBWriter.close() rdb.close() } } diff --git a/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala index 721c3b722f..3fc7277ab4 100644 --- a/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala @@ -22,7 +22,7 @@ object RollbackBenchmark extends ScorexLogging { val settings = Application.loadApplicationConfig(Some(new File(args(0)))) val rdb = RDB.open(settings.dbSettings) val time = new NTP(settings.ntpServer) - val rocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) + val rocksDBWriter = RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) val issuer = KeyPair(new Array[Byte](32)) @@ -111,6 +111,7 @@ object RollbackBenchmark extends ScorexLogging { rocksDBWriter.rollbackTo(1) val end = System.nanoTime() log.info(f"Rollback took ${(end - start) * 1e-6}%.3f ms") + rocksDBWriter.close() rdb.close() } } diff --git a/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala b/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala index 1f6348ed50..37803fa5fd 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/BaseState.scala @@ -104,6 +104,7 @@ trait BaseState { @TearDown def close(): Unit = { + state.close() rdb.close() } } diff --git a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBIteratorBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBIteratorBenchmark.scala index 386e69770c..58e944f0fb 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBIteratorBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBIteratorBenchmark.scala @@ -1,8 +1,5 @@ package com.wavesplatform.state -import java.nio.file.Files -import java.util.concurrent.TimeUnit - import com.google.common.primitives.Ints import com.typesafe.config.ConfigFactory import com.wavesplatform.database.RDB @@ -12,6 +9,10 @@ import org.openjdk.jmh.annotations.* import org.openjdk.jmh.infra.Blackhole import org.rocksdb.{ReadOptions, WriteBatch, WriteOptions} +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.util.Using + @OutputTimeUnit(TimeUnit.NANOSECONDS) @BenchmarkMode(Array(Mode.AverageTime)) @Threads(1) @@ -60,7 +61,7 @@ object RocksDBIteratorBenchmark { RDB.open(wavesSettings.dbSettings.copy(directory = dir)) } - val keysPrefix = "keysPrefix" + val keysPrefix = "keysPrefix" // Must have 10 or more bytes, see RDB.newColumnFamilyOptions val firstKey: Array[Byte] = keysPrefix.getBytes ++ Ints.toByteArray(1) val lastKey: Array[Byte] = keysPrefix.getBytes ++ Ints.toByteArray(10000) @@ -70,14 +71,18 @@ object RocksDBIteratorBenchmark { val readOptions: ReadOptions = new ReadOptions().setTotalOrderSeek(false).setPrefixSameAsStart(true) - private val wb: WriteBatch = new WriteBatch() - kvs.foreach { case (key, value) => - wb.put(key, value) + Using.Manager { use => + val wb = use(new WriteBatch()) + val wo = use(new WriteOptions()) + kvs.foreach { case (key, value) => + wb.put(key, value) + } + rdb.db.write(wo, wb) } - rdb.db.write(new WriteOptions(), wb) @TearDown def close(): Unit = { + readOptions.close() rdb.close() } } diff --git a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBSeekForPrevBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBSeekForPrevBenchmark.scala index a5b284e302..06680ca86f 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBSeekForPrevBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBSeekForPrevBenchmark.scala @@ -1,11 +1,7 @@ package com.wavesplatform.state -import java.nio.file.Files -import java.util.concurrent.TimeUnit - import com.google.common.primitives.{Bytes, Shorts} import com.typesafe.config.ConfigFactory -import com.wavesplatform.account.Address import com.wavesplatform.database.{ AddressId, CurrentData, @@ -24,6 +20,10 @@ import org.openjdk.jmh.annotations.* import org.openjdk.jmh.infra.Blackhole import org.rocksdb.{ReadOptions, WriteBatch, WriteOptions} +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.util.Using + @OutputTimeUnit(TimeUnit.NANOSECONDS) @BenchmarkMode(Array(Mode.AverageTime)) @Threads(1) @@ -63,11 +63,10 @@ object RocksDBSeekForPrevBenchmark { RDB.open(wavesSettings.dbSettings.copy(directory = dir)) } - val address: Address = Address(Array.fill(20)(1.toByte)) val addressId: AddressId = AddressId(1L) val keyString = "key" - val currentDataKey: Array[Byte] = Keys.data(address, keyString).keyBytes + val currentDataKey: Array[Byte] = Keys.data(addressId, keyString).keyBytes val dataNodeKey: Height => Array[Byte] = Keys.dataAt(addressId, "key")(_).keyBytes val dataNodeKeyPrefix: Array[Byte] = Bytes.concat(Shorts.toByteArray(KeyTags.DataHistory.id.toShort), addressId.toByteArray, keyString.getBytes) @@ -75,15 +74,18 @@ object RocksDBSeekForPrevBenchmark { val readOptions: ReadOptions = new ReadOptions() - private val wb: WriteBatch = new WriteBatch() - wb.put(currentDataKey, writeCurrentData(CurrentData(dataEntry, Height(10000), Height(9999)))) - (1 to 1000).foreach { h => - wb.put(dataNodeKey(Height(h)), writeDataNode(DataNode(dataEntry, Height(h - 1)))) + Using.Manager { use => + val wb = use(new WriteBatch()) + wb.put(currentDataKey, writeCurrentData(CurrentData(dataEntry, Height(10000), Height(9999)))) + (1 to 1000).foreach { h => + wb.put(dataNodeKey(Height(h)), writeDataNode(DataNode(dataEntry, Height(h - 1)))) + } + rdb.db.write(use(new WriteOptions()), wb) } - rdb.db.write(new WriteOptions(), wb) @TearDown def close(): Unit = { + readOptions.close() rdb.close() } } diff --git a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala index fa0d6f5472..bd8d04995a 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/RocksDBWriterBenchmark.scala @@ -1,8 +1,5 @@ package com.wavesplatform.state -import java.io.File -import java.util.concurrent.{ThreadLocalRandom, TimeUnit} - import com.typesafe.config.ConfigFactory import com.wavesplatform.account.* import com.wavesplatform.api.BlockMeta @@ -17,7 +14,10 @@ import com.wavesplatform.transaction.Transaction import org.openjdk.jmh.annotations.* import org.openjdk.jmh.infra.Blackhole +import java.io.File +import java.util.concurrent.{ThreadLocalRandom, TimeUnit} import scala.io.Codec +import scala.util.Using /** Tests over real database. How to test: * 1. Download a database 2. Import it: @@ -87,7 +87,7 @@ object RocksDBWriterBenchmark { RDB.open(wavesSettings.dbSettings) } - val db = new RocksDBWriter(rawDB, wavesSettings.blockchainSettings, wavesSettings.dbSettings, wavesSettings.enableLightMode) + val db = RocksDBWriter(rawDB, wavesSettings.blockchainSettings, wavesSettings.dbSettings, wavesSettings.enableLightMode) def loadBlockInfoAt(height: Int): Option[(BlockMeta, Seq[(TxMeta, Transaction)])] = loadBlockMetaAt(height).map { meta => @@ -102,15 +102,12 @@ object RocksDBWriterBenchmark { @TearDown def close(): Unit = { + db.close() rawDB.close() } protected def load[T](label: String, absolutePath: String)(f: String => T): Vector[T] = { - scala.io.Source - .fromFile(absolutePath)(Codec.UTF8) - .getLines() - .map(f) - .toVector + Using.resource(scala.io.Source.fromFile(absolutePath)(Codec.UTF8))(_.getLines().map(f).toVector) } } diff --git a/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala index 8e831b286c..4ff497de59 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/WavesEnvironmentBenchmark.scala @@ -21,6 +21,7 @@ import scodec.bits.BitVector import java.io.File import java.util.concurrent.{ThreadLocalRandom, TimeUnit} import scala.io.Codec +import scala.util.Using /** Tests over real database. How to test: * 1. Download a database 2. Import it: @@ -134,8 +135,8 @@ object WavesEnvironmentBenchmark { RDB.open(wavesSettings.dbSettings) } + val state = RocksDBWriter(rdb, wavesSettings.blockchainSettings, wavesSettings.dbSettings, wavesSettings.enableLightMode) val environment: Environment[Id] = { - val state = new RocksDBWriter(rdb, wavesSettings.blockchainSettings, wavesSettings.dbSettings, wavesSettings.enableLightMode) WavesEnvironment( AddressScheme.current.chainId, Coeval.raiseError(new NotImplementedError("`tx` is not implemented")), @@ -149,15 +150,12 @@ object WavesEnvironmentBenchmark { @TearDown def close(): Unit = { + state.close() rdb.close() } protected def load[T](label: String, absolutePath: String)(f: String => T): Vector[T] = { - scala.io.Source - .fromFile(absolutePath)(Codec.UTF8) - .getLines() - .map(f) - .toVector + Using.resource(scala.io.Source.fromFile(absolutePath)(Codec.UTF8))(_.getLines().map(f).toVector) } } diff --git a/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala b/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala index ebc35669cd..945e1d1439 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/BaseTargetChecker.scala @@ -23,11 +23,11 @@ object BaseTargetChecker { .withFallback(defaultReference()) .resolve() - val settings = WavesSettings.fromRootConfig(sharedConfig) - val db = RDB.open(settings.dbSettings.copy(directory = "/tmp/tmp-db")) - val ntpTime = new NTP("ntp.pool.org") - val (blockchainUpdater, _) = StorageFactory(settings, db, ntpTime, BlockchainUpdateTriggers.noop) - val poSSelector = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) + val settings = WavesSettings.fromRootConfig(sharedConfig) + val db = RDB.open(settings.dbSettings.copy(directory = "/tmp/tmp-db")) + val ntpTime = new NTP("ntp.pool.org") + val (blockchainUpdater, rdbWriter) = StorageFactory(settings, db, ntpTime, BlockchainUpdateTriggers.noop) + val poSSelector = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) try { val genesisBlock = @@ -51,6 +51,10 @@ object BaseTargetChecker { f"$address: ${timeDelay * 1e-3}%10.3f s" } - } finally ntpTime.close() + } finally { + ntpTime.close() + rdbWriter.close() + db.close() + } } } diff --git a/node-it/src/test/scala/com/wavesplatform/it/Docker.scala b/node-it/src/test/scala/com/wavesplatform/it/Docker.scala index dd78309747..a5c6761987 100644 --- a/node-it/src/test/scala/com/wavesplatform/it/Docker.scala +++ b/node-it/src/test/scala/com/wavesplatform/it/Docker.scala @@ -349,6 +349,7 @@ class Docker( client.startContainer(id) nodes.asScala.find(_.containerId == id).foreach { node => node.nodeInfo = getNodeInfo(node.containerId, node.settings) + Await.result(node.waitForStartup(), 3.minutes) } } diff --git a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala index 06f59f3d10..d66ce80616 100644 --- a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala +++ b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala @@ -1,6 +1,5 @@ package com.wavesplatform.test -import com.wavesplatform.{Exporter, checkGenesis, crypto} import com.wavesplatform.Exporter.IO import com.wavesplatform.account.KeyPair import com.wavesplatform.block.{Block, BlockHeader} @@ -17,30 +16,16 @@ import com.wavesplatform.settings.{DBSettings, WavesSettings} import com.wavesplatform.state.appender.BlockAppender import com.wavesplatform.test.BlockchainGenerator.{GenBlock, GenTx} import com.wavesplatform.transaction.TxValidationError.GenericError -import com.wavesplatform.transaction.assets.{ - BurnTransaction, - IssueTransaction, - ReissueTransaction, - SetAssetScriptTransaction, - SponsorFeeTransaction, - UpdateAssetInfoTransaction -} +import com.wavesplatform.transaction.assets.* import com.wavesplatform.transaction.assets.exchange.ExchangeTransaction import com.wavesplatform.transaction.lease.{LeaseCancelTransaction, LeaseTransaction} import com.wavesplatform.transaction.smart.{InvokeScriptTransaction, SetScriptTransaction} import com.wavesplatform.transaction.transfer.{MassTransferTransaction, TransferTransaction} -import com.wavesplatform.transaction.{ - CreateAliasTransaction, - DataTransaction, - EthTxGenerator, - EthereumTransaction, - PaymentTransaction, - Transaction, - TxHelpers -} +import com.wavesplatform.transaction.{CreateAliasTransaction, DataTransaction, EthTxGenerator, EthereumTransaction, PaymentTransaction, Transaction, TxHelpers} import com.wavesplatform.utils.{Schedulers, ScorexLogging, Time} import com.wavesplatform.utx.UtxPoolImpl import com.wavesplatform.wallet.Wallet +import com.wavesplatform.{Exporter, checkGenesis, crypto} import io.netty.channel.group.DefaultChannelGroup import monix.execution.Scheduler.Implicits.global import monix.reactive.subjects.ConcurrentSubject @@ -118,86 +103,87 @@ class BlockchainGenerator(wavesSettings: WavesSettings) extends ScorexLogging { var time: Long = startTime override def correctedTime(): Long = time - override def getTimestamp(): Long = time - } - Using.resource(RDB.open(dbSettings)) { db => - val (blockchain, _) = - StorageFactory(settings, db, time, BlockchainUpdateTriggers.noop) - Using.resource(new UtxPoolImpl(time, blockchain, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable)) { utxPool => - val pos = PoSSelector(blockchain, settings.synchronizationSettings.maxBaseTarget) - val extAppender = BlockAppender(blockchain, time, utxPool, pos, scheduler)(_, None) - val utxEvents = ConcurrentSubject.publish[UtxEvent] - - val miner = new MinerImpl( - new DefaultChannelGroup("", null), - blockchain, - settings, - time, - utxPool, - Wallet(settings.walletSettings), - PoSSelector(blockchain, None), - scheduler, - scheduler, - utxEvents.collect { case _: UtxEvent.TxAdded => - () - } - ) - checkGenesis(settings, blockchain, Miner.Disabled) - val result = genBlocks.foldLeft[Either[ValidationError, Unit]](Right(())) { - case (res @ Left(_), _) => res - case (_, genBlock) => - time.time = miner.nextBlockGenerationTime(blockchain, blockchain.height, blockchain.lastBlockHeader.get, genBlock.signer).explicitGet() - val correctedTimeTxs = genBlock.txs.map(correctTxTimestamp(_, time)) - - miner.forgeBlock(genBlock.signer) match { - case Right((block, _)) => - for { - blockWithTxs <- Block.buildAndSign( - block.header.version, - block.header.timestamp, - block.header.reference, - block.header.baseTarget, - block.header.generationSignature, - correctedTimeTxs, - genBlock.signer, - block.header.featureVotes, - block.header.rewardVote, - block.header.stateHash, - block.header.challengedHeader - ) - _ <- Await - .result(extAppender(blockWithTxs).runAsyncLogErr, Duration.Inf) - } yield exportToFile(blockWithTxs) - - case Left(err) => Left(GenericError(err)) - } + override def getTimestamp(): Long = time + } + Using.Manager { use => + val db = use(RDB.open(dbSettings)) + val (blockchain, rdbWriterRaw) = StorageFactory(settings, db, time, BlockchainUpdateTriggers.noop) + use(rdbWriterRaw) + val utxPool = use(new UtxPoolImpl(time, blockchain, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable)) + val pos = PoSSelector(blockchain, settings.synchronizationSettings.maxBaseTarget) + val extAppender = BlockAppender(blockchain, time, utxPool, pos, scheduler)(_, None) + val utxEvents = ConcurrentSubject.publish[UtxEvent] + + val miner = new MinerImpl( + new DefaultChannelGroup("", null), + blockchain, + settings, + time, + utxPool, + Wallet(settings.walletSettings), + PoSSelector(blockchain, None), + scheduler, + scheduler, + utxEvents.collect { case _: UtxEvent.TxAdded => + () } - result match { - case Right(_) => - if (blockchain.isFeatureActivated(BlockchainFeatures.NG) && blockchain.liquidBlockMeta.nonEmpty) { - val lastHeader = blockchain.lastBlockHeader.get.header - val pseudoBlock = Block( - BlockHeader( - blockchain.blockVersionAt(blockchain.height), - time.getTimestamp() + settings.blockchainSettings.genesisSettings.averageBlockDelay.toMillis, - blockchain.lastBlockId.get, - lastHeader.baseTarget, - lastHeader.generationSignature, - lastHeader.generator, - Nil, - 0, - ByteStr.empty, - None, - None - ), + ) + + checkGenesis(settings, blockchain, Miner.Disabled) + val result = genBlocks.foldLeft[Either[ValidationError, Unit]](Right(())) { + case (res@Left(_), _) => res + case (_, genBlock) => + time.time = miner.nextBlockGenerationTime(blockchain, blockchain.height, blockchain.lastBlockHeader.get, genBlock.signer).explicitGet() + val correctedTimeTxs = genBlock.txs.map(correctTxTimestamp(_, time)) + + miner.forgeBlock(genBlock.signer) match { + case Right((block, _)) => + for { + blockWithTxs <- Block.buildAndSign( + block.header.version, + block.header.timestamp, + block.header.reference, + block.header.baseTarget, + block.header.generationSignature, + correctedTimeTxs, + genBlock.signer, + block.header.featureVotes, + block.header.rewardVote, + block.header.stateHash, + block.header.challengedHeader + ) + _ <- Await + .result(extAppender(blockWithTxs).runAsyncLogErr, Duration.Inf) + } yield exportToFile(blockWithTxs) + + case Left(err) => Left(GenericError(err)) + } + } + result match { + case Right(_) => + if (blockchain.isFeatureActivated(BlockchainFeatures.NG) && blockchain.liquidBlockMeta.nonEmpty) { + val lastHeader = blockchain.lastBlockHeader.get.header + val pseudoBlock = Block( + BlockHeader( + blockchain.blockVersionAt(blockchain.height), + time.getTimestamp() + settings.blockchainSettings.genesisSettings.averageBlockDelay.toMillis, + blockchain.lastBlockId.get, + lastHeader.baseTarget, + lastHeader.generationSignature, + lastHeader.generator, + Nil, + 0, ByteStr.empty, - Nil - ) - blockchain.processBlock(pseudoBlock, ByteStr.empty, None, verify = false) - } - case Left(err) => log.error(s"Error appending block: $err") - } + None, + None + ), + ByteStr.empty, + Nil + ) + blockchain.processBlock(pseudoBlock, ByteStr.empty, None, verify = false) + } + case Left(err) => log.error(s"Error appending block: $err") } } } diff --git a/node/src/main/resources/application.conf b/node/src/main/resources/application.conf index e89a571e00..f642aa6f81 100644 --- a/node/src/main/resources/application.conf +++ b/node/src/main/resources/application.conf @@ -25,6 +25,21 @@ waves { max-rollback-depth = 2000 remember-blocks = 3h + # Delete old history entries (Data, WAVES and Asset balances) in this interval before a safe rollback height. + # Comment to disable. + # Affects: + # REST API: + # GET /addresses/balance/$addr/$confirmations - confirmations should be <= 1000 to get a guaranteed result. + # GET /addresses/balance + # POST /addresses/balance + # GET /assets/$asset/distribution/$height/limit/$limit + # GET /debug/balances/history/$addr + # GET /debug/stateWaves/$height + # Explorer tool: + # A WAVES balance history for address. + # AA Asset balance history for address. + # cleanup-interval = 500 # Optimal for Xmx2G + use-bloom-filter = false rocksdb { diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index a9804b9340..5d65421c70 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -516,6 +516,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con shutdownAndWait(appenderScheduler, "Appender", 5.minutes.some) log.info("Closing storage") + rocksDB.close() rdb.close() // extensions should be shut down last, after all node functionality, to guarantee no data loss diff --git a/node/src/main/scala/com/wavesplatform/Explorer.scala b/node/src/main/scala/com/wavesplatform/Explorer.scala index f034331b1e..96bae8692f 100644 --- a/node/src/main/scala/com/wavesplatform/Explorer.scala +++ b/node/src/main/scala/com/wavesplatform/Explorer.scala @@ -1,7 +1,7 @@ package com.wavesplatform import com.google.common.hash.{Funnels, BloomFilter as GBloomFilter} -import com.google.common.primitives.Longs +import com.google.common.primitives.{Ints, Longs, Shorts} import com.wavesplatform.account.Address import com.wavesplatform.api.common.{AddressPortfolio, CommonAccountsApi} import com.wavesplatform.common.state.ByteStr @@ -12,12 +12,11 @@ import com.wavesplatform.lang.script.ContractScript import com.wavesplatform.lang.script.v1.ExprScript import com.wavesplatform.settings.Constants import com.wavesplatform.state.diffs.{DiffsCommon, SetScriptTransactionDiff} -import com.wavesplatform.state.SnapshotBlockchain -import com.wavesplatform.state.{Blockchain, Height, Portfolio, StateSnapshot, TransactionId} +import com.wavesplatform.state.{Blockchain, Height, Portfolio, SnapshotBlockchain, StateSnapshot, TransactionId} import com.wavesplatform.transaction.Asset.IssuedAsset import com.wavesplatform.utils.ScorexLogging import monix.execution.{ExecutionModel, Scheduler} -import org.rocksdb.RocksDB +import org.rocksdb.{ReadOptions, RocksDB} import play.api.libs.json.Json import java.io.File @@ -29,7 +28,7 @@ import scala.collection.mutable import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} import scala.jdk.CollectionConverters.* -import scala.util.Using +import scala.util.{Try, Using} //noinspection ScalaStyle object Explorer extends ScorexLogging { @@ -69,7 +68,7 @@ object Explorer extends ScorexLogging { log.info(s"Data directory: ${settings.dbSettings.directory}") val rdb = RDB.open(settings.dbSettings) - val reader = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) + val reader = RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) val blockchainHeight = reader.height log.info(s"Blockchain height is $blockchainHeight") @@ -94,19 +93,27 @@ object Explorer extends ScorexLogging { flag match { case "WB" => - val balances = mutable.Map[BigInt, Long]() + var accountsBaseTotalBalance = 0L + var wavesBalanceRecords = 0 rdb.db.iterateOver(KeyTags.WavesBalance) { e => - val addressId = BigInt(e.getKey.drop(6)) - val balance = Longs.fromByteArray(e.getValue) - balances += (addressId -> balance) + val addressId = AddressId(Longs.fromByteArray(e.getKey.drop(Shorts.BYTES))) + val key = Keys.wavesBalance(addressId) + accountsBaseTotalBalance += key.parse(e.getValue).balance + wavesBalanceRecords += 1 } var actualTotalReward = 0L - rdb.db.iterateOver(KeyTags.BlockReward) { e => - actualTotalReward += Longs.fromByteArray(e.getValue) + var blocksRecords = 0 + rdb.db.iterateOver(KeyTags.BlockInfoAtHeight) { e => + val height = Height(Ints.fromByteArray(e.getKey.drop(Shorts.BYTES))) + val key = Keys.blockMetaAt(height) + actualTotalReward += key.parse(e.getValue).fold(0L)(_.reward) + blocksRecords += 1 } - val actualTotalBalance = balances.values.sum + reader.carryFee(None) + log.info(s"Found $wavesBalanceRecords waves balance records and $blocksRecords block records") + + val actualTotalBalance = accountsBaseTotalBalance + reader.carryFee(None) val expectedTotalBalance = Constants.UnitsInWave * Constants.TotalWaves + actualTotalReward val byKeyTotalBalance = reader.wavesAmount(blockchainHeight) @@ -223,7 +230,9 @@ object Explorer extends ScorexLogging { val result = new util.HashMap[Short, Stats] Seq(rdb.db.getDefaultColumnFamily, rdb.txHandle.handle, rdb.txSnapshotHandle.handle, rdb.txMetaHandle.handle).foreach { cf => - Using(rdb.db.newIterator(cf)) { iterator => + Using.Manager { use => + val ro = use(new ReadOptions().setTotalOrderSeek(true)) + val iterator = use(rdb.db.newIterator(cf, ro)) iterator.seekToFirst() while (iterator.isValid) { @@ -245,7 +254,7 @@ object Explorer extends ScorexLogging { log.info("key-space,entry-count,total-key-size,total-value-size") for ((prefix, stats) <- result.asScala) { - log.info(s"${KeyTags(prefix)},${stats.entryCount},${stats.totalKeySize},${stats.totalValueSize}") + log.info(s"${Try(KeyTags(prefix)).getOrElse(prefix.toString)},${stats.entryCount},${stats.totalKeySize},${stats.totalValueSize}") } case "TXBH" => @@ -363,10 +372,12 @@ object Explorer extends ScorexLogging { case "CTI" => log.info("Counting transaction IDs") var counter = 0 - Using(rdb.db.newIterator(rdb.txMetaHandle.handle)) { iter => + Using.Manager { use => + val ro = use(new ReadOptions().setTotalOrderSeek(true)) + val iter = use(rdb.db.newIterator(rdb.txMetaHandle.handle, ro)) iter.seekToFirst() -// iter.seek(KeyTags.TransactionMetaById.prefixBytes) - log.info(iter.key().mkString(",")) + // iter.seek(KeyTags.TransactionMetaById.prefixBytes) // Doesn't work, because of CappedPrefixExtractor(10) + while (iter.isValid && iter.key().startsWith(KeyTags.TransactionMetaById.prefixBytes)) { counter += 1 iter.next() @@ -380,6 +391,9 @@ object Explorer extends ScorexLogging { val meta = rdb.db.get(Keys.transactionMetaById(TransactionId(ByteStr.decodeBase58(id).get), rdb.txMetaHandle)) log.info(s"Meta: $meta") } - } finally rdb.close() + } finally { + reader.close() + rdb.close() + } } } diff --git a/node/src/main/scala/com/wavesplatform/Exporter.scala b/node/src/main/scala/com/wavesplatform/Exporter.scala index f001dff3c6..3675fdbe43 100644 --- a/node/src/main/scala/com/wavesplatform/Exporter.scala +++ b/node/src/main/scala/com/wavesplatform/Exporter.scala @@ -18,6 +18,7 @@ import org.rocksdb.{ColumnFamilyHandle, ReadOptions, RocksDB} import scopt.OParser import java.io.{BufferedOutputStream, File, FileOutputStream, OutputStream} +import scala.annotation.tailrec import scala.concurrent.Await import scala.concurrent.duration.* import scala.jdk.CollectionConverters.* @@ -45,9 +46,9 @@ object Exporter extends ScorexLogging { new NTP(settings.ntpServer), RDB.open(settings.dbSettings) ) { (time, rdb) => - val (blockchain, _) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.noop) - val blockchainHeight = blockchain.height - val height = Math.min(blockchainHeight, exportHeight.getOrElse(blockchainHeight)) + val (blockchain, rdbWriter) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.noop) + val blockchainHeight = blockchain.height + val height = Math.min(blockchainHeight, exportHeight.getOrElse(blockchainHeight)) log.info(s"Blockchain height is $blockchainHeight exporting to $height") val blocksOutputFilename = s"$blocksOutputFileNamePrefix-$height" log.info(s"Blocks output file: $blocksOutputFilename") @@ -66,8 +67,9 @@ object Exporter extends ScorexLogging { Using.resources( createOutputFile(blocksOutputFilename), - snapshotsOutputFilename.map(createOutputFile) - ) { case (blocksOutput, snapshotsOutput) => + snapshotsOutputFilename.map(createOutputFile), + rdbWriter + ) { case (blocksOutput, snapshotsOutput, _) => Using.resources(createBufferedOutputStream(blocksOutput, 10), snapshotsOutput.map(createBufferedOutputStream(_, 100))) { case (blocksStream, snapshotsStream) => var exportedBlocksBytes = 0L @@ -139,7 +141,8 @@ object Exporter extends ScorexLogging { ) } - def loadTxData[A](acc: Seq[A], height: Int, iterator: DataIterator[A], updateNextEntryF: (Int, A) => Unit): Seq[A] = { + @tailrec + private def loadTxData[A](acc: Seq[A], height: Int, iterator: DataIterator[A], updateNextEntryF: (Int, A) => Unit): Seq[A] = { if (iterator.hasNext) { val (h, txData) = iterator.next() if (h == height) { @@ -151,7 +154,8 @@ object Exporter extends ScorexLogging { } else acc.reverse } - override def computeNext(): (Int, Block, Seq[Array[Byte]]) = { + @tailrec + override final def computeNext(): (Int, Block, Seq[Array[Byte]]) = { if (blockMetaIterator.hasNext) { val (h, meta) = blockMetaIterator.next() if (h <= targetHeight) { @@ -172,8 +176,10 @@ object Exporter extends ScorexLogging { } } else Seq.empty createBlock(PBBlocks.vanilla(meta.getHeader), meta.signature.toByteStr, txs).toOption - .map(block => (h, block, snapshots)) - .getOrElse(computeNext()) + .map(block => (h, block, snapshots)) match { + case Some(r) => r + case None => computeNext() + } } else { closeResources() endOfData() @@ -204,7 +210,8 @@ object Exporter extends ScorexLogging { dbIterator.seek(prefixBytes) - override def computeNext(): (Int, A) = { + @tailrec + override final def computeNext(): (Int, A) = { if (dbIterator.isValid && dbIterator.key().startsWith(prefixBytes)) { val h = Ints.fromByteArray(heightFromKeyF(dbIterator.key())) if (h > 1) { diff --git a/node/src/main/scala/com/wavesplatform/Importer.scala b/node/src/main/scala/com/wavesplatform/Importer.scala index 818191ef53..7f20c716e2 100644 --- a/node/src/main/scala/com/wavesplatform/Importer.scala +++ b/node/src/main/scala/com/wavesplatform/Importer.scala @@ -347,7 +347,7 @@ object Importer extends ScorexLogging { val actorSystem = ActorSystem("wavesplatform-import") val rdb = RDB.open(settings.dbSettings) - val (blockchainUpdater, _) = + val (blockchainUpdater, rdbWriter) = StorageFactory(settings, rdb, time, BlockchainUpdateTriggers.combined(triggers)) val utxPool = new UtxPoolImpl(time, blockchainUpdater, settings.utxSettings, settings.maxTxErrorLogSize, settings.minerSettings.enable) val pos = PoSSelector(blockchainUpdater, settings.synchronizationSettings.maxBaseTarget) @@ -423,6 +423,7 @@ object Importer extends ScorexLogging { utxPool.close() blockchainUpdater.shutdown() + rdbWriter.close() rdb.close() } blocksInputStream.close() diff --git a/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala b/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala index 9c770ef2fa..812e38d430 100644 --- a/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala +++ b/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala @@ -6,10 +6,9 @@ import com.wavesplatform.account.{Address, Alias} import com.wavesplatform.api.common.AddressPortfolio.{assetBalanceIterator, nftIterator} import com.wavesplatform.api.common.lease.AddressLeaseInfo import com.wavesplatform.common.state.ByteStr -import com.wavesplatform.database.{DBExt, DBResource, KeyTags, Keys, RDB} +import com.wavesplatform.database.{AddressId, DBExt, DBResource, KeyTags, Keys, RDB} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.lang.ValidationError -import com.wavesplatform.protobuf.transaction.PBRecipients import com.wavesplatform.state.{AccountScriptInfo, AssetDescription, Blockchain, DataEntry, SnapshotBlockchain} import com.wavesplatform.transaction.Asset.IssuedAsset import monix.eval.Task @@ -113,11 +112,13 @@ object CommonAccountsApi { .fold(Array.empty[DataEntry[?]])(_.filter { case (k, _) => pattern.forall(_.matcher(k).matches()) }.values.toArray.sortBy(_.key)) rdb.db.resourceObservable.flatMap { dbResource => - Observable - .fromIterator( - Task(new AddressDataIterator(dbResource, address, entriesFromDiff, pattern).asScala) - ) - .filterNot(_.isEmpty) + dbResource.get(Keys.addressId(address)).fold(Observable.fromIterable(entriesFromDiff)) { addressId => + Observable + .fromIterator( + Task(new AddressDataIterator(dbResource, addressId, entriesFromDiff, pattern).asScala) + ) + .filterNot(_.isEmpty) + } } } @@ -132,11 +133,11 @@ object CommonAccountsApi { private class AddressDataIterator( db: DBResource, - address: Address, + addressId: AddressId, entriesFromDiff: Array[DataEntry[?]], pattern: Option[Pattern] ) extends AbstractIterator[DataEntry[?]] { - private val prefix: Array[Byte] = KeyTags.Data.prefixBytes ++ PBRecipients.publicKeyHash(address) + private val prefix: Array[Byte] = KeyTags.Data.prefixBytes ++ addressId.toByteArray private val length: Int = entriesFromDiff.length @@ -145,7 +146,7 @@ object CommonAccountsApi { private var nextIndex = 0 private var nextDbEntry: Option[DataEntry[?]] = None - private def matches(key: String): Boolean = pattern.forall(_.matcher(key).matches()) + private def matches(dataKey: String): Boolean = pattern.forall(_.matcher(dataKey).matches()) @tailrec private def doComputeNext(iter: RocksIterator): DataEntry[?] = @@ -177,10 +178,10 @@ object CommonAccountsApi { endOfData() } } else { - val key = new String(iter.key().drop(2 + Address.HashLength), Charsets.UTF_8) - if (matches(key)) { + val dataKey = new String(iter.key().drop(prefix.length), Charsets.UTF_8) + if (matches(dataKey)) { nextDbEntry = Option(iter.value()).map { arr => - Keys.data(address, key).parse(arr).entry + Keys.data(addressId, dataKey).parse(arr).entry } } iter.next() diff --git a/node/src/main/scala/com/wavesplatform/database/Caches.scala b/node/src/main/scala/com/wavesplatform/database/Caches.scala index a640144d9f..38a51d6f37 100644 --- a/node/src/main/scala/com/wavesplatform/database/Caches.scala +++ b/node/src/main/scala/com/wavesplatform/database/Caches.scala @@ -183,7 +183,7 @@ abstract class Caches extends Blockchain with Storage { protected def discardAccountData(addressWithKey: (Address, String)): Unit = accountDataCache.invalidate(addressWithKey) protected def loadAccountData(acc: Address, key: String): CurrentData - protected def loadEntryHeights(keys: Iterable[(Address, String)]): Map[(Address, String), Height] + protected def loadEntryHeights(keys: Iterable[(Address, String)], addressIdOf: Address => AddressId): Map[(Address, String), Height] private[database] def addressId(address: Address): Option[AddressId] = addressIdCache.get(address) private[database] def addressIds(addresses: Seq[Address]): Map[Address, Option[AddressId]] = @@ -299,8 +299,8 @@ abstract class Caches extends Blockchain with Storage { (key, entry) <- entries } yield ((address, key), entry) - val cachedEntries = accountDataCache.getAllPresent(newEntries.keys.asJava).asScala - val loadedPrevEntries = loadEntryHeights(newEntries.keys.filterNot(cachedEntries.contains)) + val cachedEntries = accountDataCache.getAllPresent(newEntries.keys.asJava).asScala + val loadedPrevEntries = loadEntryHeights(newEntries.keys.filterNot(cachedEntries.contains), addressIdWithFallback(_, newAddressIds)) val updatedDataWithNodes = (for { (k, currentEntry) <- cachedEntries.view.mapValues(_.height) ++ loadedPrevEntries diff --git a/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala b/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala index a0fadfa64a..74bd3fe903 100644 --- a/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala +++ b/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala @@ -1,9 +1,10 @@ package com.wavesplatform.database -import java.nio.ByteBuffer - import com.google.common.primitives.{Bytes, Ints, Longs, Shorts} -import com.wavesplatform.state.TxNum +import com.wavesplatform.state +import com.wavesplatform.state.{Height, TxNum} + +import java.nio.ByteBuffer object KeyHelpers { def h(height: Int): Array[Byte] = Ints.toByteArray(height) @@ -23,6 +24,14 @@ object KeyHelpers { def longKey(keyTag: KeyTags.KeyTag, default: Long = 0): Key[Long] = Key(keyTag, Array.emptyByteArray, v => if (v != null && v.length >= Longs.BYTES) Longs.fromByteArray(v) else default, Longs.toByteArray) + def heightKey(keyTag: KeyTags.KeyTag, default: Int = 0): Key[Height] = + Key( + keyTag, + Array.emptyByteArray, + v => state.Height @@ (if (v != null && v.length >= Ints.BYTES) Ints.fromByteArray(v) else default), + Ints.toByteArray + ) + def bytesSeqNr(keyTag: KeyTags.KeyTag, suffix: Array[Byte], default: Int = 0): Key[Int] = Key(keyTag, suffix, v => if (v != null && v.length >= Ints.BYTES) Ints.fromByteArray(v) else default, Ints.toByteArray) diff --git a/node/src/main/scala/com/wavesplatform/database/KeyTags.scala b/node/src/main/scala/com/wavesplatform/database/KeyTags.scala index 53d45434ef..b115499504 100644 --- a/node/src/main/scala/com/wavesplatform/database/KeyTags.scala +++ b/node/src/main/scala/com/wavesplatform/database/KeyTags.scala @@ -6,7 +6,6 @@ object KeyTags extends Enumeration { type KeyTag = Value val Version, Height, - Score, HeightOf, WavesBalance, WavesBalanceHistory, @@ -21,6 +20,7 @@ object KeyTags extends Enumeration { FilledVolumeAndFeeHistory, FilledVolumeAndFee, ChangedAddresses, + ChangedWavesBalances, ChangedAssetBalances, ChangedDataKeys, AddressIdOfAlias, @@ -39,21 +39,17 @@ object KeyTags extends Enumeration { AssetScriptHistory, AssetScript, SafeRollbackHeight, + LastCleanupHeight, BlockInfoAtHeight, NthTransactionInfoAtHeight, AddressTransactionSeqNr, AddressTransactionHeightTypeAndNums, TransactionMetaById, - BlockTransactionsFee, InvokeScriptResult, - BlockReward, - WavesAmount, - HitSource, DisabledAliases, AssetStaticInfo, NftCount, NftPossession, - BloomFilterChecksum, IssuedAssets, UpdatedAssets, SponsoredAssets, diff --git a/node/src/main/scala/com/wavesplatform/database/Keys.scala b/node/src/main/scala/com/wavesplatform/database/Keys.scala index 05660deeab..064366ace2 100644 --- a/node/src/main/scala/com/wavesplatform/database/Keys.scala +++ b/node/src/main/scala/com/wavesplatform/database/Keys.scala @@ -6,8 +6,6 @@ import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 import com.wavesplatform.database.protobuf.{EthereumTransactionMeta, StaticAssetInfo, TransactionMeta, BlockMeta as PBBlockMeta} import com.wavesplatform.protobuf.snapshot.TransactionStateSnapshot -import com.wavesplatform.protobuf.transaction.PBRecipients -import com.wavesplatform.state import com.wavesplatform.state.* import com.wavesplatform.transaction.Asset.IssuedAsset import com.wavesplatform.transaction.{ERC20Address, Transaction} @@ -21,6 +19,7 @@ object CurrentBalance { case class BalanceNode(balance: Long, prevHeight: Height) object BalanceNode { val Empty: BalanceNode = BalanceNode(0, Height(0)) + val SizeInBytes: Int = 12 } case class CurrentVolumeAndFee(volume: Long, fee: Long, height: Height, prevHeight: Height) @@ -57,9 +56,8 @@ object Keys { import KeyHelpers.* import KeyTags.{AddressId as AddressIdTag, EthereumTransactionMeta as EthereumTransactionMetaTag, InvokeScriptResult as InvokeScriptResultTag, LeaseDetails as LeaseDetailsTag, *} - val version: Key[Int] = intKey(Version, default = 1) - val height: Key[Height] = - Key(Height, Array.emptyByteArray, v => state.Height @@ (if (v != null && v.length >= Ints.BYTES) Ints.fromByteArray(v) else 0), Ints.toByteArray) + val version: Key[Int] = intKey(Version, default = 1) + val height: Key[Height] = heightKey(Height) def heightOf(blockId: ByteStr): Key[Option[Int]] = Key.opt[Int](HeightOf, blockId.arr, Ints.fromByteArray, Ints.toByteArray) @@ -103,9 +101,14 @@ object Keys { def changedAddresses(height: Int): Key[Seq[AddressId]] = Key(ChangedAddresses, h(height), readAddressIds, writeAddressIds) + def changedWavesBalances(height: Int): Key[Seq[AddressId]] = + Key(ChangedWavesBalances, h(height), readAddressIds, writeAddressIds) + def changedBalances(height: Int, asset: IssuedAsset): Key[Seq[AddressId]] = Key(ChangedAssetBalances, h(height) ++ asset.id.arr, readAddressIds, writeAddressIds) + def changedBalancesAtPrefix(height: Int): Array[Byte] = KeyTags.ChangedAssetBalances.prefixBytes ++ h(height) + def addressIdOfAlias(alias: Alias): Key[Option[AddressId]] = Key.opt(AddressIdOfAlias, alias.bytes, AddressId.fromByteArray, _.toByteArray) val lastAddressId: Key[Option[Long]] = Key.opt(LastAddressId, Array.emptyByteArray, Longs.fromByteArray, _.toByteArray) @@ -120,9 +123,8 @@ object Keys { val approvedFeatures: Key[Map[Short, Int]] = Key(ApprovedFeatures, Array.emptyByteArray, readFeatureMap, writeFeatureMap) val activatedFeatures: Key[Map[Short, Int]] = Key(ActivatedFeatures, Array.emptyByteArray, readFeatureMap, writeFeatureMap) - // public key hash is used here so it's possible to populate bloom filter by just scanning all the history keys - def data(address: Address, key: String): Key[CurrentData] = - Key(Data, PBRecipients.publicKeyHash(address) ++ key.utf8Bytes, readCurrentData(key), writeCurrentData) + def data(addressId: AddressId, key: String): Key[CurrentData] = + Key(Data, addressId.toByteArray ++ key.utf8Bytes, readCurrentData(key), writeCurrentData) def dataAt(addressId: AddressId, key: String)(height: Int): Key[DataNode] = Key(DataHistory, hBytes(addressId.toByteArray ++ key.utf8Bytes, height), readDataNode(key), writeDataNode) @@ -139,7 +141,8 @@ object Keys { def assetScriptPresent(asset: IssuedAsset)(height: Int): Key[Option[Unit]] = Key.opt(AssetScript, hBytes(asset.id.arr, height), _ => (), _ => Array[Byte]()) - val safeRollbackHeight: Key[Int] = intKey(SafeRollbackHeight) + val safeRollbackHeight: Key[Int] = intKey(SafeRollbackHeight) + val lastCleanupHeight: Key[Height] = heightKey(LastCleanupHeight) def changedDataKeys(height: Int, addressId: AddressId): Key[Seq[String]] = Key(ChangedDataKeys, hBytes(addressId.toByteArray, height), readStrings, writeStrings) @@ -204,14 +207,6 @@ object Keys { Some(cfh.handle) ) - def blockTransactionsFee(height: Int): Key[Long] = - Key( - BlockTransactionsFee, - h(height), - Longs.fromByteArray, - Longs.toByteArray - ) - def invokeScriptResult(height: Int, txNum: TxNum): Key[Option[InvokeScriptResult]] = Key.opt(InvokeScriptResultTag, hNum(height, txNum), InvokeScriptResult.fromBytes, InvokeScriptResult.toBytes) diff --git a/node/src/main/scala/com/wavesplatform/database/RDB.scala b/node/src/main/scala/com/wavesplatform/database/RDB.scala index 9f7501e63a..00a3e70077 100644 --- a/node/src/main/scala/com/wavesplatform/database/RDB.scala +++ b/node/src/main/scala/com/wavesplatform/database/RDB.scala @@ -40,8 +40,11 @@ object RDB extends StrictLogging { val dbDir = file.getAbsoluteFile dbDir.getParentFile.mkdirs() - val handles = new util.ArrayList[ColumnFamilyHandle]() - val defaultCfOptions = newColumnFamilyOptions(12.0, 16 << 10, settings.rocksdb.mainCacheSize, 0.6, settings.rocksdb.writeBufferSize) + val handles = new util.ArrayList[ColumnFamilyHandle]() + val defaultCfOptions = newColumnFamilyOptions(12.0, 16 << 10, settings.rocksdb.mainCacheSize, 0.6, settings.rocksdb.writeBufferSize) + val defaultCfCompressionForLevels = CompressionType.NO_COMPRESSION :: // Disable compaction for L0, because it is predictable and small + List.fill(defaultCfOptions.options.numLevels() - 1)(CompressionType.LZ4_COMPRESSION) + val txMetaCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txMetaCacheSize, 0.9, settings.rocksdb.writeBufferSize) val txCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txCacheSize, 0.9, settings.rocksdb.writeBufferSize) val txSnapshotCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txSnapshotCacheSize, 0.9, settings.rocksdb.writeBufferSize) @@ -52,12 +55,14 @@ object RDB extends StrictLogging { new ColumnFamilyDescriptor( RocksDB.DEFAULT_COLUMN_FAMILY, defaultCfOptions.options + .setMaxWriteBufferNumber(3) + .setCompressionPerLevel(defaultCfCompressionForLevels.asJava) .setCfPaths(Seq(new DbPath(new File(dbDir, "default").toPath, 0L)).asJava) ), new ColumnFamilyDescriptor( "tx-meta".utf8Bytes, txMetaCfOptions.options - .optimizeForPointLookup(16 << 20) + .optimizeForPointLookup(16 << 20) // Iterators might not work with this option .setDisableAutoCompactions(true) .setCfPaths(Seq(new DbPath(new File(dbDir, "tx-meta").toPath, 0L)).asJava) ), @@ -111,7 +116,11 @@ object RDB extends StrictLogging { .setDataBlockHashTableUtilRatio(0.5) ) .setWriteBufferSize(writeBufferSize) + .setCompactionStyle(CompactionStyle.LEVEL) .setLevelCompactionDynamicLevelBytes(true) + // Defines the prefix. + // Improves an iterator performance for keys with prefixes of 10 or more bytes. + // If specified key has less than 10 bytes: iterator finds the exact key for seek(key) and becomes invalid after next(). .useCappedPrefixExtractor(10) .setMemtablePrefixBloomSizeRatio(0.25) .setCompressionType(CompressionType.LZ4_COMPRESSION) @@ -124,11 +133,11 @@ object RDB extends StrictLogging { val dbOptions = new DBOptions() .setCreateIfMissing(true) .setParanoidChecks(true) - .setIncreaseParallelism(4) + .setIncreaseParallelism(6) .setBytesPerSync(2 << 20) - .setMaxBackgroundJobs(4) .setCreateMissingColumnFamilies(true) .setMaxOpenFiles(100) + .setMaxSubcompactions(2) // Write stalls expected without this option. Can lead to max_background_jobs * max_subcompactions background threads if (settings.rocksdb.enableStatistics) { val statistics = new Statistics() diff --git a/node/src/main/scala/com/wavesplatform/database/RW.scala b/node/src/main/scala/com/wavesplatform/database/RW.scala index aac07a5547..5fbe5ddf60 100644 --- a/node/src/main/scala/com/wavesplatform/database/RW.scala +++ b/node/src/main/scala/com/wavesplatform/database/RW.scala @@ -21,6 +21,13 @@ class RW(db: RocksDB, readOptions: ReadOptions, batch: WriteBatch) extends ReadO def delete[V](key: Key[V]): Unit = batch.delete(key.columnFamilyHandle.getOrElse(db.getDefaultColumnFamily), key.keyBytes) + def deleteRange[V](fromInclusive: Key[V], toExclusive: Key[V]): Unit = deleteRange(fromInclusive.keyBytes, toExclusive.keyBytes) + + // Deletes in range [from, to) + // Keep in mind, that bytes in Java are signed. + // So fromInclusive=[0, ...] removes all keys, but [Byte.MinValue, ...] can skip some keys. + def deleteRange(fromInclusive: Array[Byte], toExclusive: Array[Byte]): Unit = batch.deleteRange(fromInclusive, toExclusive) + def filterHistory(key: Key[Seq[Int]], heightToRemove: Int): Unit = { val newValue = get(key).filterNot(_ == heightToRemove) if (newValue.nonEmpty) put(key, newValue) diff --git a/node/src/main/scala/com/wavesplatform/database/ReadOnlyDB.scala b/node/src/main/scala/com/wavesplatform/database/ReadOnlyDB.scala index 09451bca78..7752069835 100644 --- a/node/src/main/scala/com/wavesplatform/database/ReadOnlyDB.scala +++ b/node/src/main/scala/com/wavesplatform/database/ReadOnlyDB.scala @@ -5,7 +5,6 @@ import com.wavesplatform.metrics.RocksDBStats import com.wavesplatform.metrics.RocksDBStats.DbHistogramExt import org.rocksdb.{ColumnFamilyHandle, ReadOptions, RocksDB, RocksIterator} -import scala.annotation.tailrec import scala.util.Using class ReadOnlyDB(db: RocksDB, readOptions: ReadOptions) { @@ -15,16 +14,16 @@ class ReadOnlyDB(db: RocksDB, readOptions: ReadOptions) { key.parse(bytes) } - def multiGetOpt[V](keys: IndexedSeq[Key[Option[V]]], valBufferSize: Int): Seq[Option[V]] = + def multiGetOpt[V](keys: collection.IndexedSeq[Key[Option[V]]], valBufferSize: Int): Seq[Option[V]] = db.multiGetOpt(readOptions, keys, valBufferSize) - def multiGet[V](keys: IndexedSeq[Key[V]], valBufferSize: Int): Seq[Option[V]] = + def multiGet[V](keys: collection.IndexedSeq[Key[V]], valBufferSize: Int): Seq[Option[V]] = db.multiGet(readOptions, keys, valBufferSize) - def multiGetOpt[V](keys: IndexedSeq[Key[Option[V]]], valBufSizes: IndexedSeq[Int]): Seq[Option[V]] = + def multiGetOpt[V](keys: collection.IndexedSeq[Key[Option[V]]], valBufSizes: collection.IndexedSeq[Int]): Seq[Option[V]] = db.multiGetOpt(readOptions, keys, valBufSizes) - def multiGetInts(keys: IndexedSeq[Key[Int]]): Seq[Option[Int]] = + def multiGetInts(keys: collection.IndexedSeq[Key[Int]]): Seq[Option[Int]] = db.multiGetInts(readOptions, keys) def has[V](key: Key[V]): Boolean = { @@ -35,24 +34,15 @@ class ReadOnlyDB(db: RocksDB, readOptions: ReadOptions) { def newIterator: RocksIterator = db.newIterator(readOptions.setTotalOrderSeek(true)) - def iterateOverPrefix(tag: KeyTags.KeyTag)(f: DBEntry => Unit): Unit = iterateOverPrefix(tag.prefixBytes)(f) - - def iterateOverPrefix(prefix: Array[Byte])(f: DBEntry => Unit): Unit = { - @tailrec - def loop(iter: RocksIterator): Unit = { - val key = iter.key() - if (iter.isValid) { - f(Maps.immutableEntry(key, iter.value())) - iter.next() - loop(iter) - } else () - } - - Using.resource(db.newIterator(readOptions.setTotalOrderSeek(false).setPrefixSameAsStart(true))) { iter => - iter.seek(prefix) - loop(iter) + def iterateOverWithSeek(prefix: Array[Byte], seek: Array[Byte], cfh: Option[ColumnFamilyHandle] = None)(f: DBEntry => Boolean): Unit = + Using.resource(db.newIterator(cfh.getOrElse(db.getDefaultColumnFamily), readOptions.setTotalOrderSeek(true))) { iter => + iter.seek(seek) + var continue = true + while (iter.isValid && iter.key().startsWith(prefix) && continue) { + continue = f(Maps.immutableEntry(iter.key(), iter.value())) + if (continue) iter.next() + } } - } def iterateOver(prefix: Array[Byte], cfh: Option[ColumnFamilyHandle] = None)(f: DBEntry => Unit): Unit = Using.resource(db.newIterator(cfh.getOrElse(db.getDefaultColumnFamily), readOptions.setTotalOrderSeek(true))) { iter => @@ -63,6 +53,10 @@ class ReadOnlyDB(db: RocksDB, readOptions: ReadOptions) { } } + /** Tries to find the exact key if prefix.length < 10. + * @see + * RDB.newColumnFamilyOptions + */ def prefixExists(prefix: Array[Byte]): Boolean = Using.resource(db.newIterator(readOptions.setTotalOrderSeek(false).setPrefixSameAsStart(true))) { iter => iter.seek(prefix) diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index caabfe52d5..aab88daa93 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -5,6 +5,7 @@ import com.google.common.cache.CacheBuilder import com.google.common.collect.MultimapBuilder import com.google.common.hash.{BloomFilter, Funnels} import com.google.common.primitives.Ints +import com.google.common.util.concurrent.MoreExecutors import com.wavesplatform.account.{Address, Alias} import com.wavesplatform.api.common.WavesBalanceIterator import com.wavesplatform.block.Block.BlockId @@ -31,14 +32,20 @@ import com.wavesplatform.transaction.lease.{LeaseCancelTransaction, LeaseTransac import com.wavesplatform.transaction.smart.{InvokeExpressionTransaction, InvokeScriptTransaction, SetScriptTransaction} import com.wavesplatform.transaction.transfer.* import com.wavesplatform.utils.{LoggerFacade, ScorexLogging} +import io.netty.util.concurrent.DefaultThreadFactory import org.rocksdb.{RocksDB, Status} import org.slf4j.LoggerFactory import sun.nio.ch.Util +import java.nio.ByteBuffer import java.util +import java.util.concurrent.* import scala.annotation.tailrec +import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters.* +import scala.util.Using +import scala.util.Using.Releasable import scala.util.control.NonFatal object RocksDBWriter extends ScorexLogging { @@ -100,6 +107,38 @@ object RocksDBWriter extends ScorexLogging { recMergeFixed(wbh.head, wbh.tail, lbh.head, lbh.tail, ArrayBuffer.empty).toSeq } + + private implicit val buffersReleaseable: Releasable[collection.IndexedSeq[ByteBuffer]] = _.foreach(Util.releaseTemporaryDirectBuffer) + + def apply( + rdb: RDB, + settings: BlockchainSettings, + dbSettings: DBSettings, + isLightMode: Boolean, + bfBlockInsertions: Int = 10000, + forceCleanupExecutorService: Option[ExecutorService] = None + ): RocksDBWriter = new RocksDBWriter( + rdb, + settings, + dbSettings, + isLightMode, + bfBlockInsertions, + dbSettings.cleanupInterval match { + case None => MoreExecutors.newDirectExecutorService() // We don't care if disabled + case Some(_) => + forceCleanupExecutorService.getOrElse { + new ThreadPoolExecutor( + 1, + 1, + 0, + TimeUnit.SECONDS, + new LinkedBlockingQueue[Runnable](1), // Only one task at time + new DefaultThreadFactory("rocksdb-cleanup", true), + { (_: Runnable, _: ThreadPoolExecutor) => /* Ignore new jobs, because TPE is busy, we will clean the data next time */ } + ) + } + } + ) } //noinspection UnstableApiUsage @@ -108,8 +147,10 @@ class RocksDBWriter( val settings: BlockchainSettings, val dbSettings: DBSettings, isLightMode: Boolean, - bfBlockInsertions: Int = 10000 -) extends Caches { + bfBlockInsertions: Int = 10000, + cleanupExecutorService: ExecutorService +) extends Caches + with AutoCloseable { import rdb.db as writableDB private[this] val log = LoggerFacade(LoggerFactory.getLogger(classOf[RocksDBWriter])) @@ -118,6 +159,12 @@ class RocksDBWriter( import RocksDBWriter.* + override def close(): Unit = { + cleanupExecutorService.shutdownNow() + if (!cleanupExecutorService.awaitTermination(20, TimeUnit.SECONDS)) + log.warn("Not enough time for a cleanup task, try to increase the limit") + } + private[database] def readOnly[A](f: ReadOnlyDB => A): A = writableDB.readOnly(f) private[this] def readWrite[A](f: RW => A): A = writableDB.readWrite(f) @@ -164,11 +211,13 @@ class RocksDBWriter( override def carryFee(refId: Option[ByteStr]): Long = writableDB.get(Keys.carryFee(height)) override protected def loadAccountData(address: Address, key: String): CurrentData = - writableDB.get(Keys.data(address, key)) + addressId(address).fold(CurrentData.empty(key)) { addressId => + writableDB.get(Keys.data(addressId, key)) + } - override protected def loadEntryHeights(keys: Iterable[(Address, String)]): Map[(Address, String), Height] = { - val keyBufs = database.getKeyBuffersFromKeys(keys.map { case (addr, k) => Keys.data(addr, k) }.toVector) - val valBufs = database.getValueBuffers(keys.size, 8) + override protected def loadEntryHeights(keys: Iterable[(Address, String)], addressIdOf: Address => AddressId): Map[(Address, String), Height] = { + val keyBufs = database.getKeyBuffersFromKeys(keys.map { case (addr, k) => Keys.data(addressIdOf(addr), k) }.toVector) + val valBufs = database.getValueBuffers(keys.size, 8) val valueBuf = new Array[Byte](8) val result = rdb.db @@ -176,12 +225,13 @@ class RocksDBWriter( .asScala .view .zip(keys) - .map { case (status, k@(_, key)) => + .map { case (status, k @ (_, key)) => if (status.status.getCode == Status.Code.Ok) { status.value.get(valueBuf) k -> readCurrentData(key)(valueBuf).height } else k -> Height(0) - }.toMap + } + .toMap keyBufs.foreach(Util.releaseTemporaryDirectBuffer) valBufs.foreach(Util.releaseTemporaryDirectBuffer) @@ -192,7 +242,7 @@ class RocksDBWriter( override def hasData(address: Address): Boolean = { writableDB.readOnly { ro => ro.get(Keys.addressId(address)).fold(false) { addressId => - ro.prefixExists(KeyTags.ChangedDataKeys.prefixBytes ++ addressId.toByteArray) + ro.prefixExists(KeyTags.Data.prefixBytes ++ addressId.toByteArray) } } } @@ -324,12 +374,14 @@ class RocksDBWriter( assetStatics: Map[IssuedAsset, (AssetStaticInfo, Int)], rw: RW ): Unit = { + var changedWavesBalances = List.empty[AddressId] val changedAssetBalances = MultimapBuilder.hashKeys().hashSetValues().build[IssuedAsset, java.lang.Long]() val updatedNftLists = MultimapBuilder.hashKeys().linkedHashSetValues().build[java.lang.Long, IssuedAsset]() for (((addressId, asset), (currentBalance, balanceNode)) <- balances) { asset match { case Waves => + changedWavesBalances = addressId :: changedWavesBalances rw.put(Keys.wavesBalance(addressId), currentBalance) rw.put(Keys.wavesBalanceAt(addressId, currentBalance.height), balanceNode) case a: IssuedAsset => @@ -355,6 +407,7 @@ class RocksDBWriter( } } + rw.put(Keys.changedWavesBalances(height), changedWavesBalances) changedAssetBalances.asMap().forEach { (asset, addresses) => rw.put(Keys.changedBalances(height, asset), addresses.asScala.map(id => AddressId(id.toLong)).toSeq) } @@ -367,7 +420,7 @@ class RocksDBWriter( val addressId = addressIdWithFallback(address, newAddresses) changedKeys.put(addressId, key) - val kdh = Keys.data(address, key) + val kdh = Keys.data(addressId, key) rw.put(kdh, currentData) rw.put(Keys.dataAt(addressId, key)(height), dataNode) } @@ -431,6 +484,9 @@ class RocksDBWriter( if (previousSafeRollbackHeight < newSafeRollbackHeight) { rw.put(Keys.safeRollbackHeight, newSafeRollbackHeight) + dbSettings.cleanupInterval.foreach { cleanupInterval => + runCleanupTask(newSafeRollbackHeight - 1, cleanupInterval) // -1 because we haven't appended this block + } } rw.put(Keys.blockMetaAt(Height(height)), Some(blockMeta)) @@ -679,6 +735,206 @@ class RocksDBWriter( log.trace(s"Finished persisting block ${blockMeta.id} at height $height") } + @volatile private var lastCleanupHeight = writableDB.get(Keys.lastCleanupHeight) + private def runCleanupTask(newLastSafeHeightForDeletion: Int, cleanupInterval: Int): Unit = + if (lastCleanupHeight + cleanupInterval < newLastSafeHeightForDeletion) { + cleanupExecutorService.submit(new Runnable { + override def run(): Unit = { + val firstDirtyHeight = Height(lastCleanupHeight + 1) + val toHeightExclusive = Height(firstDirtyHeight + cleanupInterval) + val startTs = System.nanoTime() + + rdb.db.withOptions { (ro, wo) => + rdb.db.readWriteWithOptions(ro, wo.setLowPri(true)) { rw => + batchCleanupWavesBalances( + fromInclusive = firstDirtyHeight, + toExclusive = toHeightExclusive, + rw = rw + ) + + batchCleanupAssetBalances( + fromInclusive = firstDirtyHeight, + toExclusive = toHeightExclusive, + rw = rw + ) + + batchCleanupAccountData( + fromInclusive = firstDirtyHeight, + toExclusive = toHeightExclusive, + rw = rw + ) + + lastCleanupHeight = Height(toHeightExclusive - 1) + rw.put(Keys.lastCleanupHeight, lastCleanupHeight) + } + } + + log.debug(s"Cleanup in [$firstDirtyHeight; $toHeightExclusive) took ${(System.nanoTime() - startTs) / 1_000_000}ms") + } + }) + } + + private def batchCleanupWavesBalances(fromInclusive: Height, toExclusive: Height, rw: RW): Unit = { + val lastUpdateAt = mutable.LongMap.empty[Height] + + val updateAt = new ArrayBuffer[(AddressId, Height)]() // AddressId -> First height of update in this range + val updateAtKeys = new ArrayBuffer[Key[BalanceNode]]() + + val changedKeyPrefix = KeyTags.ChangedWavesBalances.prefixBytes + val changedFromKey = Keys.changedWavesBalances(fromInclusive) // fromInclusive doesn't affect the parsing result + rw.iterateOverWithSeek(changedKeyPrefix, changedFromKey.keyBytes) { e => + val currHeight = Height(Ints.fromByteArray(e.getKey.drop(changedKeyPrefix.length))) + val continue = currHeight < toExclusive + if (continue) + changedFromKey.parse(e.getValue).foreach { addressId => + lastUpdateAt.updateWith(addressId) { orig => + if (orig.isEmpty) { + updateAt.addOne(addressId -> currHeight) + updateAtKeys.addOne(Keys.wavesBalanceAt(addressId, currHeight)) + } + Some(currHeight) + } + } + continue + } + + rw.multiGet(updateAtKeys, BalanceNode.SizeInBytes) + .view + .zip(updateAt) + .foreach { case (prevBalanceNode, (addressId, firstHeight)) => + // We have changes on: previous period = 1000, 1200, 1900, current period = 2000, 2500. + // Removed on a previous period: 1100, 1200. We need to remove on a current period: 1900, 2000. + // We doesn't know about 1900, so we should delete all keys from 1. + // But there is an issue in RocksDB: https://github.com/facebook/rocksdb/issues/11407 that leads to stopped writes. + // So we need to issue non-overlapping delete ranges and we have to read changes on 2000 to know 1900. + // Also note: memtable_max_range_deletions doesn't have any effect. + // TODO Use deleteRange(1, height) after RocksDB's team solves the overlapping deleteRange issue. + val firstDeleteHeight = prevBalanceNode.fold(firstHeight) { x => + if (x.prevHeight == 0) firstHeight // There is no previous record + else x.prevHeight + } + + val lastDeleteHeight = lastUpdateAt(addressId) + if (firstDeleteHeight != lastDeleteHeight) + rw.deleteRange( + Keys.wavesBalanceAt(addressId, firstDeleteHeight), + Keys.wavesBalanceAt(addressId, lastDeleteHeight) // Deletes exclusively + ) + } + + rw.deleteRange(Keys.changedWavesBalances(fromInclusive), Keys.changedWavesBalances(toExclusive)) + } + + private def batchCleanupAssetBalances(fromInclusive: Height, toExclusive: Height, rw: RW): Unit = { + val lastUpdateAt = mutable.AnyRefMap.empty[(AddressId, IssuedAsset), Height] + + val updateAt = new ArrayBuffer[(AddressId, IssuedAsset, Height)]() // First height of update in this range + val updateAtKeys = new ArrayBuffer[Key[BalanceNode]]() + + val changedKeyPrefix = KeyTags.ChangedAssetBalances.prefixBytes + val changedKey = Keys.changedBalances(Int.MaxValue, IssuedAsset(ByteStr.empty)) + rw.iterateOverWithSeek(changedKeyPrefix, Keys.changedBalancesAtPrefix(fromInclusive)) { e => + val currHeight = Height(Ints.fromByteArray(e.getKey.drop(changedKeyPrefix.length))) + val continue = currHeight < toExclusive + if (continue) { + val asset = IssuedAsset(ByteStr(e.getKey.takeRight(AssetIdLength))) + changedKey.parse(e.getValue).foreach { addressId => + lastUpdateAt.updateWith((addressId, asset)) { orig => + if (orig.isEmpty) { + updateAt.addOne((addressId, asset, currHeight)) + updateAtKeys.addOne(Keys.assetBalanceAt(addressId, asset, currHeight)) + } + Some(currHeight) + } + } + } + continue + } + + rw.multiGet(updateAtKeys, BalanceNode.SizeInBytes) + .view + .zip(updateAt) + .foreach { case (prevBalanceNode, (addressId, asset, firstHeight)) => + val firstDeleteHeight = prevBalanceNode.fold(firstHeight) { x => + if (x.prevHeight == 0) firstHeight + else x.prevHeight + } + + val lastDeleteHeight = lastUpdateAt((addressId, asset)) + if (firstDeleteHeight != lastDeleteHeight) + rw.deleteRange( + Keys.assetBalanceAt(addressId, asset, firstDeleteHeight), + Keys.assetBalanceAt(addressId, asset, lastDeleteHeight) + ) + } + + rw.deleteRange(Keys.changedBalancesAtPrefix(fromInclusive), Keys.changedBalancesAtPrefix(toExclusive)) + } + + private def batchCleanupAccountData(fromInclusive: Height, toExclusive: Height, rw: RW): Unit = { + val changedDataAddresses = mutable.Set.empty[AddressId] + val lastUpdateAt = mutable.AnyRefMap.empty[(AddressId, String), Height] + + val updateAt = new ArrayBuffer[(AddressId, String, Height)]() // First height of update in this range + val updateAtKeys = new ArrayBuffer[Key[DataNode]]() + + val changedAddressesPrefix = KeyTags.ChangedAddresses.prefixBytes + val changedAddressesFromKey = Keys.changedAddresses(fromInclusive) + rw.iterateOverWithSeek(changedAddressesPrefix, changedAddressesFromKey.keyBytes) { e => + val currHeight = Height(Ints.fromByteArray(e.getKey.drop(changedAddressesPrefix.length))) + val continue = currHeight < toExclusive + if (continue) + changedAddressesFromKey.parse(e.getValue).foreach { addressId => + val changedDataKeys = rw.get(Keys.changedDataKeys(currHeight, addressId)) + if (changedDataKeys.nonEmpty) { + changedDataAddresses.addOne(addressId) + changedDataKeys.foreach { accountDataKey => + lastUpdateAt.updateWith((addressId, accountDataKey)) { orig => + if (orig.isEmpty) { + updateAt.addOne((addressId, accountDataKey, currHeight)) + updateAtKeys.addOne(Keys.dataAt(addressId, accountDataKey)(currHeight)) + } + Some(currHeight) + } + } + } + } + + continue + } + + val valueBuff = new Array[Byte](Ints.BYTES) // height of DataNode + Using.resources( + database.getKeyBuffersFromKeys(updateAtKeys), + database.getValueBuffers(updateAtKeys.size, valueBuff.length) + ) { (keyBuffs, valBuffs) => + rdb.db + .multiGetByteBuffers(keyBuffs.asJava, valBuffs.asJava) + .asScala + .view + .zip(updateAt) + .foreach { case (status, (addressId, accountDataKey, firstHeight)) => + val firstDeleteHeight = if (status.status.getCode == Status.Code.Ok) { + status.value.get(valueBuff) + val r = readDataNode(accountDataKey)(valueBuff).prevHeight + if (r == 0) firstHeight else r + } else firstHeight + + val lastDeleteHeight = lastUpdateAt((addressId, accountDataKey)) + if (firstDeleteHeight != lastDeleteHeight) + rw.deleteRange( + Keys.dataAt(addressId, accountDataKey)(firstDeleteHeight), + Keys.dataAt(addressId, accountDataKey)(lastDeleteHeight) + ) + } + } + + rw.deleteRange(Keys.changedAddresses(fromInclusive), Keys.changedAddresses(toExclusive)) + changedDataAddresses.foreach { addressId => + rw.deleteRange(Keys.changedDataKeys(fromInclusive, addressId), Keys.changedDataKeys(toExclusive, addressId)) + } + } + override protected def doRollback(targetHeight: Int): DiscardedBlocks = { val targetBlockId = readOnly(_.get(Keys.blockMetaAt(Height @@ targetHeight))) .map(_.id) @@ -709,8 +965,8 @@ class RocksDBWriter( addressId <- rw.get(Keys.changedAddresses(currentHeight)) } yield addressId -> rw.get(Keys.idToAddress(addressId)) - rw.iterateOver(KeyTags.ChangedAssetBalances.prefixBytes ++ Ints.toByteArray(currentHeight)) { e => - val assetId = IssuedAsset(ByteStr(e.getKey.takeRight(32))) + rw.iterateOver(KeyTags.ChangedAssetBalances.prefixBytes ++ KeyHelpers.h(currentHeight)) { e => + val assetId = IssuedAsset(ByteStr(e.getKey.takeRight(AssetIdLength))) for ((addressId, address) <- changedAddresses) { balancesToInvalidate += address -> assetId rollbackBalanceHistory(rw, Keys.assetBalance(addressId, assetId), Keys.assetBalanceAt(addressId, assetId, _), currentHeight) @@ -723,7 +979,7 @@ class RocksDBWriter( accountDataToInvalidate += (address -> k) rw.delete(Keys.dataAt(addressId, k)(currentHeight)) - rollbackDataHistory(rw, Keys.data(address, k), Keys.dataAt(addressId, k)(_), currentHeight) + rollbackDataHistory(rw, Keys.data(addressId, k), Keys.dataAt(addressId, k)(_), currentHeight) } rw.delete(Keys.changedDataKeys(currentHeight, addressId)) @@ -827,11 +1083,11 @@ class RocksDBWriter( rw.delete(Keys.blockMetaAt(currentHeight)) rw.delete(Keys.changedAddresses(currentHeight)) + rw.delete(Keys.changedWavesBalances(currentHeight)) rw.delete(Keys.heightOf(discardedMeta.id)) blockHeightsToInvalidate.addOne(discardedMeta.id) rw.delete(Keys.carryFee(currentHeight)) rw.delete(Keys.blockStateHash(currentHeight)) - rw.delete(Keys.blockTransactionsFee(currentHeight)) rw.delete(Keys.stateHash(currentHeight)) if (DisableHijackedAliases.height == currentHeight) { @@ -881,7 +1137,10 @@ class RocksDBWriter( if (currentData.height == currentHeight) { val prevDataNode = rw.get(dataNodeKey(currentData.prevHeight)) rw.delete(dataNodeKey(currentHeight)) - rw.put(currentDataKey, CurrentData(prevDataNode.entry, currentData.prevHeight, prevDataNode.prevHeight)) + prevDataNode.entry match { + case _: EmptyDataEntry => rw.delete(currentDataKey) + case _ => rw.put(currentDataKey, CurrentData(prevDataNode.entry, currentData.prevHeight, prevDataNode.prevHeight)) + } } } @@ -960,7 +1219,7 @@ class RocksDBWriter( .get(Keys.transactionAt(Height(tm.height), TxNum(tm.num.toShort), rdb.txHandle)) .collect { case (tm, t: TransferTransaction) if tm.status == TxMeta.Status.Succeeded => t - case (m, e @ EthereumTransaction(transfer: Transfer, _, _, _)) if tm.status == PBStatus.SUCCEEDED => + case (_, e @ EthereumTransaction(transfer: Transfer, _, _, _)) if tm.status == PBStatus.SUCCEEDED => val asset = transfer.tokenAddress.fold[Asset](Waves)(resolveERC20Address(_).get) e.toTransferLike(TxPositiveAmount.unsafeFrom(transfer.amount), transfer.recipient, asset) } @@ -971,10 +1230,13 @@ class RocksDBWriter( override def transactionInfos(ids: Seq[ByteStr]): Seq[Option[(TxMeta, Transaction)]] = readOnly { db => val tms = db.multiGetOpt(ids.view.map(id => Keys.transactionMetaById(TransactionId(id), rdb.txMetaHandle)).toVector, 36) - val (keys, sizes) = tms.view.map { - case Some(tm) => Keys.transactionAt(Height(tm.height), TxNum(tm.num.toShort), rdb.txHandle) -> tm.size - case None => Keys.transactionAt(Height(0), TxNum(0.toShort), rdb.txHandle) -> 0 - }.toVector.unzip + val (keys, sizes) = tms.view + .map { + case Some(tm) => Keys.transactionAt(Height(tm.height), TxNum(tm.num.toShort), rdb.txHandle) -> tm.size + case None => Keys.transactionAt(Height(0), TxNum(0.toShort), rdb.txHandle) -> 0 + } + .toVector + .unzip db.multiGetOpt(keys, sizes) } diff --git a/node/src/main/scala/com/wavesplatform/database/package.scala b/node/src/main/scala/com/wavesplatform/database/package.scala index 1b8dbe76f4..b147da9fa5 100644 --- a/node/src/main/scala/com/wavesplatform/database/package.scala +++ b/node/src/main/scala/com/wavesplatform/database/package.scala @@ -37,7 +37,6 @@ import scala.annotation.tailrec import scala.collection.mutable.ArrayBuffer import scala.collection.{View, mutable} import scala.jdk.CollectionConverters.* -import scala.util.Using //noinspection UnstableApiUsage package object database { @@ -309,13 +308,15 @@ package object database { } private def readDataEntry(key: String)(bs: Array[Byte]): DataEntry[?] = - if (bs == null || bs.length == 0) EmptyDataEntry(key) else pb.DataEntry.parseFrom(bs).value match { - case Value.Empty => EmptyDataEntry(key) - case Value.IntValue(value) => IntegerDataEntry(key, value) - case Value.BoolValue(value) => BooleanDataEntry(key, value) - case Value.BinaryValue(value) => BinaryDataEntry(key, value.toByteStr) - case Value.StringValue(value) => StringDataEntry(key, value) - } + if (bs == null || bs.length == 0) EmptyDataEntry(key) + else + pb.DataEntry.parseFrom(bs).value match { + case Value.Empty => EmptyDataEntry(key) + case Value.IntValue(value) => IntegerDataEntry(key, value) + case Value.BoolValue(value) => BooleanDataEntry(key, value) + case Value.BinaryValue(value) => BinaryDataEntry(key, value.toByteStr) + case Value.StringValue(value) => StringDataEntry(key, value) + } private def writeDataEntry(e: DataEntry[?]): Array[Byte] = pb.DataEntry(e match { @@ -351,7 +352,7 @@ package object database { def writeCurrentBalance(balance: CurrentBalance): Array[Byte] = Longs.toByteArray(balance.balance) ++ Ints.toByteArray(balance.height) ++ Ints.toByteArray(balance.prevHeight) - def readBalanceNode(bs: Array[Byte]): BalanceNode = if (bs != null && bs.length == 12) + def readBalanceNode(bs: Array[Byte]): BalanceNode = if (bs != null && bs.length == BalanceNode.SizeInBytes) BalanceNode(Longs.fromByteArray(bs.take(8)), Height(Ints.fromByteArray(bs.takeRight(4)))) else BalanceNode.Empty @@ -390,39 +391,54 @@ package object database { implicit class DBExt(val db: RocksDB) extends AnyVal { - def readOnly[A](f: ReadOnlyDB => A): A = { - Using.resource(db.getSnapshot) { s => - Using.resource(new ReadOptions().setSnapshot(s).setVerifyChecksums(false)) { ro => - f(new ReadOnlyDB(db, ro)) - } - }(db.releaseSnapshot(_)) - } + def readOnly[A](f: ReadOnlyDB => A): A = withReadOptions { ro => f(new ReadOnlyDB(db, ro)) } /** @note * Runs operations in batch, so keep in mind, that previous changes don't appear lately in f */ - def readWrite[A](f: RW => A): A = { - val snapshot = db.getSnapshot - val readOptions = new ReadOptions().setSnapshot(snapshot).setVerifyChecksums(false) - val batch = new WriteBatch() - val rw = new RW(db, readOptions, batch) - val writeOptions = new WriteOptions() + def readWrite[A](f: RW => A): A = withOptions { (ro, wo) => readWriteWithOptions(ro, wo)(f) } + + def readWriteWithOptions[A](readOptions: ReadOptions, writeOptions: WriteOptions)(f: RW => A): A = { + val batch = new WriteBatch() + val rw = new RW(db, readOptions, batch) try { val r = f(rw) db.write(writeOptions, batch) r - } finally { - readOptions.close() - writeOptions.close() - batch.close() + } finally batch.close() + } + + def withOptions[A](f: (ReadOptions, WriteOptions) => A): A = + withReadOptions { ro => + withWriteOptions { wo => + f(ro, wo) + } + } + + def withWriteOptions[A](f: WriteOptions => A): A = { + val wo = new WriteOptions() + try f(wo) + finally wo.close() + } + + def withReadOptions[A](f: ReadOptions => A): A = { + val snapshot = db.getSnapshot + val ro = new ReadOptions().setSnapshot(snapshot).setVerifyChecksums(false) + try f(ro) + finally { + ro.close() db.releaseSnapshot(snapshot) } } - def multiGetOpt[A](readOptions: ReadOptions, keys: IndexedSeq[Key[Option[A]]], valBufSize: Int): Seq[Option[A]] = + def multiGetOpt[A](readOptions: ReadOptions, keys: collection.IndexedSeq[Key[Option[A]]], valBufSize: Int): Seq[Option[A]] = multiGetOpt(readOptions, keys, getKeyBuffersFromKeys(keys), getValueBuffers(keys.size, valBufSize)) - def multiGetOpt[A](readOptions: ReadOptions, keys: IndexedSeq[Key[Option[A]]], valBufSizes: IndexedSeq[Int]): Seq[Option[A]] = + def multiGetOpt[A]( + readOptions: ReadOptions, + keys: collection.IndexedSeq[Key[Option[A]]], + valBufSizes: collection.IndexedSeq[Int] + ): Seq[Option[A]] = multiGetOpt(readOptions, keys, getKeyBuffersFromKeys(keys), getValueBuffers(valBufSizes)) def multiGet[A](readOptions: ReadOptions, keys: ArrayBuffer[Key[A]], valBufSizes: ArrayBuffer[Int]): View[A] = @@ -431,7 +447,7 @@ package object database { def multiGet[A](readOptions: ReadOptions, keys: ArrayBuffer[Key[A]], valBufSize: Int): View[A] = multiGet(readOptions, keys, getKeyBuffersFromKeys(keys), getValueBuffers(keys.size, valBufSize)) - def multiGet[A](readOptions: ReadOptions, keys: IndexedSeq[Key[A]], valBufSize: Int): Seq[Option[A]] = { + def multiGet[A](readOptions: ReadOptions, keys: collection.IndexedSeq[Key[A]], valBufSize: Int): Seq[Option[A]] = { val keyBufs = getKeyBuffersFromKeys(keys) val valBufs = getValueBuffers(keys.size, valBufSize) @@ -452,7 +468,7 @@ package object database { result } - def multiGetInts(readOptions: ReadOptions, keys: IndexedSeq[Key[Int]]): Seq[Option[Int]] = { + def multiGetInts(readOptions: ReadOptions, keys: collection.IndexedSeq[Key[Int]]): Seq[Option[Int]] = { val keyBytes = keys.map(_.keyBytes) val keyBufs = getKeyBuffers(keyBytes) val valBufs = getValueBuffers(keyBytes.size, 4) diff --git a/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala b/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala index 6ee437a27b..3c3ece9bf7 100644 --- a/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala +++ b/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala @@ -19,7 +19,7 @@ object StorageFactory extends ScorexLogging { miner: Miner = _ => () ): (BlockchainUpdaterImpl, RocksDBWriter) = { checkVersion(rdb.db) - val rocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) + val rocksDBWriter = RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode) val bui = new BlockchainUpdaterImpl( rocksDBWriter, settings, diff --git a/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala b/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala index 7123a53e86..511bd6861d 100644 --- a/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala +++ b/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala @@ -9,6 +9,7 @@ case class DBSettings( storeStateHashes: Boolean, maxCacheSize: Int, maxRollbackDepth: Int, + cleanupInterval: Option[Int] = None, rememberBlocks: FiniteDuration, useBloomFilter: Boolean, rocksdb: RocksDBSettings diff --git a/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala b/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala index 99a2174e78..fe592d1246 100644 --- a/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala +++ b/node/src/main/scala/com/wavesplatform/utils/generator/BlockchainGeneratorApp.scala @@ -118,11 +118,12 @@ object BlockchainGeneratorApp extends ScorexLogging { val blockchain = { val rdb = RDB.open(wavesSettings.dbSettings) - val (blockchainUpdater, rocksdb) = + val (blockchainUpdater, rdbWriter) = StorageFactory(wavesSettings, rdb, fakeTime, BlockchainUpdateTriggers.noop) com.wavesplatform.checkGenesis(wavesSettings, blockchainUpdater, Miner.Disabled) sys.addShutdownHook(synchronized { blockchainUpdater.shutdown() + rdbWriter.close() rdb.close() }) blockchainUpdater diff --git a/node/src/main/scala/com/wavesplatform/utils/generator/MinerChallengeSimulator.scala b/node/src/main/scala/com/wavesplatform/utils/generator/MinerChallengeSimulator.scala index ef816ab0a8..aede189071 100644 --- a/node/src/main/scala/com/wavesplatform/utils/generator/MinerChallengeSimulator.scala +++ b/node/src/main/scala/com/wavesplatform/utils/generator/MinerChallengeSimulator.scala @@ -186,7 +186,7 @@ object MinerChallengeSimulator { val dbSettings = wavesSettings.dbSettings.copy(directory = correctBlockchainDbDir) val fixedWavesSettings = wavesSettings.copy(dbSettings = dbSettings) val rdb = RDB.open(dbSettings) - val rocksDBWriter = new RocksDBWriter(rdb, fixedWavesSettings.blockchainSettings, fixedWavesSettings.dbSettings, false) + val rocksDBWriter = RocksDBWriter(rdb, fixedWavesSettings.blockchainSettings, fixedWavesSettings.dbSettings, false) val fakeTime = createFakeTime(rocksDBWriter.lastBlockTimestamp.get) val blockchainUpdater = new BlockchainUpdaterImpl( rocksDBWriter, diff --git a/node/src/test/resources/application.conf b/node/src/test/resources/application.conf index 3ce8fc25d9..1604353306 100644 --- a/node/src/test/resources/application.conf +++ b/node/src/test/resources/application.conf @@ -3,11 +3,15 @@ waves { utx-synchronizer.network-tx-cache-size = 20 wallet.password = "some string as password" - db.rocksdb { - main-cache-size = 1K - tx-cache-size = 1K - tx-meta-cache-size = 1K - tx-snapshot-cache-size = 1K - write-buffer-size = 1M + db { + cleanup-interval = null # Disable in tests by default + + rocksdb { + main-cache-size = 1K + tx-cache-size = 1K + tx-meta-cache-size = 1K + tx-snapshot-cache-size = 1K + write-buffer-size = 1M + } } } diff --git a/node/src/test/scala/com/wavesplatform/api/common/CommonAccountApiSpec.scala b/node/src/test/scala/com/wavesplatform/api/common/CommonAccountApiSpec.scala index 6676e7792b..411c2be103 100644 --- a/node/src/test/scala/com/wavesplatform/api/common/CommonAccountApiSpec.scala +++ b/node/src/test/scala/com/wavesplatform/api/common/CommonAccountApiSpec.scala @@ -35,10 +35,7 @@ class CommonAccountApiSpec extends FreeSpec with WithDomain with BlocksTransacti val data5 = data(acc, Seq(EmptyDataEntry("test2"), entry1, entry2), version = V2) withDomain(RideV4) { d => - val commonAccountsApi = - CommonAccountsApi(() => d.blockchainUpdater.snapshotBlockchain, d.rdb, d.blockchainUpdater) - def dataList(): Set[DataEntry[?]] = - commonAccountsApi.dataStream(acc.toAddress, None).toListL.runSyncUnsafe().toSet + def dataList(): Set[DataEntry[?]] = d.accountsApi.dataStream(acc.toAddress, None).toListL.runSyncUnsafe().toSet d.appendBlock(genesis) d.appendMicroBlock(data1) @@ -73,12 +70,7 @@ class CommonAccountApiSpec extends FreeSpec with WithDomain with BlocksTransacti forAll(preconditions) { case (acc, block1, mb1, block2, mb2) => withDomain(domainSettingsWithFS(TestFunctionalitySettings.withFeatures(BlockchainFeatures.NG, BlockchainFeatures.DataTransaction))) { d => - val commonAccountsApi = CommonAccountsApi( - () => d.blockchainUpdater.snapshotBlockchain, - d.rdb, - d.blockchainUpdater - ) - def dataList(): Set[DataEntry[?]] = commonAccountsApi.dataStream(acc.toAddress, Some("test_.*")).toListL.runSyncUnsafe().toSet + def dataList(): Set[DataEntry[?]] = d.accountsApi.dataStream(acc.toAddress, Some("test_.*")).toListL.runSyncUnsafe().toSet d.appendBlock(block1) dataList() shouldBe empty @@ -151,7 +143,6 @@ class CommonAccountApiSpec extends FreeSpec with WithDomain with BlocksTransacti invoke ) - val api = CommonAccountsApi(() => d.blockchain.snapshotBlockchain, d.rdb, d.blockchain) val leaseId = Lease.calculateId( Lease( Recipient.Address(ByteStr(TxHelpers.defaultAddress.bytes)), @@ -160,7 +151,7 @@ class CommonAccountApiSpec extends FreeSpec with WithDomain with BlocksTransacti ), invoke.id() ) - api.leaseInfo(leaseId) shouldBe Some( + d.accountsApi.leaseInfo(leaseId) shouldBe Some( LeaseInfo(leaseId, invoke.id(), TxHelpers.secondAddress, TxHelpers.defaultAddress, 1, 2, LeaseInfo.Status.Active) ) } diff --git a/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala b/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala index 543e8faa19..f3dc9a5dfa 100644 --- a/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala +++ b/node/src/test/scala/com/wavesplatform/consensus/FPPoSSelectorTest.scala @@ -295,6 +295,7 @@ class FPPoSSelectorTest extends FreeSpec with WithNewDBForEachTest with DBCacheS bcu.shutdown() } finally { bcu.shutdown() + defaultWriter.close() rdb.close() TestHelpers.deleteRecursively(path) } diff --git a/node/src/test/scala/com/wavesplatform/database/RocksDBWriterSpec.scala b/node/src/test/scala/com/wavesplatform/database/RocksDBWriterSpec.scala index 331d470521..64b96624e8 100644 --- a/node/src/test/scala/com/wavesplatform/database/RocksDBWriterSpec.scala +++ b/node/src/test/scala/com/wavesplatform/database/RocksDBWriterSpec.scala @@ -1,10 +1,15 @@ package com.wavesplatform.database -import com.wavesplatform.account.KeyPair +import com.google.common.primitives.{Ints, Longs, Shorts} +import com.wavesplatform.TestValues +import com.wavesplatform.account.{Address, KeyPair} +import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.EitherExt2 +import com.wavesplatform.database.KeyTags.KeyTag import com.wavesplatform.db.WithDomain import com.wavesplatform.db.WithState.AddrWithBalance import com.wavesplatform.features.BlockchainFeatures +import com.wavesplatform.history.Domain import com.wavesplatform.lang.directives.values.{V2, V5} import com.wavesplatform.lang.v1.compiler.Terms.CONST_BOOLEAN import com.wavesplatform.lang.v1.compiler.TestCompiler @@ -12,9 +17,12 @@ import com.wavesplatform.settings.{GenesisTransactionSettings, WavesSettings} import com.wavesplatform.state.TxMeta.Status import com.wavesplatform.test.* import com.wavesplatform.test.DomainPresets.* -import com.wavesplatform.transaction.TxHelpers import com.wavesplatform.transaction.TxValidationError.AliasDoesNotExist import com.wavesplatform.transaction.smart.SetScriptTransaction +import com.wavesplatform.transaction.{TxHelpers, TxPositiveAmount} +import org.rocksdb.{ReadOptions, RocksIterator} + +import scala.util.{Random, Using} class RocksDBWriterSpec extends FreeSpec with WithDomain { "Slice" - { @@ -152,4 +160,226 @@ class RocksDBWriterSpec extends FreeSpec with WithDomain { // check if caches are updated after rollback d.blockchain.resolveAlias(createAlias.alias) shouldEqual Left(AliasDoesNotExist(createAlias.alias)) } + + "cleanup" - { + val settings = { + val s = DomainPresets.RideV6 + s.copy(dbSettings = s.dbSettings.copy(maxRollbackDepth = 4, cleanupInterval = Some(4))) + } + + val alice = TxHelpers.signer(1) + val aliceAddress = alice.toAddress + + val bob = TxHelpers.signer(2) + val bobAddress = bob.toAddress + + val carl = TxHelpers.signer(3) + val carlAddress = carl.toAddress + + val userAddresses = Seq(aliceAddress, bobAddress, carlAddress) + val minerAddresses = Seq(TxHelpers.defaultAddress) + val allAddresses = userAddresses ++ minerAddresses + + def transferWavesTx = TxHelpers.massTransfer( + to = Seq( + aliceAddress -> 100.waves, + bobAddress -> 100.waves + ), + fee = 1.waves + ) + + val issueTx = TxHelpers.issue(issuer = alice, amount = 100) + def transferAssetTx = TxHelpers.transfer(from = alice, to = carlAddress, asset = issueTx.asset, amount = 1) + + val dataKey = "test" + val dataTxFee = TxPositiveAmount.unsafeFrom(TestValues.fee) + def dataTx = TxHelpers.dataSingle(account = bob, key = dataKey, value = Random.nextInt().toString, fee = dataTxFee.value) + + "doesn't delete if disabled" in withDomain( + settings.copy(dbSettings = settings.dbSettings.copy(cleanupInterval = None)), + Seq(AddrWithBalance(TxHelpers.defaultSigner.toAddress)) + ) { d => + d.appendBlock(transferWavesTx, issueTx, transferAssetTx, dataTx) + (3 to 10).foreach(_ => d.appendBlock()) + d.blockchain.height shouldBe 10 + + d.rdb.db.get(Keys.lastCleanupHeight) shouldBe 0 + withClue("All data exists: ") { + checkHistoricalDataOnlySinceHeight(d, allAddresses, 1) + } + } + + "doesn't delete sole data" in withDomain(settings, Seq(AddrWithBalance(TxHelpers.defaultSigner.toAddress))) { d => + d.appendBlock(transferWavesTx, issueTx, transferAssetTx, dataTx) // Last user data + d.blockchain.height shouldBe 2 + + (3 to 11).foreach(_ => d.appendBlock()) + d.blockchain.height shouldBe 11 + + d.rdb.db.get(Keys.lastCleanupHeight) shouldBe 4 + withClue("No data before: ") { + checkHistoricalDataOnlySinceHeight(d, userAddresses, 2) + checkHistoricalDataOnlySinceHeight(d, minerAddresses, 4) // Updated on each height + } + } + + "deletes old data and doesn't delete recent data" in withDomain(settings, Seq(AddrWithBalance(TxHelpers.defaultSigner.toAddress))) { d => + d.appendBlock(transferWavesTx, issueTx, transferAssetTx, dataTx) + + d.appendBlock() + + d.appendBlock( + transferWavesTx, + transferAssetTx, + dataTx + ) // Last user data + d.blockchain.height shouldBe 4 + + (5 to 11).foreach(_ => d.appendBlock()) + d.blockchain.height shouldBe 11 + + d.rdb.db.get(Keys.lastCleanupHeight) shouldBe 4 + withClue("No data before: ") { + checkHistoricalDataOnlySinceHeight(d, allAddresses, 4) + } + } + + "deletes old data from previous intervals" in withDomain(settings, Seq(AddrWithBalance(TxHelpers.defaultSigner.toAddress))) { d => + (2 to 3).foreach(_ => d.appendBlock()) + + d.appendBlock( + transferWavesTx, + issueTx, + transferAssetTx, + dataTx + ) + d.blockchain.height shouldBe 4 + + d.appendBlock() + + d.appendBlock( + transferWavesTx, + transferAssetTx, + dataTx + ) // Last user data + d.blockchain.height shouldBe 6 + + (7 to 15).foreach(_ => d.appendBlock()) + d.blockchain.height shouldBe 15 + + d.rdb.db.get(Keys.lastCleanupHeight) shouldBe 8 + withClue("No data before: ") { + checkHistoricalDataOnlySinceHeight(d, userAddresses, 6) + checkHistoricalDataOnlySinceHeight(d, minerAddresses, 8) // Updated on each height + } + } + + "doesn't affect other sequences" in { + def appendBlocks(d: Domain): Unit = { + (2 to 3).foreach(_ => d.appendBlock()) + d.appendBlock(transferWavesTx, issueTx, transferAssetTx, dataTx) + + d.appendBlock() + d.appendBlock(transferWavesTx, transferAssetTx, dataTx) + + (7 to 14).foreach(_ => d.appendBlock()) + } + + var nonHistoricalKeysWithoutCleanup: CollectedKeys = Vector.empty + withDomain( + settings.copy(dbSettings = settings.dbSettings.copy(cleanupInterval = None)), + Seq(AddrWithBalance(TxHelpers.defaultSigner.toAddress)) + ) { d => + appendBlocks(d) + nonHistoricalKeysWithoutCleanup = collectNonHistoricalKeys(d) + } + + withDomain(settings, Seq(AddrWithBalance(TxHelpers.defaultSigner.toAddress))) { d => + appendBlocks(d) + val nonHistoricalKeys = collectNonHistoricalKeys(d) + nonHistoricalKeys should contain theSameElementsInOrderAs nonHistoricalKeysWithoutCleanup + } + } + } + + private val HistoricalKeyTags = Seq( + KeyTags.ChangedAssetBalances, + KeyTags.ChangedWavesBalances, + KeyTags.WavesBalanceHistory, + KeyTags.AssetBalanceHistory, + KeyTags.ChangedDataKeys, + KeyTags.DataHistory, + KeyTags.ChangedAddresses + ) + + private type CollectedKeys = Vector[(ByteStr, String)] + private def collectNonHistoricalKeys(d: Domain): CollectedKeys = { + var xs: CollectedKeys = Vector.empty + withGlobalIterator(d.rdb) { iter => + iter.seekToFirst() + while (iter.isValid) { + val k = iter.key() + if ( + !(HistoricalKeyTags.exists(kt => k.startsWith(kt.prefixBytes)) || k + .startsWith(KeyTags.HeightOf.prefixBytes) || k.startsWith(KeyTags.LastCleanupHeight.prefixBytes)) + ) { + val description = KeyTags(Shorts.fromByteArray(k)).toString + xs = xs.appended(ByteStr(k) -> description) + } + iter.next() + } + } + xs + } + + private def checkHistoricalDataOnlySinceHeight(d: Domain, addresses: Seq[Address], sinceHeight: Int): Unit = { + val addressIds = addresses.map(getAddressId(d, _)) + HistoricalKeyTags.foreach { keyTag => + withClue(s"$keyTag:") { + d.rdb.db.iterateOver(keyTag) { e => + val (affectedHeight, affectedAddressIds) = getHeightAndAddressIds(keyTag, e) + if (affectedAddressIds.exists(addressIds.contains)) { + withClue(s"$addresses: ") { + affectedHeight should be >= sinceHeight + } + } + } + } + } + } + + private def getHeightAndAddressIds(tag: KeyTag, bytes: DBEntry): (Int, Seq[AddressId]) = { + val (heightBytes, addresses) = tag match { + case KeyTags.ChangedAddresses | KeyTags.ChangedAssetBalances | KeyTags.ChangedWavesBalances => + ( + bytes.getKey.drop(Shorts.BYTES), + readAddressIds(bytes.getValue) + ) + + case KeyTags.WavesBalanceHistory | KeyTags.AssetBalanceHistory | KeyTags.ChangedDataKeys => + ( + bytes.getKey.takeRight(Ints.BYTES), + Seq(AddressId.fromByteArray(bytes.getKey.dropRight(Ints.BYTES).takeRight(Longs.BYTES))) + ) + + case KeyTags.DataHistory => + ( + bytes.getKey.takeRight(Ints.BYTES), + Seq(AddressId.fromByteArray(bytes.getKey.drop(Shorts.BYTES))) + ) + + case _ => throw new IllegalArgumentException(s"$tag") + } + + (Ints.fromByteArray(heightBytes), addresses) + } + + private def getAddressId(d: Domain, address: Address): AddressId = + d.rdb.db.get(Keys.addressId(address)).getOrElse(throw new RuntimeException(s"Can't find address id for $address")) + + private def withGlobalIterator(rdb: RDB)(f: RocksIterator => Unit): Unit = { + Using(new ReadOptions().setTotalOrderSeek(true)) { ro => + Using(rdb.db.newIterator(ro))(f).get + }.get + } } diff --git a/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala b/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala index 4666c0a79a..5bc42e0dfb 100644 --- a/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala +++ b/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala @@ -1,5 +1,6 @@ package com.wavesplatform.database +import com.google.common.util.concurrent.MoreExecutors import com.wavesplatform.events.BlockchainUpdateTriggers import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state.BlockchainUpdaterImpl @@ -12,7 +13,14 @@ object TestStorageFactory { time: Time, blockchainUpdateTriggers: BlockchainUpdateTriggers ): (BlockchainUpdaterImpl, RocksDBWriter) = { - val rocksDBWriter: RocksDBWriter = new RocksDBWriter(rdb, settings.blockchainSettings, settings.dbSettings, settings.enableLightMode, 100) + val rocksDBWriter: RocksDBWriter = RocksDBWriter( + rdb, + settings.blockchainSettings, + settings.dbSettings, + settings.enableLightMode, + 100, + Some(MoreExecutors.newDirectExecutorService()) + ) ( new BlockchainUpdaterImpl(rocksDBWriter, settings, time, blockchainUpdateTriggers, loadActiveLeases(rdb, _, _)), rocksDBWriter diff --git a/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala b/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala index e1d08d1d7a..520c0db373 100644 --- a/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala +++ b/node/src/test/scala/com/wavesplatform/db/ScriptCacheTest.scala @@ -147,9 +147,9 @@ class ScriptCacheTest extends FreeSpec with WithNewDBForEachTest { } f(accounts, bcu) - bcu.shutdown() } finally { bcu.shutdown() + defaultWriter.close() } } } diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index 229d1c8e0f..0544679434 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -34,6 +34,7 @@ import org.scalatest.{BeforeAndAfterAll, Suite} import java.nio.file.Files import scala.concurrent.duration.* +import scala.util.Using trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers with NTPTime { _: Suite => protected val ignoreBlockchainUpdateTriggers: BlockchainUpdateTriggers = BlockchainUpdateTriggers.noop @@ -69,7 +70,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit ntpTime, ignoreBlockchainUpdateTriggers ) - test(rdw) + Using.resource(rdw)(test) } finally { Seq(rdb.db.getDefaultColumnFamily, rdb.txHandle.handle, rdb.txMetaHandle.handle).foreach { cfh => rdb.db.deleteRange(cfh, MinKey, MaxKey) @@ -91,7 +92,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit ntpTime, ignoreBlockchainUpdateTriggers ) - test(bcu, rdw) + Using.resource(rdw)(test(bcu, _)) } finally { Seq(rdb.db.getDefaultColumnFamily, rdb.txHandle.handle, rdb.txMetaHandle.handle).foreach { cfh => rdb.db.deleteRange(cfh, MinKey, MaxKey) diff --git a/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala b/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala index bdf68e8dce..a38fc58367 100644 --- a/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala +++ b/node/src/test/scala/com/wavesplatform/mining/BlockWithMaxBaseTargetTest.scala @@ -166,6 +166,7 @@ class BlockWithMaxBaseTargetTest extends FreeSpec with WithNewDBForEachTest with schedulerService.shutdown() utxPoolStub.close() bcu.shutdown() + defaultWriter.close() } } } diff --git a/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala b/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala index e9a62a4cf0..f3b489571f 100644 --- a/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala +++ b/node/src/test/scala/com/wavesplatform/mining/MiningWithRewardSuite.scala @@ -1,10 +1,7 @@ package com.wavesplatform.mining -import scala.concurrent.Future -import scala.concurrent.duration.* import cats.effect.Resource import com.typesafe.config.ConfigFactory -import com.wavesplatform.{TransactionGen, WithNewDBForEachTest} import com.wavesplatform.account.KeyPair import com.wavesplatform.block.Block import com.wavesplatform.common.state.ByteStr @@ -15,13 +12,14 @@ import com.wavesplatform.db.DBCacheSettings import com.wavesplatform.features.{BlockchainFeature, BlockchainFeatures} import com.wavesplatform.lagonaki.mocks.TestBlock import com.wavesplatform.settings.* -import com.wavesplatform.state.{Blockchain, BlockchainUpdaterImpl, NG} import com.wavesplatform.state.diffs.ENOUGH_AMT -import com.wavesplatform.transaction.{BlockchainUpdater, GenesisTransaction, Transaction} +import com.wavesplatform.state.{Blockchain, BlockchainUpdaterImpl, NG} import com.wavesplatform.transaction.Asset.Waves import com.wavesplatform.transaction.transfer.TransferTransaction +import com.wavesplatform.transaction.{BlockchainUpdater, GenesisTransaction, Transaction} import com.wavesplatform.utx.UtxPoolImpl import com.wavesplatform.wallet.Wallet +import com.wavesplatform.{TransactionGen, WithNewDBForEachTest} import io.netty.channel.group.DefaultChannelGroup import io.netty.util.concurrent.GlobalEventExecutor import monix.eval.Task @@ -32,6 +30,9 @@ import org.scalatest.compatible.Assertion import org.scalatest.flatspec.AsyncFlatSpec import org.scalatest.matchers.should.Matchers +import scala.concurrent.Future +import scala.concurrent.duration.* + class MiningWithRewardSuite extends AsyncFlatSpec with Matchers with WithNewDBForEachTest with TransactionGen with DBCacheSettings { import MiningWithRewardSuite.* @@ -142,14 +143,17 @@ class MiningWithRewardSuite extends AsyncFlatSpec with Matchers with WithNewDBFo private def forgeBlock(miner: MinerImpl)(account: KeyPair): Either[String, (Block, MiningConstraint)] = miner.forgeBlock(account) private def resources(settings: WavesSettings): Resource[Task, (BlockchainUpdaterImpl, RDB)] = - Resource.make { - val (bcu, _) = TestStorageFactory(settings, db, ntpTime, ignoreBlockchainUpdateTriggers) - Task.now((bcu, db)) - } { case (blockchainUpdater, _) => - Task { - blockchainUpdater.shutdown() + Resource + .make { + val (bcu, rdbWriter) = TestStorageFactory(settings, db, ntpTime, ignoreBlockchainUpdateTriggers) + Task.now((bcu, rdbWriter, db)) + } { case (blockchainUpdater, rdbWriter, _) => + Task { + blockchainUpdater.shutdown() + rdbWriter.close() + } } - } + .map { case (blockchainUpdater, _, db) => (blockchainUpdater, db) } } object MiningWithRewardSuite { diff --git a/node/src/test/scala/com/wavesplatform/test/SharedDomain.scala b/node/src/test/scala/com/wavesplatform/test/SharedDomain.scala index cedd61b6de..fb2d69aed5 100644 --- a/node/src/test/scala/com/wavesplatform/test/SharedDomain.scala +++ b/node/src/test/scala/com/wavesplatform/test/SharedDomain.scala @@ -32,8 +32,9 @@ trait SharedDomain extends BeforeAndAfterAll with NTPTime with DBCacheSettings { override protected def afterAll(): Unit = { super.afterAll() - rdb.close() bui.shutdown() + ldb.close() + rdb.close() TestHelpers.deleteRecursively(path) } } diff --git a/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala b/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala index 9adaaa720d..aa056d23da 100644 --- a/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala +++ b/node/src/test/scala/com/wavesplatform/utx/UtxPoolSpecification.scala @@ -57,6 +57,7 @@ private object UtxPoolSpecification { val writer: RocksDBWriter = TestRocksDB.withFunctionalitySettings(rdb, fs) override def close(): Unit = { + writer.close() rdb.close() TestHelpers.deleteRecursively(path) } @@ -91,17 +92,19 @@ class UtxPoolSpecification extends FreeSpec with MockFactory with BlocksTransact ) Using.resource(TempDB(settings.blockchainSettings.functionalitySettings, settings.dbSettings)) { dbContext => - val (bcu, _) = TestStorageFactory(settings, dbContext.rdb, new TestTime, ignoreBlockchainUpdateTriggers) - bcu.processBlock( - Block - .genesis( - genesisSettings, - bcu.isFeatureActivated(BlockchainFeatures.RideV6), - bcu.isFeatureActivated(BlockchainFeatures.LightNode) - ) - .explicitGet() - ) should beRight - test(bcu) + val (bcu, rdbWriter) = TestStorageFactory(settings, dbContext.rdb, new TestTime, ignoreBlockchainUpdateTriggers) + Using.resource(rdbWriter) { _ => + bcu.processBlock( + Block + .genesis( + genesisSettings, + bcu.isFeatureActivated(BlockchainFeatures.RideV6), + bcu.isFeatureActivated(BlockchainFeatures.LightNode) + ) + .explicitGet() + ) should beRight + test(bcu) + } } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 3fd849c9c0..ced4ae8a3f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -73,6 +73,7 @@ object Dependencies { lazy val it = scalaTest +: Seq( logback, + "com.github.jnr" % "jnr-unixsocket" % "0.38.21", // To support Apple ARM "com.spotify" % "docker-client" % "8.16.0", "com.fasterxml.jackson.dataformat" % "jackson-dataformat-properties" % "2.16.0", asyncHttpClient @@ -97,7 +98,7 @@ object Dependencies { akkaModule("slf4j") % Runtime ) - private val rocksdb = "org.rocksdb" % "rocksdbjni" % "8.8.1" + private val rocksdb = "org.rocksdb" % "rocksdbjni" % "8.9.1" private val scalapbJson = "com.thesamet.scalapb" %% "scalapb-json4s" % "0.12.1" diff --git a/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/DBResource.scala b/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/DBResource.scala index 46f67e252c..6c3eca3c06 100644 --- a/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/DBResource.scala +++ b/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/DBResource.scala @@ -35,6 +35,11 @@ object DBResource { def multiGet[A](keys: ArrayBuffer[Key[A]], valBufferSize: Int): View[A] = db.multiGet(readOptions, keys, valBufferSize) + /** + * Finds the exact key for iter.seek(key) if key.length < 10 and becomes invalid on iter.next(). + * Works as intended if prefix(key).length >= 10. + * @see RDB.newColumnFamilyOptions + */ override lazy val prefixIterator: RocksIterator = db.newIterator(readOptions.setTotalOrderSeek(false).setPrefixSameAsStart(true)) override lazy val fullIterator: RocksIterator = db.newIterator(readOptions.setTotalOrderSeek(true)) From b9d603b718d95be4cbdd33f913b17e634599b334 Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Tue, 16 Jan 2024 15:37:55 +0400 Subject: [PATCH 4/8] Bumped dependencies (#3933) --- build.sbt | 2 +- node/build.sbt | 3 ++- project/Dependencies.scala | 27 +++++++++++++-------------- project/build.properties | 2 +- project/plugins.sbt | 8 ++++---- ride-runner/build.sbt | 3 ++- 6 files changed, 23 insertions(+), 22 deletions(-) diff --git a/build.sbt b/build.sbt index 7f4f492311..fbe44e4f3a 100644 --- a/build.sbt +++ b/build.sbt @@ -213,7 +213,7 @@ checkPRRaw := Def (`repl-jvm` / Test / test).value (`lang-js` / Compile / fastOptJS).value (`lang-tests-js` / Test / test).value - (`grpc-server` / Test / test).value +// (`grpc-server` / Test / test).value (node / Test / test).value (`repl-js` / Compile / fastOptJS).value (`node-it` / Test / compile).value diff --git a/node/build.sbt b/node/build.sbt index 334c0ba084..7c9e0642b4 100644 --- a/node/build.sbt +++ b/node/build.sbt @@ -41,7 +41,8 @@ inTask(assembly)( case p if p.endsWith(".proto") || p.endsWith("module-info.class") || - p.endsWith("io.netty.versions.properties") => + p.endsWith("io.netty.versions.properties") || + p.endsWith(".kotlin_module") => MergeStrategy.discard case "scala-collection-compat.properties" => diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ced4ae8a3f..8cce403945 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,7 +6,7 @@ import scalapb.compiler.Version.scalapbVersion object Dependencies { // Node protobuf schemas private[this] val protoSchemasLib = - "com.wavesplatform" % "protobuf-schemas" % "1.5.2-86-SNAPSHOT" classifier "protobuf-src" intransitive () + "com.wavesplatform" % "protobuf-schemas" % "1.5.2" classifier "protobuf-src" intransitive () private def akkaModule(module: String) = "com.typesafe.akka" %% s"akka-$module" % "2.6.21" @@ -20,17 +20,19 @@ object Dependencies { def monixModule(module: String): Def.Initialize[ModuleID] = Def.setting("io.monix" %%% s"monix-$module" % "3.4.1") + private def grpcModule(module: String) = "io.grpc" % module % "1.61.0" + val kindProjector = compilerPlugin("org.typelevel" % "kind-projector" % "0.13.2" cross CrossVersion.full) val akkaHttp = akkaHttpModule("akka-http") - val googleGuava = "com.google.guava" % "guava" % "32.1.3-jre" + val googleGuava = "com.google.guava" % "guava" % "33.0.0-jre" val kamonCore = kamonModule("core") val machinist = "org.typelevel" %% "machinist" % "0.6.8" val logback = "ch.qos.logback" % "logback-classic" % "1.4.14" val janino = "org.codehaus.janino" % "janino" % "3.1.11" val asyncHttpClient = "org.asynchttpclient" % "async-http-client" % "2.12.3" val curve25519 = "com.wavesplatform" % "curve25519-java" % "0.6.6" - val nettyHandler = "io.netty" % "netty-handler" % "4.1.101.Final" + val nettyHandler = "io.netty" % "netty-handler" % "4.1.104.Final" val shapeless = Def.setting("com.chuusai" %%% "shapeless" % "2.3.10") @@ -75,7 +77,7 @@ object Dependencies { logback, "com.github.jnr" % "jnr-unixsocket" % "0.38.21", // To support Apple ARM "com.spotify" % "docker-client" % "8.16.0", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-properties" % "2.16.0", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-properties" % "2.16.1", asyncHttpClient ).map(_ % Test) @@ -89,7 +91,7 @@ object Dependencies { lazy val qaseReportDeps = Seq( playJson, - ("io.qase" % "qase-api" % "3.1.1").excludeAll(ExclusionRule(organization = "javax.ws.rs")) + ("io.qase" % "qase-api" % "3.2.0").excludeAll(ExclusionRule(organization = "javax.ws.rs")) ).map(_ % Test) lazy val logDeps = Seq( @@ -100,8 +102,6 @@ object Dependencies { private val rocksdb = "org.rocksdb" % "rocksdbjni" % "8.9.1" - private val scalapbJson = "com.thesamet.scalapb" %% "scalapb-json4s" % "0.12.1" - lazy val node = Def.setting( Seq( rocksdb, @@ -115,7 +115,7 @@ object Dependencies { kamonModule("influxdb"), kamonModule("akka-http"), kamonModule("executors"), - "org.influxdb" % "influxdb-java" % "2.23", + "org.influxdb" % "influxdb-java" % "2.24", googleGuava, "com.google.code.findbugs" % "jsr305" % "3.0.2" % Compile, // javax.annotation stubs playJson, @@ -128,7 +128,7 @@ object Dependencies { nettyHandler, "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", "eu.timepit" %% "refined" % "0.11.0" exclude ("org.scala-lang.modules", "scala-xml_2.13"), - "com.esaulpaugh" % "headlong" % "10.0.1", + "com.esaulpaugh" % "headlong" % "10.0.2", "com.github.jbellis" % "jamm" % "0.4.0", // Weighing caches web3jModule("abi"), akkaModule("testkit") % Test, @@ -136,7 +136,7 @@ object Dependencies { ) ++ test ++ console ++ logDeps ++ protobuf.value ++ langCompilerPlugins.value ) - val gProto = "com.google.protobuf" % "protobuf-java" % "3.25.1" + val gProto = "com.google.protobuf" % "protobuf-java" % "3.25.2" lazy val scalapbRuntime = Def.setting( Seq( @@ -152,8 +152,8 @@ object Dependencies { } lazy val grpc: Seq[ModuleID] = Seq( - "io.grpc" % "grpc-netty" % scalapb.compiler.Version.grpcJavaVersion, - "io.grpc" % "grpc-services" % scalapb.compiler.Version.grpcJavaVersion, + grpcModule("grpc-netty"), + grpcModule("grpc-services"), "com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapbVersion, protoSchemasLib % "protobuf" ) @@ -161,7 +161,6 @@ object Dependencies { lazy val rideRunner = Def.setting( Seq( rocksdb, - scalapbJson, // https://github.com/netty/netty/wiki/Native-transports // "io.netty" % "netty-transport-native-epoll" % "4.1.79.Final" classifier "linux-x86_64", "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8", @@ -174,7 +173,7 @@ object Dependencies { akkaHttpModule("akka-http-testkit") % Test, "com.softwaremill.diffx" %% "diffx-core" % "0.9.0" % Test, "com.softwaremill.diffx" %% "diffx-scalatest-should" % "0.9.0" % Test, - "io.grpc" % "grpc-inprocess" % "1.60.0" % Test + grpcModule("grpc-inprocess") % Test ) ++ Dependencies.console ++ Dependencies.logDeps ++ Dependencies.test ) diff --git a/project/build.properties b/project/build.properties index 27430827bc..abbbce5da4 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.6 +sbt.version=1.9.8 diff --git a/project/plugins.sbt b/project/plugins.sbt index 0d0b022093..7cb18557cc 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -12,18 +12,18 @@ Seq( "com.eed3si9n" % "sbt-assembly" % "2.1.5", "com.github.sbt" % "sbt-native-packager" % "1.9.16", "se.marcuslonnberg" % "sbt-docker" % "1.11.0", - "org.scala-js" % "sbt-scalajs" % "1.14.0", + "org.scala-js" % "sbt-scalajs" % "1.15.0", "org.portable-scala" % "sbt-scalajs-crossproject" % "1.3.2", - "pl.project13.scala" % "sbt-jmh" % "0.4.6", + "pl.project13.scala" % "sbt-jmh" % "0.4.7", "com.github.sbt" % "sbt-ci-release" % "1.5.12", "com.lightbend.sbt" % "sbt-javaagent" % "0.1.6" ).map(addSbtPlugin) libraryDependencies ++= Seq( - "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.16.0", + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.16.1", "org.hjson" % "hjson" % "3.1.0", "org.vafer" % "jdeb" % "1.10" artifacts Artifact("jdeb", "jar", "jar"), - "org.slf4j" % "jcl-over-slf4j" % "2.0.9", + "org.slf4j" % "jcl-over-slf4j" % "2.0.11", ("com.spotify" % "docker-client" % "8.16.0") .exclude("commons-logging", "commons-logging") ) diff --git a/ride-runner/build.sbt b/ride-runner/build.sbt index 6b7d11a732..7c739a2f5f 100644 --- a/ride-runner/build.sbt +++ b/ride-runner/build.sbt @@ -94,7 +94,8 @@ inTask(assembly)( case p if p.endsWith(".proto") || p.endsWith("module-info.class") || - p.endsWith("io.netty.versions.properties") => + p.endsWith("io.netty.versions.properties") || + p.endsWith(".kotlin_module") => MergeStrategy.discard case "scala-collection-compat.properties" => From 993c7c2fa8ec7dc139e1ad22e1d4987f17eb7ab8 Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Tue, 16 Jan 2024 17:34:14 +0400 Subject: [PATCH 5/8] Version 1.5.2 (Mainnet + Testnet + Stagenet) (#3936) --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index be45222151..01efb98af8 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -git.baseVersion := "1.5.1" +git.baseVersion := "1.5.2" From c94bf22b6f2ab269a6dddc3a63e232aea79cd269 Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Mon, 5 Feb 2024 14:21:58 +0400 Subject: [PATCH 6/8] Stability improvements (#3938) --- .github/workflows/publish-docker-node.yaml | 2 +- build.sbt | 13 +- .../events/BlockchainUpdates.scala | 7 +- .../com/wavesplatform/crypto/Curve25519.scala | 7 +- .../src/test/resources/logback-test.xml | 14 -- .../wavesplatform/report/QaseReporter.scala | 11 +- .../tests/src/test/resources/logback-test.xml | 14 -- node-it/build.sbt | 4 +- .../test/BlockchainGenerator.scala | 8 +- node/src/main/resources/application.conf | 6 +- .../scala/com/wavesplatform/Explorer.scala | 86 ++++++++++ .../wavesplatform/GenesisBlockGenerator.scala | 2 +- .../scala/com/wavesplatform/Importer.scala | 15 +- .../api/common/AddressTransactions.scala | 69 ++++---- .../api/common/CommonAccountsApi.scala | 2 +- .../api/common/lease/AddressLeaseInfo.scala | 8 +- .../common/lease/LeaseByAddressIterator.scala | 11 +- .../wavesplatform/api/common/package.scala | 8 +- .../com/wavesplatform/database/Caches.scala | 16 +- .../wavesplatform/database/DBResource.scala | 12 +- .../wavesplatform/database/KeyHelpers.scala | 5 +- .../com/wavesplatform/database/Keys.scala | 36 +++-- .../com/wavesplatform/database/RDB.scala | 32 ++-- .../database/RocksDBWriter.scala | 147 ++++++++++-------- .../com/wavesplatform/database/package.scala | 14 +- .../history/StorageFactory.scala | 2 +- .../wavesplatform/settings/DBSettings.scala | 5 +- .../settings/RocksDBSettings.scala | 4 +- node/src/test/resources/application.conf | 3 +- .../database/TestStorageFactory.scala | 1 - .../com/wavesplatform/db/InterferableDB.scala | 9 +- .../wavesplatform/db/TxBloomFilterSpec.scala | 34 ++++ .../com/wavesplatform/db/WithState.scala | 4 +- .../history/BlockchainUpdaterNFTTest.scala | 3 +- .../com/wavesplatform/history/Domain.scala | 2 +- .../wavesplatform/state/DataKeyRollback.scala | 60 +++++++ project/Dependencies.scala | 20 ++- project/plugins.sbt | 2 +- repl/jvm/src/test/logback-test.xml | 14 -- .../database/rocksdb/KeyTags.scala | 1 - 40 files changed, 457 insertions(+), 256 deletions(-) delete mode 100644 lang/testkit/src/test/resources/logback-test.xml delete mode 100644 lang/tests/src/test/resources/logback-test.xml create mode 100644 node/src/test/scala/com/wavesplatform/db/TxBloomFilterSpec.scala create mode 100644 node/src/test/scala/com/wavesplatform/state/DataKeyRollback.scala delete mode 100644 repl/jvm/src/test/logback-test.xml diff --git a/.github/workflows/publish-docker-node.yaml b/.github/workflows/publish-docker-node.yaml index 804c39d8b5..8fddb878c9 100644 --- a/.github/workflows/publish-docker-node.yaml +++ b/.github/workflows/publish-docker-node.yaml @@ -31,7 +31,7 @@ jobs: - name: Build sources run: | - sbt --mem 2048 -J-XX:+UseG1GC -Dcoursier.cache=~/.cache/coursier -Dsbt.boot.directory=~/.sbt buildTarballsForDocker + sbt --mem 2048 -J-XX:+UseG1GC -Dcoursier.cache=~/.cache/coursier -Dsbt.boot.directory=~/.sbt ;buildTarballsForDocker;buildRIDERunnerForDocker - name: Setup Docker buildx uses: docker/setup-buildx-action@v2 diff --git a/build.sbt b/build.sbt index fbe44e4f3a..f56a44abf2 100644 --- a/build.sbt +++ b/build.sbt @@ -84,10 +84,7 @@ lazy val repl = crossProject(JSPlatform, JVMPlatform) libraryDependencies ++= Dependencies.protobuf.value ++ Dependencies.langCompilerPlugins.value ++ - Dependencies.circe.value ++ - Seq( - "org.scala-js" %%% "scala-js-macrotask-executor" % "1.0.0" - ), + Dependencies.circe.value, inConfig(Compile)( Seq( PB.targets += scalapb.gen(flatPackage = true) -> sourceManaged.value, @@ -109,6 +106,9 @@ lazy val `repl-jvm` = repl.jvm ) lazy val `repl-js` = repl.js.dependsOn(`lang-js`) + .settings( + libraryDependencies += "org.scala-js" %%% "scala-js-macrotask-executor" % "1.1.1" + ) lazy val `curve25519-test` = project.dependsOn(node) @@ -198,6 +198,10 @@ buildTarballsForDocker := { (`grpc-server` / Universal / packageZipTarball).value, baseDirectory.value / "docker" / "target" / "waves-grpc-server.tgz" ) +} + +lazy val buildRIDERunnerForDocker = taskKey[Unit]("Package RIDE Runner tarball and copy it to docker/target") +buildRIDERunnerForDocker := { IO.copyFile( (`ride-runner` / Universal / packageZipTarball).value, (`ride-runner` / baseDirectory).value / "docker" / "target" / s"${(`ride-runner` / name).value}.tgz" @@ -244,7 +248,6 @@ lazy val buildDebPackages = taskKey[Unit]("Build debian packages") buildDebPackages := { (`grpc-server` / Debian / packageBin).value (node / Debian / packageBin).value - (`ride-runner` / Debian / packageBin).value } def buildPackages: Command = Command("buildPackages")(_ => Network.networkParser) { (state, args) => diff --git a/grpc-server/src/main/scala/com/wavesplatform/events/BlockchainUpdates.scala b/grpc-server/src/main/scala/com/wavesplatform/events/BlockchainUpdates.scala index 166f41d5a4..270a99986f 100644 --- a/grpc-server/src/main/scala/com/wavesplatform/events/BlockchainUpdates.scala +++ b/grpc-server/src/main/scala/com/wavesplatform/events/BlockchainUpdates.scala @@ -2,7 +2,6 @@ package com.wavesplatform.events import com.wavesplatform.block.{Block, MicroBlock} import com.wavesplatform.common.state.ByteStr -import com.wavesplatform.database.RDB import com.wavesplatform.events.api.grpc.protobuf.BlockchainUpdatesApiGrpc import com.wavesplatform.events.settings.BlockchainUpdatesSettings import com.wavesplatform.extensions.{Context, Extension} @@ -14,6 +13,7 @@ import io.grpc.{Metadata, Server, ServerStreamTracer, Status} import monix.execution.schedulers.SchedulerService import monix.execution.{ExecutionModel, Scheduler, UncaughtExceptionReporter} import net.ceedubs.ficus.Ficus.* +import org.rocksdb.RocksDB import java.net.InetSocketAddress import java.util.concurrent.TimeUnit @@ -31,9 +31,8 @@ class BlockchainUpdates(private val context: Context) extends Extension with Sco ) private[this] val settings = context.settings.config.as[BlockchainUpdatesSettings]("waves.blockchain-updates") - // todo: no need to open column families here - private[this] val rdb = RDB.open(context.settings.dbSettings.copy(directory = context.settings.directory + "/blockchain-updates")) - private[this] val repo = new Repo(rdb.db, context.blocksApi) + private[this] val rdb = RocksDB.open(context.settings.directory + "/blockchain-updates") + private[this] val repo = new Repo(rdb, context.blocksApi) private[this] val grpcServer: Server = NettyServerBuilder .forAddress(new InetSocketAddress("0.0.0.0", settings.grpcPort)) diff --git a/lang/jvm/src/main/scala/com/wavesplatform/crypto/Curve25519.scala b/lang/jvm/src/main/scala/com/wavesplatform/crypto/Curve25519.scala index 40f876471f..5c31009e71 100644 --- a/lang/jvm/src/main/scala/com/wavesplatform/crypto/Curve25519.scala +++ b/lang/jvm/src/main/scala/com/wavesplatform/crypto/Curve25519.scala @@ -21,6 +21,9 @@ object Curve25519 { def sign(privateKey: Array[Byte], message: Array[Byte]): Array[Byte] = provider.calculateSignature(provider.getRandom(SignatureLength), privateKey, message) - def verify(signature: Array[Byte], message: Array[Byte], publicKey: Array[Byte]): Boolean = provider.verifySignature(publicKey, message, signature) - + def verify(signature: Array[Byte], message: Array[Byte], publicKey: Array[Byte]): Boolean = + signature != null && signature.length == SignatureLength && + publicKey != null && publicKey.length == KeyLength && + message != null && + provider.verifySignature(publicKey, message, signature) } diff --git a/lang/testkit/src/test/resources/logback-test.xml b/lang/testkit/src/test/resources/logback-test.xml deleted file mode 100644 index 471a19efaf..0000000000 --- a/lang/testkit/src/test/resources/logback-test.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - %date %-5level [%.15thread] %logger{26} - %msg%n - - - - - - - - - diff --git a/lang/testkit/src/test/scala/com/wavesplatform/report/QaseReporter.scala b/lang/testkit/src/test/scala/com/wavesplatform/report/QaseReporter.scala index fc3dbc834c..92d692a1a7 100644 --- a/lang/testkit/src/test/scala/com/wavesplatform/report/QaseReporter.scala +++ b/lang/testkit/src/test/scala/com/wavesplatform/report/QaseReporter.scala @@ -1,9 +1,10 @@ package com.wavesplatform.report import com.wavesplatform.report.QaseReporter.{CaseIdPattern, QaseProjects, TestResult} -import io.qase.api.QaseClient +import io.qase.api.config.QaseConfig import io.qase.api.utils.IntegrationUtils import io.qase.client.model.ResultCreate +import org.aeonbits.owner.ConfigFactory import org.scalatest.Reporter import org.scalatest.events.* import play.api.libs.json.{Format, Json} @@ -45,7 +46,7 @@ class QaseReporter extends Reporter { msgOpt: Option[String], duration: Option[Long] ): Unit = - if (QaseClient.isEnabled) { + if (QaseReporter.isEnabled) { val errMsg = msgOpt.map(msg => s"\n\n**Error**\n$msg").getOrElse("") val comment = s"$testName$errMsg" val stacktrace = throwable.map(IntegrationUtils.getStacktrace) @@ -55,7 +56,7 @@ class QaseReporter extends Reporter { } private def saveRunResults(): Unit = - if (QaseClient.isEnabled) { + if (QaseReporter.isEnabled) { results.asScala.foreach { case (projectCode, results) => if (results.nonEmpty) { val writer = new FileWriter(s"./$projectCode-${System.currentTimeMillis()}") @@ -73,6 +74,10 @@ class QaseReporter extends Reporter { } object QaseReporter { + // this hack prevents QaseClient class from being initialized, which in turn initializes Logback with malformed config + // and prints a warning about unused appender to stdout + private[QaseReporter] val isEnabled = ConfigFactory.create(classOf[QaseConfig]).isEnabled + val RunIdKeyPrefix = "QASE_RUN_ID_" val CheckPRRunIdKey = "CHECKPR_RUN_ID" val QaseProjects = Seq("NODE", "RIDE", "BU", "SAPI") diff --git a/lang/tests/src/test/resources/logback-test.xml b/lang/tests/src/test/resources/logback-test.xml deleted file mode 100644 index b1f394b2e4..0000000000 --- a/lang/tests/src/test/resources/logback-test.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - %date %-5level [%.15thread] %logger{26} - %msg%n - - - - - - - - - diff --git a/node-it/build.sbt b/node-it/build.sbt index d2d0af510c..7e73688140 100644 --- a/node-it/build.sbt +++ b/node-it/build.sbt @@ -11,5 +11,5 @@ inTask(docker)( ) ) -val packageAll = taskKey[Unit]("build all packages") -docker := docker.dependsOn(LocalProject("waves-node") / packageAll).value +val buildTarballsForDocker = taskKey[Unit]("build all packages") +docker := docker.dependsOn(LocalProject("waves-node") / buildTarballsForDocker).value diff --git a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala index d66ce80616..a8932b09bc 100644 --- a/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala +++ b/node-it/src/test/scala/com/wavesplatform/test/BlockchainGenerator.scala @@ -64,10 +64,10 @@ class BlockchainGenerator(wavesSettings: WavesSettings) extends ScorexLogging { private val settings: WavesSettings = wavesSettings.copy(minerSettings = wavesSettings.minerSettings.copy(quorum = 0)) - def generateDb(genBlocks: Seq[GenBlock], dbDirPath: String = settings.dbSettings.directory): Unit = + def generateDb(genBlocks: Iterator[GenBlock], dbDirPath: String = settings.dbSettings.directory): Unit = generateBlockchain(genBlocks, settings.dbSettings.copy(directory = dbDirPath)) - def generateBinaryFile(genBlocks: Seq[GenBlock]): Unit = { + def generateBinaryFile(genBlocks: Iterator[GenBlock]): Unit = { val targetHeight = genBlocks.size + 1 log.info(s"Exporting to $targetHeight") val outputFilename = s"blockchain-$targetHeight" @@ -94,7 +94,7 @@ class BlockchainGenerator(wavesSettings: WavesSettings) extends ScorexLogging { } } - private def generateBlockchain(genBlocks: Seq[GenBlock], dbSettings: DBSettings, exportToFile: Block => Unit = _ => ()): Unit = { + private def generateBlockchain(genBlocks: Iterator[GenBlock], dbSettings: DBSettings, exportToFile: Block => Unit = _ => ()): Unit = { val scheduler = Schedulers.singleThread("appender") val time = new Time { val startTime: Long = settings.blockchainSettings.genesisSettings.timestamp @@ -185,7 +185,7 @@ class BlockchainGenerator(wavesSettings: WavesSettings) extends ScorexLogging { } case Left(err) => log.error(s"Error appending block: $err") } - } + }.get } private def correctTxTimestamp(genTx: GenTx, time: Time): Transaction = diff --git a/node/src/main/resources/application.conf b/node/src/main/resources/application.conf index f642aa6f81..8501dbf4b1 100644 --- a/node/src/main/resources/application.conf +++ b/node/src/main/resources/application.conf @@ -23,7 +23,6 @@ waves { max-cache-size = 100000 max-rollback-depth = 2000 - remember-blocks = 3h # Delete old history entries (Data, WAVES and Asset balances) in this interval before a safe rollback height. # Comment to disable. @@ -40,15 +39,16 @@ waves { # AA Asset balance history for address. # cleanup-interval = 500 # Optimal for Xmx2G - use-bloom-filter = false - rocksdb { main-cache-size = 512M tx-cache-size = 16M tx-meta-cache-size = 16M tx-snapshot-cache-size = 16M + api-cache-size=16M write-buffer-size = 128M enable-statistics = false + # When enabled, after writing every SST file of the default column family, reopen it and read all the keys. + paranoid-checks = off } } diff --git a/node/src/main/scala/com/wavesplatform/Explorer.scala b/node/src/main/scala/com/wavesplatform/Explorer.scala index 96bae8692f..b51f99e5e9 100644 --- a/node/src/main/scala/com/wavesplatform/Explorer.scala +++ b/node/src/main/scala/com/wavesplatform/Explorer.scala @@ -390,6 +390,92 @@ object Explorer extends ScorexLogging { log.info(s"Load meta for $id") val meta = rdb.db.get(Keys.transactionMetaById(TransactionId(ByteStr.decodeBase58(id).get), rdb.txMetaHandle)) log.info(s"Meta: $meta") + case "DH" => + val address = Address.fromString(argument(1, "address")).explicitGet() + val key = argument(2, "key") + val requestedHeight = argument(3, "height").toInt + log.info(s"Loading address ID for $address") + val addressId = rdb.db.get(Keys.addressId(address)).get + log.info(s"Collecting data history for key $key on $address ($addressId)") + val currentEntry = rdb.db.get(Keys.data(addressId, key)) + log.info(s"Current entry: $currentEntry") + val problematicEntry = rdb.db.get(Keys.dataAt(addressId, key)(requestedHeight)) + log.info(s"Entry at $requestedHeight: $problematicEntry") + case "DHC" => + log.info("Looking for data entry history corruptions") + var thisAddressId = 0L + var prevHeight = 0 + var key = "" + var addressCount = 0 + rdb.db.iterateOver(KeyTags.DataHistory.prefixBytes, None) { e => + val addressIdFromKey = Longs.fromByteArray(e.getKey.slice(2, 10)) + val heightFromKey = Ints.fromByteArray(e.getKey.takeRight(4)) + val keyFromKey = new String(e.getKey.drop(10).dropRight(4), "utf-8") + if (addressIdFromKey != thisAddressId) { + thisAddressId = addressIdFromKey + key = keyFromKey + addressCount += 1 + } else if (key != keyFromKey) { + key = keyFromKey + } else { + val node = readDataNode(key)(e.getValue) + if (node.prevHeight != prevHeight) { + val address = rdb.db.get(Keys.idToAddress(AddressId(thisAddressId))) + log.warn(s"$address/$key@$heightFromKey: node.prevHeight=${node.prevHeight}, actual=$prevHeight") + + } + } + prevHeight = heightFromKey + } + log.info(s"Checked $addressCount addresses") + case "ABHC" => + log.info("Looking for asset balance history corruptions") + var thisAddressId = 0L + var prevHeight = 0 + var key = IssuedAsset(ByteStr(new Array[Byte](32))) + var addressCount = 0 + rdb.db.iterateOver(KeyTags.AssetBalanceHistory.prefixBytes, None) { e => + val addressIdFromKey = Longs.fromByteArray(e.getKey.slice(34, 42)) + val heightFromKey = Ints.fromByteArray(e.getKey.takeRight(4)) + val keyFromKey = IssuedAsset(ByteStr(e.getKey.slice(2, 34))) + if (keyFromKey != key) { + thisAddressId = addressIdFromKey + key = keyFromKey + addressCount += 1 + } else if (thisAddressId != addressIdFromKey) { + thisAddressId = addressIdFromKey + } else { + val node = readBalanceNode(e.getValue) + if (node.prevHeight != prevHeight) { + val address = rdb.db.get(Keys.idToAddress(AddressId(thisAddressId))) + log.warn(s"$key/$address@$heightFromKey: node.prevHeight=${node.prevHeight}, actual=$prevHeight") + + } + } + prevHeight = heightFromKey + } + log.info(s"Checked $addressCount assets") + case "BHC" => + log.info("Looking for balance history corruptions") + var thisAddressId = 0L + var prevHeight = 0 + var addressCount = 0 + rdb.db.iterateOver(KeyTags.WavesBalanceHistory.prefixBytes, None) { e => + val addressIdFromKey = Longs.fromByteArray(e.getKey.slice(2, 10)) + val heightFromKey = Ints.fromByteArray(e.getKey.takeRight(4)) + if (addressIdFromKey != thisAddressId) { + thisAddressId = addressIdFromKey + addressCount += 1 + } else { + val node = readBalanceNode(e.getValue) + if (node.prevHeight != prevHeight) { + val address = rdb.db.get(Keys.idToAddress(AddressId(thisAddressId))) + log.warn(s"$address@$heightFromKey: node.prevHeight=${node.prevHeight}, actual=$prevHeight") + } + } + prevHeight = heightFromKey + } + log.info(s"Checked $addressCount addresses") } } finally { reader.close() diff --git a/node/src/main/scala/com/wavesplatform/GenesisBlockGenerator.scala b/node/src/main/scala/com/wavesplatform/GenesisBlockGenerator.scala index f41d5874a4..51444fddf5 100644 --- a/node/src/main/scala/com/wavesplatform/GenesisBlockGenerator.scala +++ b/node/src/main/scala/com/wavesplatform/GenesisBlockGenerator.scala @@ -96,7 +96,7 @@ object GenesisBlockGenerator { .headOption .map(new File(_).getAbsoluteFile.ensuring(f => !f.isDirectory && f.getParentFile.isDirectory || f.getParentFile.mkdirs())) - val settings = parseSettings(ConfigFactory.parseFile(inputConfFile)) + val settings = parseSettings(ConfigFactory.parseFile(inputConfFile).resolve()) val confBody = createConfig(settings) outputConfFile.foreach(ocf => Files.write(ocf.toPath, confBody.utf8Bytes)) } diff --git a/node/src/main/scala/com/wavesplatform/Importer.scala b/node/src/main/scala/com/wavesplatform/Importer.scala index 7f20c716e2..2b959f8030 100644 --- a/node/src/main/scala/com/wavesplatform/Importer.scala +++ b/node/src/main/scala/com/wavesplatform/Importer.scala @@ -4,7 +4,7 @@ import akka.actor.ActorSystem import cats.implicits.catsSyntaxOption import cats.syntax.apply.* import com.google.common.io.ByteStreams -import com.google.common.primitives.Ints +import com.google.common.primitives.{Ints, Longs} import com.wavesplatform.Exporter.Formats import com.wavesplatform.api.common.{CommonAccountsApi, CommonAssetsApi, CommonBlocksApi, CommonTransactionsApi} import com.wavesplatform.block.{Block, BlockHeader} @@ -217,6 +217,8 @@ object Importer extends ScorexLogging { val maxSize = importOptions.maxQueueSize val queue = new mutable.Queue[(VanillaBlock, Option[BlockSnapshotResponse])](maxSize) + val CurrentTS = System.currentTimeMillis() + @tailrec def readBlocks(queue: mutable.Queue[(VanillaBlock, Option[BlockSnapshotResponse])], remainCount: Int, maxCount: Int): Unit = { if (remainCount == 0) () @@ -247,11 +249,12 @@ object Importer extends ScorexLogging { if (blocksToSkip > 0) { blocksToSkip -= 1 } else { - val blockV5 = blockchain.isFeatureActivated(BlockchainFeatures.BlockV5, blockchain.height + (maxCount - remainCount) + 1) val rideV6 = blockchain.isFeatureActivated(BlockchainFeatures.RideV6, blockchain.height + (maxCount - remainCount) + 1) lazy val parsedProtoBlock = PBBlocks.vanilla(PBBlocks.addChainId(protobuf.block.PBBlock.parseFrom(blockBytes)), unsafe = true) - - val block = (if (!blockV5) Block.parseBytes(blockBytes) else parsedProtoBlock).orElse(parsedProtoBlock).get + val block = (if (1 < blockBytes.head && blockBytes.head < 5 && Longs.fromByteArray(blockBytes.slice(1, 9)) < CurrentTS) + Block.parseBytes(blockBytes).orElse(parsedProtoBlock) + else + parsedProtoBlock).get val blockSnapshot = snapshotsBytes.map { bytes => BlockSnapshotResponse( block.id(), @@ -308,7 +311,7 @@ object Importer extends ScorexLogging { case _ => counter = counter + 1 } - } else { + } else if (!quit){ log.warn(s"Block $block is not a child of the last block ${blockchain.lastBlockId.get}") } } @@ -360,7 +363,7 @@ object Importer extends ScorexLogging { val blocksFileOffset = importOptions.format match { case Formats.Binary => - var blocksOffset = 0 + var blocksOffset = 0L rdb.db.iterateOver(KeyTags.BlockInfoAtHeight) { e => e.getKey match { case Array(_, _, 0, 0, 0, 1) => // Skip genesis diff --git a/node/src/main/scala/com/wavesplatform/api/common/AddressTransactions.scala b/node/src/main/scala/com/wavesplatform/api/common/AddressTransactions.scala index 4fe340eac3..bfbc56bd26 100644 --- a/node/src/main/scala/com/wavesplatform/api/common/AddressTransactions.scala +++ b/node/src/main/scala/com/wavesplatform/api/common/AddressTransactions.scala @@ -33,28 +33,33 @@ object AddressTransactions { } .toSeq - private def loadInvokeScriptResult(resource: DBResource, txMetaHandle: RDB.TxMetaHandle, txId: ByteStr): Option[InvokeScriptResult] = + private def loadInvokeScriptResult( + resource: DBResource, + txMetaHandle: RDB.TxMetaHandle, + apiHandle: RDB.ApiHandle, + txId: ByteStr + ): Option[InvokeScriptResult] = for { tm <- resource.get(Keys.transactionMetaById(TransactionId(txId), txMetaHandle)) - scriptResult <- resource.get(Keys.invokeScriptResult(tm.height, TxNum(tm.num.toShort))) + scriptResult <- resource.get(Keys.invokeScriptResult(tm.height, TxNum(tm.num.toShort), apiHandle)) } yield scriptResult - def loadInvokeScriptResult(db: RocksDB, txMetaHandle: RDB.TxMetaHandle, txId: ByteStr): Option[InvokeScriptResult] = - db.withResource(r => loadInvokeScriptResult(r, txMetaHandle, txId)) + def loadInvokeScriptResult(db: RocksDB, txMetaHandle: RDB.TxMetaHandle, apiHandle: RDB.ApiHandle, txId: ByteStr): Option[InvokeScriptResult] = + db.withResource(r => loadInvokeScriptResult(r, txMetaHandle, apiHandle, txId)) - def loadInvokeScriptResult(db: RocksDB, height: Height, txNum: TxNum): Option[InvokeScriptResult] = - db.get(Keys.invokeScriptResult(height, txNum)) + def loadInvokeScriptResult(db: RocksDB, apiHandle: RDB.ApiHandle, height: Height, txNum: TxNum): Option[InvokeScriptResult] = + db.get(Keys.invokeScriptResult(height, txNum, apiHandle)) - def loadEthereumMetadata(db: RocksDB, txMetaHandle: RDB.TxMetaHandle, txId: ByteStr): Option[EthereumTransactionMeta] = db.withResource { - resource => + def loadEthereumMetadata(db: RocksDB, txMetaHandle: RDB.TxMetaHandle, apiHandle: RDB.ApiHandle, txId: ByteStr): Option[EthereumTransactionMeta] = + db.withResource { resource => for { tm <- resource.get(Keys.transactionMetaById(TransactionId(txId), txMetaHandle)) - m <- resource.get(Keys.ethereumTransactionMeta(Height(tm.height), TxNum(tm.num.toShort))) + m <- resource.get(Keys.ethereumTransactionMeta(Height(tm.height), TxNum(tm.num.toShort), apiHandle)) } yield m - } + } - def loadEthereumMetadata(db: RocksDB, height: Height, txNum: TxNum): Option[EthereumTransactionMeta] = - db.get(Keys.ethereumTransactionMeta(height, txNum)) + def loadEthereumMetadata(db: RocksDB, apiHandle: RDB.ApiHandle, height: Height, txNum: TxNum): Option[EthereumTransactionMeta] = + db.get(Keys.ethereumTransactionMeta(height, txNum, apiHandle)) def allAddressTransactions( rdb: RDB, @@ -82,24 +87,25 @@ object AddressTransactions { sender: Option[Address], types: Set[Transaction.Type], fromId: Option[ByteStr] - ): Observable[(TxMeta, Transaction, Option[TxNum])] = rdb.db.resourceObservable.flatMap { dbResource => - dbResource - .get(Keys.addressId(subject)) - .fold(Observable.empty[(TxMeta, Transaction, Option[TxNum])]) { addressId => - val (maxHeight, maxTxNum) = - fromId - .flatMap(id => rdb.db.get(Keys.transactionMetaById(TransactionId(id), rdb.txMetaHandle))) - .fold[(Height, TxNum)](Height(Int.MaxValue) -> TxNum(Short.MaxValue)) { tm => - Height(tm.height) -> TxNum(tm.num.toShort) - } + ): Observable[(TxMeta, Transaction, Option[TxNum])] = + rdb.db.resourceObservable(rdb.apiHandle.handle).flatMap { dbResource => + dbResource + .get(Keys.addressId(subject)) + .fold(Observable.empty[(TxMeta, Transaction, Option[TxNum])]) { addressId => + val (maxHeight, maxTxNum) = + fromId + .flatMap(id => rdb.db.get(Keys.transactionMetaById(TransactionId(id), rdb.txMetaHandle))) + .fold[(Height, TxNum)](Height(Int.MaxValue) -> TxNum(Short.MaxValue)) { tm => + Height(tm.height) -> TxNum(tm.num.toShort) + } - Observable - .fromIterator( - Task(new TxByAddressIterator(dbResource, rdb.txHandle, addressId, maxHeight, maxTxNum, sender, types).asScala) - ) - .concatMapIterable(identity) - } - } + Observable + .fromIterator( + Task(new TxByAddressIterator(dbResource, rdb.txHandle, rdb.apiHandle, addressId, maxHeight, maxTxNum, sender, types).asScala) + ) + .concatMapIterable(identity) + } + } private def transactionsFromSnapshot( maybeSnapshot: Option[(Height, StateSnapshot)], @@ -121,14 +127,15 @@ object AddressTransactions { private class TxByAddressIterator( db: DBResource, txHandle: RDB.TxHandle, + apiHandle: RDB.ApiHandle, addressId: AddressId, maxHeight: Int, maxTxNum: Int, sender: Option[Address], types: Set[Transaction.Type] ) extends AbstractIterator[Seq[(TxMeta, Transaction, Option[TxNum])]] { - private val seqNr = db.get(Keys.addressTransactionSeqNr(addressId)) - db.withSafePrefixIterator(_.seekForPrev(Keys.addressTransactionHN(addressId, seqNr).keyBytes))() + private val seqNr = db.get(Keys.addressTransactionSeqNr(addressId, apiHandle)) + db.withSafePrefixIterator(_.seekForPrev(Keys.addressTransactionHN(addressId, seqNr, apiHandle).keyBytes))() final override def computeNext(): Seq[(TxMeta, Transaction, Option[TxNum])] = db.withSafePrefixIterator { dbIterator => val keysBuffer = new ArrayBuffer[Key[Option[(TxMeta, Transaction)]]]() diff --git a/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala b/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala index 812e38d430..4faaf1b6c1 100644 --- a/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala +++ b/node/src/main/scala/com/wavesplatform/api/common/CommonAccountsApi.scala @@ -94,7 +94,7 @@ object CommonAccountsApi { } override def nftList(address: Address, after: Option[IssuedAsset]): Observable[Seq[(IssuedAsset, AssetDescription)]] = { - rdb.db.resourceObservable.flatMap { resource => + rdb.db.resourceObservable(rdb.apiHandle.handle).flatMap { resource => Observable .fromIterator(Task(nftIterator(resource, address, compositeBlockchain().snapshot, after, blockchain.assetDescription))) } diff --git a/node/src/main/scala/com/wavesplatform/api/common/lease/AddressLeaseInfo.scala b/node/src/main/scala/com/wavesplatform/api/common/lease/AddressLeaseInfo.scala index 9bb9d62184..85b6e9cd9e 100644 --- a/node/src/main/scala/com/wavesplatform/api/common/lease/AddressLeaseInfo.scala +++ b/node/src/main/scala/com/wavesplatform/api/common/lease/AddressLeaseInfo.scala @@ -39,15 +39,15 @@ object AddressLeaseInfo { private def leasesFromDb(rdb: RDB, subject: Address): Observable[LeaseInfo] = for { - dbResource <- rdb.db.resourceObservable + dbResource <- rdb.db.resourceObservable(rdb.apiHandle.handle) (leaseId, details) <- dbResource .get(Keys.addressId(subject)) - .map(fromLeaseDbIterator(dbResource, _)) + .map(fromLeaseDbIterator(dbResource, rdb.apiHandle, _)) .getOrElse(Observable.empty) } yield LeaseInfo.fromLeaseDetails(leaseId, details) - private def fromLeaseDbIterator(dbResource: DBResource, addressId: AddressId): Observable[(ByteStr, LeaseDetails)] = + private def fromLeaseDbIterator(dbResource: DBResource, apiHandle: RDB.ApiHandle, addressId: AddressId): Observable[(ByteStr, LeaseDetails)] = Observable - .fromIterator(Task(new LeaseByAddressIterator(dbResource, addressId).asScala)) + .fromIterator(Task(new LeaseByAddressIterator(dbResource, apiHandle, addressId).asScala)) .concatMapIterable(identity) } diff --git a/node/src/main/scala/com/wavesplatform/api/common/lease/LeaseByAddressIterator.scala b/node/src/main/scala/com/wavesplatform/api/common/lease/LeaseByAddressIterator.scala index 6f18da0c21..f9821a7d21 100644 --- a/node/src/main/scala/com/wavesplatform/api/common/lease/LeaseByAddressIterator.scala +++ b/node/src/main/scala/com/wavesplatform/api/common/lease/LeaseByAddressIterator.scala @@ -3,18 +3,19 @@ package com.wavesplatform.api.common.lease import com.google.common.collect.AbstractIterator import com.wavesplatform.common.state.ByteStr import com.wavesplatform.database -import com.wavesplatform.database.{AddressId, DBResource, Keys} +import com.wavesplatform.database.{AddressId, DBResource, Keys, RDB} import com.wavesplatform.state.LeaseDetails -private class LeaseByAddressIterator(resource: DBResource, addressId: AddressId) extends AbstractIterator[Seq[(ByteStr, LeaseDetails)]] { - private val seqNr = resource.get(Keys.addressLeaseSeqNr(addressId)) - resource.withSafePrefixIterator(_.seekForPrev(Keys.addressLeaseSeq(addressId, seqNr).keyBytes))() +private class LeaseByAddressIterator(resource: DBResource, apiHandle: RDB.ApiHandle, addressId: AddressId) + extends AbstractIterator[Seq[(ByteStr, LeaseDetails)]] { + private val seqNr = resource.get(Keys.addressLeaseSeqNr(addressId, apiHandle)) + resource.withSafePrefixIterator(_.seekForPrev(Keys.addressLeaseSeq(addressId, seqNr, apiHandle).keyBytes))() final override def computeNext(): Seq[(ByteStr, LeaseDetails)] = resource.withSafePrefixIterator { iterator => if (iterator.isValid) { val details = for { - id <- database.readLeaseIdSeq(iterator.value()) + id <- database.readLeaseIdSeq(iterator.value()) details <- database.loadLease(resource, id) if details.isActive } yield (id, details) iterator.prev() diff --git a/node/src/main/scala/com/wavesplatform/api/common/package.scala b/node/src/main/scala/com/wavesplatform/api/common/package.scala index 6b2e1e2043..a1d0dc6047 100644 --- a/node/src/main/scala/com/wavesplatform/api/common/package.scala +++ b/node/src/main/scala/com/wavesplatform/api/common/package.scala @@ -28,12 +28,12 @@ package object common { def loadISR(t: Transaction) = maybeDiff .flatMap { case (_, diff) => diff.scriptResults.get(t.id()) } - .orElse(txNumOpt.flatMap(loadInvokeScriptResult(rdb.db, m.height, _))) + .orElse(txNumOpt.flatMap(loadInvokeScriptResult(rdb.db, rdb.apiHandle, m.height, _))) def loadETM(t: Transaction) = maybeDiff .flatMap { case (_, diff) => diff.ethereumTransactionMeta.get(t.id()) } - .orElse(txNumOpt.flatMap(loadEthereumMetadata(rdb.db, m.height, _))) + .orElse(txNumOpt.flatMap(loadEthereumMetadata(rdb.db, rdb.apiHandle, m.height, _))) TransactionMeta.create( m.height, @@ -90,11 +90,11 @@ package object common { ist => maybeSnapshot .flatMap { case (_, s) => s.scriptResults.get(ist.id()) } - .orElse(loadInvokeScriptResult(rdb.db, rdb.txMetaHandle, ist.id())), + .orElse(loadInvokeScriptResult(rdb.db, rdb.txMetaHandle, rdb.apiHandle, ist.id())), et => maybeSnapshot .flatMap { case (_, s) => s.ethereumTransactionMeta.get(et.id()) } - .orElse(loadEthereumMetadata(rdb.db, rdb.txMetaHandle, et.id())) + .orElse(loadEthereumMetadata(rdb.db, rdb.txMetaHandle, rdb.apiHandle, et.id())) ) } } diff --git a/node/src/main/scala/com/wavesplatform/database/Caches.scala b/node/src/main/scala/com/wavesplatform/database/Caches.scala index 38a51d6f37..5129ce7c6b 100644 --- a/node/src/main/scala/com/wavesplatform/database/Caches.scala +++ b/node/src/main/scala/com/wavesplatform/database/Caches.scala @@ -128,7 +128,7 @@ abstract class Caches extends Blockchain with Storage { VolumeAndFee(curVf.volume, curVf.fee) } - private val memMeter = MemoryMeter.builder().build() + protected val memMeter = MemoryMeter.builder().build() private val scriptCache: LoadingCache[Address, Option[AccountScriptInfo]] = CacheBuilder @@ -183,7 +183,7 @@ abstract class Caches extends Blockchain with Storage { protected def discardAccountData(addressWithKey: (Address, String)): Unit = accountDataCache.invalidate(addressWithKey) protected def loadAccountData(acc: Address, key: String): CurrentData - protected def loadEntryHeights(keys: Iterable[(Address, String)], addressIdOf: Address => AddressId): Map[(Address, String), Height] + protected def loadEntryHeights(keys: Seq[(Address, String)], addressIdOf: Address => AddressId): Map[(Address, String), Height] private[database] def addressId(address: Address): Option[AddressId] = addressIdCache.get(address) private[database] def addressIds(addresses: Seq[Address]): Map[Address, Option[AddressId]] = @@ -299,15 +299,15 @@ abstract class Caches extends Blockchain with Storage { (key, entry) <- entries } yield ((address, key), entry) - val cachedEntries = accountDataCache.getAllPresent(newEntries.keys.asJava).asScala - val loadedPrevEntries = loadEntryHeights(newEntries.keys.filterNot(cachedEntries.contains), addressIdWithFallback(_, newAddressIds)) + val cachedEntries = accountDataCache.getAllPresent(newEntries.keys.asJava).asScala + val loadedPrevEntryHeights = loadEntryHeights(newEntries.keys.filterNot(cachedEntries.contains).toSeq, addressIdWithFallback(_, newAddressIds)) val updatedDataWithNodes = (for { - (k, currentEntry) <- cachedEntries.view.mapValues(_.height) ++ loadedPrevEntries - newEntry <- newEntries.get(k) + (k, heightOfPreviousEntry) <- cachedEntries.view.mapValues(_.height) ++ loadedPrevEntryHeights + newEntry <- newEntries.get(k) } yield k -> ( - CurrentData(newEntry, Height(height), currentEntry), - DataNode(newEntry, currentEntry) + CurrentData(newEntry, Height(height), heightOfPreviousEntry), + DataNode(newEntry, heightOfPreviousEntry) )).toMap val orderFillsWithNodes = for { diff --git a/node/src/main/scala/com/wavesplatform/database/DBResource.scala b/node/src/main/scala/com/wavesplatform/database/DBResource.scala index 5217cbbcff..7d3f17515b 100644 --- a/node/src/main/scala/com/wavesplatform/database/DBResource.scala +++ b/node/src/main/scala/com/wavesplatform/database/DBResource.scala @@ -1,6 +1,6 @@ package com.wavesplatform.database -import org.rocksdb.{ReadOptions, RocksDB, RocksIterator} +import org.rocksdb.{ColumnFamilyHandle, ReadOptions, RocksDB, RocksIterator} import scala.collection.View import scala.collection.mutable.ArrayBuffer @@ -18,9 +18,9 @@ trait DBResource extends AutoCloseable { } object DBResource { - def apply(db: RocksDB): DBResource = new DBResource { + def apply(db: RocksDB, iteratorCfHandle: Option[ColumnFamilyHandle] = None): DBResource = new DBResource { private[this] val snapshot = db.getSnapshot - private[this] val readOptions = new ReadOptions().setSnapshot(snapshot).setVerifyChecksums(false) + private[this] val readOptions = new ReadOptions().setSnapshot(snapshot) override def get[V](key: Key[V]): V = key.parse(db.get(key.columnFamilyHandle.getOrElse(db.getDefaultColumnFamily), readOptions, key.keyBytes)) @@ -35,9 +35,11 @@ object DBResource { def multiGet[A](keys: ArrayBuffer[Key[A]], valBufferSize: Int): View[A] = db.multiGet(readOptions, keys, valBufferSize) - override lazy val prefixIterator: RocksIterator = db.newIterator(readOptions.setTotalOrderSeek(false).setPrefixSameAsStart(true)) + override lazy val prefixIterator: RocksIterator = + db.newIterator(iteratorCfHandle.getOrElse(db.getDefaultColumnFamily), readOptions.setTotalOrderSeek(false).setPrefixSameAsStart(true)) - override lazy val fullIterator: RocksIterator = db.newIterator(readOptions.setTotalOrderSeek(true)) + override lazy val fullIterator: RocksIterator = + db.newIterator(iteratorCfHandle.getOrElse(db.getDefaultColumnFamily), readOptions.setTotalOrderSeek(true)) override def withSafePrefixIterator[A](ifNotClosed: RocksIterator => A)(ifClosed: => A): A = prefixIterator.synchronized { if (prefixIterator.isOwningHandle) ifNotClosed(prefixIterator) else ifClosed diff --git a/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala b/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala index 74bd3fe903..3176c59166 100644 --- a/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala +++ b/node/src/main/scala/com/wavesplatform/database/KeyHelpers.scala @@ -3,6 +3,7 @@ package com.wavesplatform.database import com.google.common.primitives.{Bytes, Ints, Longs, Shorts} import com.wavesplatform.state import com.wavesplatform.state.{Height, TxNum} +import org.rocksdb.ColumnFamilyHandle import java.nio.ByteBuffer @@ -32,8 +33,8 @@ object KeyHelpers { Ints.toByteArray ) - def bytesSeqNr(keyTag: KeyTags.KeyTag, suffix: Array[Byte], default: Int = 0): Key[Int] = - Key(keyTag, suffix, v => if (v != null && v.length >= Ints.BYTES) Ints.fromByteArray(v) else default, Ints.toByteArray) + def bytesSeqNr(keyTag: KeyTags.KeyTag, suffix: Array[Byte], default: Int = 0, cfh: Option[ColumnFamilyHandle] = None): Key[Int] = + Key(keyTag, suffix, v => if (v != null && v.length >= Ints.BYTES) Ints.fromByteArray(v) else default, Ints.toByteArray, cfh) def unsupported[A](message: String): A => Array[Byte] = _ => throw new UnsupportedOperationException(message) } diff --git a/node/src/main/scala/com/wavesplatform/database/Keys.scala b/node/src/main/scala/com/wavesplatform/database/Keys.scala index 064366ace2..0323a236da 100644 --- a/node/src/main/scala/com/wavesplatform/database/Keys.scala +++ b/node/src/main/scala/com/wavesplatform/database/Keys.scala @@ -167,7 +167,7 @@ object Keys { Some(cfHandle.handle) ) - def transactionStateSnapshotAt(height: Height, n: TxNum, cfHandle: RDB.TxHandle): Key[Option[TransactionStateSnapshot]] = + def transactionStateSnapshotAt(height: Height, n: TxNum, cfHandle: RDB.TxSnapshotHandle): Key[Option[TransactionStateSnapshot]] = Key.opt[TransactionStateSnapshot]( NthTransactionStateSnapshotAtHeight, hNum(height, n), @@ -176,26 +176,28 @@ object Keys { Some(cfHandle.handle) ) - def addressTransactionSeqNr(addressId: AddressId): Key[Int] = - bytesSeqNr(AddressTransactionSeqNr, addressId.toByteArray) + def addressTransactionSeqNr(addressId: AddressId, cfh: RDB.ApiHandle): Key[Int] = + bytesSeqNr(AddressTransactionSeqNr, addressId.toByteArray, cfh = Some(cfh.handle)) - def addressTransactionHN(addressId: AddressId, seqNr: Int): Key[Option[(Height, Seq[(Byte, TxNum, Int)])]] = + def addressTransactionHN(addressId: AddressId, seqNr: Int, cfh: RDB.ApiHandle): Key[Option[(Height, Seq[(Byte, TxNum, Int)])]] = Key.opt( AddressTransactionHeightTypeAndNums, hBytes(addressId.toByteArray, seqNr), readTransactionHNSeqAndType, - writeTransactionHNSeqAndType + writeTransactionHNSeqAndType, + Some(cfh.handle) ) - def addressLeaseSeqNr(addressId: AddressId): Key[Int] = - bytesSeqNr(AddressLeaseInfoSeqNr, addressId.toByteArray) + def addressLeaseSeqNr(addressId: AddressId, cfh: RDB.ApiHandle): Key[Int] = + bytesSeqNr(AddressLeaseInfoSeqNr, addressId.toByteArray, cfh = Some(cfh.handle)) - def addressLeaseSeq(addressId: AddressId, seqNr: Int): Key[Option[Seq[ByteStr]]] = + def addressLeaseSeq(addressId: AddressId, seqNr: Int, cfh: RDB.ApiHandle): Key[Option[Seq[ByteStr]]] = Key.opt( AddressLeaseInfoSeq, hBytes(addressId.toByteArray, seqNr), readLeaseIdSeq, - writeLeaseIdSeq + writeLeaseIdSeq, + Some(cfh.handle) ) def transactionMetaById(txId: TransactionId, cfh: RDB.TxMetaHandle): Key[Option[TransactionMeta]] = @@ -207,8 +209,8 @@ object Keys { Some(cfh.handle) ) - def invokeScriptResult(height: Int, txNum: TxNum): Key[Option[InvokeScriptResult]] = - Key.opt(InvokeScriptResultTag, hNum(height, txNum), InvokeScriptResult.fromBytes, InvokeScriptResult.toBytes) + def invokeScriptResult(height: Int, txNum: TxNum, cfh: RDB.ApiHandle): Key[Option[InvokeScriptResult]] = + Key.opt(InvokeScriptResultTag, hNum(height, txNum), InvokeScriptResult.fromBytes, InvokeScriptResult.toBytes, Some(cfh.handle)) val disabledAliases: Key[Set[Alias]] = Key( DisabledAliases, @@ -223,11 +225,11 @@ object Keys { def assetStaticInfo(addr: ERC20Address): Key[Option[StaticAssetInfo]] = Key.opt(AssetStaticInfo, addr.arr, StaticAssetInfo.parseFrom, _.toByteArray) - def nftCount(addressId: AddressId): Key[Int] = - Key(NftCount, addressId.toByteArray, Option(_).fold(0)(Ints.fromByteArray), Ints.toByteArray) + def nftCount(addressId: AddressId, cfh: RDB.ApiHandle): Key[Int] = + Key(NftCount, addressId.toByteArray, Option(_).fold(0)(Ints.fromByteArray), Ints.toByteArray, Some(cfh.handle)) - def nftAt(addressId: AddressId, index: Int, assetId: IssuedAsset): Key[Option[Unit]] = - Key.opt(NftPossession, addressId.toByteArray ++ Longs.toByteArray(index) ++ assetId.id.arr, _ => (), _ => Array.emptyByteArray) + def nftAt(addressId: AddressId, index: Int, assetId: IssuedAsset, cfh: RDB.ApiHandle): Key[Option[Unit]] = + Key.opt(NftPossession, addressId.toByteArray ++ Longs.toByteArray(index) ++ assetId.id.arr, _ => (), _ => Array.emptyByteArray, Some(cfh.handle)) def stateHash(height: Int): Key[Option[StateHash]] = Key.opt(StateHash, h(height), readStateHash, writeStateHash) @@ -235,8 +237,8 @@ object Keys { def blockStateHash(height: Int): Key[ByteStr] = Key(BlockStateHash, h(height), Option(_).fold(TxStateSnapshotHashBuilder.InitStateHash)(ByteStr(_)), _.arr) - def ethereumTransactionMeta(height: Height, txNum: TxNum): Key[Option[EthereumTransactionMeta]] = - Key.opt(EthereumTransactionMetaTag, hNum(height, txNum), EthereumTransactionMeta.parseFrom, _.toByteArray) + def ethereumTransactionMeta(height: Height, txNum: TxNum, cfh: RDB.ApiHandle): Key[Option[EthereumTransactionMeta]] = + Key.opt(EthereumTransactionMetaTag, hNum(height, txNum), EthereumTransactionMeta.parseFrom, _.toByteArray, Some(cfh.handle)) def maliciousMinerBanHeights(addressBytes: Array[Byte]): Key[Seq[Int]] = historyKey(MaliciousMinerBanHeights, addressBytes) diff --git a/node/src/main/scala/com/wavesplatform/database/RDB.scala b/node/src/main/scala/com/wavesplatform/database/RDB.scala index 00a3e70077..7c9bb13a79 100644 --- a/node/src/main/scala/com/wavesplatform/database/RDB.scala +++ b/node/src/main/scala/com/wavesplatform/database/RDB.scala @@ -1,7 +1,7 @@ package com.wavesplatform.database import com.typesafe.scalalogging.StrictLogging -import com.wavesplatform.database.RDB.{TxHandle, TxMetaHandle} +import com.wavesplatform.database.RDB.{ApiHandle, TxHandle, TxMetaHandle, TxSnapshotHandle} import com.wavesplatform.settings.DBSettings import com.wavesplatform.utils.* import org.rocksdb.* @@ -16,7 +16,8 @@ final class RDB( val db: RocksDB, val txMetaHandle: TxMetaHandle, val txHandle: TxHandle, - val txSnapshotHandle: TxHandle, + val txSnapshotHandle: TxSnapshotHandle, + val apiHandle: ApiHandle, acquiredResources: Seq[RocksObject] ) extends AutoCloseable { override def close(): Unit = { @@ -28,6 +29,9 @@ final class RDB( object RDB extends StrictLogging { final class TxMetaHandle private[RDB] (val handle: ColumnFamilyHandle) final class TxHandle private[RDB] (val handle: ColumnFamilyHandle) + final class TxSnapshotHandle private[RDB] (val handle: ColumnFamilyHandle) + final class ApiHandle private[RDB] (val handle: ColumnFamilyHandle) + case class OptionsWithResources[A](options: A, resources: Seq[RocksObject]) def open(settings: DBSettings): RDB = { @@ -36,18 +40,15 @@ object RDB extends StrictLogging { logger.debug(s"Open DB at ${settings.directory}") val dbOptions = createDbOptions(settings) - - val dbDir = file.getAbsoluteFile + val dbDir = file.getAbsoluteFile dbDir.getParentFile.mkdirs() - val handles = new util.ArrayList[ColumnFamilyHandle]() - val defaultCfOptions = newColumnFamilyOptions(12.0, 16 << 10, settings.rocksdb.mainCacheSize, 0.6, settings.rocksdb.writeBufferSize) - val defaultCfCompressionForLevels = CompressionType.NO_COMPRESSION :: // Disable compaction for L0, because it is predictable and small - List.fill(defaultCfOptions.options.numLevels() - 1)(CompressionType.LZ4_COMPRESSION) - + val handles = new util.ArrayList[ColumnFamilyHandle]() + val defaultCfOptions = newColumnFamilyOptions(12.0, 16 << 10, settings.rocksdb.mainCacheSize, 0.6, settings.rocksdb.writeBufferSize) val txMetaCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txMetaCacheSize, 0.9, settings.rocksdb.writeBufferSize) val txCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txCacheSize, 0.9, settings.rocksdb.writeBufferSize) val txSnapshotCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.txSnapshotCacheSize, 0.9, settings.rocksdb.writeBufferSize) + val apiCfOptions = newColumnFamilyOptions(10.0, 2 << 10, settings.rocksdb.apiCacheSize, 0.9, settings.rocksdb.writeBufferSize) val db = RocksDB.open( dbOptions.options, settings.directory, @@ -56,7 +57,6 @@ object RDB extends StrictLogging { RocksDB.DEFAULT_COLUMN_FAMILY, defaultCfOptions.options .setMaxWriteBufferNumber(3) - .setCompressionPerLevel(defaultCfCompressionForLevels.asJava) .setCfPaths(Seq(new DbPath(new File(dbDir, "default").toPath, 0L)).asJava) ), new ColumnFamilyDescriptor( @@ -75,6 +75,11 @@ object RDB extends StrictLogging { "tx-snapshot".utf8Bytes, txSnapshotCfOptions.options .setCfPaths(Seq(new DbPath(new File(dbDir, "tx-snapshot").toPath, 0L)).asJava) + ), + new ColumnFamilyDescriptor( + "api".utf8Bytes, + apiCfOptions.options + .setCfPaths(Seq(new DbPath(new File(dbDir, "api").toPath, 0L)).asJava) ) ).asJava, handles @@ -84,7 +89,8 @@ object RDB extends StrictLogging { db, new TxMetaHandle(handles.get(1)), new TxHandle(handles.get(2)), - new TxHandle(handles.get(3)), + new TxSnapshotHandle(handles.get(3)), + new ApiHandle(handles.get(4)), dbOptions.resources ++ defaultCfOptions.resources ++ txMetaCfOptions.resources ++ txCfOptions.resources ++ txSnapshotCfOptions.resources ) } @@ -109,7 +115,6 @@ object RDB extends StrictLogging { .setPinL0FilterAndIndexBlocksInCache(true) .setFormatVersion(5) .setBlockSize(blockSize) - .setChecksumType(ChecksumType.kNoChecksum) .setBlockCache(blockCache) .setCacheIndexAndFilterBlocksWithHighPriority(true) .setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash) @@ -132,12 +137,13 @@ object RDB extends StrictLogging { private def createDbOptions(settings: DBSettings): OptionsWithResources[DBOptions] = { val dbOptions = new DBOptions() .setCreateIfMissing(true) - .setParanoidChecks(true) + .setParanoidChecks(settings.rocksdb.paranoidChecks) .setIncreaseParallelism(6) .setBytesPerSync(2 << 20) .setCreateMissingColumnFamilies(true) .setMaxOpenFiles(100) .setMaxSubcompactions(2) // Write stalls expected without this option. Can lead to max_background_jobs * max_subcompactions background threads + .setMaxManifestFileSize(200 << 20) if (settings.rocksdb.enableStatistics) { val statistics = new Statistics() diff --git a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala index aab88daa93..c2f8361fce 100644 --- a/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala +++ b/node/src/main/scala/com/wavesplatform/database/RocksDBWriter.scala @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory import sun.nio.ch.Util import java.nio.ByteBuffer +import java.time.Duration import java.util import java.util.concurrent.* import scala.annotation.tailrec @@ -115,14 +116,12 @@ object RocksDBWriter extends ScorexLogging { settings: BlockchainSettings, dbSettings: DBSettings, isLightMode: Boolean, - bfBlockInsertions: Int = 10000, forceCleanupExecutorService: Option[ExecutorService] = None ): RocksDBWriter = new RocksDBWriter( rdb, settings, dbSettings, isLightMode, - bfBlockInsertions, dbSettings.cleanupInterval match { case None => MoreExecutors.newDirectExecutorService() // We don't care if disabled case Some(_) => @@ -147,7 +146,6 @@ class RocksDBWriter( val settings: BlockchainSettings, val dbSettings: DBSettings, isLightMode: Boolean, - bfBlockInsertions: Int = 10000, cleanupExecutorService: ExecutorService ) extends Caches with AutoCloseable { @@ -215,20 +213,18 @@ class RocksDBWriter( writableDB.get(Keys.data(addressId, key)) } - override protected def loadEntryHeights(keys: Iterable[(Address, String)], addressIdOf: Address => AddressId): Map[(Address, String), Height] = { - val keyBufs = database.getKeyBuffersFromKeys(keys.map { case (addr, k) => Keys.data(addressIdOf(addr), k) }.toVector) - val valBufs = database.getValueBuffers(keys.size, 8) - val valueBuf = new Array[Byte](8) + override protected def loadEntryHeights(keys: Seq[(Address, String)], addressIdOf: Address => AddressId): Map[(Address, String), Height] = { + val keyBufs = database.getKeyBuffersFromKeys(keys.view.map { case (addr, k) => Keys.data(addressIdOf(addr), k) }.toVector) + val valBufs = database.getValueBuffers(keys.size, 4) val result = rdb.db .multiGetByteBuffers(keyBufs.asJava, valBufs.asJava) .asScala .view .zip(keys) - .map { case (status, k @ (_, key)) => + .map { case (status, k) => if (status.status.getCode == Status.Code.Ok) { - status.value.get(valueBuf) - k -> readCurrentData(key)(valueBuf).height + k -> Height(status.value.getInt) } else k -> Height(0) } .toMap @@ -399,11 +395,11 @@ class RocksDBWriter( } for ((addressId, nftIds) <- updatedNftLists.asMap().asScala) { - val kCount = Keys.nftCount(AddressId(addressId.toLong)) + val kCount = Keys.nftCount(AddressId(addressId.toLong), rdb.apiHandle) val previousNftCount = rw.get(kCount) rw.put(kCount, previousNftCount + nftIds.size()) for ((id, idx) <- nftIds.asScala.zipWithIndex) { - rw.put(Keys.nftAt(AddressId(addressId.toLong), previousNftCount + idx, id), Some(())) + rw.put(Keys.nftAt(AddressId(addressId.toLong), previousNftCount + idx, id, rdb.apiHandle), Some(())) } } @@ -430,32 +426,52 @@ class RocksDBWriter( } } - // todo: instead of fixed-size block batches, store fixed-time batches - private val BlockStep = 200 - private def mkFilter() = BloomFilter.create[Array[Byte]](Funnels.byteArrayFunnel(), BlockStep * bfBlockInsertions, 0.01f) - private def initFilters(): (BloomFilter[Array[Byte]], BloomFilter[Array[Byte]]) = { - def loadFilter(heights: Seq[Int]): BloomFilter[Array[Byte]] = { - val filter = mkFilter() - heights.filter(_ > 0).foreach { h => - loadTransactions(Height(h), rdb).foreach { case (_, tx) => filter.put(tx.id().arr) } + private var TxFilterResetTs = lastBlock.fold(0L)(_.header.timestamp) + private def mkFilter() = BloomFilter.create[Array[Byte]](Funnels.byteArrayFunnel(), 1_000_000, 0.001f) + private var currentTxFilter = mkFilter() + private var prevTxFilter = lastBlock match { + case Some(b) => + TxFilterResetTs = b.header.timestamp + val prevFilter = mkFilter() + + var fromHeight = height + Using(writableDB.newIterator()) { iter => + iter.seek(Keys.blockMetaAt(Height(height)).keyBytes) + var lastBlockTs = TxFilterResetTs + + while ( + iter.isValid && + iter.key().startsWith(KeyTags.BlockInfoAtHeight.prefixBytes) && + (TxFilterResetTs - lastBlockTs) < settings.functionalitySettings.maxTransactionTimeBackOffset.toMillis * 2 + ) { + lastBlockTs = readBlockMeta(iter.value()).getHeader.timestamp + fromHeight = Ints.fromByteArray(iter.key().drop(2)) + iter.prev() + } } - filter - } - val lastFilterStart = (height / BlockStep) * BlockStep + 1 - val prevFilterStart = lastFilterStart - BlockStep - val (bf0Heights, bf1Heights) = if ((height / BlockStep) % 2 == 0) { - (lastFilterStart to height, prevFilterStart until lastFilterStart) - } else { - (prevFilterStart until lastFilterStart, lastFilterStart to height) - } - (loadFilter(bf0Heights), loadFilter(bf1Heights)) - } + Using(writableDB.newIterator(rdb.txHandle.handle)) { iter => + var counter = 0 + iter.seek(Keys.transactionAt(Height(fromHeight), TxNum(0.toShort), rdb.txHandle).keyBytes) + while ( + iter.isValid && + iter.key().startsWith(KeyTags.NthTransactionInfoAtHeight.prefixBytes) && + Ints.fromByteArray(iter.key().slice(2, 6)) <= height + ) { + counter += 1 + prevFilter.put(readTransaction(Height(0))(iter.value())._2.id().arr) + iter.next() + } + log.debug(s"Loaded $counter tx IDs from [$fromHeight, $height]. Filter size is ${memMeter.measureDeep(prevFilter)} bytes") + } - private var (bf0, bf1) = initFilters() + prevFilter + case None => + mkFilter() + } override def containsTransaction(tx: Transaction): Boolean = - (bf0.mightContain(tx.id().arr) || bf1.mightContain(tx.id().arr)) && { + (prevTxFilter.mightContain(tx.id().arr) || currentTxFilter.mightContain(tx.id().arr)) && { writableDB.get(Keys.transactionMetaById(TransactionId(tx.id()), rdb.txMetaHandle)).isDefined } @@ -586,14 +602,13 @@ class RocksDBWriter( rw.put(Keys.assetScript(asset)(height), Some(script)) } - if (height % BlockStep == 1) { - if ((height / BlockStep) % 2 == 0) { - bf0 = mkFilter() - } else { - bf1 = mkFilter() - } + if (blockMeta.getHeader.timestamp - TxFilterResetTs > settings.functionalitySettings.maxTransactionTimeBackOffset.toMillis * 2) { + log.trace(s"Rotating filter at $height, prev ts = $TxFilterResetTs, new ts = ${blockMeta.getHeader.timestamp}, interval = ${Duration + .ofMillis(blockMeta.getHeader.timestamp - TxFilterResetTs)}") + TxFilterResetTs = blockMeta.getHeader.timestamp + prevTxFilter = currentTxFilter + currentTxFilter = mkFilter() } - val targetBf = if ((height / BlockStep) % 2 == 0) bf0 else bf1 val transactionsWithSize = snapshot.transactions.zipWithIndex.map { case ((id, txInfo), i) => @@ -608,14 +623,14 @@ class RocksDBWriter( Some(PBSnapshots.toProtobuf(txInfo.snapshot, txInfo.status)) ) rw.put(Keys.transactionMetaById(txId, rdb.txMetaHandle), Some(TransactionMeta(height, num, tx.tpe.id, meta.status.protobuf, 0, size))) - targetBf.put(id.arr) + currentTxFilter.put(id.arr) txId -> (num, tx, size) }.toMap if (dbSettings.storeTransactionsByAddress) { val addressTxs = addressTransactions.asScala.toSeq.map { case (aid, txIds) => - (aid, txIds, Keys.addressTransactionSeqNr(aid)) + (aid, txIds, Keys.addressTransactionSeqNr(aid, rdb.apiHandle)) } rw.multiGetInts(addressTxs.view.map(_._3).toVector) .zip(addressTxs) @@ -625,7 +640,7 @@ class RocksDBWriter( val (num, tx, size) = transactionsWithSize(txId) (tx.tpe.id.toByte, num, size) }.toSeq - rw.put(Keys.addressTransactionHN(addressId, nextSeqNr), Some((Height(height), txTypeNumSeq.sortBy(-_._2)))) + rw.put(Keys.addressTransactionHN(addressId, nextSeqNr, rdb.apiHandle), Some((Height(height), txTypeNumSeq.sortBy(-_._2)))) rw.put(txSeqNrKey, nextSeqNr) } } @@ -637,13 +652,15 @@ class RocksDBWriter( address <- Seq(details.recipientAddress, details.sender.toAddress) addressId = this.addressIdWithFallback(address, newAddresses) } yield (addressId, leaseId) - val leaseIdsByAddressId = addressIdWithLeaseIds.groupMap { case (addressId, _) => (addressId, Keys.addressLeaseSeqNr(addressId)) }(_._2).toSeq + val leaseIdsByAddressId = addressIdWithLeaseIds.groupMap { case (addressId, _) => + (addressId, Keys.addressLeaseSeqNr(addressId, rdb.apiHandle)) + }(_._2).toSeq rw.multiGetInts(leaseIdsByAddressId.view.map(_._1._2).toVector) .zip(leaseIdsByAddressId) .foreach { case (prevSeqNr, ((addressId, leaseSeqKey), leaseIds)) => val nextSeqNr = prevSeqNr.getOrElse(0) + 1 - rw.put(Keys.addressLeaseSeq(addressId, nextSeqNr), Some(leaseIds)) + rw.put(Keys.addressLeaseSeq(addressId, nextSeqNr, rdb.apiHandle), Some(leaseIds)) rw.put(leaseSeqKey, nextSeqNr) } } @@ -698,7 +715,7 @@ class RocksDBWriter( }) .getOrElse(throw new IllegalArgumentException(s"Couldn't find transaction height and num: $txId")) - try rw.put(Keys.invokeScriptResult(txHeight, txNum), Some(result)) + try rw.put(Keys.invokeScriptResult(txHeight, txNum, rdb.apiHandle), Some(result)) catch { case NonFatal(e) => throw new RuntimeException(s"Error storing invoke script result for $txId: $result", e) @@ -707,7 +724,7 @@ class RocksDBWriter( for ((txId, pbMeta) <- snapshot.ethereumTransactionMeta) { val txNum = transactionsWithSize(TransactionId @@ txId)._1 - val key = Keys.ethereumTransactionMeta(Height(height), txNum) + val key = Keys.ethereumTransactionMeta(Height(height), txNum, rdb.apiHandle) rw.put(key, Some(pbMeta)) } @@ -975,11 +992,9 @@ class RocksDBWriter( for ((addressId, address) <- changedAddresses) { for (k <- rw.get(Keys.changedDataKeys(currentHeight, addressId))) { - log.trace(s"Discarding $k for $address at $currentHeight") accountDataToInvalidate += (address -> k) - rw.delete(Keys.dataAt(addressId, k)(currentHeight)) - rollbackDataHistory(rw, Keys.data(addressId, k), Keys.dataAt(addressId, k)(_), currentHeight) + rollbackDataEntry(rw, k, address, addressId, currentHeight) } rw.delete(Keys.changedDataKeys(currentHeight, addressId)) @@ -993,9 +1008,9 @@ class RocksDBWriter( discardLeaseBalance(address) if (dbSettings.storeTransactionsByAddress) { - val kTxSeqNr = Keys.addressTransactionSeqNr(addressId) + val kTxSeqNr = Keys.addressTransactionSeqNr(addressId, rdb.apiHandle) val txSeqNr = rw.get(kTxSeqNr) - val kTxHNSeq = Keys.addressTransactionHN(addressId, txSeqNr) + val kTxHNSeq = Keys.addressTransactionHN(addressId, txSeqNr, rdb.apiHandle) rw.get(kTxHNSeq).collect { case (`currentHeight`, _) => rw.delete(kTxHNSeq) @@ -1004,9 +1019,9 @@ class RocksDBWriter( } if (dbSettings.storeLeaseStatesByAddress) { - val leaseSeqNrKey = Keys.addressLeaseSeqNr(addressId) + val leaseSeqNrKey = Keys.addressLeaseSeqNr(addressId, rdb.apiHandle) val leaseSeqNr = rw.get(leaseSeqNrKey) - val leaseSeqKey = Keys.addressLeaseSeq(addressId, leaseSeqNr) + val leaseSeqKey = Keys.addressLeaseSeq(addressId, leaseSeqNr, rdb.apiHandle) rw.get(leaseSeqKey) .flatMap(_.headOption) .flatMap(leaseDetails) @@ -1054,7 +1069,7 @@ class RocksDBWriter( case _: DataTransaction => // see changed data keys removal case _: InvokeScriptTransaction | _: InvokeExpressionTransaction => - rw.delete(Keys.invokeScriptResult(currentHeight, num)) + rw.delete(Keys.invokeScriptResult(currentHeight, num, rdb.apiHandle)) case tx: CreateAliasTransaction => rw.delete(Keys.addressIdOfAlias(tx.alias)) @@ -1063,7 +1078,7 @@ class RocksDBWriter( ordersToInvalidate += rollbackOrderFill(rw, tx.buyOrder.id(), currentHeight) ordersToInvalidate += rollbackOrderFill(rw, tx.sellOrder.id(), currentHeight) case _: EthereumTransaction => - rw.delete(Keys.ethereumTransactionMeta(currentHeight, num)) + rw.delete(Keys.ethereumTransactionMeta(currentHeight, num, rdb.apiHandle)) } if (tx.tpe != TransactionType.Genesis) { @@ -1132,14 +1147,20 @@ class RocksDBWriter( discardedBlocks.reverse } - private def rollbackDataHistory(rw: RW, currentDataKey: Key[CurrentData], dataNodeKey: Height => Key[DataNode], currentHeight: Height): Unit = { - val currentData = rw.get(currentDataKey) + private def rollbackDataEntry(rw: RW, key: String, address: Address, addressId: AddressId, currentHeight: Height): Unit = { + val currentDataKey = Keys.data(addressId, key) + val currentData = rw.get(currentDataKey) + rw.delete(Keys.dataAt(addressId, key)(currentHeight)) if (currentData.height == currentHeight) { - val prevDataNode = rw.get(dataNodeKey(currentData.prevHeight)) - rw.delete(dataNodeKey(currentHeight)) - prevDataNode.entry match { - case _: EmptyDataEntry => rw.delete(currentDataKey) - case _ => rw.put(currentDataKey, CurrentData(prevDataNode.entry, currentData.prevHeight, prevDataNode.prevHeight)) + if (currentData.prevHeight > 0) { + val prevDataNode = rw.get(Keys.dataAt(addressId, key)(currentData.prevHeight)) + log.trace( + s"PUT $address($addressId)/$key: ${currentData.entry}@$currentHeight => ${prevDataNode.entry}@${currentData.prevHeight}>${prevDataNode.prevHeight}" + ) + rw.put(currentDataKey, CurrentData(prevDataNode.entry, currentData.prevHeight, prevDataNode.prevHeight)) + } else { + log.trace(s"DEL $address($addressId)/$key: ${currentData.entry}@$currentHeight => EMPTY@${currentData.prevHeight}") + rw.delete(currentDataKey) } } } diff --git a/node/src/main/scala/com/wavesplatform/database/package.scala b/node/src/main/scala/com/wavesplatform/database/package.scala index b147da9fa5..b0fc6bc540 100644 --- a/node/src/main/scala/com/wavesplatform/database/package.scala +++ b/node/src/main/scala/com/wavesplatform/database/package.scala @@ -423,7 +423,7 @@ package object database { def withReadOptions[A](f: ReadOptions => A): A = { val snapshot = db.getSnapshot - val ro = new ReadOptions().setSnapshot(snapshot).setVerifyChecksums(false) + val ro = new ReadOptions().setSnapshot(snapshot) try f(ro) finally { ro.close() @@ -536,7 +536,11 @@ package object database { } finally iterator.close() } - def resourceObservable: Observable[DBResource] = Observable.resource(Task(DBResource(db)))(r => Task(r.close())) + def resourceObservable: Observable[DBResource] = + Observable.resource(Task(DBResource(db, None)))(r => Task(r.close())) + + def resourceObservable(iteratorCfHandle: ColumnFamilyHandle): Observable[DBResource] = + Observable.resource(Task(DBResource(db, Some(iteratorCfHandle))))(r => Task(r.close())) def withResource[A](f: DBResource => A): A = { val resource = DBResource(db) @@ -544,6 +548,12 @@ package object database { finally resource.close() } + def withResource[A](iteratorCfHandle: ColumnFamilyHandle)(f: DBResource => A): A = { + val resource = DBResource(db, Some(iteratorCfHandle)) + try f(resource) + finally resource.close() + } + private def multiGetOpt[A]( readOptions: ReadOptions, keys: collection.IndexedSeq[Key[Option[A]]], diff --git a/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala b/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala index 3c3ece9bf7..9fd841ebb9 100644 --- a/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala +++ b/node/src/main/scala/com/wavesplatform/history/StorageFactory.scala @@ -9,7 +9,7 @@ import com.wavesplatform.utils.{ScorexLogging, Time, UnsupportedFeature, forceSt import org.rocksdb.RocksDB object StorageFactory extends ScorexLogging { - private val StorageVersion = 1 + private val StorageVersion = 2 def apply( settings: WavesSettings, diff --git a/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala b/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala index 511bd6861d..906e085557 100644 --- a/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala +++ b/node/src/main/scala/com/wavesplatform/settings/DBSettings.scala @@ -1,5 +1,4 @@ package com.wavesplatform.settings -import scala.concurrent.duration.FiniteDuration case class DBSettings( directory: String, @@ -10,7 +9,5 @@ case class DBSettings( maxCacheSize: Int, maxRollbackDepth: Int, cleanupInterval: Option[Int] = None, - rememberBlocks: FiniteDuration, - useBloomFilter: Boolean, - rocksdb: RocksDBSettings + rocksdb: RocksDBSettings, ) diff --git a/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala b/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala index a31434526b..30ad8d172a 100644 --- a/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala +++ b/node/src/main/scala/com/wavesplatform/settings/RocksDBSettings.scala @@ -5,6 +5,8 @@ case class RocksDBSettings( txCacheSize: SizeInBytes, txMetaCacheSize: SizeInBytes, txSnapshotCacheSize: SizeInBytes, + apiCacheSize: SizeInBytes, writeBufferSize: SizeInBytes, - enableStatistics: Boolean + enableStatistics: Boolean, + paranoidChecks: Boolean ) diff --git a/node/src/test/resources/application.conf b/node/src/test/resources/application.conf index 1604353306..98841a94ec 100644 --- a/node/src/test/resources/application.conf +++ b/node/src/test/resources/application.conf @@ -4,8 +4,7 @@ waves { wallet.password = "some string as password" db { - cleanup-interval = null # Disable in tests by default - + max-cache-size = 1 rocksdb { main-cache-size = 1K tx-cache-size = 1K diff --git a/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala b/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala index 5bc42e0dfb..b8aab17471 100644 --- a/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala +++ b/node/src/test/scala/com/wavesplatform/database/TestStorageFactory.scala @@ -18,7 +18,6 @@ object TestStorageFactory { settings.blockchainSettings, settings.dbSettings, settings.enableLightMode, - 100, Some(MoreExecutors.newDirectExecutorService()) ) ( diff --git a/node/src/test/scala/com/wavesplatform/db/InterferableDB.scala b/node/src/test/scala/com/wavesplatform/db/InterferableDB.scala index 58bd629cef..5e07e7de21 100644 --- a/node/src/test/scala/com/wavesplatform/db/InterferableDB.scala +++ b/node/src/test/scala/com/wavesplatform/db/InterferableDB.scala @@ -1,10 +1,17 @@ package com.wavesplatform.db -import org.rocksdb.{ReadOptions, RocksDB, RocksIterator} +import org.rocksdb.{ColumnFamilyHandle, ReadOptions, RocksDB, RocksIterator} import java.util.concurrent.locks.Lock case class InterferableDB(db: RocksDB, startRead: Lock) extends RocksDB(db.getNativeHandle) { + override def getDefaultColumnFamily: ColumnFamilyHandle = db.getDefaultColumnFamily + + override def newIterator(columnFamilyHandle: ColumnFamilyHandle, readOptions: ReadOptions): RocksIterator = { + startRead.lock() + db.newIterator(columnFamilyHandle, readOptions) + } + override def newIterator(options: ReadOptions): RocksIterator = { startRead.lock() db.newIterator(options) diff --git a/node/src/test/scala/com/wavesplatform/db/TxBloomFilterSpec.scala b/node/src/test/scala/com/wavesplatform/db/TxBloomFilterSpec.scala new file mode 100644 index 0000000000..fd43e9b3e2 --- /dev/null +++ b/node/src/test/scala/com/wavesplatform/db/TxBloomFilterSpec.scala @@ -0,0 +1,34 @@ +package com.wavesplatform.db + +import com.wavesplatform.db.WithState.AddrWithBalance +import com.wavesplatform.settings.WavesSettings +import com.wavesplatform.test.* +import com.wavesplatform.transaction.TxHelpers + +class TxBloomFilterSpec extends PropSpec with SharedDomain { + private val richAccount = TxHelpers.signer(1200) + + override def settings: WavesSettings = DomainPresets.TransactionStateSnapshot + + override def genesisBalances: Seq[AddrWithBalance] = Seq(AddrWithBalance(richAccount.toAddress, 10000.waves)) + + property("Filter rotation works") { + val transfer = TxHelpers.transfer(richAccount, TxHelpers.address(1201), 10.waves) + 1 to 8 foreach { _ => domain.appendBlock() } + domain.blockchain.height shouldEqual 9 + domain.appendBlock(transfer) // transfer at height 10 + domain.appendBlock() // height = 11 + domain.appendBlock() // solid state height = 11, filters are rotated + domain.appendBlockE(transfer) should produce("AlreadyInTheState") + + domain.appendBlock() + val tf2 = TxHelpers.transfer(richAccount, TxHelpers.address(1202), 20.waves) + domain.appendBlock(tf2) + 1 to 20 foreach { _ => + withClue(s"height = ${domain.blockchain.height}") { + domain.appendBlockE(tf2) should produce("AlreadyInTheState") + } + domain.appendBlock() + } + } +} diff --git a/node/src/test/scala/com/wavesplatform/db/WithState.scala b/node/src/test/scala/com/wavesplatform/db/WithState.scala index 0544679434..cab873284e 100644 --- a/node/src/test/scala/com/wavesplatform/db/WithState.scala +++ b/node/src/test/scala/com/wavesplatform/db/WithState.scala @@ -72,7 +72,7 @@ trait WithState extends BeforeAndAfterAll with DBCacheSettings with Matchers wit ) Using.resource(rdw)(test) } finally { - Seq(rdb.db.getDefaultColumnFamily, rdb.txHandle.handle, rdb.txMetaHandle.handle).foreach { cfh => + Seq(rdb.db.getDefaultColumnFamily, rdb.txHandle.handle, rdb.txMetaHandle.handle, rdb.apiHandle.handle).foreach { cfh => rdb.db.deleteRange(cfh, MinKey, MaxKey) } } @@ -395,7 +395,7 @@ trait WithDomain extends WithState { _: Suite => try { val wrappedDb = wrapDB(rdb.db) assert(wrappedDb.getNativeHandle == rdb.db.getNativeHandle, "wrap function should not create new database instance") - domain = Domain(new RDB(wrappedDb, rdb.txMetaHandle, rdb.txHandle, rdb.txSnapshotHandle, Seq.empty), bcu, blockchain, settings) + domain = Domain(new RDB(wrappedDb, rdb.txMetaHandle, rdb.txHandle, rdb.txSnapshotHandle, rdb.apiHandle, Seq.empty), bcu, blockchain, settings) val genesis = balances.map { case AddrWithBalance(address, amount) => TxHelpers.genesis(address, amount) } diff --git a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala index 0b9ff018b2..bee4b3eb7f 100644 --- a/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala +++ b/node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterNFTTest.scala @@ -89,7 +89,7 @@ class BlockchainUpdaterNFTTest extends PropSpec with DomainScenarioDrivenPropert val persistedNfts = Seq.newBuilder[IssuedAsset] d.rdb.db.readOnly { ro => val addressId = ro.get(Keys.addressId(firstAccount)).get - ro.iterateOver(KeyTags.NftPossession.prefixBytes ++ addressId.toByteArray) { e => + ro.iterateOver(KeyTags.NftPossession.prefixBytes ++ addressId.toByteArray, Some(d.rdb.apiHandle.handle)) { e => persistedNfts += IssuedAsset(ByteStr(e.getKey.takeRight(32))) } } @@ -99,7 +99,6 @@ class BlockchainUpdaterNFTTest extends PropSpec with DomainScenarioDrivenPropert val settings = settingsWithFeatures(BlockchainFeatures.NG, BlockchainFeatures.ReduceNFTFee) withDomain(settings)(assert) - withDomain(settings.copy(dbSettings = settings.dbSettings.copy(useBloomFilter = true)))(assert) } } diff --git a/node/src/test/scala/com/wavesplatform/history/Domain.scala b/node/src/test/scala/com/wavesplatform/history/Domain.scala index 58def9c43c..ff6b315594 100644 --- a/node/src/test/scala/com/wavesplatform/history/Domain.scala +++ b/node/src/test/scala/com/wavesplatform/history/Domain.scala @@ -197,7 +197,7 @@ case class Domain(rdb: RDB, blockchainUpdater: BlockchainUpdaterImpl, rocksDBWri def balance(address: Address): Long = blockchainUpdater.balance(address) def balance(address: Address, asset: Asset): Long = blockchainUpdater.balance(address, asset) - def nftList(address: Address): Seq[(IssuedAsset, AssetDescription)] = rdb.db.withResource { resource => + def nftList(address: Address): Seq[(IssuedAsset, AssetDescription)] = rdb.db.withResource(rdb.apiHandle.handle) { resource => AddressPortfolio .nftIterator(resource, address, blockchainUpdater.bestLiquidSnapshot.orEmpty, None, blockchainUpdater.assetDescription) .toSeq diff --git a/node/src/test/scala/com/wavesplatform/state/DataKeyRollback.scala b/node/src/test/scala/com/wavesplatform/state/DataKeyRollback.scala new file mode 100644 index 0000000000..074940ba3a --- /dev/null +++ b/node/src/test/scala/com/wavesplatform/state/DataKeyRollback.scala @@ -0,0 +1,60 @@ +package com.wavesplatform.state + +import com.wavesplatform.db.WithState +import com.wavesplatform.db.WithState.AddrWithBalance +import com.wavesplatform.lang.directives.values.V7 +import com.wavesplatform.lang.v1.compiler.TestCompiler +import com.wavesplatform.settings.WavesSettings +import com.wavesplatform.test.* +import com.wavesplatform.transaction.TxHelpers + +class DataKeyRollback extends PropSpec with SharedDomain { + private val richAccount = TxHelpers.signer(1500) + + override def genesisBalances: Seq[WithState.AddrWithBalance] = Seq(AddrWithBalance(richAccount.toAddress, 10_000_000.waves)) + override def settings: WavesSettings = DomainPresets.TransactionStateSnapshot + + property("check new entries") { + val oracleAccount = TxHelpers.signer(1501) + val dappAccount = TxHelpers.signer(1502) + + val dataSenderCount = 5 + val dataEntryCount = 5 + + val dataSenders = IndexedSeq.tabulate(dataSenderCount)(i => TxHelpers.signer(1550 + i)) + domain.appendBlock( + TxHelpers + .massTransfer( + richAccount, + dataSenders.map(kp => kp.toAddress -> 100.waves) ++ + Seq(oracleAccount.toAddress -> 100.waves, dappAccount.toAddress -> 10.waves), + fee = 0.05.waves + ), + TxHelpers.setScript( + dappAccount, + TestCompiler(V7).compileContract(s""" + let oracleAddress = Address(base58'${oracleAccount.toAddress}') + @Callable(i) + func default() = [ + IntegerEntry("loadedHeight_" + height.toString() + i.transactionId.toBase58String(), oracleAddress.getIntegerValue("lastUpdatedBlock")) + ] + """) + ), + TxHelpers.data(oracleAccount, Seq(IntegerDataEntry("lastUpdatedBlock", 2))) + ) + domain.appendBlock(dataSenders.map(kp => TxHelpers.data(kp, Seq.tabulate(dataEntryCount)(i => IntegerDataEntry("kv_" + i, 501)), 0.01.waves))*) + domain.appendBlock(dataSenders.map(kp => TxHelpers.data(kp, Seq.tabulate(dataEntryCount)(i => IntegerDataEntry("kv_" + i, 503)), 0.01.waves))*) + domain.appendBlock( + (dataSenders.map(kp => TxHelpers.data(kp, Seq.tabulate(dataEntryCount)(i => IntegerDataEntry("kv_" + i, 504)), 0.01.waves)) ++ + Seq( + TxHelpers.invoke(dappAccount.toAddress, invoker = richAccount), + TxHelpers.data(oracleAccount, Seq(IntegerDataEntry("lastUpdatedBlock", 5))) + ))* + ) + domain.appendBlock() + val discardedBlocks = domain.rollbackTo(domain.blockchain.blockId(domain.blockchain.height - 2).get) + discardedBlocks.foreach { case (block, _, _) => + domain.appendBlock(block) + } + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 8cce403945..e169b40121 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -32,19 +32,19 @@ object Dependencies { val janino = "org.codehaus.janino" % "janino" % "3.1.11" val asyncHttpClient = "org.asynchttpclient" % "async-http-client" % "2.12.3" val curve25519 = "com.wavesplatform" % "curve25519-java" % "0.6.6" - val nettyHandler = "io.netty" % "netty-handler" % "4.1.104.Final" + val nettyHandler = "io.netty" % "netty-handler" % "4.1.106.Final" val shapeless = Def.setting("com.chuusai" %%% "shapeless" % "2.3.10") - val playJson = "com.typesafe.play" %% "play-json" % "2.10.3" // 2.10.x and later is built for Java 11 + val playJson = "com.typesafe.play" %% "play-json" % "2.10.4" val scalaTest = "org.scalatest" %% "scalatest" % "3.2.17" % Test val scalaJsTest = Def.setting("com.lihaoyi" %%% "utest" % "0.8.2" % Test) - val sttp3 = "com.softwaremill.sttp.client3" % "core_2.13" % "3.9.1" // 3.6.x and later is built for Java 11 - val sttp3Monix = "com.softwaremill.sttp.client3" %% "monix" % "3.9.1" + val sttp3 = "com.softwaremill.sttp.client3" % "core_2.13" % "3.9.2" + val sttp3Monix = "com.softwaremill.sttp.client3" %% "monix" % "3.9.2" - val bouncyCastleProvider = "org.bouncycastle" % s"bcprov-jdk15on" % "1.70" + val bouncyCastleProvider = "org.bouncycastle" % s"bcprov-jdk18on" % "1.77" val console = Seq("com.github.scopt" %% "scopt" % "4.1.0") @@ -69,7 +69,7 @@ object Dependencies { curve25519, bouncyCastleProvider, "com.wavesplatform" % "zwaves" % "0.2.1", - web3jModule("crypto") + web3jModule("crypto").excludeAll(ExclusionRule("org.bouncycastle", "bcprov-jdk15on")), ) ++ langCompilerPlugins.value ++ scalapbRuntime.value ++ protobuf.value ) @@ -100,7 +100,7 @@ object Dependencies { akkaModule("slf4j") % Runtime ) - private val rocksdb = "org.rocksdb" % "rocksdbjni" % "8.9.1" + private val rocksdb = "org.rocksdb" % "rocksdbjni" % "8.10.0" lazy val node = Def.setting( Seq( @@ -127,10 +127,10 @@ object Dependencies { monixModule("reactive").value, nettyHandler, "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", - "eu.timepit" %% "refined" % "0.11.0" exclude ("org.scala-lang.modules", "scala-xml_2.13"), + "eu.timepit" %% "refined" % "0.11.1" exclude ("org.scala-lang.modules", "scala-xml_2.13"), "com.esaulpaugh" % "headlong" % "10.0.2", "com.github.jbellis" % "jamm" % "0.4.0", // Weighing caches - web3jModule("abi"), + web3jModule("abi").excludeAll(ExclusionRule("org.bouncycastle", "bcprov-jdk15on")), akkaModule("testkit") % Test, akkaHttpModule("akka-http-testkit") % Test ) ++ test ++ console ++ logDeps ++ protobuf.value ++ langCompilerPlugins.value @@ -161,8 +161,6 @@ object Dependencies { lazy val rideRunner = Def.setting( Seq( rocksdb, - // https://github.com/netty/netty/wiki/Native-transports - // "io.netty" % "netty-transport-native-epoll" % "4.1.79.Final" classifier "linux-x86_64", "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8", "net.logstash.logback" % "logstash-logback-encoder" % "7.4" % Runtime, kamonModule("caffeine"), diff --git a/project/plugins.sbt b/project/plugins.sbt index 7cb18557cc..9c052045cf 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,7 +6,7 @@ resolvers ++= Seq( // Should go before Scala.js addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.6") -libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.14" +libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.15" Seq( "com.eed3si9n" % "sbt-assembly" % "2.1.5", diff --git a/repl/jvm/src/test/logback-test.xml b/repl/jvm/src/test/logback-test.xml deleted file mode 100644 index 8b4e22b2f2..0000000000 --- a/repl/jvm/src/test/logback-test.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - %date %-5level [%.15thread] %logger{26} - %msg%n - - - - - - - - - diff --git a/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/KeyTags.scala b/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/KeyTags.scala index 919ddd1bd6..fdd474f5b5 100644 --- a/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/KeyTags.scala +++ b/ride-runner/src/main/scala/com/wavesplatform/database/rocksdb/KeyTags.scala @@ -53,7 +53,6 @@ object KeyTags extends Enumeration { AssetStaticInfo, NftCount, NftPossession, - BloomFilterChecksum, IssuedAssets, UpdatedAssets, SponsoredAssets, From e318113784a2f0993c77fca31acd3bce1fe03bcf Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Mon, 5 Feb 2024 14:44:03 +0400 Subject: [PATCH 7/8] Version 1.5.3 (Mainnet + Testnet + Stagenet) (#3939) --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index 01efb98af8..73ee74eb8c 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -git.baseVersion := "1.5.2" +git.baseVersion := "1.5.3" From 979d2ea2b68959a5fe0c5d2b4901f3bce3818ae2 Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Mon, 11 Mar 2024 15:09:31 +0200 Subject: [PATCH 8/8] Sign transactions with SK in Util app (#3941) --- .../scala/com/wavesplatform/Application.scala | 2 +- .../api/http/requests/BurnRequest.scala | 2 +- .../http/requests/CreateAliasRequest.scala | 2 +- .../api/http/requests/ExchangeRequest.scala | 2 +- .../api/http/requests/IssueRequest.scala | 2 +- .../http/requests/LeaseCancelRequest.scala | 2 +- .../api/http/requests/LeaseRequest.scala | 2 +- .../api/http/requests/ReissueRequest.scala | 2 +- .../api/http/requests/TransferRequest.scala | 2 +- .../http/requests/TxBroadcastRequest.scala | 6 +- .../requests/UpdateAssetInfoRequest.scala | 2 +- .../com/wavesplatform/utils/UtilApp.scala | 56 ++++++++++++++++--- 12 files changed, 61 insertions(+), 21 deletions(-) diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index 5d65421c70..01d727aaa9 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -637,7 +637,7 @@ object Application extends ScorexLogging { case "explore" => Explorer.main(args.tail) case "util" => UtilApp.main(args.tail) case "help" | "--help" | "-h" => println("Usage: waves | export | import | explore | util") - case _ => startNode(args.headOption) // TODO: Consider adding option to specify network-name + case _ => startNode(args.headOption) } } diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/BurnRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/BurnRequest.scala index 0788e9d611..fb848af997 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/BurnRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/BurnRequest.scala @@ -19,7 +19,7 @@ case class BurnRequest( timestamp: Option[Long], signature: Option[ByteStr], proofs: Option[Proofs] -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[BurnTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, BurnTransaction] = for { validProofs <- toProofs(signature, proofs) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/CreateAliasRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/CreateAliasRequest.scala index a765e5b9bd..35dbb74940 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/CreateAliasRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/CreateAliasRequest.scala @@ -15,7 +15,7 @@ case class CreateAliasRequest( timestamp: Option[TxTimestamp] = None, signature: Option[ByteStr] = None, proofs: Option[Proofs] = None -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[CreateAliasTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, CreateAliasTransaction] = for { validProofs <- toProofs(signature, proofs) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/ExchangeRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/ExchangeRequest.scala index f7ae53c758..470bdfe489 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/ExchangeRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/ExchangeRequest.scala @@ -21,7 +21,7 @@ case class ExchangeRequest( timestamp: Option[TxTimestamp] = None, signature: Option[ByteStr] = None, proofs: Option[Proofs] = None -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[ExchangeTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, ExchangeTransaction] = for { validProofs <- toProofs(signature, proofs) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/IssueRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/IssueRequest.scala index 8f9109ad81..b30e73e8c4 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/IssueRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/IssueRequest.scala @@ -22,7 +22,7 @@ case class IssueRequest( timestamp: Option[Long], signature: Option[ByteStr], proofs: Option[Proofs] -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[IssueTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, IssueTransaction] = { val actualVersion = version.getOrElse(TxVersion.V3) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseCancelRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseCancelRequest.scala index ba12293439..7d843d2492 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseCancelRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseCancelRequest.scala @@ -17,7 +17,7 @@ case class LeaseCancelRequest( timestamp: Option[Long], signature: Option[ByteStr], proofs: Option[Proofs] -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[LeaseCancelTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, LeaseCancelTransaction] = for { validProofs <- toProofs(signature, proofs) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseRequest.scala index 2dffac1f3b..ab78fcc201 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/LeaseRequest.scala @@ -17,7 +17,7 @@ case class LeaseRequest( timestamp: Option[Long], signature: Option[ByteStr], proofs: Option[Proofs] -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[LeaseTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, LeaseTransaction] = for { validRecipient <- AddressOrAlias.fromString(recipient) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/ReissueRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/ReissueRequest.scala index b952fc9792..7869a0b747 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/ReissueRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/ReissueRequest.scala @@ -19,7 +19,7 @@ case class ReissueRequest( timestamp: Option[Long], signature: Option[ByteStr], proofs: Option[Proofs] -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[ReissueTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, ReissueTransaction] = for { validProofs <- toProofs(signature, proofs) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/TransferRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/TransferRequest.scala index a9d659b2ee..8e71299358 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/TransferRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/TransferRequest.scala @@ -20,7 +20,7 @@ case class TransferRequest( timestamp: Option[Long] = None, signature: Option[ByteStr] = None, proofs: Option[Proofs] = None -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[TransferTransaction] { def toTxFrom(sender: PublicKey): Either[ValidationError, TransferTransaction] = for { validRecipient <- AddressOrAlias.fromString(recipient) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/TxBroadcastRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/TxBroadcastRequest.scala index be8beb9d0a..13bcbe1e52 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/TxBroadcastRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/TxBroadcastRequest.scala @@ -5,13 +5,13 @@ import com.wavesplatform.lang.ValidationError import com.wavesplatform.transaction.Transaction import com.wavesplatform.transaction.TxValidationError.GenericError -trait TxBroadcastRequest { +trait TxBroadcastRequest[A <: Transaction] { def sender: Option[String] def senderPublicKey: Option[String] - def toTxFrom(sender: PublicKey): Either[ValidationError, Transaction] + def toTxFrom(sender: PublicKey): Either[ValidationError, A] - def toTx: Either[ValidationError, Transaction] = + def toTx: Either[ValidationError, A] = for { sender <- senderPublicKey match { case Some(key) => PublicKey.fromBase58String(key) diff --git a/node/src/main/scala/com/wavesplatform/api/http/requests/UpdateAssetInfoRequest.scala b/node/src/main/scala/com/wavesplatform/api/http/requests/UpdateAssetInfoRequest.scala index 1d26cf4163..59e9774f0c 100644 --- a/node/src/main/scala/com/wavesplatform/api/http/requests/UpdateAssetInfoRequest.scala +++ b/node/src/main/scala/com/wavesplatform/api/http/requests/UpdateAssetInfoRequest.scala @@ -21,7 +21,7 @@ case class UpdateAssetInfoRequest( fee: Long, feeAssetId: Option[String], proofs: Option[Proofs] -) extends TxBroadcastRequest { +) extends TxBroadcastRequest[UpdateAssetInfoTransaction] { override def toTxFrom(sender: PublicKey): Either[ValidationError, UpdateAssetInfoTransaction] = for { _assetId <- parseBase58(assetId, "invalid.assetId", AssetIdStringLength) diff --git a/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala b/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala index a243eea845..c46f87e406 100644 --- a/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala +++ b/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala @@ -1,23 +1,25 @@ package com.wavesplatform.utils -import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream} -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Paths} - import com.google.common.io.ByteStreams import com.wavesplatform.account.{KeyPair, PrivateKey, PublicKey} +import com.wavesplatform.api.http.requests.* import com.wavesplatform.common.state.ByteStr import com.wavesplatform.common.utils.{Base58, Base64, FastBase58} import com.wavesplatform.features.EstimatorProvider.* import com.wavesplatform.lang.script.{Script, ScriptReader} import com.wavesplatform.settings.WavesSettings -import com.wavesplatform.transaction.TransactionFactory +import com.wavesplatform.transaction.TxValidationError.GenericError import com.wavesplatform.transaction.smart.script.ScriptCompiler +import com.wavesplatform.transaction.{Transaction, TransactionFactory, TransactionType} import com.wavesplatform.wallet.Wallet import com.wavesplatform.{Application, Version} import play.api.libs.json.{JsObject, Json} import scopt.OParser +import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream} +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths} + //noinspection ScalaStyle // TODO: Consider remove implemented methods from REST API object UtilApp { @@ -31,6 +33,7 @@ object UtilApp { case object Hash extends Mode case object SerializeTx extends Mode case object SignTx extends Mode + case object SignTxWithSk extends Mode } case class CompileOptions(assetScript: Boolean = false) @@ -63,18 +66,19 @@ object UtilApp { def main(args: Array[String]): Unit = { OParser.parse(commandParser, args, Command()) match { case Some(cmd) => - lazy val nodeState = new NodeState(cmd) + val settings = Application.loadApplicationConfig(cmd.configFile.map(new File(_))) val inBytes = IO.readInput(cmd) val result = { val doAction = cmd.mode match { - case Command.CompileScript => Actions.doCompile(nodeState.settings) _ + case Command.CompileScript => Actions.doCompile(settings) _ case Command.DecompileScript => Actions.doDecompile _ case Command.SignBytes => Actions.doSign _ case Command.VerifySignature => Actions.doVerify _ case Command.CreateKeyPair => Actions.doCreateKeyPair _ case Command.Hash => Actions.doHash _ case Command.SerializeTx => Actions.doSerializeTx _ - case Command.SignTx => Actions.doSignTx(nodeState) _ + case Command.SignTx => Actions.doSignTx(new NodeState(cmd)) _ + case Command.SignTxWithSk => Actions.doSignTxWithSK _ } doAction(cmd, inBytes) } @@ -191,6 +195,15 @@ object UtilApp { .abbr("sa") .text("Signer address (requires corresponding key in wallet.dat)") .action((a, c) => c.copy(signTxOptions = c.signTxOptions.copy(signerAddress = a))) + ), + cmd("sign-with-sk") + .text("Sign JSON transaction with private key") + .action((_, c) => c.copy(mode = Command.SignTxWithSk)) + .children( + opt[String]("private-key") + .abbr("sk") + .text("Private key") + .action((a, c) => c.copy(signOptions = c.signOptions.copy(privateKey = PrivateKey(Base58.decode(a))))) ) ), help("help").hidden(), @@ -265,6 +278,33 @@ object UtilApp { .left .map(_.toString) .map(tx => Json.toBytes(tx.json())) + + def doSignTxWithSK(c: Command, data: Array[Byte]): ActionResult = { + import cats.syntax.either.* + import com.wavesplatform.api.http.requests.InvokeScriptRequest.signedInvokeScriptRequestReads + import com.wavesplatform.api.http.requests.SponsorFeeRequest.signedSponsorRequestFormat + import com.wavesplatform.transaction.TransactionType.* + + val json = Json.parse(data) + (TransactionType((json \ "type").as[Int]) match { + case Issue => json.as[IssueRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case Transfer => json.as[TransferRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case Reissue => json.as[ReissueRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case Burn => json.as[BurnRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case Exchange => json.as[ExchangeRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case Lease => json.as[LeaseRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case LeaseCancel => json.as[LeaseCancelRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case CreateAlias => json.as[CreateAliasRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case MassTransfer => json.as[SignedMassTransferRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case Data => json.as[SignedDataRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case SetScript => json.as[SignedSetScriptRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case SponsorFee => json.as[SignedSponsorFeeRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case SetAssetScript => json.as[SignedSetAssetScriptRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case InvokeScript => json.as[SignedInvokeScriptRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case UpdateAssetInfo => json.as[SignedUpdateAssetInfoRequest].toTx.map(_.signWith(c.signOptions.privateKey)) + case other => GenericError(s"Signing $other is not supported").asLeft[Transaction] + }).leftMap(_.toString).map(_.json().toString().getBytes()) + } } private[this] object IO {